summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKyle Gunger <kgunger12@gmail.com>2023-12-05 02:03:20 -0500
committerKyle Gunger <kgunger12@gmail.com>2023-12-05 02:03:20 -0500
commit59fefb35d825c0fa9abd837d99c51c462c6c97de (patch)
tree0d7563c80ccb145de4d0dbbe2f55a9dccb1bab8f
parentf33024799722433c55e03aad49bace89beaf99ca (diff)
tokenization flag
-rw-r--r--compiler.c66
1 files changed, 65 insertions, 1 deletions
diff --git a/compiler.c b/compiler.c
index f0a1b30..1d99f69 100644
--- a/compiler.c
+++ b/compiler.c
@@ -840,7 +840,7 @@ void parse_reserved_tokens(int *next, Vector *out, int *line, int *col, FILE *fi
while (*next != EOF) {
add = *next;
- if (!is_reserved(add)) {
+ if (!is_reserved(add) || add == '"' || add == '\'') {
break;
}
@@ -2113,6 +2113,58 @@ void compile(Artifact *path_in, Artifact *path_out) {
cdat_end(&out);
}
+char *tok_type_strs[] = {
+ "DEFWORD 0",
+ "KEYWORD 1",
+ "KEYTYPE 2",
+ "LITERAL 3",
+ "AUGMENT 4",
+ "DELIMIT",
+ "SPLITTR"
+};
+
+void write_token(FILE *out, Token *t) {
+ fprintf(out, "{line: %d, column: %d, type %s, data: \"%s\"}\n", t->line, t->col, tok_type_strs[t->type], t->data);
+}
+
+void tokenize(Artifact *path_in, Artifact *path_out) {
+ char *full_path = art_to_str(path_in, '/');
+ FILE *fin = fopen(full_path, "r");
+
+ if (fin == NULL) {
+ printf("Unable to open file %s for reading.\n\n", full_path);
+ free(full_path);
+ return;
+ }
+
+ free(full_path);
+ full_path = art_to_str(path_out, '/');
+ FILE *fout = fopen(full_path, "w");
+
+ if (fout == NULL) {
+ printf("Unable to open file %s for writing.\n\n", full_path);
+ free(full_path);
+ fclose(fin);
+ return;
+ }
+
+ free(full_path);
+
+ Vector tokens = parse_file(fin);
+ fclose(fin);
+
+ for(size_t i = 0; i < tokens.count; i++) {
+ Token *t = vect_get(&tokens, i);
+ write_token(fout, t);
+ fflush(fout);
+ free(t->data);
+ }
+
+ fclose(fout);
+
+ vect_end(&tokens);
+}
+
// Entrypoint
void help() {
@@ -2133,6 +2185,18 @@ int main(int argc, char ** argv) {
} else if (strcmp(argv[1], "-v") == 0) {
printf("C based TNSL Compiler (CTC) - version 0.1\n");
return 0;
+ } else if (strcmp(argv[1], "-t") == 0) {
+ Artifact in = art_from_str(argv[2], '/');
+ Artifact out;
+ if (argc == 3) {
+ out = art_from_str("out.asm", '/');
+ } else {
+ out = art_from_str(argv[3], '/');
+ }
+ tokenize(&in, &out);
+ art_end(&in);
+ art_end(&out);
+ return 0;
}
Artifact in = art_from_str(argv[1], '/');