diff options
Diffstat (limited to 'tnslc/src/parse')
-rw-r--r-- | tnslc/src/parse/parse.tnsl | 22 | ||||
-rw-r--r-- | tnslc/src/parse/token.tnsl | 295 | ||||
-rw-r--r-- | tnslc/src/parse/tokenizer.tnsl | 38 |
3 files changed, 0 insertions, 355 deletions
diff --git a/tnslc/src/parse/parse.tnsl b/tnslc/src/parse/parse.tnsl deleted file mode 100644 index e10ab4c..0000000 --- a/tnslc/src/parse/parse.tnsl +++ /dev/null @@ -1,22 +0,0 @@ -/## - Copyright 2021 Kyle Gunger - - This file is licensed under the CDDL 1.0 (the License) - and may only be used in accordance with the License. - You should have received a copy of the License with this - software/source code. If you did not, a copy can be found - at the following URL: - - https://opensource.org/licenses/CDDL-1.0 - - THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO - WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE - EXPRESS OR IMPLIED -#/ - -/; export module parse - /:import - "token.tnsl" - "tokenizer.tnsl" - :/ -;/
\ No newline at end of file diff --git a/tnslc/src/parse/token.tnsl b/tnslc/src/parse/token.tnsl deleted file mode 100644 index 21abe7b..0000000 --- a/tnslc/src/parse/token.tnsl +++ /dev/null @@ -1,295 +0,0 @@ -/# - Copyright 2021 Kyle Gunger - - This file is licensed under the CDDL 1.0 (the License) - and may only be used in accordance with the License. - You should have received a copy of the License with this - software/source code. If you did not, a copy can be found - at the following URL: - - https://opensource.org/licenses/CDDL-1.0 - - THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO - WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE - EXPRESS OR IMPLIED -#/ - -/# The various types of tokens #/ -; enum TOKEN_TYPE [uint] { - LINESEP = 0, - INLNSEP = 1, - DELIMIT = 2, - AUGMENT = 3, - LITERAL = 4, - KEYTYPE = 5, - PREWORD = 6, - KEYWORD = 7, - DEFWORD = 8 -} - -/# Token struct definition #/ -;raw struct Token { - uint - token_type, - line, - col, - - ~{}charp - data -} - -/; method Token - - /; operator delete - ;delete self.data - ;/ - - /; add_char (~{}charp part) - ;uint l = len self.data` - ;realloc self.data, l + len part - /;loop (uint i = 0) [i++; i < len part] - ;self.data`{l + i} = part{i} - ;/ - ;/ -;/ - -/# - Reserved words and characters, as well as - helper funcs for checking their token types. -#/ - -;const {}{}charp PREWORDS = { - "include", - "define", - "extern", - "size", - "align", - "address", - "rootfile", - "if", - "else", - "abi" -} - -;const {}{}charp KEYTYPES = { - "bool", - "char", - "charp", - - "int8", - "int16", - "int32", - "int64", - "int", - "uint8", - "uint16", - "uint32", - "uint64", - "uint", - - "float32", - "float64", - "float", - - "void", - "type" -} - -;const {}{}charp KEYWORDS = { - "struct", - "interface", - "enum", - "is", - "extends", - - "loop", - "continue", - "break", - - "match", - "case", - "default", - - "label", - "goto", - - "if", - "else", - - "const", - "static", - "volatile", - - "method", - "override", - "self", - "super", - "operator", - - "raw", - "asm", - "inline", - - "delete", - - "module", - "export", -} - -;const {}{}charp LITERALS = { - "true", - "false" -} - -;const {}charp DELIMITS = "()[]{}" -;const {}charp LINESEPS = ";:#" -;const {}charp INLNSEPS = "," -;const {}charp AUGMENTS = "~`.&|^><!+-*/%" - -;const {}{}charp MDELIMITS = { - # Code block - "/;", - ";/", - - # Comment block - "/#", - "#/", - - # Preproc block - "/:", - ":/", - - # Redef blocks - ";;", - "::", - ";#", - ":#", - "#;", - "#:" -} - -;const {}{}charp MAUGMENTS = { - # Boolean - "==", - "&&", - "||", - - # Bitwise shifts - "<<", - ">>", - - # PREaugmented augmentors - "&=", - "|=", - "^=", - "+=", - "-=", - "*=", - "/=", - "%=", - "~=", - "`=", - - # POSTaugmented augmentors - "!&", - "!|", - "!^", - "!==", - "!&&", - "!||", - "!>", - "!<", - ">==", - "<==", - - # Increment and De-increment - "++", - "--" -} - -; const uint MAX_MRESERVED = 3 - -/## - Checks if the character point p is in the string cmp - -#; is_in_string (const {}charp` cmp, charp p) [bool] - - /; loop (int i = 0; i < len cmp) [i++] - - /; if (s == cmp{i}) - ;return true - ;/ - ;/ - - ;return false -;/ - - -/## - Checks if the string s is in the list cmp - -#; is_in_string_list (const {}{}charp` cmp, {}charp` s) [bool] - - /; loop (int i = 0; i < len cmp) [i++] - - /; if (len s == len cmp{i}) - - /; loop (int j = 0; j < len s) [j++] - - /; if (s{j} !== cmp{i}{j}) - ;goto cont_outer - ;/ - ;/ - - ;return true - ;/ - - ;label cont_outer - ;/ - - ;return false -;/ - -/# - Get the token_type value for a given string of character points - -#; get_token_type ({}charp` s) [int] - - /; if (len s > 1) - - /; if (is_in_string_list(~PREWORDS, ~s)) - ;return TOKEN_TYPE.PREWORD - ;; else if (is_in_string_list(~KEYTYPES, ~s)) - ;return TOKEN_TYPE.KEYTYPE - ;; else if (is_in_string_list(~KEYWORDS, ~s)) - ;return TOKEN_TYPE.KEYWORD - ;; else if (is_in_string_list(~LITERALS, ~s)) - ;return TOKEN_TYPE.LITERAL - ;; else if (is_in_string_list(~MDELIMITS, ~s)) - ;return TOKEN_TYPE.DELIMIT - ;; else if (is_in_string_list(~MAUGMENTS, ~s)) - ;return TOKEN_TYPE.AUGMENT - ;/ - - ;return TOKEN_TYPE.DEFWORD - - ;;else if (len s == 1) - - /; if (is_in_string(~DELIMITS, s{0})) - ;return TOKEN_TYPE.DELIMIT - ;; else if (is_in_string(~LINESEPS, s{0})) - ;return TOKEN_TYPE.LINESEP - ;; else if (is_in_string(~INLNSEPS, s{0})) - ;return TOKEN_TYPE.INLNSEP - ;; else if (is_in_string(~AUGMENTS, s{0})) - ;return TOKEN_TYPE.AUGMENT - ;/ - - ;return TOKEN_TYPE.DEFWORD - ;/ - - # What, we just produce vacant tokens now? - # Something has gone wrong. - - ;return -1 -;/
\ No newline at end of file diff --git a/tnslc/src/parse/tokenizer.tnsl b/tnslc/src/parse/tokenizer.tnsl deleted file mode 100644 index 4e5eeb7..0000000 --- a/tnslc/src/parse/tokenizer.tnsl +++ /dev/null @@ -1,38 +0,0 @@ -/# - Copyright 2021 Kyle Gunger - - This file is licensed under the CDDL 1.0 (the License) - and may only be used in accordance with the License. - You should have received a copy of the License with this - software/source code. If you did not, a copy can be found - at the following URL: - - https://opensource.org/licenses/CDDL-1.0 - - THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO - WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE - EXPRESS OR IMPLIED -#/ - -:using 'tnsl' - -/## - parse.numeric_literal tokenizes the next numeric literal value in a file. - Returns a token with the proper data as well as the number of characters read - -#; numeric_literal (io.text_stream fstream) [Token, uint] - ;Token out = {token_type: TOKEN_TYPE.LITERAL} - ;uint counter = 0 - - ;bool l, d, run = false, false, true - ;~{}charp num - - ;num, run = fstream.read_number() - - /; loop (run) [num, run = fstream.read_number()] - ;out.add_data(num) - ;delete num - ;/ - - ;return out, counter -;/
\ No newline at end of file |