summaryrefslogtreecommitdiff
path: root/tnslc/parse/token.tnsl
diff options
context:
space:
mode:
authorKyle Gunger <kgunger12@gmail.com>2023-02-16 05:14:31 -0500
committerKyle Gunger <kgunger12@gmail.com>2023-02-16 05:14:31 -0500
commit09df1bb48823631228af3df647111af1eebc854b (patch)
treeef091fc942b00b224f7d66aacb0c3db0ab4d490c /tnslc/parse/token.tnsl
parentc94edbe007910755087e71cbb1a6a349d75e9b85 (diff)
Collapse tnslc into a single file
Diffstat (limited to 'tnslc/parse/token.tnsl')
-rw-r--r--tnslc/parse/token.tnsl331
1 files changed, 0 insertions, 331 deletions
diff --git a/tnslc/parse/token.tnsl b/tnslc/parse/token.tnsl
deleted file mode 100644
index fecc7f7..0000000
--- a/tnslc/parse/token.tnsl
+++ /dev/null
@@ -1,331 +0,0 @@
-/#
- Copyright 2021-2022 Kyle Gunger
-
- This file is licensed under the CDDL 1.0 (the License)
- and may only be used in accordance with the License.
- You should have received a copy of the License with this
- software/source code. If you did not, a copy can be found
- at the following URL:
-
- https://opensource.org/licenses/CDDL-1.0
-
- THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
- WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
- EXPRESS OR IMPLIED
-#/
-
-/#
- Reserved words and characters, as well as
- helper funcs for checking their token types.
-#/
-
-;{}{}uint8 PREWORDS = {
- "include",
- "define",
- "extern",
- "size",
- "align",
- "address",
- "rootfile",
- "if",
- "else",
- "abi"
-}
-
-;{}{}uint8 KEYTYPES = {
- "int8",
- "int16",
- "int32",
- "int64",
- "int",
-
- "uint8",
- "uint16",
- "uint32",
- "uint64",
- "uint",
-
- "float32",
- "float64",
- "float",
-
- "bool",
- "void",
- "type"
-}
-
-;{}{}uint8 KEYWORDS = {
- "struct",
- "interface",
- "enum",
- "extends",
-
- "loop",
- "continue",
- "break",
-
- "match",
- "case",
- "default",
-
- "label",
- "goto",
-
- "if",
- "else",
-
- "const",
- "static",
- "volatile",
-
- "method",
- "override",
- "operator",
-
- "raw",
- "asm",
- "inline",
- "virtual",
-
- "delete",
- "alloc",
- "salloc",
- "realloc",
-
- "module",
- "export"
-}
-
-;{}{}uint8 LITERALS = {
- "true",
- "false",
-
- "null",
-
- "self",
- "super"
-}
-
-;{}uint8 RESERVED = "`~!#%^&*()-=+[]{}|;:,.<>/"
-
-;{}uint8 DELIMITS = "()[]{}"
-;{}uint8 LINESEPS = ";:#"
-;{}uint8 INLNSEPS = ","
-;{}uint8 AUGMENTS = "~`.&|^><!+-*/%"
-
-;{}{}uint8 MDELIMITS = {
- # Code block
- "/;",
- ";/",
-
- # Comment block
- "/#",
- "#/",
-
- # Preproc block
- "/:",
- ":/",
-
- # Redef blocks
- ";;",
- "::",
- ";#",
- ":#",
- "#;",
- "#:"
-}
-
-;{}{}uint8 MAUGMENTS = {
- # Boolean
- "==",
- "&&",
- "||",
-
- # Bitwise shifts
- "<<",
- ">>",
-
- # PREaugmented augmentors
- "&=",
- "|=",
- "^=",
- "+=",
- "-=",
- "*=",
- "/=",
- "%=",
- "~=",
- "`=",
-
- # POSTaugmented augmentors
- "!&",
- "!|",
- "!^",
- "!==",
- "!&&",
- "!||",
- "!>",
- "!<",
- ">==",
- "<==",
-
- # Increment and De-increment
- "++",
- "--",
-
- "is",
- "len",
- "size"
-}
-
-;int MAX_MRESERVED = 3
-
-/##
- Checks if the character point p is in the string cmp
-
-#; is_in_string (~{}uint8 cmp, uint8 p) [bool]
-
- /; loop (int i = 0; i < len cmp`) [i++]
- /; if (p == cmp`{i})
- ;return true
- ;/
- ;/
-
- ;return false
-;/
-
-/##
- Checks if the string s is in the list cmp
-
-#; is_in_string_list (~{}{}uint8 cmp, ~{}uint8 s) [bool]
-
- /; loop (int i = 0; i < len cmp`) [i++]
-
- /; if (len s` == len cmp`{i})
-
- /; loop (int j = 0; j < len s`) [j++]
-
- /; if (s`{j} !== cmp`{i}{j})
- ;break 1
- ;/
- ;/
-
- ;return true
- ;/
-
- ;/
-
- ;return false
-;/
-
-/; is_numeric_literal(~{}uint8 dat) [bool]
- /; if (len dat` == 0)
- ;return false
- ;/
-
- ;bool dec = true, flt = false
-
- ;int i = 0
-
- /; if (len dat` > 1)
- /; if (dat`{0} == '0' && !is_digit(dat`{1}) && dat`{1} !== '.')
- ;dec = false
- ;i = 2
- ;/
- ;/
-
- /; loop (i < len dat`) [i++]
- /; if (!is_digit(dat`{i}) && dec)
- /; if (dat`{i} == '.')
- /; if (flt)
- ;return false
- ;/
- ;flt = true
- ;; else if (dec)
- ;return false
- ;/
- ;/
- ;/
-
- ;return true
-;/
-
-/; is_text_literal(~{}uint8 dat) [bool]
- /; if (len dat` < 1)
- ;return false
- ;/
- ;return dat`{0} == '"' || dat`{0} == '\''
-;/
-
-/; string_closed ({}uint8 dat, uint8 c) [bool]
- /; if (len dat < 2)
- ;return false
- ;/
-
- ;uint8 closing = dat{0}
- ;bool escaping = false
-
- /; loop (int i = 1; i < len dat) [i++]
- /; if (dat{i} == closing && !escaping)
- ;return true
- ;; else if (dat{i} == '\\' && !escaping)
- ;escaping = true
- ;; else
- ;escaping = false
- ;/
- ;/
-
- ;return false
-;/
-
-/#
- Get the token_type value for a given string of character points
-
-#; get_token_type (~{}uint8 s) [int]
-
- /; if (len s` > 1)
-
- /; if (is_numeric_literal(s) || s`{0} == '"' || s`{0} == '\'')
- ;return TOKEN_TYPE.LITERAL
- ;/
-
- /; if (is_in_string_list(~PREWORDS, s))
- ;return TOKEN_TYPE.PREWORD
- ;; else if (is_in_string_list(~KEYTYPES, s))
- ;return TOKEN_TYPE.KEYTYPE
- ;; else if (is_in_string_list(~KEYWORDS, s))
- ;return TOKEN_TYPE.KEYWORD
- ;; else if (is_in_string_list(~LITERALS, s))
- ;return TOKEN_TYPE.LITERAL
- ;; else if (is_in_string_list(~MDELIMITS, s))
- ;return TOKEN_TYPE.DELIMIT
- ;; else if (is_in_string_list(~MAUGMENTS, s))
- ;return TOKEN_TYPE.AUGMENT
- ;/
-
- ;return TOKEN_TYPE.DEFWORD
-
- ;;else if (len s` == 1)
-
- /; if (is_digit(s`{0}))
- ;return TOKEN_TYPE.LITERAL
- ;/
-
- /; if (is_in_string(~DELIMITS, s`{0}))
- ;return TOKEN_TYPE.DELIMIT
- ;; else if (is_in_string(~LINESEPS, s`{0}))
- ;return TOKEN_TYPE.LINESEP
- ;; else if (is_in_string(~INLNSEPS, s`{0}))
- ;return TOKEN_TYPE.INLNSEP
- ;; else if (is_in_string(~AUGMENTS, s`{0}))
- ;return TOKEN_TYPE.AUGMENT
- ;/
-
- ;return TOKEN_TYPE.DEFWORD
- ;/
-
- # What, we just produce vacant tokens now?
- # Something has gone wrong.
-
- ;return -1
-;/ \ No newline at end of file