summaryrefslogtreecommitdiff
path: root/tnslc/src/parse
diff options
context:
space:
mode:
Diffstat (limited to 'tnslc/src/parse')
-rw-r--r--tnslc/src/parse/parse.tnsl22
-rw-r--r--tnslc/src/parse/token.tnsl295
-rw-r--r--tnslc/src/parse/tokenizer.tnsl38
3 files changed, 0 insertions, 355 deletions
diff --git a/tnslc/src/parse/parse.tnsl b/tnslc/src/parse/parse.tnsl
deleted file mode 100644
index e10ab4c..0000000
--- a/tnslc/src/parse/parse.tnsl
+++ /dev/null
@@ -1,22 +0,0 @@
-/##
- Copyright 2021 Kyle Gunger
-
- This file is licensed under the CDDL 1.0 (the License)
- and may only be used in accordance with the License.
- You should have received a copy of the License with this
- software/source code. If you did not, a copy can be found
- at the following URL:
-
- https://opensource.org/licenses/CDDL-1.0
-
- THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
- WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
- EXPRESS OR IMPLIED
-#/
-
-/; export module parse
- /:import
- "token.tnsl"
- "tokenizer.tnsl"
- :/
-;/ \ No newline at end of file
diff --git a/tnslc/src/parse/token.tnsl b/tnslc/src/parse/token.tnsl
deleted file mode 100644
index 21abe7b..0000000
--- a/tnslc/src/parse/token.tnsl
+++ /dev/null
@@ -1,295 +0,0 @@
-/#
- Copyright 2021 Kyle Gunger
-
- This file is licensed under the CDDL 1.0 (the License)
- and may only be used in accordance with the License.
- You should have received a copy of the License with this
- software/source code. If you did not, a copy can be found
- at the following URL:
-
- https://opensource.org/licenses/CDDL-1.0
-
- THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
- WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
- EXPRESS OR IMPLIED
-#/
-
-/# The various types of tokens #/
-; enum TOKEN_TYPE [uint] {
- LINESEP = 0,
- INLNSEP = 1,
- DELIMIT = 2,
- AUGMENT = 3,
- LITERAL = 4,
- KEYTYPE = 5,
- PREWORD = 6,
- KEYWORD = 7,
- DEFWORD = 8
-}
-
-/# Token struct definition #/
-;raw struct Token {
- uint
- token_type,
- line,
- col,
-
- ~{}charp
- data
-}
-
-/; method Token
-
- /; operator delete
- ;delete self.data
- ;/
-
- /; add_char (~{}charp part)
- ;uint l = len self.data`
- ;realloc self.data, l + len part
- /;loop (uint i = 0) [i++; i < len part]
- ;self.data`{l + i} = part{i}
- ;/
- ;/
-;/
-
-/#
- Reserved words and characters, as well as
- helper funcs for checking their token types.
-#/
-
-;const {}{}charp PREWORDS = {
- "include",
- "define",
- "extern",
- "size",
- "align",
- "address",
- "rootfile",
- "if",
- "else",
- "abi"
-}
-
-;const {}{}charp KEYTYPES = {
- "bool",
- "char",
- "charp",
-
- "int8",
- "int16",
- "int32",
- "int64",
- "int",
- "uint8",
- "uint16",
- "uint32",
- "uint64",
- "uint",
-
- "float32",
- "float64",
- "float",
-
- "void",
- "type"
-}
-
-;const {}{}charp KEYWORDS = {
- "struct",
- "interface",
- "enum",
- "is",
- "extends",
-
- "loop",
- "continue",
- "break",
-
- "match",
- "case",
- "default",
-
- "label",
- "goto",
-
- "if",
- "else",
-
- "const",
- "static",
- "volatile",
-
- "method",
- "override",
- "self",
- "super",
- "operator",
-
- "raw",
- "asm",
- "inline",
-
- "delete",
-
- "module",
- "export",
-}
-
-;const {}{}charp LITERALS = {
- "true",
- "false"
-}
-
-;const {}charp DELIMITS = "()[]{}"
-;const {}charp LINESEPS = ";:#"
-;const {}charp INLNSEPS = ","
-;const {}charp AUGMENTS = "~`.&|^><!+-*/%"
-
-;const {}{}charp MDELIMITS = {
- # Code block
- "/;",
- ";/",
-
- # Comment block
- "/#",
- "#/",
-
- # Preproc block
- "/:",
- ":/",
-
- # Redef blocks
- ";;",
- "::",
- ";#",
- ":#",
- "#;",
- "#:"
-}
-
-;const {}{}charp MAUGMENTS = {
- # Boolean
- "==",
- "&&",
- "||",
-
- # Bitwise shifts
- "<<",
- ">>",
-
- # PREaugmented augmentors
- "&=",
- "|=",
- "^=",
- "+=",
- "-=",
- "*=",
- "/=",
- "%=",
- "~=",
- "`=",
-
- # POSTaugmented augmentors
- "!&",
- "!|",
- "!^",
- "!==",
- "!&&",
- "!||",
- "!>",
- "!<",
- ">==",
- "<==",
-
- # Increment and De-increment
- "++",
- "--"
-}
-
-; const uint MAX_MRESERVED = 3
-
-/##
- Checks if the character point p is in the string cmp
-
-#; is_in_string (const {}charp` cmp, charp p) [bool]
-
- /; loop (int i = 0; i < len cmp) [i++]
-
- /; if (s == cmp{i})
- ;return true
- ;/
- ;/
-
- ;return false
-;/
-
-
-/##
- Checks if the string s is in the list cmp
-
-#; is_in_string_list (const {}{}charp` cmp, {}charp` s) [bool]
-
- /; loop (int i = 0; i < len cmp) [i++]
-
- /; if (len s == len cmp{i})
-
- /; loop (int j = 0; j < len s) [j++]
-
- /; if (s{j} !== cmp{i}{j})
- ;goto cont_outer
- ;/
- ;/
-
- ;return true
- ;/
-
- ;label cont_outer
- ;/
-
- ;return false
-;/
-
-/#
- Get the token_type value for a given string of character points
-
-#; get_token_type ({}charp` s) [int]
-
- /; if (len s > 1)
-
- /; if (is_in_string_list(~PREWORDS, ~s))
- ;return TOKEN_TYPE.PREWORD
- ;; else if (is_in_string_list(~KEYTYPES, ~s))
- ;return TOKEN_TYPE.KEYTYPE
- ;; else if (is_in_string_list(~KEYWORDS, ~s))
- ;return TOKEN_TYPE.KEYWORD
- ;; else if (is_in_string_list(~LITERALS, ~s))
- ;return TOKEN_TYPE.LITERAL
- ;; else if (is_in_string_list(~MDELIMITS, ~s))
- ;return TOKEN_TYPE.DELIMIT
- ;; else if (is_in_string_list(~MAUGMENTS, ~s))
- ;return TOKEN_TYPE.AUGMENT
- ;/
-
- ;return TOKEN_TYPE.DEFWORD
-
- ;;else if (len s == 1)
-
- /; if (is_in_string(~DELIMITS, s{0}))
- ;return TOKEN_TYPE.DELIMIT
- ;; else if (is_in_string(~LINESEPS, s{0}))
- ;return TOKEN_TYPE.LINESEP
- ;; else if (is_in_string(~INLNSEPS, s{0}))
- ;return TOKEN_TYPE.INLNSEP
- ;; else if (is_in_string(~AUGMENTS, s{0}))
- ;return TOKEN_TYPE.AUGMENT
- ;/
-
- ;return TOKEN_TYPE.DEFWORD
- ;/
-
- # What, we just produce vacant tokens now?
- # Something has gone wrong.
-
- ;return -1
-;/ \ No newline at end of file
diff --git a/tnslc/src/parse/tokenizer.tnsl b/tnslc/src/parse/tokenizer.tnsl
deleted file mode 100644
index 4e5eeb7..0000000
--- a/tnslc/src/parse/tokenizer.tnsl
+++ /dev/null
@@ -1,38 +0,0 @@
-/#
- Copyright 2021 Kyle Gunger
-
- This file is licensed under the CDDL 1.0 (the License)
- and may only be used in accordance with the License.
- You should have received a copy of the License with this
- software/source code. If you did not, a copy can be found
- at the following URL:
-
- https://opensource.org/licenses/CDDL-1.0
-
- THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
- WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
- EXPRESS OR IMPLIED
-#/
-
-:using 'tnsl'
-
-/##
- parse.numeric_literal tokenizes the next numeric literal value in a file.
- Returns a token with the proper data as well as the number of characters read
-
-#; numeric_literal (io.text_stream fstream) [Token, uint]
- ;Token out = {token_type: TOKEN_TYPE.LITERAL}
- ;uint counter = 0
-
- ;bool l, d, run = false, false, true
- ;~{}charp num
-
- ;num, run = fstream.read_number()
-
- /; loop (run) [num, run = fstream.read_number()]
- ;out.add_data(num)
- ;delete num
- ;/
-
- ;return out, counter
-;/ \ No newline at end of file