diff options
| -rw-r--r-- | tnslc/dummy.tnsl | 2 | ||||
| -rw-r--r-- | tnslc/parse/token.tnsl | 36 | ||||
| -rw-r--r-- | tnslc/parse/tokenizer.tnsl | 17 | ||||
| -rw-r--r-- | tnslc/tnslc.tnsl | 2 | ||||
| -rw-r--r-- | tnslc/util.tnsl | 46 |
5 files changed, 94 insertions, 9 deletions
diff --git a/tnslc/dummy.tnsl b/tnslc/dummy.tnsl index da9003a..bb09483 100644 --- a/tnslc/dummy.tnsl +++ b/tnslc/dummy.tnsl @@ -1,3 +1,3 @@ /; main [float] - ;return .342 + ;return a.b ;/ diff --git a/tnslc/parse/token.tnsl b/tnslc/parse/token.tnsl index 668312b..6c402fd 100644 --- a/tnslc/parse/token.tnsl +++ b/tnslc/parse/token.tnsl @@ -250,6 +250,38 @@ ;return false ;/ +/; is_numeric_literal(~{}charp dat) [bool] + /; if (len dat` == 0) + ;return false + ;/ + + ;bool dec = true, flt = false + + ;int i = 0 + + /; if (len dat` > 1) + /; if (dat`{0} == '0' && !is_digit(dat`{1}) && dat`{1} !== '.') + ;dec = false + ;i = 2 + ;/ + ;/ + + /; loop (i < len dat`) [i++] + /; if (!is_digit(dat`{i}) && dec) + /; if (dat`{i} == '.') + /; if (flt || !dec) + ;return false + ;/ + ;flt = true + ;; else if (dec) + ;return false + ;/ + ;/ + ;/ + + ;return true +;/ + /# Get the token_type value for a given string of character points @@ -257,6 +289,10 @@ /; if (len s` > 1) + /; if (is_numeric_literal(s)) + ;return TOKEN_TYPE.LITERAL + ;/ + /; if (is_in_string_list(~PREWORDS, s)) ;return TOKEN_TYPE.PREWORD ;; else if (is_in_string_list(~KEYTYPES, s)) diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl index 0b80a15..b57c766 100644 --- a/tnslc/parse/tokenizer.tnsl +++ b/tnslc/parse/tokenizer.tnsl @@ -14,12 +14,8 @@ EXPRESS OR IMPLIED #/ -/; is_space (charp c) [bool] - ;return c == '\t' || c == '\n' || c == ' ' -;/ - -/; is_digit (charp c) [bool] - ;return c !< '0' && c !> '9' +/; is_float (~{}charp dat) [bool] + ;return is_numeric_literal(dat) && is_in_string(dat, '.') ;/ /; break_token ({}charp dat, charp c) [bool] @@ -34,8 +30,13 @@ ;return !is_digit(c) ;/ ;return true + ;; else if (is_in_string(~RESERVED, c)) + /; if (is_numeric_literal(~dat) && !is_float(~dat) && c == '.') + ;return false + ;/ + ;return true ;/ - ;return is_space(c) + ;return is_whitespace(c) ;/ /; parse_reserved ({}charp dat) [{}Token] @@ -55,7 +56,7 @@ ;tdat = {} ;/ ;/ - /; if ( !is_space(i) ) + /; if ( !is_whitespace(i) ) ;tdat.append(i) ;/ ;/ diff --git a/tnslc/tnslc.tnsl b/tnslc/tnslc.tnsl index b1c4b0a..90a1020 100644 --- a/tnslc/tnslc.tnsl +++ b/tnslc/tnslc.tnsl @@ -14,6 +14,8 @@ EXPRESS OR IMPLIED #/ +:include "util.tnsl" + /; module tnslc :include "parse/parse.tnsl" ;/ diff --git a/tnslc/util.tnsl b/tnslc/util.tnsl new file mode 100644 index 0000000..a29d544 --- /dev/null +++ b/tnslc/util.tnsl @@ -0,0 +1,46 @@ +/## + Copyright 2021 Kyle Gunger + + This file is licensed under the CDDL 1.0 (the License) + and may only be used in accordance with the License. + You should have received a copy of the License with this + software/source code. If you did not, a copy can be found + at the following URL: + + https://opensource.org/licenses/CDDL-1.0 + + THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO + WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE + EXPRESS OR IMPLIED +#/ + +/# util.tnsl + Utility functions that may be useful in many places. +#/ + +/; string_equate(~{}charp s1, s2) [bool] + /; if (len s1` !== len s2`) + ;return false + ;/ + + /; loop (int i = 0; i < len s1`) [i++] + /; if (s1`{i} !== s2`{i}) + ;return false + ;/ + ;/ + + ;return true +;/ + +/; is_whitespace (charp c) [bool] + ;return c == '\t' || c == '\n' || c == ' ' +;/ + +/; is_digit (charp c) [bool] + ;return c !< '0' && c !> '9' +;/ + +/; is_alpha (charp c) [bool] + ;bool low = c !< 'A' && c !> 'Z', high = c !< 'a' && c >! 'z' + ;return low || high +;/ |