path: root/tnslc/parse/tokenizer.tnsl

uint TTYPE_DELIM = 0
uint TTYPE_SEP   = 1
uint TTYPE_KEYWD = 2
uint TTYPE_KEYTP = 3
uint TTYPE_LITRL = 4
uint TTYPE_AUG   = 5
uint TTYPE_USRWD = 6
uint TTYPE_COMNT = 7

uint TTYPE_UNKNOWN = 998
uint TTYPE_ERR   = 999

struct Token {
	uint _type,
	~uint8 data,
	uint
		line,
		col
}

/; method Token
	/; eq (~uint8 str) [bool]
		return utils.strcmp(self.data, str)
	;/

	/; end
		_delete(self.data)
	;/
;/

/; _in_csv (~uint8 csv, ~uint8 str) [bool]
	int along = 0

	/; loop (csv` !== 0) [csv++]
		/; if (csv` == ',')
			/; if (along !< 0 && str{along} == 0)
				return true
			;/
			along = 0
		;; else if (along !< 0 && str{along} == csv`)
			along++
		;; else
			along = 0
			along--
		;/
	;/

	return along !< 0 && str{along} == 0
;/

/; _str_contains (~uint8 str, uint8 ch) [bool]
	/; loop (str` !== 0) [str++]
		/; if (str` == ch)
			return true
		;/
	;/
	return false
;/


~uint8 KEYWORDS   = "import,using,module,export,struct,method,implements,interface,operator,enum,if,else,loop,continue,break,return,label,goto,asm\0"
~uint8 KEYTYPES   = "uint8,uint16,uint32,uint64,uint,int8,int16,int32,int64,int,float32,float64,float,bool,void,vect,type\0"
~uint8 LITERALS   = "false,true,null\0"

~uint8 RESERVED   = "~`!@#$%^&*()[]{}-+=\"\'\\|;:/?.>,<\0"

~uint8 OP         = "`~!%^&|*-=+./><\0"
~uint8 MULTI_OP   = "==,&&,||,^^,!==,!&&,!||,!^^,!<,!>,<<,>>,!&,!|,!^,++,--,>==,<==\0"
uint   MAX_MULTI  = 3
~uint8 MULTI_OP_W = "is,len\0"

~uint8 DELIMS     = "()[]{}\0"


/; produce_word_token (~utils.File fin, Token prev) [Token]
	Token out
	out.line = prev.line
	out.col = prev.col
	out._type = TTYPE_USRWD

	utils.Vector tmp
	tmp.init(1)

	uint8 ch = fin`.read()

	/; loop (ch !== 0 && is_reserved(ch) == false && is_whitespace(ch) == false)
		tmp.push(~ch)
		ch = fin`.read()
	;/

	/; if (ch !== 0)
		fin`.unread()
	;/

	out.data = tmp.as_cstr()
	/; if (_in_csv(KEYWORDS, out.data) == true)
		out._type = TTYPE_KEYWD
	;; else if (_in_csv(KEYTYPES, out.data) == true)
		out._type = TTYPE_KEYTP
	;; else if (_in_csv(LITERALS, out.data) == true)
		out._type = TTYPE_LITRL
	;; else if (_in_csv(MULTI_OP_W) == true)
		out._type = TTYPE_AUG
	;/

	return out
;/

/; produce_string_token (~utils.File fin, Token prev) [Token]
	Token out
	out.line = prev.line
	out.col = prev.col
	out._type = TTYPE_LITRL

	utils.Vector tmp
	tmp.init(1)

	uint8 delim = fin`.read()
	tmp.push(~delim)

	/; loop (fin`.at_end == false && delim !== 0)
		uint8 ch = fin`.read()
		/; if (ch == '\\')
			tmp.push(~ch)
			ch = fin`.read()
		;; else if (ch == '\n')
			out.line++
		;; else if (ch == delim)
			delim = 0
		;/
		
		/; if (ch !== 0)
			tmp.push(~ch)
		;/
	;/

	out.data = tmp.as_cstr()
	return out
;/

/; comment_line (~utils.File fin)
	uint8 ch = fin`.read()

	/; loop (fin`.at_end == false && ch !== '\n')
		ch = fin`.read()
	;/

	/; if (fin`.at_end == false)
		fin`.unread()
	;/
;/

/; comment_block (~utils.File fin, ~Token out)
	uint8 ch = 1
	/; loop (fin`.at_end == false && ch !== 0)
		ch = fin`.read()
		/; if (ch == '#')
			ch = fin`.read()
			/; if (ch == '/')
				ch = 0
			;; else
				comment_line(fin)
			;/
		;/

		/; if (ch == '\n')
			out`.line++
		;/
	;/
;/

/; is_comment_block (~uint8 str) [bool]
	return utils.strcmp(str, "/#\0")
;/

/; is_multi_delim(~uint8 str) [bool]
	/; if (utils.strcmp(str, "/;\0") == true)
		return true
	;; else if (utils.strcmp(str, ";;\0") == true)
		return true
	;; else if (utils.strcmp(str, ";/\0") == true)
		return true
	;/
	return false
;/

/; produce_reserved_token (~utils.File fin, Token prev) [Token]
	Token out
	out.line = prev.line
	out.col = prev.col
	out._type = TTYPE_USRWD

	utils.Vector tmp
	tmp.init(1)

	uint8 ch = fin`.read()

	/; if (ch == '#')
		tmp.push(~ch)
		out._type = TTYPE_COMNT
		out.data = tmp.as_cstr()
		comment_line(fin)
		return out
	;/

	tmp.push(~ch)
	/; loop (int i = 1; i < MAX_MULTI) [i++]
		ch = fin`.read()
		/; if (is_reserved(ch) == false)
			i = MAX_MULTI
			fin`.unread()
		;; else
			tmp.push(~ch)
		;/
	;/

	/; loop (bool run = true; run == true)
		~uint8 str = tmp.as_cstr()
		/; if (tmp.count == 1)
			/; if (str` == ',' || str` == ';')
				out._type = TTYPE_SEP
			;; else if (_str_contains(OP, str`))
				out._type = TTYPE_AUG
			;; else if (_str_contains(DELIMS, str`))
				out._type = TTYPE_DELIM
			;; else
				out._type = TTYPE_UNKNOWN
			;/
			run = false
		;; else if (_in_csv(MULTI_OP, str) == true)
			out._type = TTYPE_AUG
			run = false
		;; else if (is_comment_block(str) == true)
			out._type = TTYPE_COMNT
			comment_block(fin, ~out)
			run = false
		;; else if (is_multi_delim(str) == true)
			out._type = TTYPE_DELIM
			run = false
		;; else
			tmp.pop()
			fin`.unread()
		;/
	;/

	out.data = tmp.as_cstr()
	return out
;/

/; produce_numeric_token (~utils.File fin, Token prev) [Token]
	Token out
	out.line = prev.line
	out.col = prev.col
	out._type = TTYPE_LITRL

	utils.Vector tmp
	tmp.init(1)
	
	uint8 ch = fin`.read()
	tmp.push(~ch)
	bool base = false
	/; if (ch == '0')
		ch = fin`.read()
		/; if (ch == 0)
			out.data = tmp.as_cstr()
			return out
		;; else if (is_reserved(ch) == false && is_whitespace(ch) == false && is_numeric(ch) == false)
			base = true
			tmp.push(~ch)
		;/
	;/

	bool decimal = false
	/; loop (bool run = true; run == true && fin`.at_end == false)
		ch = fin`.read()
		/; if (ch == 0)
			run = false
		;; else if (decimal == false && ch == '.')
			decimal = true
			tmp.push(~ch)
		;; else if (is_reserved(ch) == true || is_whitespace(ch) == true)
			fin`.unread()
			run = false
		;; else if (is_numeric(ch) == false && base == false)
			fin`.unread()
			run = false
		;; else if (ch !== 0)
			tmp.push(~ch)
		;/
	;/

	out.data = tmp.as_cstr()
	return out
;/

/; is_whitespace (uint8 ch) [bool]
	/; if (ch > 8 && ch < 14)
		return true
	;; else if (ch == ' ')
		return true
	;/
	return false
;/

/; is_reserved (uint8 ch) [bool]
	return _str_contains(RESERVED, ch) == true
;/

/; is_numeric (uint8 ch) [bool]
	/; if (ch !< '0' && ch !> '9')
		return true
	;/
	return false
;/

/; produce_next_token (~utils.File fin, Token prev) [Token]
	/; if (prev._type !== TTYPE_ERR)
		prev.col = prev.col + utils.strlen(prev.data)
	;/

	uint8 first = fin`.read()
	/; loop (is_whitespace(first) == true)
		/; if (first == '\n')
			prev.line++
			prev.col = 0
		;/
		first = fin`.read()
		prev.col++
	;/
	fin`.unread()
	
	/; if (first == '\'' || first == '\"')
		return produce_string_token(fin, prev)
	;; else if (is_reserved(first) == true)
		Token out = produce_reserved_token(fin, prev)
		/; loop (out._type == TTYPE_COMNT)
			Token tmp = produce_next_token(fin, out)
			out.end()
			out = tmp
		;/
		return out
	;; else if (is_numeric(first) == true)
		return produce_numeric_token(fin, prev)
	;; else if (first !== 0)
		return produce_word_token(fin, prev)
	;/

	Token out
	out.line = prev.line
	out.col = prev.col
	out._type = TTYPE_ERR
	out.data = utils.strcpy("\0")
	return out
;/

/; produce_first_token (~utils.File fin) [Token]
	Token tmp
	tmp.line = 1
	tmp.col = 1
	tmp._type = TTYPE_ERR
	
	return produce_next_token(fin, tmp)
;/

/; gen_token_list (~utils.File fin) [utils.Vector]
	utils.Vector out
	Token tmp
	out.init(len tmp)
	
	fin`.open()
	tmp = produce_first_token(fin)
	/; loop (tmp._type !== TTYPE_ERR)
		/; if (tmp._type !== TTYPE_COMNT)
			out.push(~tmp)
			tmp = produce_next_token(fin, tmp)
		;; else
			Token com = tmp
			tmp = produce_next_token(fin, com)
			com.end()
		;/
	;/
	tmp.end()
	fin`.close()

	return out
;/

/; print_token_type(Token t)
	
	/; if (t._type == TTYPE_DELIM)
		_printf("DELIM\0")
	;; else if (t._type == TTYPE_SEP)
		_printf("SEP\0")
	;; else if (t._type == TTYPE_KEYWD)
		_printf("KEYWD\0")
	;; else if (t._type ==TTYPE_KEYTP)
		_printf("KEYTP\0")
	;; else if (t._type == TTYPE_LITRL)
		_printf("LITRL\0")
	;; else if (t._type == TTYPE_AUG)
		_printf("AUG\0")
	;; else if (t._type == TTYPE_USRWD)
		_printf("USRWD\0")
	;; else if (t._type == TTYPE_COMNT)
		_printf("COMNT\0")
	;; else if (t._type == TTYPE_UNKNOWN)
		_printf("UNKNOWN\0")
	;; else if (t._type == TTYPE_ERR)
		_printf("ERR\0")
	;/

;/

/; print_token (Token t)
	_printf("Token {\0")
	_printf(t.data)
	_print_num(", line: %u\0", t.line)
	_print_num(", col: %u, type: \0", t.col)
	print_token_type(t)
	_printf("}\n\0")
;/

/; print_token_list (~utils.Vector vec)
	~Token tok
	/; loop (uint i = 0; i < vec`.count) [i++]
		tok = vec.get(i)
		print_token(tok`)
	;/
;/

/; end_token_list (~utils.Vector vec)
	~Token tok
	
	/; loop (uint i = 0; i < vec`.count) [i++]
		tok = vec`.get(i)
		tok`.end()
	;/
	vec`.end()
;/