From 678d992e64e9b9adaf4df92326f08c954bff128f Mon Sep 17 00:00:00 2001 From: Lewin Kelly Date: Wed, 14 Feb 2024 02:18:44 +0000 Subject: [PATCH] Output tokens from Luau version --- Script/main.go | 10 ++--- Script/main.luau | 98 ++++++++++++++++++++++++++++-------------------- 2 files changed, 61 insertions(+), 47 deletions(-) diff --git a/Script/main.go b/Script/main.go index 0343168..a5007a8 100644 --- a/Script/main.go +++ b/Script/main.go @@ -71,7 +71,6 @@ type BlockExpr struct { } const ( - EOF = "EOF" INDENT = "INDENT" SPACE = "SPACE" NEWLINE = "NEWLINE" @@ -273,6 +272,7 @@ func lex(source string) []Token { for i := 0; i < len(source); i++ { char := source[i] column++ + switch char { case '=': addToken(EQUALS, "=") @@ -322,8 +322,6 @@ func lex(source string) []Token { os.Exit(1) } - fmt.Println("got a string literal", stringLiteral, startLine, startColumn) - addToken(STRING, stringLiteral, startLine, startColumn) case '+': @@ -379,7 +377,7 @@ func lex(source string) []Token { startColumn := column var identifierOrKeyword string - + // keep going until we hit a non-letter for i < len(source) && (source[i] >= 'a' && source[i] <= 'z' || @@ -454,10 +452,10 @@ func main() { tokens := lex(sourceString) for _, token := range tokens { - if token.kind == "SPACE" { + if token.kind == SPACE { continue } - if token.kind == "NEWLINE" { + if token.kind == NEWLINE { fmt.Println("────────────────┼───────────────┼─────────────────────────────") continue } diff --git a/Script/main.luau b/Script/main.luau index 6b5bfd1..2eb5b60 100644 --- a/Script/main.luau +++ b/Script/main.luau @@ -4,7 +4,6 @@ local process = require "@lune/process" local exit = process.exit local colour = require "colour" -local EOF = "EOF" local INDENT = "INDENT" local SPACE = "SPACE" local NEWLINE = "NEWLINE" @@ -61,17 +60,11 @@ type Token = { -- return {} -- end -local function s(str: string) - return function(n: number) - return string.sub(str, n, n) - end -end - -local function lex(source: string): { Token } +local function lex(source: { string }): { Token } local tokens: { Token } = {} local function last(n: number): Token - return tokens[#tokens - n] + return tokens[#tokens - (n - 1)] end local line, column = 1, 0 @@ -89,13 +82,17 @@ local function lex(source: string): { Token } }) end - for i = 1, #source do - local char = s(source)(i) + local len = #source + 1 + + local i = 0 + while i < len - 1 do + i += 1 + local char = source[i] column += 1 if char == "=" then addToken(EQUALS, "=") - elseif char == "\n" then + elseif char == "\n" then -- newline dont work for some reason addToken(NEWLINE, "\n") line += 1 column = 0 @@ -114,8 +111,8 @@ local function lex(source: string): { Token } local startColumn = column i += 1 -- skip the semicolon local comment = "" - while i < #source and s(source)(i) ~= "\n" do - comment ..= s(source)(i) + while i < len and source[i] ~= "\n" do + comment ..= source[i] column += 1 i += 1 end @@ -129,28 +126,25 @@ local function lex(source: string): { Token } column += 1 i += 1 -- skip the first quote - - while i < #source and s(source)(i) ~= '"' do - stringLiteral ..= s(source)(i) + while i < len and source[i] ~= '"' do + stringLiteral ..= source[i] column += 1 i += 1 end - if i == #source then - print(colour.red "unclosed string literal") + if i == len then + print(colour.red "unclosed string literal", stringLiteral) exit(1) - end - - print("got a string literal", stringLiteral, startLine, startColumn) + end addToken(STRING, stringLiteral, startLine, startColumn) elseif char == "+" then -- check if it's a ++ or a += or just a + - if i + 1 < #source and s(source)(i + 1) == "+" then + if i + 1 < len and source[i + 1] == "+" then addToken(PLUSPLUS, "++") i += 1 column += 1 - elseif i + 1 < #source and s(source)(i + 1) == "=" then + elseif i + 1 < len and source[i + 1] == "=" then addToken(PLUSEQUALS, "+=") i += 1 column += 1 @@ -159,11 +153,11 @@ local function lex(source: string): { Token } end elseif char == "-" then -- check if it's a -- or a -= or just a - - if i + 1 < #source and s(source)(i + 1) == "-" then + if i + 1 < len and source[i + 1] == "-" then addToken(MINUSMINUS, "--") i += 1 column += 1 - elseif i + 1 < #source and s(source)(i + 1) == "=" then + elseif i + 1 < len and source[i + 1] == "=" then addToken(MINUSEQUALS, "-=") i += 1 column += 1 @@ -183,12 +177,8 @@ local function lex(source: string): { Token } local number = "" -- keep going until we hit a non-number - while - i < #source - and s(source)(i) >= "0" - and s(source)(i) <= "9" - do - number ..= s(source)(i) + while i < len and source[i] >= "0" and source[i] <= "9" do + number ..= source[i] column += 1 i += 1 end @@ -204,19 +194,19 @@ local function lex(source: string): { Token } -- keep going until we hit a non-letter while - i < #source + i < len and ( - s(source)(i) >= "a" and s(source)(i) <= "z" - or s(source)(i) >= "A" and s(source)(i) <= "Z" - or s(source)(i) >= "0" and s(source)(i) <= "9" + source[i] >= "a" and source[i] <= "z" + or source[i] >= "A" and source[i] <= "Z" + or source[i] >= "0" and source[i] <= "9" ) do - identifierOrKeyword ..= s(source)(i) + identifierOrKeyword ..= source[i] column += 1 i += 1 end - if i == #source then + if i == len then -- you can't end a program with an identifier print(colour.red "cant end program with identifier") exit(1) @@ -254,7 +244,10 @@ local function lex(source: string): { Token } startColumn ) else - print(colour.red "that isnt a valid character", colour.yellow(char)) + print( + colour.red "that isnt a valid character", + colour.yellow(char) + ) exit(1) end end @@ -295,9 +288,32 @@ local function main() -- remove trailing newlines source = string.gsub(source, "\n+$", "") - local tokens = lex(source) + local split = string.split(source, "") - print(tokens) + local tokens = lex(split) + + for _, token in tokens do + if token.kind == SPACE then + continue + end + if token.kind == NEWLINE then + print "────────────────┼───────────────┼─────────────────────────────" + continue + end + + -- print in a nice format + local function pad(str: string, len: number): string + return str .. string.rep(" ", len - #str) + end + + print( + pad(`{target}:{token.line}:{token.column}`, 15), + "│", + pad(colour.yellow(token.kind), 22), + "│", + colour.purple(token.value) + ) + end -- local program = parse(tokens) -- local out = generate(program)