From 117b69f10f95ee0493ee041fde5ade01ac1ab108 Mon Sep 17 00:00:00 2001 From: Christopher Schleiden Date: Tue, 31 Jan 2023 15:31:16 -0800 Subject: [PATCH] Add range to tokens --- actions-expressions/src/errors.ts | 2 +- actions-expressions/src/lexer.test.ts | 194 ++++++++++++++++++-------- actions-expressions/src/lexer.ts | 29 +++- 3 files changed, 157 insertions(+), 68 deletions(-) diff --git a/actions-expressions/src/errors.ts b/actions-expressions/src/errors.ts index 0a2db07..72202a9 100644 --- a/actions-expressions/src/errors.ts +++ b/actions-expressions/src/errors.ts @@ -20,7 +20,7 @@ export class ExpressionError extends Error { constructor(private typ: ErrorType, private tok: Token) { super(`${errorDescription(typ)}: '${tokenString(tok)}'`); - this.pos = this.tok.pos; + this.pos = this.tok.range.start; } public pos: Pos; diff --git a/actions-expressions/src/lexer.test.ts b/actions-expressions/src/lexer.test.ts index 749f721..3145534 100644 --- a/actions-expressions/src/lexer.test.ts +++ b/actions-expressions/src/lexer.test.ts @@ -3,32 +3,32 @@ import {Lexer, Token, TokenType} from "./lexer"; describe("lexer", () => { const tests: { input: string; - tokenType: TokenType[]; - token?: Token; + tokenTypes: TokenType[]; + tokens?: Token[]; }[] = [ - {input: "<", tokenType: [TokenType.LESS]}, - {input: ">", tokenType: [TokenType.GREATER]}, + {input: "<", tokenTypes: [TokenType.LESS]}, + {input: ">", tokenTypes: [TokenType.GREATER]}, - {input: "!=", tokenType: [TokenType.BANG_EQUAL]}, - {input: "==", tokenType: [TokenType.EQUAL_EQUAL]}, - {input: "<=", tokenType: [TokenType.LESS_EQUAL]}, - {input: ">=", tokenType: [TokenType.GREATER_EQUAL]}, + {input: "!=", tokenTypes: [TokenType.BANG_EQUAL]}, + {input: "==", tokenTypes: [TokenType.EQUAL_EQUAL]}, + {input: "<=", tokenTypes: [TokenType.LESS_EQUAL]}, + {input: ">=", tokenTypes: [TokenType.GREATER_EQUAL]}, - {input: "&&", tokenType: [TokenType.AND]}, - {input: "||", tokenType: [TokenType.OR]}, + {input: "&&", tokenTypes: [TokenType.AND]}, + {input: "||", tokenTypes: [TokenType.OR]}, // Numbers - {input: "12", tokenType: [TokenType.NUMBER]}, - {input: "12.0", tokenType: [TokenType.NUMBER]}, - {input: "0", tokenType: [TokenType.NUMBER]}, - {input: "-0", tokenType: [TokenType.NUMBER]}, - {input: "-12.0", tokenType: [TokenType.NUMBER]}, + {input: "12", tokenTypes: [TokenType.NUMBER]}, + {input: "12.0", tokenTypes: [TokenType.NUMBER]}, + {input: "0", tokenTypes: [TokenType.NUMBER]}, + {input: "-0", tokenTypes: [TokenType.NUMBER]}, + {input: "-12.0", tokenTypes: [TokenType.NUMBER]}, // Strings - {input: "'It''s okay'", tokenType: [TokenType.STRING]}, + {input: "'It''s okay'", tokenTypes: [TokenType.STRING]}, { input: "format('{0} == ''queued''', needs)", - tokenType: [ + tokenTypes: [ TokenType.IDENTIFIER, TokenType.LEFT_PAREN, TokenType.STRING, @@ -41,87 +41,161 @@ describe("lexer", () => { // Arrays { input: "[1,2]", - tokenType: [TokenType.LEFT_BRACKET, TokenType.NUMBER, TokenType.COMMA, TokenType.NUMBER, TokenType.RIGHT_BRACKET] + tokenTypes: [TokenType.LEFT_BRACKET, TokenType.NUMBER, TokenType.COMMA, TokenType.NUMBER, TokenType.RIGHT_BRACKET] }, // Simple expressions { input: "1 == 2", - tokenType: [TokenType.NUMBER, TokenType.EQUAL_EQUAL, TokenType.NUMBER] + tokenTypes: [TokenType.NUMBER, TokenType.EQUAL_EQUAL, TokenType.NUMBER] }, { input: "1== 1", - tokenType: [TokenType.NUMBER, TokenType.EQUAL_EQUAL, TokenType.NUMBER] + tokenTypes: [TokenType.NUMBER, TokenType.EQUAL_EQUAL, TokenType.NUMBER] }, { input: "1< 1", - tokenType: [TokenType.NUMBER, TokenType.LESS, TokenType.NUMBER] + tokenTypes: [TokenType.NUMBER, TokenType.LESS, TokenType.NUMBER] }, // Identifiers { input: "github", - tokenType: [TokenType.IDENTIFIER], - token: { - type: TokenType.IDENTIFIER, - lexeme: "github", - pos: { - line: 0, - column: 0 + tokenTypes: [TokenType.IDENTIFIER], + tokens: [ + { + type: TokenType.IDENTIFIER, + lexeme: "github", + range: { + start: { + line: 0, + column: 0 + }, + end: { + line: 0, + column: 6 + } + } } - } + ] }, // Keywords { input: "true", - tokenType: [TokenType.TRUE], - token: { - type: TokenType.TRUE, - lexeme: "true", - pos: { - line: 0, - column: 0 + tokenTypes: [TokenType.TRUE], + tokens: [ + { + type: TokenType.TRUE, + lexeme: "true", + range: { + start: { + line: 0, + column: 0 + }, + end: { + line: 0, + column: 4 + } + } } - } + ] }, { input: "false", - tokenType: [TokenType.FALSE], - token: { - type: TokenType.FALSE, - lexeme: "false", - pos: { - line: 0, - column: 0 + tokenTypes: [TokenType.FALSE], + tokens: [ + { + type: TokenType.FALSE, + lexeme: "false", + range: { + start: { + line: 0, + column: 0 + }, + end: { + line: 0, + column: 5 + } + } } - } + ] }, { input: "null", - tokenType: [TokenType.NULL], - token: { - type: TokenType.NULL, - lexeme: "null", - pos: { - line: 0, - column: 0 + tokenTypes: [TokenType.NULL], + tokens: [ + { + type: TokenType.NULL, + lexeme: "null", + range: { + start: { + line: 0, + column: 0 + }, + end: { + line: 0, + column: 4 + } + } } - } + ] + }, + + { + input: "github\n ==", + tokenTypes: [TokenType.IDENTIFIER, TokenType.EQUAL_EQUAL], + tokens: [ + { + type: TokenType.IDENTIFIER, + lexeme: "github", + range: { + start: { + line: 0, + column: 0 + }, + end: { + line: 0, + column: 6 + } + } + }, + { + type: TokenType.EQUAL_EQUAL, + lexeme: "==", + range: { + start: { + line: 1, + column: 1 + }, + end: { + line: 1, + column: 3 + } + } + } + ] } ]; - test.each(tests)("$input", ({input, tokenType, token}: {input: string; tokenType: TokenType[]; token?: Token}) => { - const l = new Lexer(input); + test.each(tests)( + "$input", + ({input, tokenTypes, tokens}: {input: string; tokenTypes: TokenType[]; tokens?: Token[]}) => { + const l = new Lexer(input); - const r = l.lex(); + const r = l.lex(); - const want = r.tokens.map(t => t.type); + const got = r.tokens.map(t => t.type); - tokenType.push(TokenType.EOF); + tokenTypes.push(TokenType.EOF); + expect(got).toEqual(tokenTypes); - expect(want).toEqual(tokenType); - }); + if (tokens) { + // Ignore the last EOF token + expect(r.tokens.slice(0, r.tokens.length - 1)).toEqual(tokens); + } + } + ); }); diff --git a/actions-expressions/src/lexer.ts b/actions-expressions/src/lexer.ts index 2d9c0fa..c52bbf2 100644 --- a/actions-expressions/src/lexer.ts +++ b/actions-expressions/src/lexer.ts @@ -36,13 +36,18 @@ export type Pos = { column: number; }; +export type Range = { + start: Pos; + end: Pos; +}; + export type Token = { type: TokenType; lexeme: string; value?: string | number | boolean; - pos: Pos; + range: Range; }; export function tokenString(tok: Token): string { @@ -192,7 +197,7 @@ export class Lexer { this.tokens.push({ type: TokenType.EOF, lexeme: "", - pos: this.pos() + range: this.range() }); return { @@ -207,6 +212,20 @@ export class Lexer { }; } + private endPos(): Pos { + return { + line: this.line, + column: this.offset - this.lastLineOffset + }; + } + + private range(): Range { + return { + start: this.pos(), + end: this.endPos() + }; + } + private atEnd(): boolean { return this.offset >= this.input.length; } @@ -240,10 +259,6 @@ export class Lexer { return this.input[this.offset++]; } - private reverse(): string { - return this.input[--this.offset]; - } - private match(expected: string): boolean { if (this.atEnd()) { return false; @@ -260,7 +275,7 @@ export class Lexer { this.tokens.push({ type, lexeme: this.input.substring(this.start, this.offset), - pos: this.pos(), + range: this.range(), value }); }