actions/languageservices
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add range to tokens

Browse Source
This commit is contained in:
Christopher Schleiden
2023-01-31 15:31:16 -08:00
parent 0aacfc0294
commit 117b69f10f
3 changed files with 157 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
actions-expressions/src/errors.ts
+1 -1
View File
@@ -20,7 +20,7 @@ export class ExpressionError extends Error {
constructor(private typ: ErrorType, private tok: Token) {
super(`${errorDescription(typ)}: '${tokenString(tok)}'`);
this.pos = this.tok.pos;
this.pos = this.tok.range.start;
}
public pos: Pos;
actions-expressions/src/lexer.test.ts
+134 -60
View File
@@ -3,32 +3,32 @@ import {Lexer, Token, TokenType} from "./lexer";
describe("lexer", () => {
const tests: {
input: string;
tokenType: TokenType[];
token?: Token;
tokenTypes: TokenType[];
tokens?: Token[];
}[] = [
{input: "<", tokenType: [TokenType.LESS]},
{input: ">", tokenType: [TokenType.GREATER]},
{input: "<", tokenTypes: [TokenType.LESS]},
{input: ">", tokenTypes: [TokenType.GREATER]},
{input: "!=", tokenType: [TokenType.BANG_EQUAL]},
{input: "==", tokenType: [TokenType.EQUAL_EQUAL]},
{input: "<=", tokenType: [TokenType.LESS_EQUAL]},
{input: ">=", tokenType: [TokenType.GREATER_EQUAL]},
{input: "!=", tokenTypes: [TokenType.BANG_EQUAL]},
{input: "==", tokenTypes: [TokenType.EQUAL_EQUAL]},
{input: "<=", tokenTypes: [TokenType.LESS_EQUAL]},
{input: ">=", tokenTypes: [TokenType.GREATER_EQUAL]},
{input: "&&", tokenType: [TokenType.AND]},
{input: "||", tokenType: [TokenType.OR]},
{input: "&&", tokenTypes: [TokenType.AND]},
{input: "||", tokenTypes: [TokenType.OR]},
// Numbers
{input: "12", tokenType: [TokenType.NUMBER]},
{input: "12.0", tokenType: [TokenType.NUMBER]},
{input: "0", tokenType: [TokenType.NUMBER]},
{input: "-0", tokenType: [TokenType.NUMBER]},
{input: "-12.0", tokenType: [TokenType.NUMBER]},
{input: "12", tokenTypes: [TokenType.NUMBER]},
{input: "12.0", tokenTypes: [TokenType.NUMBER]},
{input: "0", tokenTypes: [TokenType.NUMBER]},
{input: "-0", tokenTypes: [TokenType.NUMBER]},
{input: "-12.0", tokenTypes: [TokenType.NUMBER]},
// Strings
{input: "'It''s okay'", tokenType: [TokenType.STRING]},
{input: "'It''s okay'", tokenTypes: [TokenType.STRING]},
{
input: "format('{0} == ''queued''', needs)",
tokenType: [
tokenTypes: [
TokenType.IDENTIFIER,
TokenType.LEFT_PAREN,
TokenType.STRING,
@@ -41,87 +41,161 @@ describe("lexer", () => {
// Arrays
{
input: "[1,2]",
tokenType: [TokenType.LEFT_BRACKET, TokenType.NUMBER, TokenType.COMMA, TokenType.NUMBER, TokenType.RIGHT_BRACKET]
tokenTypes: [TokenType.LEFT_BRACKET, TokenType.NUMBER, TokenType.COMMA, TokenType.NUMBER, TokenType.RIGHT_BRACKET]
},
// Simple expressions
{
input: "1 == 2",
tokenType: [TokenType.NUMBER, TokenType.EQUAL_EQUAL, TokenType.NUMBER]
tokenTypes: [TokenType.NUMBER, TokenType.EQUAL_EQUAL, TokenType.NUMBER]
},
{
input: "1== 1",
tokenType: [TokenType.NUMBER, TokenType.EQUAL_EQUAL, TokenType.NUMBER]
tokenTypes: [TokenType.NUMBER, TokenType.EQUAL_EQUAL, TokenType.NUMBER]
},
{
input: "1< 1",
tokenType: [TokenType.NUMBER, TokenType.LESS, TokenType.NUMBER]
tokenTypes: [TokenType.NUMBER, TokenType.LESS, TokenType.NUMBER]
},
// Identifiers
{
input: "github",
tokenType: [TokenType.IDENTIFIER],
token: {
type: TokenType.IDENTIFIER,
lexeme: "github",
pos: {
line: 0,
column: 0
tokenTypes: [TokenType.IDENTIFIER],
tokens: [
{
type: TokenType.IDENTIFIER,
lexeme: "github",
range: {
start: {
line: 0,
column: 0
},
end: {
line: 0,
column: 6
}
}
}
}
]
},
// Keywords
{
input: "true",
tokenType: [TokenType.TRUE],
token: {
type: TokenType.TRUE,
lexeme: "true",
pos: {
line: 0,
column: 0
tokenTypes: [TokenType.TRUE],
tokens: [
{
type: TokenType.TRUE,
lexeme: "true",
range: {
start: {
line: 0,
column: 0
},
end: {
line: 0,
column: 4
}
}
}
}
]
},
{
input: "false",
tokenType: [TokenType.FALSE],
token: {
type: TokenType.FALSE,
lexeme: "false",
pos: {
line: 0,
column: 0
tokenTypes: [TokenType.FALSE],
tokens: [
{
type: TokenType.FALSE,
lexeme: "false",
range: {
start: {
line: 0,
column: 0
},
end: {
line: 0,
column: 5
}
}
}
}
]
},
{
input: "null",
tokenType: [TokenType.NULL],
token: {
type: TokenType.NULL,
lexeme: "null",
pos: {
line: 0,
column: 0
tokenTypes: [TokenType.NULL],
tokens: [
{
type: TokenType.NULL,
lexeme: "null",
range: {
start: {
line: 0,
column: 0
},
end: {
line: 0,
column: 4
}
}
}
}
]
},
{
input: "github\n ==",
tokenTypes: [TokenType.IDENTIFIER, TokenType.EQUAL_EQUAL],
tokens: [
{
type: TokenType.IDENTIFIER,
lexeme: "github",
range: {
start: {
line: 0,
column: 0
},
end: {
line: 0,
column: 6
}
}
},
{
type: TokenType.EQUAL_EQUAL,
lexeme: "==",
range: {
start: {
line: 1,
column: 1
},
end: {
line: 1,
column: 3
}
}
}
]
}
];
test.each(tests)("$input", ({input, tokenType, token}: {input: string; tokenType: TokenType[]; token?: Token}) => {
const l = new Lexer(input);
test.each(tests)(
"$input",
({input, tokenTypes, tokens}: {input: string; tokenTypes: TokenType[]; tokens?: Token[]}) => {
const l = new Lexer(input);
const r = l.lex();
const r = l.lex();
const want = r.tokens.map(t => t.type);
const got = r.tokens.map(t => t.type);
tokenType.push(TokenType.EOF);
tokenTypes.push(TokenType.EOF);
expect(got).toEqual(tokenTypes);
expect(want).toEqual(tokenType);
});
if (tokens) {
// Ignore the last EOF token
expect(r.tokens.slice(0, r.tokens.length - 1)).toEqual(tokens);
}
}
);
});
actions-expressions/src/lexer.ts
+22 -7
View File
@@ -36,13 +36,18 @@ export type Pos = {
column: number;
};
export type Range = {
start: Pos;
end: Pos;
};
export type Token = {
type: TokenType;
lexeme: string;
value?: string | number | boolean;
pos: Pos;
range: Range;
};
export function tokenString(tok: Token): string {
@@ -192,7 +197,7 @@ export class Lexer {
this.tokens.push({
type: TokenType.EOF,
lexeme: "",
pos: this.pos()
range: this.range()
});
return {
@@ -207,6 +212,20 @@ export class Lexer {
};
}
private endPos(): Pos {
return {
line: this.line,
column: this.offset - this.lastLineOffset
};
}
private range(): Range {
return {
start: this.pos(),
end: this.endPos()
};
}
private atEnd(): boolean {
return this.offset >= this.input.length;
}
@@ -240,10 +259,6 @@ export class Lexer {
return this.input[this.offset++];
}
private reverse(): string {
return this.input[--this.offset];
}
private match(expected: string): boolean {
if (this.atEnd()) {
return false;
@@ -260,7 +275,7 @@ export class Lexer {
this.tokens.push({
type,
lexeme: this.input.substring(this.start, this.offset),
pos: this.pos(),
range: this.range(),
value
});
}