Merge parser and expressions into repository
This commit is contained in:
Christopher Schleiden
@@ -0,0 +1,4 @@
|
||||
export interface File {
|
||||
name: string
|
||||
content: string
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export const WORKFLOW_ROOT = "workflow-root-strict"
|
||||
export const STRATEGY = "strategy"
|
||||
@@ -0,0 +1,51 @@
|
||||
import { Dictionary } from "@github/actions-expressions/data/dictionary"
|
||||
import {
|
||||
TemplateContext,
|
||||
TemplateValidationErrors,
|
||||
} from "../templates/template-context"
|
||||
import * as templateEvaluator from "../templates/template-evaluator"
|
||||
import { TemplateValidationError } from "../templates/template-validation-error"
|
||||
import { TemplateToken } from "../templates/tokens/template-token"
|
||||
import { TraceWriter } from "../templates/trace-writer"
|
||||
import { STRATEGY } from "./workflow-constants"
|
||||
import { getWorkflowSchema } from "./workflow-schema"
|
||||
|
||||
export interface EvaluateWorkflowResult {
|
||||
value: TemplateToken | undefined
|
||||
errors: TemplateValidationError[]
|
||||
}
|
||||
|
||||
export function evaluateStrategy(
|
||||
fileTable: string[],
|
||||
context: Dictionary,
|
||||
token: TemplateToken,
|
||||
trace: TraceWriter
|
||||
): EvaluateWorkflowResult {
|
||||
const templateContext = new TemplateContext(
|
||||
new TemplateValidationErrors(),
|
||||
getWorkflowSchema(),
|
||||
trace
|
||||
)
|
||||
|
||||
// Add each file name
|
||||
for (const fileName of fileTable) {
|
||||
templateContext.getFileId(fileName)
|
||||
}
|
||||
|
||||
// Add expression named contexts
|
||||
for (const pair of context.pairs()) {
|
||||
templateContext.expressionNamedContexts.push(pair.key)
|
||||
}
|
||||
|
||||
const value = templateEvaluator.evaluateTemplate(
|
||||
templateContext,
|
||||
STRATEGY,
|
||||
token,
|
||||
0,
|
||||
undefined
|
||||
)
|
||||
return <EvaluateWorkflowResult>{
|
||||
value: value,
|
||||
errors: templateContext.errors.getErrors(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { parseWorkflow } from "./workflow-parser"
|
||||
import { nullTrace } from "../test-utils/null-trace"
|
||||
|
||||
it("The template is not read when there are YAML errors", () => {
|
||||
const content = `
|
||||
on: push
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Hello \${{ fromJSON('test') == inputs.name }}'
|
||||
run: echo Hello, world!`
|
||||
|
||||
const result = parseWorkflow(
|
||||
"main.yaml",
|
||||
[{ name: "main.yaml", content: content }],
|
||||
nullTrace
|
||||
)
|
||||
|
||||
expect(result.context.errors.count).toBe(1)
|
||||
expect(result.value).toBeUndefined()
|
||||
})
|
||||
@@ -0,0 +1,52 @@
|
||||
import {
|
||||
TemplateContext,
|
||||
TemplateValidationErrors,
|
||||
} from "../templates/template-context"
|
||||
import * as templateReader from "../templates/template-reader"
|
||||
import { TemplateToken } from "../templates/tokens/template-token"
|
||||
import { TraceWriter } from "../templates/trace-writer"
|
||||
import { File } from "./file"
|
||||
import { WORKFLOW_ROOT } from "./workflow-constants"
|
||||
import { getWorkflowSchema } from "./workflow-schema"
|
||||
import { YamlObjectReader } from "./yaml-object-reader"
|
||||
export interface ParseWorkflowResult {
|
||||
context: TemplateContext
|
||||
value: TemplateToken | undefined
|
||||
}
|
||||
|
||||
export function parseWorkflow(
|
||||
entryFileName: string,
|
||||
files: File[],
|
||||
trace: TraceWriter
|
||||
): ParseWorkflowResult {
|
||||
const context = new TemplateContext(
|
||||
new TemplateValidationErrors(),
|
||||
getWorkflowSchema(),
|
||||
trace
|
||||
)
|
||||
|
||||
// Add file ids
|
||||
files.forEach((x) => context.getFileId(x.name))
|
||||
|
||||
const fileId = context.getFileId(entryFileName)
|
||||
const fileContent = files[fileId - 1].content
|
||||
const reader = new YamlObjectReader(context, fileId, fileContent)
|
||||
if (context.errors.count > 0) {
|
||||
// The file is not valid YAML, template errors could be misleading
|
||||
return {
|
||||
context,
|
||||
value: undefined,
|
||||
}
|
||||
}
|
||||
const result = templateReader.readTemplate(
|
||||
context,
|
||||
WORKFLOW_ROOT,
|
||||
reader,
|
||||
fileId
|
||||
)
|
||||
|
||||
return <ParseWorkflowResult>{
|
||||
context,
|
||||
value: result,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { JSONObjectReader } from "../templates/json-object-reader"
|
||||
import { TemplateSchema } from "../templates/schema"
|
||||
import WorkflowSchema from "./workflow-v1.0.json"
|
||||
|
||||
let schema: TemplateSchema
|
||||
|
||||
export function getWorkflowSchema(): TemplateSchema {
|
||||
if (schema === undefined) {
|
||||
const json = JSON.stringify(WorkflowSchema)
|
||||
schema = TemplateSchema.load(new JSONObjectReader(undefined, json))
|
||||
}
|
||||
return schema
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,102 @@
|
||||
import {
|
||||
TemplateContext,
|
||||
TemplateValidationErrors,
|
||||
} from "../templates/template-context"
|
||||
import { TemplateToken } from "../templates/tokens"
|
||||
import { TokenType } from "../templates/tokens/types"
|
||||
import { nullTrace } from "../test-utils/null-trace"
|
||||
import { parseWorkflow } from "./workflow-parser"
|
||||
import { getWorkflowSchema } from "./workflow-schema"
|
||||
import { YamlObjectReader } from "./yaml-object-reader"
|
||||
|
||||
describe("getLiteralToken", () => {
|
||||
it("non-zero number", () => {
|
||||
const result = parseAsWorkflow("1")
|
||||
|
||||
expect(result).not.toBeUndefined()
|
||||
expect(result?.templateTokenType).toEqual(TokenType.Number)
|
||||
expect(result?.toString()).toEqual("1")
|
||||
})
|
||||
|
||||
it("zero", () => {
|
||||
const result = parseAsWorkflow("0")
|
||||
|
||||
expect(result).not.toBeUndefined()
|
||||
expect(result?.templateTokenType).toEqual(TokenType.Number)
|
||||
expect(result?.toString()).toEqual("0")
|
||||
})
|
||||
|
||||
it("true", () => {
|
||||
const result = parseAsWorkflow("true")
|
||||
|
||||
expect(result).not.toBeUndefined()
|
||||
expect(result?.templateTokenType).toEqual(TokenType.Boolean)
|
||||
expect(result?.toString()).toEqual("true")
|
||||
})
|
||||
|
||||
it("false", () => {
|
||||
const result = parseAsWorkflow("false")
|
||||
|
||||
expect(result).not.toBeUndefined()
|
||||
expect(result?.templateTokenType).toEqual(TokenType.Boolean)
|
||||
expect(result?.toString()).toEqual("false")
|
||||
})
|
||||
|
||||
it("string", () => {
|
||||
const result = parseAsWorkflow("test")
|
||||
|
||||
expect(result).not.toBeUndefined()
|
||||
expect(result?.templateTokenType).toEqual(TokenType.String)
|
||||
expect(result?.toString()).toEqual("test")
|
||||
})
|
||||
|
||||
it("null", () => {
|
||||
const result = parseAsWorkflow("null")
|
||||
|
||||
expect(result).not.toBeUndefined()
|
||||
expect(result?.templateTokenType).toEqual(TokenType.Null)
|
||||
expect(result?.toString()).toEqual("")
|
||||
})
|
||||
})
|
||||
|
||||
it("YAML errors include range information", () => {
|
||||
const content = `
|
||||
on: push
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Hello \${{ fromJSON('test') == inputs.name }}'
|
||||
run: echo Hello, world!`
|
||||
|
||||
const context = new TemplateContext(
|
||||
new TemplateValidationErrors(),
|
||||
getWorkflowSchema(),
|
||||
nullTrace
|
||||
)
|
||||
const fileId = context.getFileId("test.yaml")
|
||||
new YamlObjectReader(context, fileId, content)
|
||||
|
||||
expect(context.errors.count).toBe(1)
|
||||
|
||||
const error = context.errors.getErrors()[0]
|
||||
expect(error.range).toEqual({
|
||||
start: [7, 38],
|
||||
end: [7, 63],
|
||||
})
|
||||
})
|
||||
|
||||
function parseAsWorkflow(content: string): TemplateToken | undefined {
|
||||
const result = parseWorkflow(
|
||||
"test.yaml",
|
||||
[
|
||||
{
|
||||
name: "test.yaml",
|
||||
content: content,
|
||||
},
|
||||
],
|
||||
nullTrace
|
||||
)
|
||||
|
||||
return result.value
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
import {
|
||||
isCollection,
|
||||
isDocument,
|
||||
isMap,
|
||||
isPair,
|
||||
isScalar,
|
||||
isSeq,
|
||||
LineCounter,
|
||||
parseDocument,
|
||||
Scalar,
|
||||
} from "yaml"
|
||||
import { LinePos } from "yaml/dist/errors"
|
||||
import { NodeBase } from "yaml/dist/nodes/Node"
|
||||
import { ObjectReader } from "../templates/object-reader"
|
||||
import { EventType, ParseEvent } from "../templates/parse-event"
|
||||
import { TemplateContext } from "../templates/template-context"
|
||||
import {
|
||||
BooleanToken,
|
||||
LiteralToken,
|
||||
MappingToken,
|
||||
NullToken,
|
||||
NumberToken,
|
||||
SequenceToken,
|
||||
StringToken,
|
||||
} from "../templates/tokens/index"
|
||||
import { Position, TokenRange } from "../templates/tokens/token-range"
|
||||
|
||||
export class YamlObjectReader implements ObjectReader {
|
||||
private readonly _generator: Generator<ParseEvent>
|
||||
private _current!: IteratorResult<ParseEvent>
|
||||
private fileId?: number
|
||||
private lineCounter = new LineCounter()
|
||||
|
||||
constructor(
|
||||
context: TemplateContext,
|
||||
fileId: number | undefined,
|
||||
content: string
|
||||
) {
|
||||
const doc = parseDocument(content, {
|
||||
lineCounter: this.lineCounter,
|
||||
keepSourceTokens: true,
|
||||
uniqueKeys: false, // Uniqueness is validated by the template reader
|
||||
})
|
||||
for (const err of doc.errors) {
|
||||
context.error(fileId, err.message, rangeFromLinePos(err.linePos))
|
||||
}
|
||||
this._generator = this.getNodes(doc)
|
||||
this.fileId = fileId
|
||||
}
|
||||
|
||||
private *getNodes(node: unknown): Generator<ParseEvent, void> {
|
||||
let range = this.getRange(node as NodeBase | undefined)
|
||||
|
||||
if (isDocument(node)) {
|
||||
yield new ParseEvent(EventType.DocumentStart)
|
||||
for (const item of this.getNodes(node.contents)) {
|
||||
yield item
|
||||
}
|
||||
yield new ParseEvent(EventType.DocumentEnd)
|
||||
}
|
||||
|
||||
if (isCollection(node)) {
|
||||
if (isSeq(node)) {
|
||||
yield new ParseEvent(
|
||||
EventType.SequenceStart,
|
||||
new SequenceToken(this.fileId, range, undefined)
|
||||
)
|
||||
} else if (isMap(node)) {
|
||||
yield new ParseEvent(
|
||||
EventType.MappingStart,
|
||||
new MappingToken(this.fileId, range, undefined)
|
||||
)
|
||||
}
|
||||
|
||||
for (const item of node.items) {
|
||||
for (const child of this.getNodes(item)) {
|
||||
yield child
|
||||
}
|
||||
}
|
||||
if (isSeq(node)) {
|
||||
yield new ParseEvent(EventType.SequenceEnd)
|
||||
} else if (isMap(node)) {
|
||||
yield new ParseEvent(EventType.MappingEnd)
|
||||
}
|
||||
}
|
||||
|
||||
if (isScalar(node)) {
|
||||
yield new ParseEvent(
|
||||
EventType.Literal,
|
||||
YamlObjectReader.getLiteralToken(this.fileId, range, node as Scalar)
|
||||
)
|
||||
}
|
||||
|
||||
if (isPair(node)) {
|
||||
const scalarKey = node.key as Scalar
|
||||
range = this.getRange(scalarKey)
|
||||
const key = scalarKey.value as string
|
||||
yield new ParseEvent(
|
||||
EventType.Literal,
|
||||
new StringToken(this.fileId, range, key, undefined)
|
||||
)
|
||||
for (const child of this.getNodes(node.value)) {
|
||||
yield child
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private getRange(node: NodeBase | undefined): TokenRange | undefined {
|
||||
const range = node?.range ?? []
|
||||
const startPos = range[0]
|
||||
const endPos = range[1]
|
||||
|
||||
if (startPos !== undefined && endPos !== undefined) {
|
||||
const slp = this.lineCounter.linePos(startPos)
|
||||
const elp = this.lineCounter.linePos(endPos)
|
||||
|
||||
return {
|
||||
start: [slp.line, slp.col],
|
||||
end: [elp.line, elp.col],
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
private static getLiteralToken(
|
||||
fileId: number | undefined,
|
||||
range: TokenRange | undefined,
|
||||
token: Scalar
|
||||
) {
|
||||
const value = token.value
|
||||
|
||||
if (value === null || value === undefined) {
|
||||
return new NullToken(fileId, range, undefined)
|
||||
}
|
||||
|
||||
switch (typeof value) {
|
||||
case "number":
|
||||
return new NumberToken(fileId, range, value, undefined)
|
||||
case "boolean":
|
||||
return new BooleanToken(fileId, range, value, undefined)
|
||||
case "string": {
|
||||
// If the string is a YAML block string, include the original source
|
||||
let source: string | undefined
|
||||
if (
|
||||
(token.type === "BLOCK_LITERAL" || // | multi-line strings
|
||||
token.type === "BLOCK_FOLDED") && // > multi-line strings
|
||||
token.srcToken &&
|
||||
token.srcToken.type === "block-scalar"
|
||||
) {
|
||||
source = token.srcToken.source
|
||||
}
|
||||
|
||||
return new StringToken(fileId, range, value, undefined, source)
|
||||
}
|
||||
default:
|
||||
throw new Error(
|
||||
`Unexpected value type '${typeof value}' when reading object`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
public allowLiteral(): LiteralToken | undefined {
|
||||
if (!this._current.done) {
|
||||
const parseEvent = this._current.value
|
||||
if (parseEvent.type === EventType.Literal) {
|
||||
this._current = this._generator.next()
|
||||
return parseEvent.token as LiteralToken
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
public allowSequenceStart(): SequenceToken | undefined {
|
||||
if (!this._current.done) {
|
||||
const parseEvent = this._current.value
|
||||
if (parseEvent.type === EventType.SequenceStart) {
|
||||
this._current = this._generator.next()
|
||||
return parseEvent.token as SequenceToken
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
public allowSequenceEnd(): boolean {
|
||||
if (!this._current.done) {
|
||||
const parseEvent = this._current.value
|
||||
if (parseEvent.type === EventType.SequenceEnd) {
|
||||
this._current = this._generator.next()
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
public allowMappingStart(): MappingToken | undefined {
|
||||
if (!this._current.done) {
|
||||
const parseEvent = this._current.value
|
||||
if (parseEvent.type === EventType.MappingStart) {
|
||||
this._current = this._generator.next()
|
||||
return parseEvent.token as MappingToken
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
public allowMappingEnd(): boolean {
|
||||
if (!this._current.done) {
|
||||
const parseEvent = this._current.value
|
||||
if (parseEvent.type === EventType.MappingEnd) {
|
||||
this._current = this._generator.next()
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
public validateEnd(): void {
|
||||
if (!this._current.done) {
|
||||
const parseEvent = this._current.value as ParseEvent
|
||||
if (parseEvent.type === EventType.DocumentEnd) {
|
||||
this._current = this._generator.next()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error("Expected end of reader")
|
||||
}
|
||||
|
||||
public validateStart(): void {
|
||||
if (!this._current) {
|
||||
this._current = this._generator.next()
|
||||
}
|
||||
|
||||
if (!this._current.done) {
|
||||
const parseEvent = this._current.value as ParseEvent
|
||||
if (parseEvent.type === EventType.DocumentStart) {
|
||||
this._current = this._generator.next()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error("Expected start of reader")
|
||||
}
|
||||
}
|
||||
|
||||
function rangeFromLinePos(
|
||||
linePos: [LinePos] | [LinePos, LinePos] | undefined
|
||||
): TokenRange | undefined {
|
||||
if (linePos === undefined) {
|
||||
return
|
||||
}
|
||||
// TokenRange and linePos are both 1-based
|
||||
const start: Position = [linePos[0].line, linePos[0].col]
|
||||
const end: Position =
|
||||
linePos.length == 2 ? [linePos[1].line, linePos[1].col] : start
|
||||
return { start, end }
|
||||
}
|
||||
Reference in New Issue
Block a user