Rename folders

workflow-parser/src/workflows/file-provider.ts

@@ -0,0 +1,6 @@
+import {File} from "./file";
+import {FileReference} from "./file-reference";
+
+export interface FileProvider {
+  getFileContent(ref: FileReference): Promise<File>;
+}

workflow-parser/src/workflows/file-reference.test.ts

@@ -0,0 +1,55 @@
+import {parseFileReference} from "./file-reference";
+
+describe("parseFileReference", () => {
+  it("parses local file reference", () => {
+    const ref = parseFileReference("./workflow/path");
+    expect(ref).toEqual({
+      path: "workflow/path"
+    });
+  });
+
+  it("parses local file references with an empty path", () => {
+    const ref = parseFileReference("./");
+    expect(ref).toEqual({
+      path: ""
+    });
+  });
+
+  it("parses remote file reference", () => {
+    const ref = parseFileReference("owner/repo/path@version");
+    expect(ref).toEqual({
+      owner: "owner",
+      repository: "repo",
+      path: "path",
+      version: "version"
+    });
+  });
+
+  it("parses remote file reference with an empty path", () => {
+    const ref = parseFileReference("owner/repo@version");
+    expect(ref).toEqual({
+      owner: "owner",
+      repository: "repo",
+      path: "",
+      version: "version"
+    });
+  });
+
+  it("parses remote file reference with slashes in the version", () => {
+    const ref = parseFileReference("owner/repo@feature-branch/dev");
+    expect(ref).toEqual({
+      owner: "owner",
+      repository: "repo",
+      path: "",
+      version: "feature-branch/dev"
+    });
+  });
+
+  it("throws for malformed remote file references", () => {
+    expect(() => parseFileReference("owner/repo/path")).toThrowError("Invalid file reference: owner/repo/path");
+
+    expect(() => parseFileReference("owner/repo/path@")).toThrowError("Invalid file reference: owner/repo/path@");
+
+    expect(() => parseFileReference("owner@")).toThrowError("Invalid file reference: owner@");
+  });
+});

workflow-parser/src/workflows/file-reference.ts

@@ -0,0 +1,42 @@
+export type FileReference = LocalFileReference | RemoteFileReference;
+
+export type LocalFileReference = {
+  path: string;
+};
+
+export type RemoteFileReference = {
+  repository: string;
+  owner: string;
+  path: string;
+  version: string;
+};
+
+export function parseFileReference(ref: string): FileReference {
+  if (ref.startsWith("./")) {
+    return {
+      path: ref.substring(2)
+    };
+  }
+
+  const [remotePath, version] = ref.split("@");
+  const [owner, repository, ...pathSegments] = remotePath.split("/").filter(s => s.length > 0);
+
+  if (!owner || !repository || !version) {
+    throw new Error(`Invalid file reference: ${ref}`);
+  }
+
+  return {
+    repository,
+    owner,
+    path: pathSegments.join("/"),
+    version
+  };
+}
+
+export function fileIdentifier(ref: FileReference): string {
+  if (!("repository" in ref)) {
+    return "./" + ref.path;
+  }
+
+  return `${ref.owner}/${ref.repository}/${ref.path}@${ref.version}`;
+}

workflow-parser/src/workflows/file.ts

@@ -0,0 +1,4 @@
+export interface File {
+  name: string;
+  content: string;
+}

workflow-parser/src/workflows/workflow-constants.ts

@@ -0,0 +1 @@
+export const WORKFLOW_ROOT = "workflow-root-strict";

workflow-parser/src/workflows/workflow-parser.test.ts

@@ -0,0 +1,18 @@
+import {parseWorkflow} from "./workflow-parser";
+import {nullTrace} from "../test-utils/null-trace";
+
+it("The template is not read when there are YAML errors", () => {
+  const content = `
+on: push
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Hello \${{ fromJSON('test') == inputs.name }}'
+        run: echo Hello, world!`;
+
+  const result = parseWorkflow({name: "main.yaml", content: content}, nullTrace);
+
+  expect(result.context.errors.count).toBe(1);
+  expect(result.value).toBeUndefined();
+});

workflow-parser/src/workflows/workflow-parser.ts

@@ -0,0 +1,40 @@
+import {TemplateContext, TemplateValidationErrors} from "../templates/template-context";
+import * as templateReader from "../templates/template-reader";
+import {TemplateToken} from "../templates/tokens/template-token";
+import {TraceWriter} from "../templates/trace-writer";
+import {File} from "./file";
+import {WORKFLOW_ROOT} from "./workflow-constants";
+import {getWorkflowSchema} from "./workflow-schema";
+import {YamlObjectReader} from "./yaml-object-reader";
+export interface ParseWorkflowResult {
+  context: TemplateContext;
+  value: TemplateToken | undefined;
+}
+
+export function parseWorkflow(entryFile: File, trace: TraceWriter): ParseWorkflowResult;
+export function parseWorkflow(entryFile: File, context: TemplateContext): ParseWorkflowResult;
+export function parseWorkflow(entryFile: File, contextOrTrace: TraceWriter | TemplateContext): ParseWorkflowResult {
+  const context =
+    contextOrTrace instanceof TemplateContext
+      ? contextOrTrace
+      : new TemplateContext(new TemplateValidationErrors(), getWorkflowSchema(), contextOrTrace);
+
+  const fileId = context.getFileId(entryFile.name);
+  const reader = new YamlObjectReader(fileId, entryFile.content);
+  if (reader.errors.length > 0) {
+    // The file is not valid YAML, template errors could be misleading
+    for (const err of reader.errors) {
+      context.error(fileId, err.message, err.range);
+    }
+    return {
+      context,
+      value: undefined
+    };
+  }
+  const result = templateReader.readTemplate(context, WORKFLOW_ROOT, reader, fileId);
+
+  return <ParseWorkflowResult>{
+    context,
+    value: result
+  };
+}

workflow-parser/src/workflows/workflow-schema.ts

@@ -0,0 +1,13 @@
+import {JSONObjectReader} from "../templates/json-object-reader";
+import {TemplateSchema} from "../templates/schema";
+import WorkflowSchema from "../workflow-v1.0.json" assert {type: "json"};
+
+let schema: TemplateSchema;
+
+export function getWorkflowSchema(): TemplateSchema {
+  if (schema === undefined) {
+    const json = JSON.stringify(WorkflowSchema);
+    schema = TemplateSchema.load(new JSONObjectReader(undefined, json));
+  }
+  return schema;
+}

workflow-parser/src/workflows/yaml-object-reader.test.ts

@@ -0,0 +1,92 @@
+import {TemplateContext, TemplateValidationErrors} from "../templates/template-context";
+import {TemplateToken} from "../templates/tokens";
+import {TokenType} from "../templates/tokens/types";
+import {nullTrace} from "../test-utils/null-trace";
+import {parseWorkflow} from "./workflow-parser";
+import {getWorkflowSchema} from "./workflow-schema";
+import {YamlObjectReader} from "./yaml-object-reader";
+
+describe("getLiteralToken", () => {
+  it("non-zero number", () => {
+    const result = parseAsWorkflow("1");
+
+    expect(result).not.toBeUndefined();
+    expect(result?.templateTokenType).toEqual(TokenType.Number);
+    expect(result?.toString()).toEqual("1");
+  });
+
+  it("zero", () => {
+    const result = parseAsWorkflow("0");
+
+    expect(result).not.toBeUndefined();
+    expect(result?.templateTokenType).toEqual(TokenType.Number);
+    expect(result?.toString()).toEqual("0");
+  });
+
+  it("true", () => {
+    const result = parseAsWorkflow("true");
+
+    expect(result).not.toBeUndefined();
+    expect(result?.templateTokenType).toEqual(TokenType.Boolean);
+    expect(result?.toString()).toEqual("true");
+  });
+
+  it("false", () => {
+    const result = parseAsWorkflow("false");
+
+    expect(result).not.toBeUndefined();
+    expect(result?.templateTokenType).toEqual(TokenType.Boolean);
+    expect(result?.toString()).toEqual("false");
+  });
+
+  it("string", () => {
+    const result = parseAsWorkflow("test");
+
+    expect(result).not.toBeUndefined();
+    expect(result?.templateTokenType).toEqual(TokenType.String);
+    expect(result?.toString()).toEqual("test");
+  });
+
+  it("null", () => {
+    const result = parseAsWorkflow("null");
+
+    expect(result).not.toBeUndefined();
+    expect(result?.templateTokenType).toEqual(TokenType.Null);
+    expect(result?.toString()).toEqual("");
+  });
+});
+
+it("YAML errors include range information", () => {
+  const content = `
+  on: push
+  jobs:
+    build:
+      runs-on: ubuntu-latest
+      steps:
+        - name: 'Hello \${{ fromJSON('test') == inputs.name }}'
+          run: echo Hello, world!`;
+
+  const context = new TemplateContext(new TemplateValidationErrors(), getWorkflowSchema(), nullTrace);
+  const fileId = context.getFileId("test.yaml");
+  const reader = new YamlObjectReader(fileId, content);
+
+  expect(reader.errors.length).toBe(1);
+
+  const error = reader.errors[0];
+  expect(error.range).toEqual({
+    start: {line: 7, column: 38},
+    end: {line: 7, column: 63}
+  });
+});
+
+function parseAsWorkflow(content: string): TemplateToken | undefined {
+  const result = parseWorkflow(
+    {
+      name: "test.yaml",
+      content: content
+    },
+    nullTrace
+  );
+
+  return result.value;
+}

workflow-parser/src/workflows/yaml-object-reader.ts

@@ -0,0 +1,229 @@
+import {isCollection, isDocument, isMap, isPair, isScalar, isSeq, LineCounter, parseDocument, Scalar} from "yaml";
+import type {LinePos} from "yaml/dist/errors";
+import type {NodeBase} from "yaml/dist/nodes/Node";
+import {ObjectReader} from "../templates/object-reader";
+import {EventType, ParseEvent} from "../templates/parse-event";
+import {TemplateContext} from "../templates/template-context";
+import {
+  BooleanToken,
+  LiteralToken,
+  MappingToken,
+  NullToken,
+  NumberToken,
+  SequenceToken,
+  StringToken
+} from "../templates/tokens/index";
+import {Position, TokenRange} from "../templates/tokens/token-range";
+
+export type YamlError = {
+  message: string;
+  range: TokenRange | undefined;
+};
+
+export class YamlObjectReader implements ObjectReader {
+  private readonly _generator: Generator<ParseEvent>;
+  private _current!: IteratorResult<ParseEvent>;
+  private fileId?: number;
+  private lineCounter = new LineCounter();
+
+  public errors: YamlError[] = [];
+
+  constructor(fileId: number | undefined, content: string) {
+    const doc = parseDocument(content, {
+      lineCounter: this.lineCounter,
+      keepSourceTokens: true,
+      uniqueKeys: false // Uniqueness is validated by the template reader
+    });
+    for (const err of doc.errors) {
+      this.errors.push({message: err.message, range: rangeFromLinePos(err.linePos)});
+    }
+    this._generator = this.getNodes(doc);
+    this.fileId = fileId;
+  }
+
+  private *getNodes(node: unknown): Generator<ParseEvent, void> {
+    let range = this.getRange(node as NodeBase | undefined);
+
+    if (isDocument(node)) {
+      yield new ParseEvent(EventType.DocumentStart);
+      for (const item of this.getNodes(node.contents)) {
+        yield item;
+      }
+      yield new ParseEvent(EventType.DocumentEnd);
+    }
+
+    if (isCollection(node)) {
+      if (isSeq(node)) {
+        yield new ParseEvent(EventType.SequenceStart, new SequenceToken(this.fileId, range, undefined));
+      } else if (isMap(node)) {
+        yield new ParseEvent(EventType.MappingStart, new MappingToken(this.fileId, range, undefined));
+      }
+
+      for (const item of node.items) {
+        for (const child of this.getNodes(item)) {
+          yield child;
+        }
+      }
+      if (isSeq(node)) {
+        yield new ParseEvent(EventType.SequenceEnd);
+      } else if (isMap(node)) {
+        yield new ParseEvent(EventType.MappingEnd);
+      }
+    }
+
+    if (isScalar(node)) {
+      yield new ParseEvent(EventType.Literal, YamlObjectReader.getLiteralToken(this.fileId, range, node as Scalar));
+    }
+
+    if (isPair(node)) {
+      const scalarKey = node.key as Scalar;
+      range = this.getRange(scalarKey);
+      const key = scalarKey.value as string;
+      yield new ParseEvent(EventType.Literal, new StringToken(this.fileId, range, key, undefined));
+      for (const child of this.getNodes(node.value)) {
+        yield child;
+      }
+    }
+  }
+
+  private getRange(node: NodeBase | undefined): TokenRange | undefined {
+    const range = node?.range ?? [];
+    const startPos = range[0];
+    const endPos = range[1];
+
+    if (startPos !== undefined && endPos !== undefined) {
+      const slp = this.lineCounter.linePos(startPos);
+      const elp = this.lineCounter.linePos(endPos);
+
+      return {
+        start: {line: slp.line, column: slp.col},
+        end: {line: elp.line, column: elp.col}
+      };
+    }
+
+    return undefined;
+  }
+
+  private static getLiteralToken(fileId: number | undefined, range: TokenRange | undefined, token: Scalar) {
+    const value = token.value;
+
+    if (value === null || value === undefined) {
+      return new NullToken(fileId, range, undefined);
+    }
+
+    switch (typeof value) {
+      case "number":
+        return new NumberToken(fileId, range, value, undefined);
+      case "boolean":
+        return new BooleanToken(fileId, range, value, undefined);
+      case "string": {
+        let source: string | undefined;
+        if (token.srcToken && "source" in token.srcToken) {
+          source = token.srcToken.source;
+        }
+
+        return new StringToken(fileId, range, value, undefined, source);
+      }
+      default:
+        throw new Error(`Unexpected value type '${typeof value}' when reading object`);
+    }
+  }
+
+  public allowLiteral(): LiteralToken | undefined {
+    if (!this._current.done) {
+      const parseEvent = this._current.value;
+      if (parseEvent.type === EventType.Literal) {
+        this._current = this._generator.next();
+        return parseEvent.token as LiteralToken;
+      }
+    }
+
+    return undefined;
+  }
+
+  public allowSequenceStart(): SequenceToken | undefined {
+    if (!this._current.done) {
+      const parseEvent = this._current.value;
+      if (parseEvent.type === EventType.SequenceStart) {
+        this._current = this._generator.next();
+        return parseEvent.token as SequenceToken;
+      }
+    }
+
+    return undefined;
+  }
+
+  public allowSequenceEnd(): boolean {
+    if (!this._current.done) {
+      const parseEvent = this._current.value;
+      if (parseEvent.type === EventType.SequenceEnd) {
+        this._current = this._generator.next();
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  public allowMappingStart(): MappingToken | undefined {
+    if (!this._current.done) {
+      const parseEvent = this._current.value;
+      if (parseEvent.type === EventType.MappingStart) {
+        this._current = this._generator.next();
+        return parseEvent.token as MappingToken;
+      }
+    }
+
+    return undefined;
+  }
+
+  public allowMappingEnd(): boolean {
+    if (!this._current.done) {
+      const parseEvent = this._current.value;
+      if (parseEvent.type === EventType.MappingEnd) {
+        this._current = this._generator.next();
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  public validateEnd(): void {
+    if (!this._current.done) {
+      const parseEvent = this._current.value as ParseEvent;
+      if (parseEvent.type === EventType.DocumentEnd) {
+        this._current = this._generator.next();
+        return;
+      }
+    }
+
+    throw new Error("Expected end of reader");
+  }
+
+  public validateStart(): void {
+    if (!this._current) {
+      this._current = this._generator.next();
+    }
+
+    if (!this._current.done) {
+      const parseEvent = this._current.value as ParseEvent;
+      if (parseEvent.type === EventType.DocumentStart) {
+        this._current = this._generator.next();
+        return;
+      }
+    }
+
+    throw new Error("Expected start of reader");
+  }
+}
+
+function rangeFromLinePos(linePos: [LinePos] | [LinePos, LinePos] | undefined): TokenRange | undefined {
+  if (linePos === undefined) {
+    return;
+  }
+  // TokenRange and linePos are both 1-based
+  const start: Position = {line: linePos[0].line, column: linePos[0].col};
+  const end: Position = linePos.length == 2 ? {line: linePos[1].line, column: linePos[1].col} : start;
+  return {start, end};
+}