actions/languageservices
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into elbrenn/expression-complete

Browse Source
This commit is contained in:
Beth Brennan
2023-02-07 17:23:37 -05:00
committed by GitHub
parent b432867548 05605c80c7
commit 928e7e3b11
121 changed files with 3265 additions and 1204 deletions
Download Patch File Download Diff File Expand all files Collapse all files
actions-workflow-parser/src/expressions.test.ts
+31 -38
View File
@@ -6,19 +6,16 @@ import {parseWorkflow} from "./workflows/workflow-parser";
describe("Workflow Expression Parsing", () => {
it("preserves original expressions when building format", () => {
const result = parseWorkflow(
"test.yaml",
[
{
name: "test.yaml",
content: `on: push
{
name: "test.yaml",
content: `on: push
run-name: Test \${{ github.event_name }} \${{ github.ref }}
jobs:
build:
runs-on: ubuntu-latest
steps:
- run: echo 'hello'`
}
],
},
nullTrace
);
@@ -44,20 +41,17 @@ jobs:
it("preserves original expressions when building format for multi-line strings", () => {
const result = parseWorkflow(
"test.yaml",
[
{
name: "test.yaml",
content: `on: push
{
name: "test.yaml",
content: `on: push
jobs:
build:
runs-on: ubuntu-latest
steps:
- run: |
echo \${{ github.event_name }}
echo 'hello' \${{ github.ref }}`
}
],
echo 'hello' \${{github.ref }}`
},
nullTrace
);
@@ -81,19 +75,27 @@ jobs:
}
expect(stepRun.originalExpressions).toHaveLength(2);
expect(stepRun.originalExpressions!.map(x => [x.toDisplayString(), x.range])).toEqual([
expect(stepRun.originalExpressions!.map(x => [x.toDisplayString(), x.range, x.expressionRange])).toEqual([
[
"${{ github.event_name }}",
{
start: {line: 7, column: 16},
end: {line: 7, column: 40}
},
{
start: {line: 7, column: 20},
end: {line: 7, column: 37}
}
],
[
"${{ github.ref }}",
{
start: {line: 8, column: 24},
end: {line: 8, column: 41}
end: {line: 8, column: 40}
},
{
start: {line: 8, column: 27},
end: {line: 8, column: 37}
}
]
]);
@@ -101,11 +103,9 @@ jobs:
it("return errors and string token with preserved expressions for (multiple) expression errors", () => {
const result = parseWorkflow(
"test.yaml",
[
{
name: "test.yaml",
content: `on: push
{
name: "test.yaml",
content: `on: push
jobs:
build:
runs-on: ubuntu-latest
@@ -113,8 +113,7 @@ jobs:
- run: |
echo \${{ abc }}
echo 'hello' \${{ gith }}`
}
],
},
nullTrace
);
@@ -139,11 +138,9 @@ jobs:
it("reports all errors for multi-line expressions at the correct locations", () => {
const result = parseWorkflow(
"test.yaml",
[
{
name: "test.yaml",
content: `on: push
{
name: "test.yaml",
content: `on: push
jobs:
build:
runs-on: ubuntu-latest
@@ -151,8 +148,7 @@ jobs:
- run: |
echo \${{ fromJSON2('test') }}
echo 'hello' \${{ toJSON2(inputs.test) }}`
}
],
},
nullTrace
);
@@ -178,19 +174,16 @@ jobs:
it("parses isExpression strings into expression tokens", () => {
const result = parseWorkflow(
"test.yaml",
[
{
name: "test.yaml",
content: `on: push
{
name: "test.yaml",
content: `on: push
jobs:
build:
runs-on: ubuntu-latest
if: github.event_name == 'push'
steps:
- run: echo 'hello'`
}
],
},
nullTrace
);
actions-workflow-parser/src/index.test.ts
+17 -29
View File
@@ -5,13 +5,10 @@ import {parseWorkflow} from "./workflows/workflow-parser";
describe("parseWorkflow", () => {
it("parses valid workflow", () => {
const result = parseWorkflow(
"test.yaml",
[
{
name: "test.yaml",
content: "on: push\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - run: echo 'hello'"
}
],
{
name: "test.yaml",
content: "on: push\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - run: echo 'hello'"
},
nullTrace
);
@@ -20,13 +17,10 @@ describe("parseWorkflow", () => {
it("contains range for error", () => {
const result = parseWorkflow(
"test.yaml",
[
{
name: "test.yaml",
content: "on: push\njobs:\n build:\n steps:\n - run: echo 'hello'"
}
],
{
name: "test.yaml",
content: "on: push\njobs:\n build:\n steps:\n - run: echo 'hello'"
},
nullTrace
);
@@ -40,19 +34,16 @@ describe("parseWorkflow", () => {
it("error range for expression is constrained to scalar node", () => {
const result = parseWorkflow(
"test.yaml",
[
{
name: "test.yaml",
content: `on: push
{
name: "test.yaml",
content: `on: push
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: \${{ github.event = 12 }}
run: echo 'hello'`
}
],
},
nullTrace
);
@@ -71,14 +62,11 @@ jobs:
it("tokens contain descriptions", () => {
const result = parseWorkflow(
"test.yaml",
[
{
name: "test.yaml",
content:
"on: push\nname: hello\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - run: echo 'hello'"
}
],
{
name: "test.yaml",
content:
"on: push\nname: hello\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - run: echo 'hello'"
},
nullTrace
);
actions-workflow-parser/src/model/convert.test.ts
+36 -54
View File
@@ -7,22 +7,19 @@ function serializeTemplate(template: unknown): unknown {
}
describe("convertWorkflowTemplate", () => {
it("converts workflow with one job", () => {
it("converts workflow with one job", async () => {
const result = parseWorkflow(
"wf.yaml",
[
{
name: "wf.yaml",
content: `on: push
{
name: "wf.yaml",
content: `on: push
jobs:
build:
runs-on: ubuntu-latest`
}
],
},
nullTrace
);
const template = convertWorkflowTemplate(result.context, result.value!, ErrorPolicy.TryConversion);
const template = await convertWorkflowTemplate(result.context, result.value!, ErrorPolicy.TryConversion);
expect(serializeTemplate(template)).toEqual({
events: {
@@ -46,13 +43,11 @@ jobs:
});
});
it("converts workflow if expressions", () => {
it("converts workflow if expressions", async () => {
const result = parseWorkflow(
"wf.yaml",
[
{
name: "wf.yaml",
content: `on: push
{
name: "wf.yaml",
content: `on: push
jobs:
build:
if: \${{ true }}
@@ -60,12 +55,11 @@ jobs:
deploy:
if: true
runs-on: ubuntu-latest`
}
],
},
nullTrace
);
const template = convertWorkflowTemplate(result.context, result.value!, ErrorPolicy.TryConversion);
const template = await convertWorkflowTemplate(result.context, result.value!, ErrorPolicy.TryConversion);
expect(serializeTemplate(template)).toEqual({
events: {
@@ -98,23 +92,20 @@ jobs:
});
});
it("converts workflow with empty needs", () => {
it("converts workflow with empty needs", async () => {
const result = parseWorkflow(
"wf.yaml",
[
{
name: "wf.yaml",
content: `on: push
{
name: "wf.yaml",
content: `on: push
jobs:
build:
needs: # comment to preserve whitespace in test
runs-on: ubuntu-latest`
}
],
},
nullTrace
);
const template = convertWorkflowTemplate(result.context, result.value!, ErrorPolicy.TryConversion);
const template = await convertWorkflowTemplate(result.context, result.value!, ErrorPolicy.TryConversion);
expect(serializeTemplate(template)).toEqual({
errors: [
@@ -143,13 +134,11 @@ jobs:
});
});
it("converts workflow with needs errors", () => {
it("converts workflow with needs errors", async () => {
const result = parseWorkflow(
"wf.yaml",
[
{
name: "wf.yaml",
content: `on: push
{
name: "wf.yaml",
content: `on: push
jobs:
job1:
needs: [unknown-job, job3]
@@ -159,12 +148,11 @@ jobs:
job3:
needs: job1
runs-on: ubuntu-latest`
}
],
},
nullTrace
);
const template = convertWorkflowTemplate(result.context, result.value!, ErrorPolicy.TryConversion);
const template = await convertWorkflowTemplate(result.context, result.value!, ErrorPolicy.TryConversion);
expect(serializeTemplate(template)).toEqual({
errors: [
@@ -223,13 +211,11 @@ jobs:
});
});
it("converts workflow with invalid on", () => {
it("converts workflow with invalid on", async () => {
const result = parseWorkflow(
"wf.yaml",
[
{
name: "wf.yaml",
content: `on:
{
name: "wf.yaml",
content: `on:
workflow_dispatch:
inputs:
test:
@@ -239,12 +225,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- run: echo hello`
}
],
},
nullTrace
);
const template = convertWorkflowTemplate(result.context, result.value!, ErrorPolicy.TryConversion);
const template = await convertWorkflowTemplate(result.context, result.value!, ErrorPolicy.TryConversion);
expect(template.jobs).not.toBeUndefined();
expect(template.jobs).toHaveLength(1);
@@ -286,21 +271,18 @@ jobs:
});
});
it("converts workflow with invalid jobs", () => {
it("converts workflow with invalid jobs", async () => {
const result = parseWorkflow(
"wf.yaml",
[
{
name: "wf.yaml",
content: `on: push
{
name: "wf.yaml",
content: `on: push
jobs:
build:`
}
],
},
nullTrace
);
const template = convertWorkflowTemplate(result.context, result.value!, ErrorPolicy.TryConversion);
const template = await convertWorkflowTemplate(result.context, result.value!, ErrorPolicy.TryConversion);
expect(template.jobs).not.toBeUndefined();
expect(template.jobs).toHaveLength(0);
actions-workflow-parser/src/model/convert.ts
+76 -3
View File
@@ -1,9 +1,14 @@
import {TemplateContext} from "../templates/template-context";
import {TemplateToken, TemplateTokenError} from "../templates/tokens/template-token";
import {FileProvider} from "../workflows/file-provider";
import {parseFileReference} from "../workflows/file-reference";
import {parseWorkflow} from "../workflows/workflow-parser";
import {convertConcurrency} from "./converter/concurrency";
import {convertOn} from "./converter/events";
import {handleTemplateTokenErrors} from "./converter/handle-errors";
import {convertJobs} from "./converter/jobs";
import {convertReferencedWorkflow} from "./converter/referencedWorkflow";
import {isReusableWorkflowJob} from "./type-guards";
import {WorkflowTemplate} from "./workflow-template";
export enum ErrorPolicy {
@@ -11,11 +16,35 @@ export enum ErrorPolicy {
TryConversion
}
export function convertWorkflowTemplate(
export type WorkflowTemplateConverterOptions = {
/**
* The maximum depth of reusable workflows allowed in the workflow.
* If this depth is exceeded, an error will be reported.
* If {@link fetchReusableWorkflowDepth} is less than this value, the maximum depth
* won't be enforced.
* Default: 4
*/
maxReusableWorkflowDepth?: number;
/**
* The depth to fetch reusable workflows, up to {@link maxReusableWorkflowDepth}.
* Currently only a fetch depth of 0 or 1 is supported.
* Default: 0
*/
fetchReusableWorkflowDepth?: number;
};
const defaultOptions: Required<WorkflowTemplateConverterOptions> = {
maxReusableWorkflowDepth: 4,
fetchReusableWorkflowDepth: 0
};
export async function convertWorkflowTemplate(
context: TemplateContext,
root: TemplateToken,
errorPolicy: ErrorPolicy = ErrorPolicy.ReturnErrorsOnly
): WorkflowTemplate {
errorPolicy: ErrorPolicy = ErrorPolicy.ReturnErrorsOnly,
fileProvider?: FileProvider,
options: WorkflowTemplateConverterOptions = defaultOptions
): Promise<WorkflowTemplate> {
const result = {} as WorkflowTemplate;
if (context.errors.getErrors().length > 0 && errorPolicy === ErrorPolicy.ReturnErrorsOnly) {
@@ -25,6 +54,12 @@ export function convertWorkflowTemplate(
return result;
}
const opts = getOptionsWithDefaults(options);
if (fileProvider === undefined && opts.fetchReusableWorkflowDepth > 0) {
context.error(root, new Error("A file provider is required to fetch reusable workflows"));
}
try {
const rootMapping = root.assertMapping("root");
@@ -49,6 +84,31 @@ export function convertWorkflowTemplate(
break;
}
}
// Load referenced workflows
for (const job of result.jobs || []) {
if (isReusableWorkflowJob(job)) {
if (opts.maxReusableWorkflowDepth === 0) {
context.error(job.ref, new Error("Reusable workflows are not allowed"));
continue;
}
if (opts.fetchReusableWorkflowDepth === 0 || fileProvider === undefined) {
continue;
}
try {
const file = await fileProvider.getFileContent(parseFileReference(job.ref.value));
const workflow = parseWorkflow(file, context);
if (!workflow.value) {
continue;
}
convertReferencedWorkflow(context, workflow.value, job);
} catch {
context.error(job.ref, new Error("Unable to find reusable workflow"));
}
}
}
} catch (err) {
if (err instanceof TemplateTokenError) {
context.error(err.token, err);
@@ -66,3 +126,16 @@ export function convertWorkflowTemplate(
return result;
}
function getOptionsWithDefaults(options: WorkflowTemplateConverterOptions): Required<WorkflowTemplateConverterOptions> {
return {
maxReusableWorkflowDepth:
options.maxReusableWorkflowDepth !== undefined
? options.maxReusableWorkflowDepth
: defaultOptions.maxReusableWorkflowDepth,
fetchReusableWorkflowDepth:
options.fetchReusableWorkflowDepth !== undefined
? options.fetchReusableWorkflowDepth
: defaultOptions.fetchReusableWorkflowDepth
};
}
actions-workflow-parser/src/model/converter/job.ts
+145
View File
@@ -0,0 +1,145 @@
import {TemplateContext} from "../../templates/template-context";
import {StringToken, MappingToken, BasicExpressionToken, TemplateToken, ScalarToken} from "../../templates/tokens";
import {isSequence, isString} from "../../templates/tokens/type-guards";
import {WorkflowJob, Step} from "../workflow-template";
import {convertConcurrency} from "./concurrency";
import {convertToJobContainer, convertToJobServices} from "./container";
import {handleTemplateTokenErrors} from "./handle-errors";
import {IdBuilder} from "./id-builder";
import {convertToActionsEnvironmentRef} from "./job/environment";
import {convertRunsOn} from "./job/runs-on";
import {convertSteps} from "./steps";
export function convertJob(context: TemplateContext, jobKey: StringToken, token: MappingToken): WorkflowJob {
const error = new IdBuilder().tryAddKnownId(jobKey.value);
if (error) {
context.error(jobKey, error);
}
let concurrency, container, env, environment, name, outputs, runsOn, services, strategy: TemplateToken | undefined;
let needs: StringToken[] | undefined = undefined;
let steps: Step[] = [];
let workflowJobRef: StringToken | undefined;
let workflowJobInputs: MappingToken | undefined;
for (const item of token) {
const propertyName = item.key.assertString("job property name");
switch (propertyName.value) {
case "concurrency":
handleTemplateTokenErrors(item.value, context, undefined, () => convertConcurrency(context, item.value));
concurrency = item.value;
break;
case "container":
convertToJobContainer(context, item.value);
container = item.value;
break;
case "env":
env = item.value.assertMapping("job env");
break;
case "environment":
handleTemplateTokenErrors(item.value, context, undefined, () =>
convertToActionsEnvironmentRef(context, item.value)
);
environment = item.value;
break;
case "name":
name = item.value.assertScalar("job name");
break;
case "needs": {
needs = [];
if (isString(item.value)) {
const jobNeeds = item.value.assertString("job needs id");
needs.push(jobNeeds);
}
if (isSequence(item.value)) {
for (const seqItem of item.value) {
const jobNeeds = seqItem.assertString("job needs id");
needs.push(jobNeeds);
}
}
break;
}
case "outputs":
outputs = item.value.assertMapping("job outputs");
break;
case "runs-on":
handleTemplateTokenErrors(item.value, context, undefined, () => convertRunsOn(context, item.value));
runsOn = item.value;
break;
case "services":
convertToJobServices(context, item.value);
services = item.value;
break;
case "steps":
steps = convertSteps(context, item.value);
break;
case "strategy":
strategy = item.value;
break;
case "uses":
workflowJobRef = item.value.assertString("job uses value");
break;
case "with":
workflowJobInputs = item.value.assertMapping("uses-with value");
break;
}
}
if (workflowJobRef !== undefined) {
return {
type: "reusableWorkflowJob",
id: jobKey,
name: jobName(name, jobKey),
needs: needs ?? [],
if: new BasicExpressionToken(undefined, undefined, "success()", undefined, undefined, undefined),
ref: workflowJobRef,
"input-definitions": undefined,
"input-values": workflowJobInputs,
outputs: undefined,
concurrency,
strategy
};
} else {
return {
type: "job",
id: jobKey,
name: jobName(name, jobKey),
needs,
if: new BasicExpressionToken(undefined, undefined, "success()", undefined, undefined, undefined),
env,
concurrency,
environment,
strategy,
"runs-on": runsOn,
container,
services,
outputs,
steps
};
}
}
function jobName(name: ScalarToken | undefined, jobKey: StringToken): ScalarToken {
if (name === undefined) {
return jobKey;
}
if (isString(name) && name.value === "") {
return jobKey;
}
return name;
}
actions-workflow-parser/src/model/converter/job/inputs.ts
+54
View File
@@ -0,0 +1,54 @@
import {TemplateContext} from "../../../templates/template-context";
import {MappingToken, TemplateToken} from "../../../templates/tokens";
import {ReusableWorkflowJob} from "../../workflow-template";
type TokenMap = Map<string, [key: string, value: TemplateToken]>;
export function convertWorkflowJobInputs(context: TemplateContext, job: ReusableWorkflowJob) {
const inputDefinitions = createTokenMap(
job["input-definitions"]?.assertMapping("workflow job input definitions"),
"inputs"
);
const inputValues = createTokenMap(job["input-values"]?.assertMapping("workflow job input values"), "with");
if (inputDefinitions !== undefined) {
for (const [_, [name, value]] of inputDefinitions) {
const inputSpec = createTokenMap(value.assertMapping(`input ${name}`), `input ${name} key`)!;
const inputTypeToken = inputSpec.get("type")?.[1];
if (!inputTypeToken) {
// This should be validated by the template reader per the schema
continue;
}
const inputSet = inputValues !== undefined && inputValues.has(name.toLowerCase());
const required = inputSpec.get("required")?.[1].assertBoolean(`input ${name} required`).value;
if (required && !inputSet) {
context.error(job.ref, `Input ${name} is required, but not provided while calling.`);
}
}
}
if (inputValues !== undefined) {
for (const [_, [name, value]] of inputValues) {
if (!inputDefinitions?.has(name.toLowerCase())) {
context.error(value, `Invalid input, ${name} is not defined in the referenced workflow.`);
}
}
}
}
function createTokenMap(mapping: MappingToken | undefined, description: string): TokenMap | undefined {
if (!mapping) {
return undefined;
}
const result = new Map<string, [key: string, value: TemplateToken]>();
for (const item of mapping) {
const name = item.key.assertString(`${description} key`);
result.set(name.value.toLowerCase(), [name.value, item.value]);
}
return result;
}
actions-workflow-parser/src/model/converter/job/runs-on.ts
+102
View File
@@ -0,0 +1,102 @@
import {TemplateContext} from "../../../templates/template-context";
import {TemplateToken} from "../../../templates/tokens";
import {isMapping, isString, isSequence} from "../../../templates/tokens/type-guards";
type RunsOn = {
labels: Set<string>;
group: string;
};
export function convertRunsOn(context: TemplateContext, token: TemplateToken): RunsOn {
const labels = convertRunsOnLabels(token);
if (!isMapping(token)) {
return {
labels,
group: ""
};
}
let group = "";
for (const item of token) {
const key = item.key.assertString("job runs-on property name");
switch (key.value) {
case "group": {
if (item.value.isExpression) {
continue;
}
const groupName = item.value.assertString("job runs-on group name").value;
const names = groupName.split("/");
switch (names.length) {
case 1: {
group = groupName;
break;
}
case 2: {
if (!["org", "organization", "ent", "enterprise"].includes(names[0])) {
context.error(
item.value,
`Invalid runs-on group name '${groupName}. Please use 'organization/' or 'enterprise/' prefix to target a single runner group.'`
);
continue;
}
if (!names[1]) {
context.error(item.value, `Invalid runs-on group name '${groupName}'.`);
continue;
}
group = groupName;
break;
}
default: {
context.error(
item.value,
`Invalid runs-on group name '${groupName}. Please use 'organization/' or 'enterprise/' prefix to target a single runner group.'`
);
break;
}
}
break;
}
case "labels": {
const mapLabels = convertRunsOnLabels(item.value);
for (const label of mapLabels) {
labels.add(label);
}
break;
}
}
}
return {
labels,
group
};
}
function convertRunsOnLabels(token: TemplateToken): Set<string> {
const labels = new Set<string>();
if (token.isExpression) {
return labels;
}
if (isString(token)) {
labels.add(token.value);
return labels;
}
if (isSequence(token)) {
for (const item of token) {
if (item.isExpression) {
continue;
}
const label = item.assertString("job runs-on label sequence item");
labels.add(label.value);
}
}
return labels;
}
actions-workflow-parser/src/model/converter/jobs.ts
+7 -207
View File
@@ -1,23 +1,19 @@
import {TemplateContext} from "../../templates/template-context";
import {BasicExpressionToken, MappingToken, StringToken} from "../../templates/tokens";
import {StringToken} from "../../templates/tokens";
import {TemplateToken} from "../../templates/tokens/template-token";
import {isMapping, isSequence, isString} from "../../templates/tokens/type-guards";
import {Job} from "../workflow-template";
import {convertConcurrency} from "./concurrency";
import {convertToJobContainer, convertToJobServices} from "./container";
import {isMapping} from "../../templates/tokens/type-guards";
import {WorkflowJob} from "../workflow-template";
import {handleTemplateTokenErrors} from "./handle-errors";
import {IdBuilder} from "./id-builder";
import {convertToActionsEnvironmentRef} from "./job/environment";
import {convertSteps} from "./steps";
import {convertJob} from "./job";
type nodeInfo = {
name: string;
needs: StringToken[];
};
export function convertJobs(context: TemplateContext, token: TemplateToken): Job[] {
export function convertJobs(context: TemplateContext, token: TemplateToken): WorkflowJob[] {
if (isMapping(token)) {
const result: Job[] = [];
const result: WorkflowJob[] = [];
const jobsWithSatisfiedNeeds: nodeInfo[] = [];
const alljobsWithUnsatisfiedNeeds: nodeInfo[] = [];
@@ -53,7 +49,7 @@ export function convertJobs(context: TemplateContext, token: TemplateToken): Job
function validateNeeds(
token: TemplateToken,
context: TemplateContext,
result: Job[],
result: WorkflowJob[],
jobsWithSatisfiedNeeds: nodeInfo[],
alljobsWithUnsatisfiedNeeds: nodeInfo[]
) {
@@ -101,199 +97,3 @@ function validateNeeds(
}
}
function convertJob(context: TemplateContext, jobKey: StringToken, token: MappingToken): Job {
const error = new IdBuilder().tryAddKnownId(jobKey.value);
if (error) {
context.error(jobKey, error);
}
const result: Job = {
type: "job",
id: jobKey,
name: undefined,
needs: undefined,
if: new BasicExpressionToken(undefined, undefined, "success()", undefined, undefined, "success()"),
env: undefined,
concurrency: undefined,
environment: undefined,
strategy: undefined,
"runs-on": undefined,
container: undefined,
services: undefined,
outputs: undefined,
steps: []
};
for (const item of token) {
const propertyName = item.key.assertString("job property name");
switch (propertyName.value) {
case "concurrency":
handleTemplateTokenErrors(item.value, context, undefined, () => convertConcurrency(context, item.value));
result.concurrency = item.value;
break;
case "container":
// Do early validation, but don't convert
convertToJobContainer(context, item.value);
result.container = item.value;
break;
case "env":
result.env = item.value.assertMapping("job env");
break;
case "environment":
handleTemplateTokenErrors(item.value, context, undefined, () =>
convertToActionsEnvironmentRef(context, item.value)
);
result.environment = item.value;
break;
case "name":
result.name = item.value.assertScalar("job name");
break;
case "needs":
result.needs = [];
if (isString(item.value)) {
const jobNeeds = item.value.assertString("job needs id");
result.needs.push(jobNeeds);
}
if (isSequence(item.value)) {
for (const seqItem of item.value) {
const jobNeeds = seqItem.assertString("job needs id");
result.needs.push(jobNeeds);
}
}
break;
case "outputs":
result.outputs = item.value.assertMapping("job outputs");
break;
case "runs-on":
handleTemplateTokenErrors(item.value, context, undefined, () => convertRunsOn(context, item.value));
result["runs-on"] = item.value;
break;
case "services":
// Do early validation, but don't convert
convertToJobServices(context, item.value);
result.services = item.value;
break;
case "steps":
result.steps = convertSteps(context, item.value);
break;
case "strategy":
result.strategy = item.value;
break;
}
}
if (!result.name) {
result.name = result.id;
}
return result;
}
type RunsOn = {
labels: Set<string>;
group: string;
};
function convertRunsOn(context: TemplateContext, token: TemplateToken): RunsOn {
const labels = convertRunsOnLabels(token);
if (!isMapping(token)) {
return {
labels,
group: ""
};
}
let group = "";
for (const item of token) {
const key = item.key.assertString("job runs-on property name");
switch (key.value) {
case "group": {
if (item.value.isExpression) {
continue;
}
const groupName = item.value.assertString("job runs-on group name").value;
const names = groupName.split("/");
switch (names.length) {
case 1: {
group = groupName;
break;
}
case 2: {
if (!["org", "organization", "ent", "enterprise"].includes(names[0])) {
context.error(
item.value,
`Invalid runs-on group name '${groupName}. Please use 'organization/' or 'enterprise/' prefix to target a single runner group.'`
);
continue;
}
if (!names[1]) {
context.error(item.value, `Invalid runs-on group name '${groupName}'.`);
continue;
}
group = groupName;
break;
}
default: {
context.error(
item.value,
`Invalid runs-on group name '${groupName}. Please use 'organization/' or 'enterprise/' prefix to target a single runner group.'`
);
break;
}
}
break;
}
case "labels": {
const mapLabels = convertRunsOnLabels(item.value);
for (const label of mapLabels) {
labels.add(label);
}
break;
}
}
}
return {
labels,
group
};
}
function convertRunsOnLabels(token: TemplateToken): Set<string> {
const labels = new Set<string>();
if (token.isExpression) {
return labels;
}
if (isString(token)) {
labels.add(token.value);
return labels;
}
if (isSequence(token)) {
for (const item of token) {
if (item.isExpression) {
continue;
}
const label = item.assertString("job runs-on label sequence item");
labels.add(label.value);
}
}
return labels;
}
actions-workflow-parser/src/model/converter/referencedWorkflow.ts
+97
View File
@@ -0,0 +1,97 @@
import {TemplateContext} from "../../templates/template-context";
import {TemplateToken} from "../../templates/tokens";
import {TokenType} from "../../templates/tokens/types";
import {ReusableWorkflowJob} from "../workflow-template";
import {handleTemplateTokenErrors} from "./handle-errors";
import {convertWorkflowJobInputs} from "./job/inputs";
import {convertJobs} from "./jobs";
export function convertReferencedWorkflow(
context: TemplateContext,
referencedWorkflow: TemplateToken,
job: ReusableWorkflowJob
) {
const mapping = referencedWorkflow.assertMapping("root");
// The language service doesn't currently handles on other documents,
// So use the ref in the original workflow as the error location
const tokenForErrors = job.ref;
for (const pair of mapping) {
const key = pair.key.assertString("root key");
switch (key.value) {
case "on": {
handleTemplateTokenErrors(tokenForErrors, context, undefined, () =>
convertReferencedWorkflowOn(context, pair.value, job)
);
break;
}
case "jobs": {
job.jobs = handleTemplateTokenErrors(tokenForErrors, context, [], () => convertJobs(context, pair.value));
break;
}
}
}
}
function convertReferencedWorkflowOn(context: TemplateContext, on: TemplateToken, job: ReusableWorkflowJob) {
const tokenForErrors = job.ref;
switch (on.templateTokenType) {
case TokenType.String: {
const event = on.assertString("Reference workflow on value").value;
if (event === "workflow_call") {
handleTemplateTokenErrors(tokenForErrors, context, undefined, () => convertWorkflowJobInputs(context, job));
return;
}
break;
}
case TokenType.Sequence: {
const events = on.assertSequence("Reference workflow on value");
for (const eventToken of events) {
const event = eventToken.assertString(`Reference workflow on value ${eventToken}`).value;
if (event === "workflow_call") {
handleTemplateTokenErrors(tokenForErrors, context, undefined, () => convertWorkflowJobInputs(context, job));
return;
}
}
break;
}
case TokenType.Mapping: {
const eventMapping = on.assertMapping("Reference workflow on value");
for (const pair of eventMapping) {
const event = pair.key.assertString(`Reference workflow on value ${pair.key}`).value;
if (event !== "workflow_call") {
continue;
}
if (pair.value.templateTokenType === TokenType.Null) {
handleTemplateTokenErrors(tokenForErrors, context, undefined, () => convertWorkflowJobInputs(context, job));
return;
}
const definitions = pair.value.assertMapping(`Reference workflow on value ${pair.key}`);
for (const definition of definitions) {
const definitionKey = definition.key.assertString(`on-workflow_call-${definition.key}`).value;
switch (definitionKey) {
case "inputs":
job["input-definitions"] = definition.value.assertMapping(`on-workflow_call-${definition.key}`);
break;
case "outputs":
job.outputs = definition.value.assertMapping(`on-workflow_call-${definition.key}`);
break;
}
}
handleTemplateTokenErrors(tokenForErrors, context, undefined, () => convertWorkflowJobInputs(context, job));
return;
}
break;
}
}
context.error(tokenForErrors, "workflow_call key is not defined in the referenced workflow.");
}
actions-workflow-parser/src/model/converter/steps.ts
+1 -1
View File
@@ -52,7 +52,7 @@ function convertStep(context: TemplateContext, idBuilder: IdBuilder, step: Templ
let uses: StringToken | undefined;
let continueOnError: boolean | undefined;
let env: MappingToken | undefined;
const ifCondition = new BasicExpressionToken(undefined, undefined, "success()", undefined, undefined, "success()");
const ifCondition = new BasicExpressionToken(undefined, undefined, "success()", undefined, undefined, undefined);
for (const item of mapping) {
const key = item.key.assertString("steps item key");
switch (key.value) {
actions-workflow-parser/src/model/type-guards.ts
+9 -1
View File
@@ -1,4 +1,4 @@
import {ActionStep, RunStep, Step} from "./workflow-template";
import {ActionStep, Job, ReusableWorkflowJob, RunStep, Step, WorkflowJob} from "./workflow-template";
export function isRunStep(step: Step): step is RunStep {
return (step as RunStep).run !== undefined;
@@ -7,3 +7,11 @@ export function isRunStep(step: Step): step is RunStep {
export function isActionStep(step: Step): step is ActionStep {
return (step as ActionStep).uses !== undefined;
}
export function isJob(job: WorkflowJob): job is Job {
return job.type === "job";
}
export function isReusableWorkflowJob(job: WorkflowJob): job is ReusableWorkflowJob {
return job.type === "reusableWorkflowJob";
}
actions-workflow-parser/src/model/workflow-template.ts
+24 -6
View File
@@ -9,7 +9,7 @@ import {
export type WorkflowTemplate = {
events: EventsConfig;
jobs: Job[];
jobs: WorkflowJob[];
concurrency: TemplateToken;
env: TemplateToken;
@@ -28,23 +28,41 @@ export type ActionsEnvironmentReference = {
url?: TemplateToken;
};
export type Job = {
type: string;
export type WorkflowJob = Job | ReusableWorkflowJob;
export type JobType = "job" | "reusableWorkflowJob";
export type BaseJob = {
type: JobType;
id: StringToken;
name?: ScalarToken;
needs?: StringToken[];
if: BasicExpressionToken;
env?: MappingToken;
concurrency?: TemplateToken;
environment?: TemplateToken;
strategy?: TemplateToken;
outputs?: MappingToken;
};
// `job-factory` in the schema
export type Job = BaseJob & {
type: "job";
env?: MappingToken;
environment?: TemplateToken;
"runs-on"?: TemplateToken;
container?: TemplateToken;
services?: TemplateToken;
outputs?: MappingToken;
steps: Step[];
};
// `workflow-job` in the schema
export type ReusableWorkflowJob = BaseJob & {
type: "reusableWorkflowJob";
ref: StringToken;
"input-definitions"?: MappingToken;
"input-values"?: MappingToken;
jobs?: WorkflowJob[];
};
export type Container = {
image: StringToken;
credentials?: Credential;
actions-workflow-parser/src/templates/template-reader.ts
+27 -4
View File
@@ -616,10 +616,10 @@ class TemplateReader {
tr: TokenRange,
rawExpression: string,
allowedContext: string[],
token: TemplateToken,
token: StringToken,
definitionInfo: DefinitionInfo | undefined
): ExpressionToken | undefined {
const parseExpressionResult = this.parseExpression(tr, rawExpression, allowedContext, definitionInfo);
const parseExpressionResult = this.parseExpression(tr, token, rawExpression, allowedContext, definitionInfo);
// Check for error
if (parseExpressionResult.error) {
@@ -631,7 +631,8 @@ class TemplateReader {
}
private parseExpression(
range: TokenRange | undefined,
range: TokenRange,
token: StringToken,
value: string,
allowedContext: string[],
definitionInfo: DefinitionInfo | undefined
@@ -666,9 +667,31 @@ class TemplateReader {
};
}
const startTrim = value.length - value.trimStart().length;
const endTrim = value.length - value.trimEnd().length;
const expressionRange: TokenRange = {
start: {
...range.start,
column: range.start.column + OPEN_EXPRESSION.length + startTrim
},
end: {
...range.end,
column: range.end.column - CLOSE_EXPRESSION.length - endTrim
}
};
// Return the expression
return <ParseExpressionResult>{
expression: new BasicExpressionToken(this._fileId, range, trimmed, definitionInfo, undefined, value),
expression: new BasicExpressionToken(
this._fileId,
range,
trimmed,
definitionInfo,
undefined,
token.source,
expressionRange
),
error: undefined
};
}
actions-workflow-parser/src/templates/tokens/basic-expression-token.ts
+18 -4
View File
@@ -11,10 +11,20 @@ import {TokenType} from "./types";
export class BasicExpressionToken extends ExpressionToken {
private readonly expr: string;
public readonly source: string | undefined;
public readonly originalExpressions: BasicExpressionToken[] | undefined;
public readonly source: string;
/**
* The range of the expression within the source string.
*
* `range` is the range of the entire expression, including the `${{` and `}}`. `expression` is only the expression
* without any ${{ }} markers. `expressionRange` is the range of just the expression within the document.
*/
public readonly expressionRange: TokenRange | undefined;
/**
* @param originalExpressions If the basic expression was transformed from individual expressions, these will be the original ones
*/
@@ -24,12 +34,14 @@ export class BasicExpressionToken extends ExpressionToken {
expression: string,
definitionInfo: DefinitionInfo | undefined,
originalExpressions: BasicExpressionToken[] | undefined,
source: string
source: string | undefined,
expressionRange?: TokenRange | undefined
) {
super(TokenType.BasicExpression, file, range, undefined, definitionInfo);
this.expr = expression;
this.originalExpressions = originalExpressions;
this.source = source;
this.originalExpressions = originalExpressions;
this.expressionRange = expressionRange;
}
public get expression(): string {
@@ -44,7 +56,8 @@ export class BasicExpressionToken extends ExpressionToken {
this.expr,
this.definitionInfo,
this.originalExpressions,
this.source
this.source,
this.expressionRange
)
: new BasicExpressionToken(
this.file,
@@ -52,7 +65,8 @@ export class BasicExpressionToken extends ExpressionToken {
this.expr,
this.definitionInfo,
this.originalExpressions,
this.source
this.source,
this.expressionRange
);
}
actions-workflow-parser/src/templates/tokens/template-token.test.ts
+4 -7
View File
@@ -6,13 +6,10 @@ import {TemplateToken} from "./template-token";
describe("traverse", () => {
it("returns parent token and key", () => {
const workflow = parseWorkflow(
"wf.yaml",
[
{
name: "wf.yaml",
content: `on: push`
}
],
{
name: "wf.yaml",
content: `on: push`
},
nullTrace
);
actions-workflow-parser/src/templates/tokens/token-range.ts
+1
View File
@@ -2,6 +2,7 @@
export type Position = {
/** The one-based line value */
line: number;
/** The one-based column value */
column: number;
};
actions-workflow-parser/src/workflows/file-provider.ts
+6
View File
@@ -0,0 +1,6 @@
import {File} from "./file";
import {FileReference} from "./file-reference";
export interface FileProvider {
getFileContent(ref: FileReference): Promise<File>;
}
actions-workflow-parser/src/workflows/file-reference.test.ts
+55
View File
@@ -0,0 +1,55 @@
import {parseFileReference} from "./file-reference";
describe("parseFileReference", () => {
it("parses local file reference", () => {
const ref = parseFileReference("./workflow/path");
expect(ref).toEqual({
path: "workflow/path"
});
});
it("parses local file references with an empty path", () => {
const ref = parseFileReference("./");
expect(ref).toEqual({
path: ""
});
});
it("parses remote file reference", () => {
const ref = parseFileReference("owner/repo/path@version");
expect(ref).toEqual({
owner: "owner",
repository: "repo",
path: "path",
version: "version"
});
});
it("parses remote file reference with an empty path", () => {
const ref = parseFileReference("owner/repo@version");
expect(ref).toEqual({
owner: "owner",
repository: "repo",
path: "",
version: "version"
});
});
it("parses remote file reference with slashes in the version", () => {
const ref = parseFileReference("owner/repo@feature-branch/dev");
expect(ref).toEqual({
owner: "owner",
repository: "repo",
path: "",
version: "feature-branch/dev"
});
});
it("throws for malformed remote file references", () => {
expect(() => parseFileReference("owner/repo/path")).toThrowError("Invalid file reference: owner/repo/path");
expect(() => parseFileReference("owner/repo/path@")).toThrowError("Invalid file reference: owner/repo/path@");
expect(() => parseFileReference("owner@")).toThrowError("Invalid file reference: owner@");
});
});
actions-workflow-parser/src/workflows/file-reference.ts
+42
View File
@@ -0,0 +1,42 @@
export type FileReference = LocalFileReference | RemoteFileReference;
export type LocalFileReference = {
path: string;
};
export type RemoteFileReference = {
repository: string;
owner: string;
path: string;
version: string;
};
export function parseFileReference(ref: string): FileReference {
if (ref.startsWith("./")) {
return {
path: ref.substring(2)
};
}
const [remotePath, version] = ref.split("@");
const [owner, repository, ...pathSegments] = remotePath.split("/").filter(s => s.length > 0);
if (!owner || !repository || !version) {
throw new Error(`Invalid file reference: ${ref}`);
}
return {
repository,
owner,
path: pathSegments.join("/"),
version
};
}
export function fileIdentifier(ref: FileReference): string {
if (!("repository" in ref)) {
return "./" + ref.path;
}
return `${ref.owner}/${ref.repository}/${ref.path}@${ref.version}`;
}
actions-workflow-parser/src/workflows/workflow-parser.test.ts
+1 -1
View File
@@ -11,7 +11,7 @@ jobs:
- name: 'Hello \${{ fromJSON('test') == inputs.name }}'
run: echo Hello, world!`;
const result = parseWorkflow("main.yaml", [{name: "main.yaml", content: content}], nullTrace);
const result = parseWorkflow({name: "main.yaml", content: content}, nullTrace);
expect(result.context.errors.count).toBe(1);
expect(result.value).toBeUndefined();
actions-workflow-parser/src/workflows/workflow-parser.ts
+9 -8
View File
@@ -11,15 +11,16 @@ export interface ParseWorkflowResult {
value: TemplateToken | undefined;
}
export function parseWorkflow(entryFileName: string, files: File[], trace: TraceWriter): ParseWorkflowResult {
const context = new TemplateContext(new TemplateValidationErrors(), getWorkflowSchema(), trace);
export function parseWorkflow(entryFile: File, trace: TraceWriter): ParseWorkflowResult;
export function parseWorkflow(entryFile: File, context: TemplateContext): ParseWorkflowResult;
export function parseWorkflow(entryFile: File, contextOrTrace: TraceWriter | TemplateContext): ParseWorkflowResult {
const context =
contextOrTrace instanceof TemplateContext
? contextOrTrace
: new TemplateContext(new TemplateValidationErrors(), getWorkflowSchema(), contextOrTrace);
// Add file ids
files.forEach(x => context.getFileId(x.name));
const fileId = context.getFileId(entryFileName);
const fileContent = files[fileId - 1].content;
const reader = new YamlObjectReader(context, fileId, fileContent);
const fileId = context.getFileId(entryFile.name);
const reader = new YamlObjectReader(context, fileId, entryFile.content);
if (context.errors.count > 0) {
// The file is not valid YAML, template errors could be misleading
return {
actions-workflow-parser/src/workflows/yaml-object-reader.test.ts
+4 -7
View File
@@ -81,13 +81,10 @@ it("YAML errors include range information", () => {
function parseAsWorkflow(content: string): TemplateToken | undefined {
const result = parseWorkflow(
"test.yaml",
[
{
name: "test.yaml",
content: content
}
],
{
name: "test.yaml",
content: content
},
nullTrace
);
actions-workflow-parser/src/xlang.test.ts
+41 -16
View File
@@ -3,6 +3,9 @@ import * as path from "path";
import * as YAML from "yaml";
import {convertWorkflowTemplate} from "./model/convert";
import {TraceWriter} from "./templates/trace-writer";
import {File} from "./workflows/file";
import {FileProvider} from "./workflows/file-provider";
import {fileIdentifier, FileReference} from "./workflows/file-reference";
import {parseWorkflow} from "./workflows/workflow-parser";
interface TestOptions {
@@ -41,31 +44,53 @@ describe("x-lang tests", () => {
const testOptions: TestOptions = YAML.parse(testDocs[0]);
const unsupportedTest = contains(testOptions.skip, "TypeScript");
const test = () => {
let testFileName = ".github/workflows" + fileName.substring(fileName.lastIndexOf("/"));
let testInput = testDocs[1];
let expectedTemplate = testDocs[2].trim();
// TODO: when reusable workflows are implemented, implement correctly
const test = async () => {
const testFileName = ".github/workflows" + fileName.substring(fileName.lastIndexOf("/"));
const testInput = testDocs[1];
const expectedTemplate = testDocs[testDocs.length - 1].trim();
// For reusable workflow tests, additional workflows are passed in as pairs of
// file names and file contents
const reusableWorkflows: Record<string, File> = {};
if (fileName.indexOf("reusable") !== -1) {
testFileName = testDocs[1];
testInput = testDocs[2];
expectedTemplate = testDocs[3].trim();
for (let i = 2; i < testDocs.length - 1; i = i + 2) {
reusableWorkflows[testDocs[i]] = {
name: testDocs[i],
content: testDocs[i + 1]
};
}
}
const parseResult = parseWorkflow(
testFileName,
[
{
name: testFileName,
content: testInput
const testFileProvider: FileProvider = {
getFileContent: async (ref: FileReference) => {
const file = reusableWorkflows[fileIdentifier(ref)];
if (file) {
return file;
}
],
throw new Error("File not found: " + fileName);
}
};
const parseResult = parseWorkflow(
{
name: testFileName,
content: testInput
},
nullTrace
);
expect(parseResult.value).not.toBeUndefined();
const workflowTemplate = convertWorkflowTemplate(parseResult.context, parseResult.value!);
const workflowTemplate = await convertWorkflowTemplate(
parseResult.context,
parseResult.value!,
undefined,
testFileProvider,
{
fetchReusableWorkflowDepth: 1
}
);
// Unless this tests is only used by TypeScript, remove the events for now.
// TODO: Remove this once we parse events everywhere