Merge pull request #172 from github/thyeggman/cache-workflow-results

Cache parsed workflow and template results

languageserver/src/connection.ts

@@ -28,6 +28,7 @@ import {fetchActionMetadata} from "./utils/action-metadata";
 import {TTLCache} from "./utils/cache";
 import {timeOperation} from "./utils/timer";
 import {valueProviders} from "./value-providers";
+import {clearCacheEntry, clearCache} from "@github/actions-languageservice/utils/workflow-cache";
 
 export function initConnection(connection: Connection) {
   const documents: TextDocuments<TextDocument> = new TextDocuments(TextDocument);
@@ -91,9 +92,18 @@ export function initConnection(connection: Connection) {
     return result;
   });
 
+  connection.onInitialized(() => {
+    if (hasWorkspaceFolderCapability) {
+      connection.workspace.onDidChangeWorkspaceFolders(_event => {
+        clearCache();
+      });
+    }
+  });
+
   // The content of a text document has changed. This event is emitted
   // when the text document first opened or when its content has changed.
   documents.onDidChangeContent(change => {
+    clearCacheEntry(change.document.uri);
     return timeOperation("validation", async () => await validateTextDocument(change.document));
   });
 

languageservice/src/complete.expressions.test.ts

@@ -6,6 +6,7 @@ import {registerLogger} from "./log";
 import {getPositionFromCursor} from "./test-utils/cursor-position";
 import {TestLogger} from "./test-utils/logger";
 import {testFileProvider} from "./test-utils/test-file-provider";
+import {clearCache} from "./utils/workflow-cache";
 
 const contextProviderConfig: ContextProviderConfig = {
   getContext: async (context: string) => {
@@ -24,6 +25,10 @@ const contextProviderConfig: ContextProviderConfig = {
 
 registerLogger(new TestLogger());
 
+beforeEach(() => {
+  clearCache();
+});
+
 describe("expressions", () => {
   it("input extraction", () => {
     const test = (input: string) => {

languageservice/src/complete.reusable-workflows.test.ts

@@ -2,6 +2,7 @@ import {CompletionItem, MarkupContent} from "vscode-languageserver-types";
 import {complete} from "./complete";
 import {getPositionFromCursor} from "./test-utils/cursor-position";
 import {testFileProvider} from "./test-utils/test-file-provider";
+import {clearCache} from "./utils/workflow-cache";
 
 function mapResult(result: CompletionItem[]) {
   return result.map(x => {
@@ -9,6 +10,10 @@ function mapResult(result: CompletionItem[]) {
   });
 }
 
+beforeEach(() => {
+  clearCache();
+});
+
 describe("completion with reusable workflows", () => {
   it("completes job inputs", async () => {
     const input = `

languageservice/src/complete.test.ts

@@ -5,9 +5,14 @@ import {registerLogger} from "./log";
 import {getPositionFromCursor} from "./test-utils/cursor-position";
 import {TestLogger} from "./test-utils/logger";
 import {ValueProviderConfig, ValueProviderKind} from "./value-providers/config";
+import {clearCache} from "./utils/workflow-cache";
 
 registerLogger(new TestLogger());
 
+beforeEach(() => {
+  clearCache();
+});
+
 describe("completion", () => {
   it("runs-on", async () => {
     const input = "on: push\njobs:\n  build:\n    runs-on: |";

languageservice/src/complete.ts

@@ -1,12 +1,6 @@
 import {complete as completeExpression, DescriptionDictionary} from "@github/actions-expressions";
 import {CompletionItem as ExpressionCompletionItem} from "@github/actions-expressions/completion";
-import {
-  convertWorkflowTemplate,
-  isBasicExpression,
-  isSequence,
-  isString,
-  parseWorkflow
-} from "@github/actions-workflow-parser";
+import {isBasicExpression, isSequence, isString} from "@github/actions-workflow-parser";
 import {ErrorPolicy} from "@github/actions-workflow-parser/model/convert";
 import {OPEN_EXPRESSION} from "@github/actions-workflow-parser/templates/template-constants";
 import {TemplateToken} from "@github/actions-workflow-parser/templates/tokens/index";
@@ -21,7 +15,6 @@ import {getContext, Mode} from "./context-providers/default";
 import {getWorkflowContext, WorkflowContext} from "./context/workflow-context";
 import {validatorFunctions} from "./expression-validation/functions";
 import {error} from "./log";
-import {nullTrace} from "./nulltrace";
 import {isPotentiallyExpression} from "./utils/expression-detection";
 import {findToken} from "./utils/find-token";
 import {guessIndentation} from "./utils/indentation-guesser";
@@ -31,6 +24,7 @@ import {isPlaceholder, transform} from "./utils/transform";
 import {Value, ValueProviderConfig} from "./value-providers/config";
 import {defaultValueProviders} from "./value-providers/default";
 import {definitionValues} from "./value-providers/definition";
+import {fetchOrParseWorkflow, fetchOrConvertWorkflowTemplate} from "./utils/workflow-cache";
 
 export function getExpressionInput(input: string, pos: number): string {
   // Find start marker around the cursor position
@@ -66,21 +60,22 @@ export async function complete(
 
   // Fix the input to work around YAML parsing issues
   const [newDoc, newPos] = transform(textDocument, position);
-
   const file: File = {
     name: textDocument.uri,
     content: newDoc.getText()
   };
-  const result = parseWorkflow(file, nullTrace);
-  if (!result.value) {
+
+  const parsedWorkflow = fetchOrParseWorkflow(file, textDocument.uri);
+  if (!parsedWorkflow) {
     return [];
   }
 
-  const {token, keyToken, parent, path} = findToken(newPos, result.value);
-  const template = await convertWorkflowTemplate(result.context, result.value, config?.fileProvider, {
+  const template = await fetchOrConvertWorkflowTemplate(parsedWorkflow, textDocument.uri, config, {
     fetchReusableWorkflowDepth: config?.fileProvider ? 1 : 0,
     errorPolicy: ErrorPolicy.TryConversion
   });
+
+  const {token, keyToken, parent, path} = findToken(newPos, parsedWorkflow.value);
   const workflowContext = getWorkflowContext(textDocument.uri, template, path);
 
   // If we are inside an expression, take a different code-path. The workflow parser does not correctly create

languageservice/src/document-links.test.ts

@@ -1,5 +1,10 @@
 import {documentLinks} from "./document-links";
 import {createDocument} from "./test-utils/document";
+import {clearCache} from "./utils/workflow-cache";
+
+beforeEach(() => {
+  clearCache();
+});
 
 describe("documentLinks", () => {
   it("no links without actions", async () => {

languageservice/src/document-links.ts

@@ -1,12 +1,11 @@
-import {convertWorkflowTemplate, parseWorkflow} from "@github/actions-workflow-parser";
 import {ErrorPolicy} from "@github/actions-workflow-parser/model/convert";
 import {isJob} from "@github/actions-workflow-parser/model/type-guards";
 import {File} from "@github/actions-workflow-parser/workflows/file";
 import {TextDocument} from "vscode-languageserver-textdocument";
 import {DocumentLink} from "vscode-languageserver-types";
 import {actionUrl, parseActionReference} from "./action";
-import {nullTrace} from "./nulltrace";
 import {mapRange} from "./utils/range";
+import {fetchOrParseWorkflow, fetchOrConvertWorkflowTemplate} from "./utils/workflow-cache";
 
 export async function documentLinks(document: TextDocument): Promise<DocumentLink[]> {
   const file: File = {
@@ -14,12 +13,12 @@ export async function documentLinks(document: TextDocument): Promise<DocumentLin
     content: document.getText()
   };
 
-  const result = parseWorkflow(file, nullTrace);
-  if (!result.value) {
+  const parsedWorkflow = fetchOrParseWorkflow(file, document.uri);
+  if (!parsedWorkflow) {
     return [];
   }
 
-  const template = await convertWorkflowTemplate(result.context, result.value!, undefined, {
+  const template = await fetchOrConvertWorkflowTemplate(parsedWorkflow, document.uri, undefined, {
     errorPolicy: ErrorPolicy.TryConversion
   });
 

languageservice/src/hover.expressions.test.ts

@@ -5,6 +5,7 @@ import {hover} from "./hover";
 import {registerLogger} from "./log";
 import {getPositionFromCursor} from "./test-utils/cursor-position";
 import {TestLogger} from "./test-utils/logger";
+import {clearCache} from "./utils/workflow-cache";
 
 const contextProviderConfig: ContextProviderConfig = {
   getContext: async (context: string) => {
@@ -34,6 +35,10 @@ const contextProviderConfig: ContextProviderConfig = {
 
 registerLogger(new TestLogger());
 
+beforeEach(() => {
+  clearCache();
+});
+
 describe("hover.expressions", () => {
   it("context access", async () => {
     const input = `on: push

languageservice/src/hover.reusable-workflow.test.ts

@@ -1,6 +1,11 @@
 import {hover} from "./hover";
 import {testHoverConfig} from "./hover.test";
 import {getPositionFromCursor} from "./test-utils/cursor-position";
+import {clearCache} from "./utils/workflow-cache";
+
+beforeEach(() => {
+  clearCache();
+});
 
 describe("hover.reusable-workflow", () => {
   it("hover on job input with description", async () => {

languageservice/src/hover.test.ts

@@ -3,6 +3,7 @@ import {StringToken} from "@github/actions-workflow-parser/templates/tokens/stri
 import {DescriptionProvider, hover, HoverConfig} from "./hover";
 import {getPositionFromCursor} from "./test-utils/cursor-position";
 import {testFileProvider} from "./test-utils/test-file-provider";
+import {clearCache} from "./utils/workflow-cache";
 
 export function testHoverConfig(tokenValue: string, tokenKey: string, description?: string) {
   return {
@@ -22,6 +23,10 @@ export function testHoverConfig(tokenValue: string, tokenKey: string, descriptio
   } satisfies HoverConfig;
 }
 
+beforeEach(() => {
+  clearCache();
+});
+
 describe("hover", () => {
   it("on a key", async () => {
     const input = `o|n: push

languageservice/src/hover.ts

@@ -1,7 +1,6 @@
 import {DescriptionDictionary, Parser} from "@github/actions-expressions";
 import {FunctionInfo} from "@github/actions-expressions/funcs/info";
 import {Lexer} from "@github/actions-expressions/lexer";
-import {convertWorkflowTemplate, parseWorkflow} from "@github/actions-workflow-parser";
 import {ErrorPolicy} from "@github/actions-workflow-parser/model/convert";
 import {getCronDescription} from "@github/actions-workflow-parser/model/converter/cron";
 import {splitAllowedContext} from "@github/actions-workflow-parser/templates/allowed-context";
@@ -23,10 +22,10 @@ import {ExpressionPos, mapToExpressionPos} from "./expression-hover/expression-p
 import {HoverVisitor} from "./expression-hover/visitor";
 import {validatorFunctions} from "./expression-validation/functions";
 import {info} from "./log";
-import {nullTrace} from "./nulltrace";
 import {isPotentiallyExpression} from "./utils/expression-detection";
 import {findToken, TokenResult} from "./utils/find-token";
 import {mapRange} from "./utils/range";
+import {fetchOrParseWorkflow, fetchOrConvertWorkflowTemplate} from "./utils/workflow-cache";
 
 export type HoverConfig = {
   descriptionProvider?: DescriptionProvider;
@@ -43,19 +42,21 @@ export async function hover(document: TextDocument, position: Position, config?:
     name: document.uri,
     content: document.getText()
   };
-  const result = parseWorkflow(file, nullTrace);
-  if (!result.value) {
+
+  const parsedWorkflow = fetchOrParseWorkflow(file, document.uri);
+  if (!parsedWorkflow) {
     return null;
   }
 
-  const tokenResult = findToken(position, result.value);
-  const {token, keyToken, parent} = tokenResult;
-
-  const tokenDefinitionInfo = (keyToken || parent || token)?.definitionInfo;
-  const template = await convertWorkflowTemplate(result.context, result.value, config?.fileProvider, {
+  const template = await fetchOrConvertWorkflowTemplate(parsedWorkflow, document.uri, config, {
     errorPolicy: ErrorPolicy.TryConversion,
     fetchReusableWorkflowDepth: config?.fileProvider ? 1 : 0
   });
+
+  const tokenResult = findToken(position, parsedWorkflow.value);
+  const {token, keyToken, parent} = tokenResult;
+  const tokenDefinitionInfo = (keyToken || parent || token)?.definitionInfo;
+
   const workflowContext = getWorkflowContext(document.uri, template, tokenResult.path);
   if (token && tokenDefinitionInfo) {
     if (isBasicExpression(token) || isPotentiallyExpression(token)) {

languageservice/src/test-utils/cursor-position.test.ts

@@ -1,5 +1,10 @@
+import {clearCache} from "../utils/workflow-cache";
 import {getPositionFromCursor} from "./cursor-position";
 
+beforeEach(() => {
+  clearCache();
+});
+
 describe("getPositionFromCursor", () => {
   it("returns the position of the cursor and the document without that cursor", () => {
     const input = "on: push\njobs:|";

languageservice/src/utils/workflow-cache.ts

@@ -0,0 +1,51 @@
+import {convertWorkflowTemplate, parseWorkflow} from "@github/actions-workflow-parser";
+import {WorkflowTemplateConverterOptions} from "@github/actions-workflow-parser/model/convert";
+import {File} from "@github/actions-workflow-parser/workflows/file";
+import {ParseWorkflowResult} from "@github/actions-workflow-parser";
+import {WorkflowTemplate} from "@github/actions-workflow-parser";
+import {nullTrace} from "../nulltrace";
+import {CompletionConfig} from "../complete";
+
+const parsedWorkflowCache = new Map<string, ParseWorkflowResult>();
+const workflowTemplateCache = new Map<string, WorkflowTemplate>();
+
+export function clearCacheEntry(uri: string) {
+  parsedWorkflowCache.delete(uri);
+  workflowTemplateCache.delete(uri);
+}
+
+export function clearCache() {
+  parsedWorkflowCache.clear();
+  workflowTemplateCache.clear();
+}
+
+export function fetchOrParseWorkflow(file: File, uri: string): ParseWorkflowResult | undefined {
+  let result = parsedWorkflowCache.get(uri);
+  if (!result?.value) {
+    result = parseWorkflow(file, nullTrace);
+    if (!result.value) {
+      return undefined;
+    }
+    parsedWorkflowCache.set(uri, result);
+  }
+  return result;
+}
+
+export async function fetchOrConvertWorkflowTemplate(
+  parsedWorkflow: ParseWorkflowResult,
+  uri: string,
+  config?: CompletionConfig,
+  options?: WorkflowTemplateConverterOptions
+): Promise<WorkflowTemplate> {
+  let template = workflowTemplateCache.get(uri);
+  if (!template) {
+    template = await convertWorkflowTemplate(
+      parsedWorkflow.context,
+      parsedWorkflow.value!,
+      config?.fileProvider,
+      options
+    );
+    workflowTemplateCache.set(uri, template);
+  }
+  return template;
+}

languageservice/src/validate.actions.test.ts

@@ -5,9 +5,14 @@ import {createDocument} from "./test-utils/document";
 import {TestLogger} from "./test-utils/logger";
 import {validate, ValidationConfig} from "./validate";
 import {ValueProviderKind} from "./value-providers/config";
+import {clearCache} from "./utils/workflow-cache";
 
 registerLogger(new TestLogger());
 
+beforeEach(() => {
+  clearCache();
+});
+
 const validationConfig: ValidationConfig = {
   fetchActionMetadata: async (ref: ActionReference) => {
     let metadata: ActionMetadata | undefined = undefined;

languageservice/src/validate.expressions.test.ts

@@ -5,9 +5,14 @@ import {registerLogger} from "./log";
 import {createDocument} from "./test-utils/document";
 import {TestLogger} from "./test-utils/logger";
 import {validate, ValidationConfig} from "./validate";
+import {clearCache} from "./utils/workflow-cache";
 
 registerLogger(new TestLogger());
 
+beforeEach(() => {
+  clearCache();
+});
+
 describe("expression validation", () => {
   it("access invalid context field", async () => {
     const result = await validate(

languageservice/src/validate.test.ts

@@ -2,6 +2,11 @@ import {Diagnostic, DiagnosticSeverity} from "vscode-languageserver-types";
 import {createDocument} from "./test-utils/document";
 import {validate} from "./validate";
 import {defaultValueProviders} from "./value-providers/default";
+import {clearCache} from "./utils/workflow-cache";
+
+beforeEach(() => {
+  clearCache();
+});
 
 describe("validation", () => {
   it("valid workflow", async () => {

languageservice/src/validate.ts

@@ -1,13 +1,6 @@
 import {Evaluator, ExpressionEvaluationError, Lexer, Parser} from "@github/actions-expressions";
 import {Expr} from "@github/actions-expressions/ast";
-import {
-  convertWorkflowTemplate,
-  isBasicExpression,
-  isString,
-  parseWorkflow,
-  ParseWorkflowResult,
-  WorkflowTemplate
-} from "@github/actions-workflow-parser";
+import {isBasicExpression, isString, ParseWorkflowResult, WorkflowTemplate} from "@github/actions-workflow-parser";
 import {ErrorPolicy} from "@github/actions-workflow-parser/model/convert";
 import {splitAllowedContext} from "@github/actions-workflow-parser/templates/allowed-context";
 import {BasicExpressionToken} from "@github/actions-workflow-parser/templates/tokens/basic-expression-token";
@@ -18,19 +11,18 @@ import {FileProvider} from "@github/actions-workflow-parser/workflows/file-provi
 import {TextDocument} from "vscode-languageserver-textdocument";
 import {Diagnostic, DiagnosticSeverity, URI} from "vscode-languageserver-types";
 import {ActionMetadata, ActionReference} from "./action";
-
 import {ContextProviderConfig} from "./context-providers/config";
 import {getContext, Mode} from "./context-providers/default";
 import {getWorkflowContext, WorkflowContext} from "./context/workflow-context";
 import {AccessError, wrapDictionary} from "./expression-validation/error-dictionary";
 import {validatorFunctions} from "./expression-validation/functions";
 import {error} from "./log";
-import {nullTrace} from "./nulltrace";
 import {findToken} from "./utils/find-token";
 import {mapRange} from "./utils/range";
 import {validateAction} from "./validate-action";
 import {ValueProviderConfig, ValueProviderKind} from "./value-providers/config";
 import {defaultValueProviders} from "./value-providers/default";
+import {fetchOrParseWorkflow, fetchOrConvertWorkflowTemplate} from "./utils/workflow-cache";
 
 export type ValidationConfig = {
   valueProviderConfig?: ValueProviderConfig;
@@ -54,10 +46,14 @@ export async function validate(textDocument: TextDocument, config?: ValidationCo
   const diagnostics: Diagnostic[] = [];
 
   try {
-    const result: ParseWorkflowResult = parseWorkflow(file, nullTrace);
+    const result: ParseWorkflowResult | undefined = fetchOrParseWorkflow(file, textDocument.uri);
+    if (!result) {
+      return [];
+    }
+
     if (result.value) {
       // Errors will be updated in the context. Attempt to do the conversion anyway in order to give the user more information
-      const template = await convertWorkflowTemplate(result.context, result.value, config?.fileProvider, {
+      const template = await fetchOrConvertWorkflowTemplate(result, textDocument.uri, config, {
         fetchReusableWorkflowDepth: config?.fileProvider ? 1 : 0,
         errorPolicy: ErrorPolicy.TryConversion
       });

languageservice/src/validate.workflow-references.test.ts

@@ -1,7 +1,12 @@
 import {createDocument} from "./test-utils/document";
 import {testFileProvider} from "./test-utils/test-file-provider";
+import {clearCache} from "./utils/workflow-cache";
 import {validate} from "./validate";
 
+beforeEach(() => {
+  clearCache();
+});
+
 describe("workflow references validation", () => {
   it("invalid workflow reference", async () => {
     const input = `