Optimize JSON data files to reduce bundle size by 90% (#229)

This commit is contained in:
eric sciple
2025-12-05 10:27:19 -06:00
committed by GitHub
parent 22c36bc946
commit 576402fc01
15 changed files with 26602 additions and 126617 deletions
+33
View File
@@ -27,3 +27,36 @@ jobs:
- run: npm run build -ws
- run: npm run lint -ws
- run: npm test -ws
check-generated:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Use Node.js 16.15
uses: actions/setup-node@v4
with:
node-version: 16.15
cache: 'npm'
registry-url: 'https://npm.pkg.github.com'
- run: npm ci
env:
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Regenerate JSON files
run: |
cd languageservice && npm run update-webhooks && cd ..
- name: Check for uncommitted changes
run: |
if ! git diff --exit-code; then
echo ""
echo "=========================================="
echo "ERROR: Generated files are out of date!"
echo "=========================================="
echo ""
echo "Please run the following commands locally and commit the changes:"
echo ""
echo " cd languageservice && npm run update-webhooks && cd .."
echo " git add -A && git commit -m 'Regenerate JSON files'"
echo ""
exit 1
fi
+9 -1
View File
@@ -2,4 +2,12 @@
*/dist
lerna-debug.log
node_modules
.DS_Store
.DS_Store
# Minified JSON (generated at build time)
*.min.json
# Intermediate JSON for size comparison (generated by update-webhooks --all)
*.all.json
*.drop.json
*.strip.json
+4
View File
@@ -8,6 +8,10 @@ This repository contains multiple npm packages for working with GitHub Actions w
- [languageserver](./languageserver) - Language Server for GitHub Actions, hosting the language service for LSP-compatible editors
- [browser-playground](./browser-playground) - Browser-based playground for the language service
## Documentation
- [JSON Data Files](./docs/json-data-files.md) - How the JSON data files are generated and maintained
### Note
Thank you for your interest in this GitHub repo, however, right now we are not taking contributions.
+150
View File
@@ -0,0 +1,150 @@
# JSON Data Files
This document describes the JSON data files used by the language service packages and how they are maintained.
## Overview
The language service uses several JSON files containing schema definitions, webhook payloads, and other metadata. To reduce bundle size, these files are:
1. **Optimized at generation time** — unused events are dropped, unused fields are stripped
2. **Minified at build time** — whitespace is removed to produce `.min.json` files
The source `.json` files are human-readable and checked into the repository. The `.min.json` files are generated during build and gitignored.
## Files
### languageservice
| File | Description |
|------|-------------|
| `src/context-providers/events/webhooks.json` | Webhook event payload schemas for autocompletion |
| `src/context-providers/events/objects.json` | Deduplicated shared object definitions referenced by webhooks |
| `src/context-providers/events/schedule.json` | Schedule event context data |
| `src/context-providers/events/workflow_call.json` | Reusable workflow call context data |
| `src/context-providers/descriptions.json` | Context variable descriptions for hover |
### workflow-parser
| File | Description |
|------|-------------|
| `src/workflow-v1.0.json` | Workflow YAML schema definition |
## Generation
### Webhooks and Objects
The `webhooks.json` and `objects.json` files are generated from the [GitHub REST API description](https://github.com/github/rest-api-description):
```bash
cd languageservice
npm run update-webhooks
```
This script:
1. Fetches webhook schemas from the GitHub API description
2. **Validates** all events are categorized (fails if new events are found)
3. **Drops** events that aren't valid workflow triggers (see [Dropped Events](#dropped-events))
4. **Strips** unused fields like `description` and `summary` (see [Stripped Fields](#stripped-fields))
5. **Deduplicates** shared object definitions into `objects.json`
6. Writes the optimized, pretty-printed JSON files
### Handling New Webhook Events
When GitHub adds a new webhook event, the script will fail with an error like:
```
ERROR: New webhook event(s) detected!
The following events are not categorized:
- new_event_name
Action required:
1. Check if the event is a valid workflow trigger
2. Add the event to DROPPED_EVENTS or KEPT_EVENTS
```
**To resolve:**
1. Check [Events that trigger workflows](https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows)
2. Edit `languageservice/script/webhooks/index.ts`:
- Add to `KEPT_EVENTS` if it's a valid workflow trigger
- Add to `DROPPED_EVENTS` if it's GitHub App or API-only
3. Run `npm run update-webhooks` and commit the changes
#### Viewing Full Unprocessed Data
To see all available fields and events before optimization:
```bash
npm run update-webhooks -- --all
```
This generates `webhooks.all.json` and `objects.all.json` (gitignored) containing the complete unprocessed data from the GitHub API.
### Other Files
The other JSON files (`schedule.json`, `workflow_call.json`, `descriptions.json`, `workflow-v1.0.json`) are manually maintained.
## Minification
At build time, all JSON files are minified (whitespace removed) to produce `.min.json` versions:
```bash
npm run minify-json
```
This runs automatically via `prebuild` and `pretest` hooks, so you don't need to run it manually.
The code imports the minified versions:
```ts
import webhooks from "./events/webhooks.min.json"
```
## CI Verification
CI verifies that generated source files are up-to-date:
1. Runs `npm run update-webhooks` to regenerate webhooks.json and objects.json
2. Checks for uncommitted changes with `git diff --exit-code`
The `.min.json` files are generated at build time and are not committed to the repository.
If the build fails, run `cd languageservice && npm run update-webhooks` locally and commit the changes.
## Dropped Events
Webhook events that aren't valid workflow `on:` triggers are dropped (e.g., `installation`, `ping`, `member`, etc.). These are GitHub App or API-only events.
See `DROPPED_EVENTS` in `script/webhooks/index.ts` for the full list.
## Stripped Fields
Unused fields are stripped to reduce bundle size. For example:
```json
// Before (from webhooks.all.json)
{
"type": "object",
"name": "issue",
"in": "body",
"description": "The issue itself.",
"isRequired": true,
"childParamsGroups": [...]
}
// After (webhooks.json)
{
"name": "issue",
"description": "The issue itself.",
"childParamsGroups": [...]
}
```
Only `name`, `description`, and `childParamsGroups` are kept — these are used for autocompletion and hover docs.
To compare all fields vs stripped, run `npm run update-webhooks -- --all` and diff the `.all.json` files against the regular ones.
See `EVENT_ACTION_FIELDS` and `BODY_PARAM_FIELDS` in `script/webhooks/index.ts` to modify what gets stripped.
+4 -1
View File
@@ -37,10 +37,13 @@
"format-check": "prettier --check '**/*.ts'",
"lint": "eslint 'src/**/*.ts'",
"lint-fix": "eslint --fix 'src/**/*.ts'",
"minify-json": "node ../script/minify-json.js src/context-providers/descriptions.json src/context-providers/events/webhooks.json src/context-providers/events/objects.json src/context-providers/events/schedule.json src/context-providers/events/workflow_call.json",
"prebuild": "npm run minify-json",
"prepublishOnly": "npm run build && npm run test",
"pretest": "npm run minify-json",
"test": "NODE_OPTIONS=\"--experimental-vm-modules\" jest",
"test-watch": "NODE_OPTIONS=\"--experimental-vm-modules\" jest --watch",
"update-webhooks": "ts-node-esm script/webhooks/index.ts",
"update-webhooks": "npx tsx script/webhooks/index.ts",
"watch": "tsc --build tsconfig.build.json --watch"
},
"dependencies": {
+273 -2
View File
@@ -7,6 +7,185 @@ const schema = schemaImport as any;
const OUTPUT_PATH = "./src/context-providers/events/webhooks.json";
const OBJECTS_PATH = "./src/context-providers/events/objects.json";
const ALL_OUTPUT_PATH = "./src/context-providers/events/webhooks.all.json";
const ALL_OBJECTS_PATH = "./src/context-providers/events/objects.all.json";
const DROP_OUTPUT_PATH = "./src/context-providers/events/webhooks.drop.json";
const DROP_OBJECTS_PATH = "./src/context-providers/events/objects.drop.json";
const STRIP_OUTPUT_PATH = "./src/context-providers/events/webhooks.strip.json";
const STRIP_OBJECTS_PATH = "./src/context-providers/events/objects.strip.json";
// Parse --all flag
const generateAll = process.argv.includes("--all");
// Events to drop - not valid workflow triggers (GitHub App or API-only events)
// See: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows
const DROPPED_EVENTS = new Set([
"branch_protection_configuration",
"code_scanning_alert",
"commit_comment",
"custom_property",
"custom_property_values",
"dependabot_alert",
"deploy_key",
"github_app_authorization",
"installation",
"installation_repositories",
"installation_target",
"marketplace_purchase",
"member",
"membership",
"merge_group",
"meta",
"org_block",
"organization",
"package",
"personal_access_token_request",
"ping",
"repository",
"repository_advisory",
"repository_ruleset",
"secret_scanning_alert",
"secret_scanning_alert_location",
"security_advisory",
"security_and_analysis",
"sponsorship",
"star",
"team",
"team_add"
]);
// Events to keep - valid workflow triggers
// See: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows
const KEPT_EVENTS = new Set([
"branch_protection_rule",
"check_run",
"check_suite",
"create",
"delete",
"deployment",
"deployment_status",
"discussion",
"discussion_comment",
"fork",
"gollum",
"issue_comment",
"issues",
"label",
"milestone",
"page_build",
"project",
"project_card",
"project_column",
"projects_v2",
"projects_v2_item",
"public",
"pull_request",
"pull_request_review",
"pull_request_review_comment",
"pull_request_review_thread",
"push",
"registry_package",
"release",
"repository_dispatch",
"repository_import",
"repository_vulnerability_alert",
"status",
"watch",
"workflow_dispatch",
"workflow_job",
"workflow_run"
]);
/**
* Fields to strip from the JSON data.
*
* EVENT_ACTION_FIELDS: stripped from each event action object (top level only)
* Example event action object before stripping:
* {
* "description": "This event is triggered when...", // <-- stripped
* "summary": "A brief summary", // <-- stripped
* "availability": ["repository"], // <-- stripped
* "category": "issues", // <-- stripped
* "action": "opened", // kept
* "bodyParameters": [...] // kept
* }
*
* BODY_PARAM_FIELDS: stripped from every bodyParameters object, recursively through childParamsGroups
* Example bodyParameter object before stripping:
* {
* "type": "object", // <-- stripped
* "name": "changes", // kept (used for property names)
* "in": "body", // <-- stripped
* "description": "The changes that were made.", // kept (used for hover docs)
* "isRequired": true, // <-- stripped
* "enum": ["a", "b"], // <-- stripped
* "default": "a", // <-- stripped
* "childParamsGroups": [ // kept (used for nested properties)
* {
* "type": "string", // <-- stripped (recursive)
* "name": "from", // kept
* "isRequired": true // <-- stripped (recursive)
* }
* ]
* }
*/
const EVENT_ACTION_FIELDS = ["description", "summary", "availability", "category"];
const BODY_PARAM_FIELDS = ["type", "in", "isRequired", "enum", "default"];
/**
* Strip fields from a bodyParameter object and recursively from childParamsGroups.
*/
function stripBodyParam(param: any): any {
if (typeof param !== "object" || param === null) {
return param;
}
const result: any = {};
for (const [key, value] of Object.entries(param)) {
if (BODY_PARAM_FIELDS.includes(key)) {
continue; // Strip this field
}
if (key === "childParamsGroups" && Array.isArray(value)) {
result[key] = value.map(stripBodyParam);
} else {
result[key] = value;
}
}
return result;
}
/**
* Strip unused fields from event action data.
*/
function stripEventActionFields(action: any): any {
const result: any = {};
for (const [key, value] of Object.entries(action)) {
if (EVENT_ACTION_FIELDS.includes(key)) {
continue; // Strip this field
}
if (key === "bodyParameters" && Array.isArray(value)) {
result[key] = value.map((p: any) => (typeof p === "number" ? p : stripBodyParam(p)));
} else {
result[key] = value;
}
}
return result;
}
/**
* Strip unused fields from all webhooks.
* Structure: { eventName: { actionName: { ...fields } } }
*/
function stripFields(webhooks: Record<string, Record<string, any>>): Record<string, Record<string, any>> {
const result: Record<string, Record<string, any>> = {};
for (const [eventName, actions] of Object.entries(webhooks)) {
result[eventName] = {};
for (const [actionName, actionData] of Object.entries(actions)) {
result[eventName][actionName] = stripEventActionFields(actionData);
}
}
return result;
}
const rawWebhooks = Object.values(schema.webhooks || schema["x-webhooks"]) as any[];
if (!rawWebhooks) {
@@ -20,11 +199,51 @@ for (const webhook of Object.values(rawWebhooks)) {
await Promise.all(webhooks.map(webhook => webhook.process()));
// Check for unknown events (not in DROPPED_EVENTS or KEPT_EVENTS)
const unknownEvents: string[] = [];
for (const webhook of webhooks) {
if (!DROPPED_EVENTS.has(webhook.category) && !KEPT_EVENTS.has(webhook.category)) {
if (!unknownEvents.includes(webhook.category)) {
unknownEvents.push(webhook.category);
}
}
}
if (unknownEvents.length > 0) {
console.error("");
console.error("══════════════════════════════════════════════════════════════════");
console.error("ERROR: New webhook event(s) detected!");
console.error("══════════════════════════════════════════════════════════════════");
console.error("");
console.error("The following events are not categorized:");
for (const event of unknownEvents.sort()) {
console.error(` - ${event}`);
}
console.error("");
console.error("Action required:");
console.error(" 1. Check if the event is a valid workflow trigger:");
console.error(
" https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows"
);
console.error("");
console.error(" 2. Add the event to DROPPED_EVENTS or KEPT_EVENTS in:");
console.error(" languageservice/script/webhooks/index.ts");
console.error("");
console.error(" 3. See docs/json-data-files.md for more details.");
console.error("");
process.exit(1);
}
// The category is the name of the webhook
const categorizedWebhooks: Record<string, Record<string, Webhook>> = {};
for (const webhook of webhooks) {
if (!webhook.action) webhook.action = "default";
// Drop unused events
if (DROPPED_EVENTS.has(webhook.category)) {
continue;
}
if (categorizedWebhooks[webhook.category]) {
categorizedWebhooks[webhook.category][webhook.action] = webhook;
} else {
@@ -33,7 +252,59 @@ for (const webhook of webhooks) {
}
}
const objectsArray = deduplicateWebhooks(categorizedWebhooks);
// Strip fields before deduplication
const strippedWebhooks = stripFields(categorizedWebhooks);
// Deduplicate after dropping and stripping
const objectsArray = deduplicateWebhooks(strippedWebhooks);
// Write optimized output
await fs.writeFile(OBJECTS_PATH, JSON.stringify(objectsArray, null, 2));
await fs.writeFile(OUTPUT_PATH, JSON.stringify(categorizedWebhooks, null, 2));
await fs.writeFile(OUTPUT_PATH, JSON.stringify(strippedWebhooks, null, 2));
console.log(`Wrote ${OUTPUT_PATH} (${Object.keys(strippedWebhooks).length} events)`);
console.log(`Wrote ${OBJECTS_PATH} (${objectsArray.length} objects)`);
// Optionally generate intermediate versions for size comparison
if (generateAll) {
// Helper to deep clone
function clone<T>(obj: T): T {
return JSON.parse(JSON.stringify(obj));
}
// Build full webhooks (no drop, no strip) from fresh data
const fullWebhooks: Record<string, Record<string, any>> = {};
for (const webhook of webhooks) {
const w = clone(webhook);
if (!w.action) w.action = "default";
fullWebhooks[w.category] ||= {};
fullWebhooks[w.category][w.action] = w;
}
// Generate all version (no drop, no strip)
const allWebhooks = clone(fullWebhooks);
const allObjects = deduplicateWebhooks(allWebhooks);
await fs.writeFile(ALL_OUTPUT_PATH, JSON.stringify(allWebhooks, null, 2));
await fs.writeFile(ALL_OBJECTS_PATH, JSON.stringify(allObjects, null, 2));
console.log(`Wrote ${ALL_OUTPUT_PATH} (${Object.keys(allWebhooks).length} events)`);
console.log(`Wrote ${ALL_OBJECTS_PATH} (${allObjects.length} objects)`);
// Generate drop-only version (drop events, no strip)
const dropWebhooks = clone(fullWebhooks);
for (const event of DROPPED_EVENTS) {
delete dropWebhooks[event];
}
const dropObjects = deduplicateWebhooks(dropWebhooks);
await fs.writeFile(DROP_OUTPUT_PATH, JSON.stringify(dropWebhooks, null, 2));
await fs.writeFile(DROP_OBJECTS_PATH, JSON.stringify(dropObjects, null, 2));
console.log(`Wrote ${DROP_OUTPUT_PATH} (${Object.keys(dropWebhooks).length} events)`);
console.log(`Wrote ${DROP_OBJECTS_PATH} (${dropObjects.length} objects)`);
// Generate strip-only version (strip fields, no drop)
const stripWebhooks = stripFields(clone(fullWebhooks));
const stripObjects = deduplicateWebhooks(stripWebhooks);
await fs.writeFile(STRIP_OUTPUT_PATH, JSON.stringify(stripWebhooks, null, 2));
await fs.writeFile(STRIP_OBJECTS_PATH, JSON.stringify(stripObjects, null, 2));
console.log(`Wrote ${STRIP_OUTPUT_PATH} (${Object.keys(stripWebhooks).length} events)`);
console.log(`Wrote ${STRIP_OBJECTS_PATH} (${stripObjects.length} objects)`);
}
@@ -1,4 +1,4 @@
import descriptions from "./descriptions.json";
import descriptions from "./descriptions.min.json";
export const RootContext = "root";
const FunctionContext = "functions";
@@ -0,0 +1,102 @@
import {DescriptionDictionary} from "@actions/expressions";
import {getEventPayload, getSupportedEventTypes} from "./eventPayloads";
describe("eventPayloads", () => {
describe("getSupportedEventTypes", () => {
it("returns action types for push event", () => {
const types = getSupportedEventTypes("push");
expect(types).toContain("default");
});
it("returns action types for issues event", () => {
const types = getSupportedEventTypes("issues");
expect(types.length).toBeGreaterThan(1);
expect(types).toContain("opened");
expect(types).toContain("closed");
});
});
describe("getEventPayload", () => {
it("returns payload for push event", () => {
const payload = getEventPayload("push", "default");
expect(payload).toBeDefined();
// Verify common fields exist
expect(payload?.get("ref")).toBeDefined();
expect(payload?.get("repository")).toBeDefined();
expect(payload?.get("sender")).toBeDefined();
});
it("returns payload for issues event", () => {
const payload = getEventPayload("issues", "opened");
expect(payload).toBeDefined();
expect(payload?.get("action")).toBeDefined();
expect(payload?.get("issue")).toBeDefined();
expect(payload?.get("repository")).toBeDefined();
});
it("preserves descriptions for hover documentation", () => {
// This test ensures bodyParameters[].description is not stripped
// during JSON optimization. The description field is used for hover
// documentation in the workflow editor.
const payload = getEventPayload("push", "default");
expect(payload).toBeDefined();
// Get the description for a well-known field
// repository should have a description like "A repository on GitHub"
const repoDescription = payload?.getDescription("repository");
expect(repoDescription).toBeDefined();
expect(repoDescription?.length).toBeGreaterThan(0);
// sender should have a description
const senderDescription = payload?.getDescription("sender");
expect(senderDescription).toBeDefined();
expect(senderDescription?.length).toBeGreaterThan(0);
});
it("preserves childParamsGroups for nested property access", () => {
// This test ensures bodyParameters[].childParamsGroups is not stripped
// during JSON optimization. childParamsGroups defines nested properties
// used for autocompletion like github.event.repository.owner.login
const payload = getEventPayload("push", "default");
expect(payload).toBeDefined();
// repository has nested properties like owner, license, etc.
const repository = payload?.get("repository") as DescriptionDictionary | undefined;
expect(repository).toBeDefined();
// repository.owner should exist (nested via childParamsGroups)
const owner = repository?.get("owner") as DescriptionDictionary | undefined;
expect(owner).toBeDefined();
// repository.owner.login should exist (deeply nested)
const login = owner?.get("login");
expect(login).toBeDefined();
});
it("preserves name fields for property identification", () => {
// This test ensures bodyParameters[].name is not stripped
// during JSON optimization. The name field identifies each property.
const payload = getEventPayload("issues", "opened");
expect(payload).toBeDefined();
// Verify well-known property names exist
expect(payload?.get("action")).toBeDefined();
expect(payload?.get("issue")).toBeDefined();
expect(payload?.get("repository")).toBeDefined();
expect(payload?.get("sender")).toBeDefined();
// Verify nested property names work
const issue = payload?.get("issue") as DescriptionDictionary | undefined;
expect(issue?.get("title")).toBeDefined();
expect(issue?.get("number")).toBeDefined();
expect(issue?.get("user")).toBeDefined();
});
it("returns undefined for unknown event", () => {
const payload = getEventPayload("not_a_real_event", "default");
expect(payload).toBeUndefined();
});
});
});
@@ -1,10 +1,10 @@
import {data, DescriptionDictionary} from "@actions/expressions";
import webhookObjects from "./objects.json";
import webhooks from "./webhooks.json";
import webhookObjects from "./objects.min.json";
import webhooks from "./webhooks.min.json";
import schedule from "./schedule.json";
import workflow_call from "./workflow_call.json";
import schedule from "./schedule.min.json";
import workflow_call from "./workflow_call.min.json";
const customEventPayloads: {[name: string]: unknown} = {
schedule,
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+1
View File
@@ -1,6 +1,7 @@
{
"name": "actions-languageservices",
"private": true,
"type": "module",
"workspaces": [
"./expressions",
"./workflow-parser",
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env node
/**
* Minifies JSON files by removing whitespace.
*
* Usage: node script/minify-json.js <file1.json> <file2.json> ...
*
* For each input file, creates a corresponding .min.json file.
* Example: src/data.json -> src/data.min.json
*/
import {promises as fs} from "fs";
import path from "path";
const files = process.argv.slice(2);
if (files.length === 0) {
console.error("Usage: node script/minify-json.js <file1.json> <file2.json> ...");
process.exit(1);
}
for (const file of files) {
try {
const content = await fs.readFile(file, "utf8");
const data = JSON.parse(content);
const minified = JSON.stringify(data);
// Replace .json with .min.json
const ext = path.extname(file);
const outputFile = file.slice(0, -ext.length) + ".min" + ext;
await fs.writeFile(outputFile, minified);
const originalSize = Buffer.byteLength(content, "utf8");
const minifiedSize = Buffer.byteLength(minified, "utf8");
const savings = ((1 - minifiedSize / originalSize) * 100).toFixed(1);
console.log(`${file} -> ${outputFile} (${savings}% smaller)`);
} catch (err) {
console.error(`Error processing ${file}:`, err);
process.exit(1);
}
}
+3
View File
@@ -38,7 +38,10 @@
"format-check": "prettier --check '**/*.ts'",
"lint": "eslint 'src/**/*.ts'",
"lint-fix": "eslint --fix 'src/**/*.ts'",
"minify-json": "node ../script/minify-json.js src/workflow-v1.0.json",
"prebuild": "npm run minify-json",
"prepublishOnly": "npm run build && npm run test",
"pretest": "npm run minify-json",
"test": "NODE_OPTIONS=\"--experimental-vm-modules\" jest",
"test-xlang": "NODE_OPTIONS=\"--experimental-vm-modules\" jest --testPathPattern xlang",
"test-watch": "NODE_OPTIONS=\"--experimental-vm-modules\" jest --watch",
@@ -1,6 +1,6 @@
import {JSONObjectReader} from "../templates/json-object-reader";
import {TemplateSchema} from "../templates/schema";
import WorkflowSchema from "../workflow-v1.0.json";
import WorkflowSchema from "../workflow-v1.0.min.json";
let schema: TemplateSchema;