Compare commits

...

4 Commits

Author SHA1 Message Date
eric sciple 38ffd53f3b . 2025-12-06 23:02:24 +00:00
eric sciple 7660f61777 Prune unused definitions from workflow schema
Remove 10 unreachable definitions from workflow-v1.0.json:
- workflow-root, on, on-mapping (non-strict variants)
- job-if-result, step-if-result
- boolean-needs-context, number-needs-context, string-needs-context
- boolean-steps-context, number-steps-context

Saves 2,039 bytes minified (2.9%), 146 bytes gzipped (1.2%).

Also adds script/prune-schema.cjs for future maintenance.
2025-12-06 22:57:10 +00:00
eric sciple c04c1b26f4 Optimize webhooks JSON with compact format and string interning
- Convert params to compact array format (type-based dispatch)
- Intern duplicate property names into string table
- Use negative indices for object references to distinguish from string indices
- Rename objects.json → webhooks.objects.json, add webhooks.strings.json
- Move event filters to JSON for maintainability
- Add CI validation job to verify optimization correctness

Reduces combined minified size by ~67% (453 KB → 148 KB), ~27% gzipped (23 KB → 17 KB).
2025-12-06 20:05:44 +00:00
eric sciple 4429c41275 Align supported Node.js engines field with dependency requirements (#231) 2025-12-05 15:28:10 -06:00
22 changed files with 25277 additions and 49886 deletions
+39 -5
View File
@@ -12,15 +12,19 @@ jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [18.x, 20.x, 22.x]
steps:
- uses: actions/checkout@v4
- name: Use Node.js 16.15
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v4
with:
node-version: 16.15
node-version: ${{ matrix.node-version }}
cache: 'npm'
registry-url: 'https://npm.pkg.github.com'
- run: npm ci
- run: npm ci --engine-strict
env:
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: npm run format-check -ws
@@ -33,10 +37,10 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Use Node.js 16.15
- name: Use Node.js 22.x
uses: actions/setup-node@v4
with:
node-version: 16.15
node-version: 22.x
cache: 'npm'
registry-url: 'https://npm.pkg.github.com'
- run: npm ci
@@ -60,3 +64,33 @@ jobs:
echo ""
exit 1
fi
validate-webhooks:
runs-on: ubuntu-latest
name: Validate webhook optimization
steps:
- uses: actions/checkout@v4
- name: Use Node.js 22.x
uses: actions/setup-node@v4
with:
node-version: 22.x
cache: 'npm'
registry-url: 'https://npm.pkg.github.com'
- run: npm ci
env:
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Build workspaces
run: npm run build -ws
- name: Generate full webhooks file
run: cd languageservice && npm run update-webhooks
- name: Run optimization validation tests
run: cd languageservice && npm test -- --testPathPattern=eventPayloads
- name: Verify validation tests ran
run: |
if [ ! -f languageservice/src/context-providers/events/webhooks.full.validation-complete ]; then
echo "ERROR: Validation tests did not run!"
echo "The webhooks.full.validation-complete marker file was not created."
exit 1
fi
echo "Validation tests completed at: $(cat languageservice/src/context-providers/events/webhooks.full.validation-complete)"
+1 -1
View File
@@ -69,7 +69,7 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: 16.x
node-version: 22.x
cache: "npm"
scope: '@actions'
+8 -4
View File
@@ -7,7 +7,11 @@ node_modules
# Minified JSON (generated at build time)
*.min.json
# Intermediate JSON for size comparison (generated by update-webhooks --all)
*.all.json
*.drop.json
*.strip.json
# Optimized workflow schema (generated by optimize-workflow-schema.js)
*.optimized.json
# Full webhooks source (generated by update-webhooks, used for validation tests)
*.full.json
# Validation marker (generated by tests)
*.validation-complete
+99 -26
View File
@@ -6,8 +6,9 @@ This document describes the JSON data files used by the language service package
The language service uses several JSON files containing schema definitions, webhook payloads, and other metadata. To reduce bundle size, these files are:
1. **Optimized at generation time** — unused events are dropped, unused fields are stripped
2. **Minified at build time** — whitespace is removed to produce `.min.json` files
1. **Optimized at generation time** — unused events are dropped, unused fields are stripped, shared objects are deduplicated, property names are interned
2. **Compacted using a space-efficient format** — params use type-based dispatch arrays instead of objects
3. **Minified at build time** — whitespace is removed to produce `.min.json` files
The source `.json` files are human-readable and checked into the repository. The `.min.json` files are generated during build and gitignored.
@@ -18,7 +19,8 @@ The source `.json` files are human-readable and checked into the repository. The
| File | Description |
|------|-------------|
| `src/context-providers/events/webhooks.json` | Webhook event payload schemas for autocompletion |
| `src/context-providers/events/objects.json` | Deduplicated shared object definitions referenced by webhooks |
| `src/context-providers/events/webhooks.objects.json` | Deduplicated shared object definitions referenced by webhooks |
| `src/context-providers/events/webhooks.strings.json` | Interned property names shared by webhooks and objects |
| `src/context-providers/events/schedule.json` | Schedule event context data |
| `src/context-providers/events/workflow_call.json` | Reusable workflow call context data |
| `src/context-providers/descriptions.json` | Context variable descriptions for hover |
@@ -33,7 +35,7 @@ The source `.json` files are human-readable and checked into the repository. The
### Webhooks and Objects
The `webhooks.json` and `objects.json` files are generated from the [GitHub REST API description](https://github.com/github/rest-api-description):
The `webhooks.json`, `webhooks.objects.json`, and `webhooks.strings.json` files are generated from the [GitHub REST API description](https://github.com/github/rest-api-description):
```bash
cd languageservice
@@ -44,9 +46,10 @@ This script:
1. Fetches webhook schemas from the GitHub API description
2. **Validates** all events are categorized (fails if new events are found)
3. **Drops** events that aren't valid workflow triggers (see [Dropped Events](#dropped-events))
4. **Strips** unused fields like `description` and `summary` (see [Stripped Fields](#stripped-fields))
5. **Deduplicates** shared object definitions into `objects.json`
6. Writes the optimized, pretty-printed JSON files
4. **Compacts** params into a space-efficient array format, keeping only `name`, `description`, and `childParamsGroups` (see [Compact Format](#compact-format))
5. **Deduplicates** shared object definitions into `webhooks.objects.json`
6. **Interns** duplicate property names into `webhooks.strings.json` (see [String Interning](#string-interning))
7. Writes the optimized, pretty-printed JSON files
### Handling New Webhook Events
@@ -67,9 +70,9 @@ Action required:
1. Check [Events that trigger workflows](https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows)
2. Edit `languageservice/script/webhooks/index.ts`:
- Add to `KEPT_EVENTS` if it's a valid workflow trigger
- Add to `DROPPED_EVENTS` if it's GitHub App or API-only
2. Edit `languageservice/src/context-providers/events/event-filters.json`:
- Add to `kept` array if it's a valid workflow trigger
- Add to `dropped` array if it's GitHub App or API-only
3. Run `npm run update-webhooks` and commit the changes
@@ -101,13 +104,15 @@ The code imports the minified versions:
```ts
import webhooks from "./events/webhooks.min.json"
import objects from "./events/webhooks.objects.min.json"
import strings from "./events/webhooks.strings.min.json"
```
## CI Verification
CI verifies that generated source files are up-to-date:
1. Runs `npm run update-webhooks` to regenerate webhooks.json and objects.json
1. Runs `npm run update-webhooks` to regenerate webhooks.json, webhooks.objects.json, and webhooks.strings.json
2. Checks for uncommitted changes with `git diff --exit-code`
The `.min.json` files are generated at build time and are not committed to the repository.
@@ -118,33 +123,101 @@ If the build fails, run `cd languageservice && npm run update-webhooks` locally
Webhook events that aren't valid workflow `on:` triggers are dropped (e.g., `installation`, `ping`, `member`, etc.). These are GitHub App or API-only events.
See `DROPPED_EVENTS` in `script/webhooks/index.ts` for the full list.
See `dropped` array in `src/context-providers/events/event-filters.json` for the full list.
## Stripped Fields
## Compact Format
Unused fields are stripped to reduce bundle size. For example:
Params are converted from verbose objects into compact arrays, keeping only the fields needed for autocompletion and hover docs (`name`, `description`, `childParamsGroups`). Unused fields like `type`, `in`, `isRequired`, `enum`, and `default` are discarded.
| Format | Meaning |
|--------|---------|
| `"name"` | Name only (no description, no children) |
| `[name, desc]` | Name + description (arr[1] is a string) |
| `[name, children]` | Name + children (arr[1] is an array) |
| `[name, desc, children]` | Name + description + children |
The reader uses `typeof arr[1]` to determine the format: if it's a string, it's a description; if it's an array, it's children.
**Example:**
```json
// Before (from webhooks.all.json)
// Before (object format)
{
"type": "object",
"name": "issue",
"in": "body",
"description": "The issue itself.",
"isRequired": true,
"childParamsGroups": [...]
"childParamsGroups": [
{ "name": "id" },
{ "name": "title", "description": "Issue title" }
]
}
// After (webhooks.json)
// After (compact format)
["issue", "The issue itself.", [
"id",
["title", "Issue title"]
]]
```
## String Interning
Property names that appear 2+ times are "interned" into a shared string table (`webhooks.strings.json`). In the compact arrays, these names are replaced with non-negative numeric indices:
```json
// webhooks.strings.json
["url", "id", "name", ...] // Index 0 = "url", 1 = "id", 2 = "name"
// webhooks.json - uses indices instead of strings
{
"name": "issue",
"description": "The issue itself.",
"childParamsGroups": [...]
"push": {
"default": {
"p": [
[0, "The URL..."], // 0 = "url" from string table
[1, "Unique ID"], // 1 = "id"
2 // 2 = "name" (name-only, no description)
]
}
}
}
```
Only `name`, `description`, and `childParamsGroups` are kept — these are used for autocompletion and hover docs.
**How to distinguish indices from other values:**
To compare all fields vs stripped, run `npm run update-webhooks -- --all` and diff the `.all.json` files against the regular ones.
- **Negative numbers** → Object indices: `-1` = object 0, `-2` = object 1, etc. (formula: `-(index + 1)`)
- **Non-negative numbers** → String indices (references into `webhooks.strings.json`)
- **Literal strings** → Singletons (names appearing only once, not interned)
See `EVENT_ACTION_FIELDS` and `BODY_PARAM_FIELDS` in `script/webhooks/index.ts` to modify what gets stripped.
Singletons are kept as literal strings for readability and to avoid the overhead of adding rarely-used names to the string table.
## Deduplication
Shared object definitions are extracted into `webhooks.objects.json` and referenced by negative index:
```json
// webhooks.objects.json
[
["url", "The URL"], // Index 0 (referenced as -1)
["id", "Unique identifier"], // Index 1 (referenced as -2)
[...]
]
// webhooks.json - negative numbers reference objects
{
"push": {
"default": {
"p": [-1, -2, ["ref", "The git ref"]] // -1 = object 0, -2 = object 1
}
}
}
```
This reduces duplication when the same object structure appears in multiple events (e.g., `repository`, `sender`, `organization`).
## Size Reduction
The optimizations achieve approximately 99% file size reduction:
| Stage | Minified | Gzip |
|-------|----------|------|
| Original (webhooks.full.json) | 15.8 MB | 968 KB |
| After optimization (combined) | 152 KB | 15.6 KB |
| **Reduction** | **99%** | **98%** |
+1 -1
View File
@@ -44,7 +44,7 @@
"watch": "tsc --build tsconfig.build.json --watch"
},
"engines": {
"node": ">= 16.15"
"node": ">= 18"
},
"files": [
"dist/**/*"
+1 -1
View File
@@ -52,7 +52,7 @@
"yaml": "^2.1.3"
},
"engines": {
"node": ">= 16.15"
"node": ">= 18"
},
"files": [
"dist/**/*"
+3 -3
View File
@@ -37,13 +37,13 @@
"format-check": "prettier --check '**/*.ts'",
"lint": "eslint 'src/**/*.ts'",
"lint-fix": "eslint --fix 'src/**/*.ts'",
"minify-json": "node ../script/minify-json.js src/context-providers/descriptions.json src/context-providers/events/webhooks.json src/context-providers/events/objects.json src/context-providers/events/schedule.json src/context-providers/events/workflow_call.json",
"minify-json": "node ../script/minify-json.js src/context-providers/descriptions.json src/context-providers/events/webhooks.json src/context-providers/events/webhooks.objects.json src/context-providers/events/webhooks.strings.json src/context-providers/events/schedule.json src/context-providers/events/workflow_call.json",
"prebuild": "npm run minify-json",
"prepublishOnly": "npm run build && npm run test",
"pretest": "npm run minify-json",
"test": "NODE_OPTIONS=\"--experimental-vm-modules\" jest",
"test-watch": "NODE_OPTIONS=\"--experimental-vm-modules\" jest --watch",
"update-webhooks": "npx tsx script/webhooks/index.ts",
"update-webhooks": "npx tsx script/webhooks/update-webhooks.ts",
"watch": "tsc --build tsconfig.build.json --watch"
},
"dependencies": {
@@ -55,7 +55,7 @@
"yaml": "^2.1.1"
},
"engines": {
"node": ">= 16.15"
"node": ">= 18"
},
"files": [
"dist/**/*"
+46 -10
View File
@@ -1,5 +1,38 @@
import Webhook from "./webhook";
/**
* Get the name from a param.
* Formats: "name" (string), or [name, ...] (array)
*/
function getParamName(param: any): string {
if (typeof param === "string") {
return param;
}
if (Array.isArray(param)) {
return param[0];
}
return param.name;
}
/**
* Get params from a webhook action.
* Uses 'p' (short key) if present, falls back to 'bodyParameters'
*/
function getParams(webhook: any): any[] {
return webhook.p || webhook.bodyParameters || [];
}
/**
* Set params on a webhook action using the short key 'p'
*/
function setParams(webhook: any, params: any[]): void {
if (webhook.p !== undefined) {
webhook.p = params;
} else {
webhook.bodyParameters = params;
}
}
// Store any repeated body parameters in an array
// and replace them in the webhook with an index in the array
export function deduplicateWebhooks(webhooks: Record<string, Record<string, Webhook>>): any[] {
@@ -10,10 +43,11 @@ export function deduplicateWebhooks(webhooks: Record<string, Record<string, Webh
const objectCount: Record<string, number> = {};
for (const webhook of iterateWebhooks(webhooks)) {
for (const param of webhook.bodyParameters) {
objectsByName[param.name] ||= [];
const index = findOrAdd(param, objectsByName[param.name]);
const key = `${param.name}:${index}`;
for (const param of getParams(webhook)) {
const name = getParamName(param);
objectsByName[name] ||= [];
const index = findOrAdd(param, objectsByName[name]);
const key = `${name}:${index}`;
objectCount[key] ||= 0;
objectCount[key]++;
}
@@ -27,18 +61,19 @@ export function deduplicateWebhooks(webhooks: Record<string, Record<string, Webh
for (const webhook of iterateWebhooks(webhooks)) {
const newParams: any[] = [];
for (const param of webhook.bodyParameters) {
const index = find(param, objectsByName[param.name]);
const key = `${param.name}:${index}`;
for (const param of getParams(webhook)) {
const name = getParamName(param);
const index = find(param, objectsByName[name]);
const key = `${name}:${index}`;
if (objectCount[key] > 1) {
newParams.push(indexForParam(param, index, bodyParamIndexMap, duplicatedBodyParams));
newParams.push(indexForParam(param, name, index, bodyParamIndexMap, duplicatedBodyParams));
} else {
// If an object is only used once, keep it inline
newParams.push(param);
}
}
webhook.bodyParameters = newParams;
setParams(webhook, newParams);
}
return duplicatedBodyParams;
@@ -74,11 +109,12 @@ function find(param: any, objects: any[]): number {
function indexForParam(
param: any,
paramName: string,
paramNameIndex: number,
objectIndexMap: Record<string, number>,
duplicatedBodyParams: any[]
): number {
const key = `${param.name}:${paramNameIndex}`;
const key = `${paramName}:${paramNameIndex}`;
const existingIndex = objectIndexMap[key];
if (existingIndex !== undefined) {
-310
View File
@@ -1,310 +0,0 @@
import {promises as fs} from "fs";
import Webhook from "./webhook.js";
import schemaImport from "rest-api-description/descriptions/api.github.com/dereferenced/api.github.com.deref.json";
import {deduplicateWebhooks} from "./deduplicate.js";
const schema = schemaImport as any;
const OUTPUT_PATH = "./src/context-providers/events/webhooks.json";
const OBJECTS_PATH = "./src/context-providers/events/objects.json";
const ALL_OUTPUT_PATH = "./src/context-providers/events/webhooks.all.json";
const ALL_OBJECTS_PATH = "./src/context-providers/events/objects.all.json";
const DROP_OUTPUT_PATH = "./src/context-providers/events/webhooks.drop.json";
const DROP_OBJECTS_PATH = "./src/context-providers/events/objects.drop.json";
const STRIP_OUTPUT_PATH = "./src/context-providers/events/webhooks.strip.json";
const STRIP_OBJECTS_PATH = "./src/context-providers/events/objects.strip.json";
// Parse --all flag
const generateAll = process.argv.includes("--all");
// Events to drop - not valid workflow triggers (GitHub App or API-only events)
// See: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows
const DROPPED_EVENTS = new Set([
"branch_protection_configuration",
"code_scanning_alert",
"commit_comment",
"custom_property",
"custom_property_values",
"dependabot_alert",
"deploy_key",
"github_app_authorization",
"installation",
"installation_repositories",
"installation_target",
"marketplace_purchase",
"member",
"membership",
"merge_group",
"meta",
"org_block",
"organization",
"package",
"personal_access_token_request",
"ping",
"repository",
"repository_advisory",
"repository_ruleset",
"secret_scanning_alert",
"secret_scanning_alert_location",
"security_advisory",
"security_and_analysis",
"sponsorship",
"star",
"team",
"team_add"
]);
// Events to keep - valid workflow triggers
// See: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows
const KEPT_EVENTS = new Set([
"branch_protection_rule",
"check_run",
"check_suite",
"create",
"delete",
"deployment",
"deployment_status",
"discussion",
"discussion_comment",
"fork",
"gollum",
"issue_comment",
"issues",
"label",
"milestone",
"page_build",
"project",
"project_card",
"project_column",
"projects_v2",
"projects_v2_item",
"public",
"pull_request",
"pull_request_review",
"pull_request_review_comment",
"pull_request_review_thread",
"push",
"registry_package",
"release",
"repository_dispatch",
"repository_import",
"repository_vulnerability_alert",
"status",
"watch",
"workflow_dispatch",
"workflow_job",
"workflow_run"
]);
/**
* Fields to strip from the JSON data.
*
* EVENT_ACTION_FIELDS: stripped from each event action object (top level only)
* Example event action object before stripping:
* {
* "description": "This event is triggered when...", // <-- stripped
* "summary": "A brief summary", // <-- stripped
* "availability": ["repository"], // <-- stripped
* "category": "issues", // <-- stripped
* "action": "opened", // kept
* "bodyParameters": [...] // kept
* }
*
* BODY_PARAM_FIELDS: stripped from every bodyParameters object, recursively through childParamsGroups
* Example bodyParameter object before stripping:
* {
* "type": "object", // <-- stripped
* "name": "changes", // kept (used for property names)
* "in": "body", // <-- stripped
* "description": "The changes that were made.", // kept (used for hover docs)
* "isRequired": true, // <-- stripped
* "enum": ["a", "b"], // <-- stripped
* "default": "a", // <-- stripped
* "childParamsGroups": [ // kept (used for nested properties)
* {
* "type": "string", // <-- stripped (recursive)
* "name": "from", // kept
* "isRequired": true // <-- stripped (recursive)
* }
* ]
* }
*/
const EVENT_ACTION_FIELDS = ["description", "summary", "availability", "category"];
const BODY_PARAM_FIELDS = ["type", "in", "isRequired", "enum", "default"];
/**
* Strip fields from a bodyParameter object and recursively from childParamsGroups.
*/
function stripBodyParam(param: any): any {
if (typeof param !== "object" || param === null) {
return param;
}
const result: any = {};
for (const [key, value] of Object.entries(param)) {
if (BODY_PARAM_FIELDS.includes(key)) {
continue; // Strip this field
}
if (key === "childParamsGroups" && Array.isArray(value)) {
result[key] = value.map(stripBodyParam);
} else {
result[key] = value;
}
}
return result;
}
/**
* Strip unused fields from event action data.
*/
function stripEventActionFields(action: any): any {
const result: any = {};
for (const [key, value] of Object.entries(action)) {
if (EVENT_ACTION_FIELDS.includes(key)) {
continue; // Strip this field
}
if (key === "bodyParameters" && Array.isArray(value)) {
result[key] = value.map((p: any) => (typeof p === "number" ? p : stripBodyParam(p)));
} else {
result[key] = value;
}
}
return result;
}
/**
* Strip unused fields from all webhooks.
* Structure: { eventName: { actionName: { ...fields } } }
*/
function stripFields(webhooks: Record<string, Record<string, any>>): Record<string, Record<string, any>> {
const result: Record<string, Record<string, any>> = {};
for (const [eventName, actions] of Object.entries(webhooks)) {
result[eventName] = {};
for (const [actionName, actionData] of Object.entries(actions)) {
result[eventName][actionName] = stripEventActionFields(actionData);
}
}
return result;
}
const rawWebhooks = Object.values(schema.webhooks || schema["x-webhooks"]) as any[];
if (!rawWebhooks) {
throw new Error("No webhooks found in schema");
}
const webhooks: Webhook[] = [];
for (const webhook of Object.values(rawWebhooks)) {
webhooks.push(new Webhook(webhook.post));
}
await Promise.all(webhooks.map(webhook => webhook.process()));
// Check for unknown events (not in DROPPED_EVENTS or KEPT_EVENTS)
const unknownEvents: string[] = [];
for (const webhook of webhooks) {
if (!DROPPED_EVENTS.has(webhook.category) && !KEPT_EVENTS.has(webhook.category)) {
if (!unknownEvents.includes(webhook.category)) {
unknownEvents.push(webhook.category);
}
}
}
if (unknownEvents.length > 0) {
console.error("");
console.error("══════════════════════════════════════════════════════════════════");
console.error("ERROR: New webhook event(s) detected!");
console.error("══════════════════════════════════════════════════════════════════");
console.error("");
console.error("The following events are not categorized:");
for (const event of unknownEvents.sort()) {
console.error(` - ${event}`);
}
console.error("");
console.error("Action required:");
console.error(" 1. Check if the event is a valid workflow trigger:");
console.error(
" https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows"
);
console.error("");
console.error(" 2. Add the event to DROPPED_EVENTS or KEPT_EVENTS in:");
console.error(" languageservice/script/webhooks/index.ts");
console.error("");
console.error(" 3. See docs/json-data-files.md for more details.");
console.error("");
process.exit(1);
}
// The category is the name of the webhook
const categorizedWebhooks: Record<string, Record<string, Webhook>> = {};
for (const webhook of webhooks) {
if (!webhook.action) webhook.action = "default";
// Drop unused events
if (DROPPED_EVENTS.has(webhook.category)) {
continue;
}
if (categorizedWebhooks[webhook.category]) {
categorizedWebhooks[webhook.category][webhook.action] = webhook;
} else {
categorizedWebhooks[webhook.category] = {};
categorizedWebhooks[webhook.category][webhook.action] = webhook;
}
}
// Strip fields before deduplication
const strippedWebhooks = stripFields(categorizedWebhooks);
// Deduplicate after dropping and stripping
const objectsArray = deduplicateWebhooks(strippedWebhooks);
// Write optimized output
await fs.writeFile(OBJECTS_PATH, JSON.stringify(objectsArray, null, 2));
await fs.writeFile(OUTPUT_PATH, JSON.stringify(strippedWebhooks, null, 2));
console.log(`Wrote ${OUTPUT_PATH} (${Object.keys(strippedWebhooks).length} events)`);
console.log(`Wrote ${OBJECTS_PATH} (${objectsArray.length} objects)`);
// Optionally generate intermediate versions for size comparison
if (generateAll) {
// Helper to deep clone
function clone<T>(obj: T): T {
return JSON.parse(JSON.stringify(obj));
}
// Build full webhooks (no drop, no strip) from fresh data
const fullWebhooks: Record<string, Record<string, any>> = {};
for (const webhook of webhooks) {
const w = clone(webhook);
if (!w.action) w.action = "default";
fullWebhooks[w.category] ||= {};
fullWebhooks[w.category][w.action] = w;
}
// Generate all version (no drop, no strip)
const allWebhooks = clone(fullWebhooks);
const allObjects = deduplicateWebhooks(allWebhooks);
await fs.writeFile(ALL_OUTPUT_PATH, JSON.stringify(allWebhooks, null, 2));
await fs.writeFile(ALL_OBJECTS_PATH, JSON.stringify(allObjects, null, 2));
console.log(`Wrote ${ALL_OUTPUT_PATH} (${Object.keys(allWebhooks).length} events)`);
console.log(`Wrote ${ALL_OBJECTS_PATH} (${allObjects.length} objects)`);
// Generate drop-only version (drop events, no strip)
const dropWebhooks = clone(fullWebhooks);
for (const event of DROPPED_EVENTS) {
delete dropWebhooks[event];
}
const dropObjects = deduplicateWebhooks(dropWebhooks);
await fs.writeFile(DROP_OUTPUT_PATH, JSON.stringify(dropWebhooks, null, 2));
await fs.writeFile(DROP_OBJECTS_PATH, JSON.stringify(dropObjects, null, 2));
console.log(`Wrote ${DROP_OUTPUT_PATH} (${Object.keys(dropWebhooks).length} events)`);
console.log(`Wrote ${DROP_OBJECTS_PATH} (${dropObjects.length} objects)`);
// Generate strip-only version (strip fields, no drop)
const stripWebhooks = stripFields(clone(fullWebhooks));
const stripObjects = deduplicateWebhooks(stripWebhooks);
await fs.writeFile(STRIP_OUTPUT_PATH, JSON.stringify(stripWebhooks, null, 2));
await fs.writeFile(STRIP_OBJECTS_PATH, JSON.stringify(stripObjects, null, 2));
console.log(`Wrote ${STRIP_OUTPUT_PATH} (${Object.keys(stripWebhooks).length} events)`);
console.log(`Wrote ${STRIP_OBJECTS_PATH} (${stripObjects.length} objects)`);
}
+291
View File
@@ -0,0 +1,291 @@
import {promises as fs} from "fs";
import Webhook from "./webhook.js";
import schemaImport from "rest-api-description/descriptions/api.github.com/dereferenced/api.github.com.deref.json";
import {deduplicateWebhooks} from "./deduplicate.js";
import eventFilters from "../../src/context-providers/events/event-filters.json";
const schema = schemaImport as any;
const DROPPED_EVENTS = new Set(eventFilters.dropped);
const KEPT_EVENTS = new Set(eventFilters.kept);
const OUTPUT_PATH = "./src/context-providers/events/webhooks.json";
const OBJECTS_PATH = "./src/context-providers/events/webhooks.objects.json";
const STRINGS_PATH = "./src/context-providers/events/webhooks.strings.json";
const FULL_OUTPUT_PATH = "./src/context-providers/events/webhooks.full.json";
/**
* Fields discarded from each event action object (top level only).
* Body parameters are compacted to only keep name, description, and childParamsGroups.
*/
const EVENT_ACTION_FIELDS = ["description", "summary", "availability", "category", "action"];
/**
* Convert a bodyParameter object to compact array format.
*
* Format (type-based dispatch):
* - "name" - name only (just a string)
* - [name, desc] - name + description (desc is string)
* - [name, [...children]] - name + children (arr[1] is array)
* - [name, desc, [...children]] - name + description + children
*
* The reader uses typeof to determine the meaning:
* - string -> name only
* - array with string arr[1] -> name + description
* - array with array arr[1] -> name + children
*/
function compactParam(param: any): any {
if (typeof param !== "object" || param === null) {
return param;
}
const name: string = param.name;
const desc: string | undefined = param.description;
const children: any[] | undefined = param.childParamsGroups;
const hasDesc = desc && desc.length > 0;
const hasChildren = children && children.length > 0;
if (hasDesc && hasChildren) {
return [name, desc, children.map(compactParam)];
} else if (hasChildren) {
return [name, children.map(compactParam)];
} else if (hasDesc) {
return [name, desc];
} else {
return name; // Just the string, not wrapped in array
}
}
/**
* Convert event action data to compact format.
*/
function compactEventAction(action: any): any {
const result: any = {};
for (const [key, value] of Object.entries(action)) {
if (EVENT_ACTION_FIELDS.includes(key)) {
continue; // Discard this field
}
if (key === "bodyParameters" && Array.isArray(value)) {
// Use short key 'p' for params
result["p"] = value.map((p: any) => (typeof p === "number" ? p : compactParam(p)));
} else {
result[key] = value;
}
}
return result;
}
/**
* Convert all webhooks to compact format.
* Structure: { eventName: { actionName: { ...fields } } }
*/
function compactWebhooks(webhooks: Record<string, Record<string, any>>): Record<string, Record<string, any>> {
const result: Record<string, Record<string, any>> = {};
for (const [eventName, actions] of Object.entries(webhooks)) {
result[eventName] = {};
for (const [actionName, actionData] of Object.entries(actions)) {
result[eventName][actionName] = compactEventAction(actionData);
}
}
return result;
}
const rawWebhooks = Object.values(schema.webhooks || schema["x-webhooks"]) as any[];
if (!rawWebhooks) {
throw new Error("No webhooks found in schema");
}
const webhooks: Webhook[] = [];
for (const webhook of Object.values(rawWebhooks)) {
webhooks.push(new Webhook(webhook.post));
}
await Promise.all(webhooks.map(webhook => webhook.process()));
// Check for unknown events (not in DROPPED_EVENTS or KEPT_EVENTS)
const unknownEvents: string[] = [];
for (const webhook of webhooks) {
if (!DROPPED_EVENTS.has(webhook.category) && !KEPT_EVENTS.has(webhook.category)) {
if (!unknownEvents.includes(webhook.category)) {
unknownEvents.push(webhook.category);
}
}
}
if (unknownEvents.length > 0) {
console.error("");
console.error("══════════════════════════════════════════════════════════════════");
console.error("ERROR: New webhook event(s) detected!");
console.error("══════════════════════════════════════════════════════════════════");
console.error("");
console.error("The following events are not categorized:");
for (const event of unknownEvents.sort()) {
console.error(` - ${event}`);
}
console.error("");
console.error("Action required:");
console.error(" 1. Check if the event is a valid workflow trigger:");
console.error(
" https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows"
);
console.error("");
console.error(" 2. Add the event to 'dropped' or 'kept' array in:");
console.error(" languageservice/src/context-providers/events/event-filters.json");
console.error("");
console.error(" 3. See docs/json-data-files.md for more details.");
console.error("");
process.exit(1);
}
// Build full webhooks (all events, no transformations) for validation tests
const fullWebhooks: Record<string, Record<string, Webhook>> = {};
for (const webhook of webhooks) {
if (!webhook.action) webhook.action = "default";
if (fullWebhooks[webhook.category]) {
fullWebhooks[webhook.category][webhook.action] = webhook;
} else {
fullWebhooks[webhook.category] = {};
fullWebhooks[webhook.category][webhook.action] = webhook;
}
}
// Write full version (before any optimizations)
await fs.writeFile(FULL_OUTPUT_PATH, JSON.stringify(fullWebhooks, null, 2));
console.log(`Wrote ${FULL_OUTPUT_PATH} (${Object.keys(fullWebhooks).length} events, unoptimized)`);
// The category is the name of the webhook
const categorizedWebhooks: Record<string, Record<string, Webhook>> = {};
for (const webhook of webhooks) {
if (!webhook.action) webhook.action = "default";
// Drop unused events
if (DROPPED_EVENTS.has(webhook.category)) {
continue;
}
if (categorizedWebhooks[webhook.category]) {
categorizedWebhooks[webhook.category][webhook.action] = webhook;
} else {
categorizedWebhooks[webhook.category] = {};
categorizedWebhooks[webhook.category][webhook.action] = webhook;
}
}
// Convert to compact format before deduplication
const compactedWebhooks = compactWebhooks(categorizedWebhooks);
// Deduplicate after compacting
const objectsArray = deduplicateWebhooks(compactedWebhooks);
// ============================================================================
// String Interning (Phase 3)
// ============================================================================
// Intern duplicate property names to reduce file size.
// Names appearing 2+ times are stored in a string table and referenced by index.
// Singleton names stay as literal strings for readability.
/**
* Collect all property names from params (for frequency counting)
*/
function collectNames(param: any, counts: Map<string, number>): void {
if (typeof param === "number") return;
if (typeof param === "string") {
counts.set(param, (counts.get(param) || 0) + 1);
return;
}
if (Array.isArray(param)) {
const name = param[0] as string;
counts.set(name, (counts.get(name) || 0) + 1);
const children = Array.isArray(param[1]) ? param[1] : param[2];
if (children) children.forEach((c: any) => collectNames(c, counts));
}
}
/**
* Replace duplicate names with indices into the string table.
* Object references use negative indices: objectIndex -> -(objectIndex + 1)
* String references use non-negative indices: stringIndex -> stringIndex
*
* @param param - The param to process
* @param nameToIndex - Map from name to string table index
*/
function internNames(param: any, nameToIndex: Map<string, number>): any {
// Object reference (already a number from deduplication) -> make negative
if (typeof param === "number") return -(param + 1);
// String -> intern if in table, otherwise keep as literal
if (typeof param === "string") {
const idx = nameToIndex.get(param);
return idx !== undefined ? idx : param;
}
if (Array.isArray(param)) {
const name = param[0] as string;
const idx = nameToIndex.get(name);
const internedName = idx !== undefined ? idx : name;
// Handle different array formats
if (typeof param[1] === "string" && !Array.isArray(param[1])) {
// [name, desc] or [name, desc, children]
if (param.length === 2) {
return [internedName, param[1]];
} else {
return [internedName, param[1], (param[2] as any[]).map((c: any) => internNames(c, nameToIndex))];
}
} else if (Array.isArray(param[1])) {
// [name, children]
return [internedName, param[1].map((c: any) => internNames(c, nameToIndex))];
}
// Shouldn't happen, but fallback
return [internedName, ...param.slice(1)];
}
return param;
}
// Pass 1: Count all names
const nameCounts = new Map<string, number>();
objectsArray.forEach((obj: any) => collectNames(obj, nameCounts));
for (const event of Object.values(compactedWebhooks)) {
for (const action of Object.values(event as Record<string, any>)) {
if (action.p) action.p.forEach((p: any) => collectNames(p, nameCounts));
}
}
// Build string table from duplicates, sorted by frequency (most common first = smaller indices)
const sortedNames = [...nameCounts.entries()].filter(([, count]) => count >= 2).sort((a, b) => b[1] - a[1]);
const stringTable = sortedNames.map(([name]) => name);
const nameToIndex = new Map(stringTable.map((name, i) => [name, i]));
console.log(
`String table: ${stringTable.length} interned names (${nameCounts.size - stringTable.length} singletons kept inline)`
);
// Pass 2: Intern names in objects and webhooks
// Objects use negative indices, strings use non-negative indices
const internedObjects = objectsArray.map((obj: any) => internNames(obj, nameToIndex));
const internedWebhooks: Record<string, Record<string, any>> = {};
for (const [eventName, actions] of Object.entries(compactedWebhooks)) {
internedWebhooks[eventName] = {};
for (const [actionName, actionData] of Object.entries(actions as Record<string, any>)) {
internedWebhooks[eventName][actionName] = {
p: actionData.p.map((p: any) => internNames(p, nameToIndex))
};
}
}
// Write optimized output with separate string table
// Format: webhooks.strings.json has string table, webhooks.json/webhooks.objects.json reference by index
const finalOutput = {
"//": "Generated file - refer to docs/json-data-files.md for format documentation",
...internedWebhooks
};
await fs.writeFile(STRINGS_PATH, JSON.stringify(stringTable, null, 2));
await fs.writeFile(OBJECTS_PATH, JSON.stringify(internedObjects, null, 2));
await fs.writeFile(OUTPUT_PATH, JSON.stringify(finalOutput, null, 2));
console.log(`Wrote ${STRINGS_PATH} (${stringTable.length} interned strings)`);
console.log(`Wrote ${OUTPUT_PATH} (${Object.keys(compactedWebhooks).length} events)`);
console.log(`Wrote ${OBJECTS_PATH} (${internedObjects.length} objects)`);
@@ -0,0 +1,75 @@
{
"dropped": [
"branch_protection_configuration",
"code_scanning_alert",
"commit_comment",
"custom_property",
"custom_property_values",
"dependabot_alert",
"deploy_key",
"github_app_authorization",
"installation",
"installation_repositories",
"installation_target",
"marketplace_purchase",
"member",
"membership",
"merge_group",
"meta",
"org_block",
"organization",
"package",
"personal_access_token_request",
"ping",
"repository",
"repository_advisory",
"repository_ruleset",
"secret_scanning_alert",
"secret_scanning_alert_location",
"security_advisory",
"security_and_analysis",
"sponsorship",
"star",
"team",
"team_add"
],
"kept": [
"branch_protection_rule",
"check_run",
"check_suite",
"create",
"delete",
"deployment",
"deployment_status",
"discussion",
"discussion_comment",
"fork",
"gollum",
"issue_comment",
"issues",
"label",
"milestone",
"page_build",
"project",
"project_card",
"project_column",
"projects_v2",
"projects_v2_item",
"public",
"pull_request",
"pull_request_review",
"pull_request_review_comment",
"pull_request_review_thread",
"push",
"registry_package",
"release",
"repository_dispatch",
"repository_import",
"repository_vulnerability_alert",
"status",
"watch",
"workflow_dispatch",
"workflow_job",
"workflow_run"
]
}
@@ -1,5 +1,34 @@
import {DescriptionDictionary} from "@actions/expressions";
import {existsSync} from "fs";
import {fileURLToPath} from "url";
import {dirname, join} from "path";
import {data, DescriptionDictionary, DescriptionPair, isDescriptionDictionary} from "@actions/expressions";
import {getEventPayload, getSupportedEventTypes} from "./eventPayloads";
import eventFilters from "./event-filters.json";
const DROPPED_EVENTS = new Set(eventFilters.dropped);
// Check if full webhooks file exists (generated by npm run update-webhooks)
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const fullWebhooksPath = join(__dirname, "webhooks.full.json");
const hasFullWebhooks = existsSync(fullWebhooksPath);
type Param = {
name: string;
description?: string;
childParamsGroups?: Param[];
};
type FullAction = {
action?: string;
bodyParameters?: Param[];
};
type FullWebhooks = {
[event: string]: {
[action: string]: FullAction;
};
};
describe("eventPayloads", () => {
describe("getSupportedEventTypes", () => {
@@ -100,3 +129,149 @@ describe("eventPayloads", () => {
});
});
});
// Optimization validation tests - only run if webhooks.full.json exists
// This file is generated by: npm run update-webhooks
// In CI, a separate job runs these tests after generating the file
const describeIfFullExists = hasFullWebhooks ? describe : describe.skip;
// Marker file path - written after validation tests complete
const fullValidationMarkerPath = join(__dirname, "webhooks.full.validation-complete");
describeIfFullExists("optimization validation", () => {
let keptWebhooks: FullWebhooks;
beforeAll(async () => {
// Dynamically import the full webhooks file using fs to avoid TS module resolution
const {readFileSync} = await import("fs");
const fullWebhooks = JSON.parse(readFileSync(fullWebhooksPath, "utf-8")) as FullWebhooks;
// Filter to only kept events
keptWebhooks = {};
for (const [event, actions] of Object.entries(fullWebhooks)) {
if (!DROPPED_EVENTS.has(event)) {
keptWebhooks[event] = actions;
}
}
});
afterAll(async () => {
// Write marker file to prove validation tests ran
const {writeFileSync} = await import("fs");
writeFileSync(fullValidationMarkerPath, new Date().toISOString());
});
/**
* Build a DescriptionDictionary from raw params (same logic as eventPayloads.ts)
*/
function buildFromParams(params: Param[]): DescriptionDictionary {
const d = new DescriptionDictionary();
for (const param of params) {
if (param.childParamsGroups && param.childParamsGroups.length > 0) {
const child = buildFromParams(param.childParamsGroups);
d.add(param.name, child, param.description);
} else {
// Match the behavior in eventPayloads.ts - don't overwrite existing
if (!d.get(param.name)) {
d.add(param.name, new data.Null(), param.description);
}
}
}
return d;
}
/**
* Compare two DescriptionDictionary structures recursively
*/
function compareStructures(full: DescriptionDictionary, optimized: DescriptionDictionary, path: string): string[] {
const errors: string[] = [];
const fullPairs = full.pairs();
const optimizedPairs = optimized.pairs();
const fullKeys = new Set(fullPairs.map((p: DescriptionPair) => p.key));
const optimizedKeys = new Set(optimizedPairs.map((p: DescriptionPair) => p.key));
// Check for missing keys in optimized
for (const key of fullKeys) {
if (!optimizedKeys.has(key)) {
errors.push(`Missing key in optimized: ${path}.${key}`);
}
}
// Check for extra keys in optimized
for (const key of optimizedKeys) {
if (!fullKeys.has(key)) {
errors.push(`Extra key in optimized: ${path}.${key}`);
}
}
// Compare descriptions and recurse into nested structures
for (const fullPair of fullPairs) {
const optimizedValue = optimized.get(fullPair.key);
if (optimizedValue === undefined) continue;
// Compare descriptions
const fullDesc = full.getDescription(fullPair.key) ?? "";
const optimizedDesc = optimized.getDescription(fullPair.key) ?? "";
if (fullDesc !== optimizedDesc) {
errors.push(
`Description mismatch at ${path}.${fullPair.key}: ` + `full="${fullDesc}" vs optimized="${optimizedDesc}"`
);
}
// Recurse into nested dictionaries
if (isDescriptionDictionary(fullPair.value) && isDescriptionDictionary(optimizedValue)) {
errors.push(...compareStructures(fullPair.value, optimizedValue, `${path}.${fullPair.key}`));
}
}
return errors;
}
it("optimized webhooks match full source for all events and actions", () => {
const allErrors: string[] = [];
for (const [event, actions] of Object.entries(keptWebhooks)) {
for (const [action, actionData] of Object.entries(actions)) {
// Build from full source (use bodyParameters, may be undefined)
const params = actionData.bodyParameters || [];
const fullPayload = buildFromParams(params);
// Get from optimized (deduplicated) source
const optimizedPayload = getEventPayload(event, action);
if (!optimizedPayload) {
allErrors.push(`Missing optimized payload for ${event}.${action}`);
continue;
}
const errors = compareStructures(fullPayload, optimizedPayload, `${event}.${action}`);
allErrors.push(...errors);
}
}
if (allErrors.length > 0) {
fail(
`Optimization validation failed:\n${allErrors.slice(0, 20).join("\n")}` +
(allErrors.length > 20 ? `\n... and ${allErrors.length - 20} more errors` : "")
);
}
});
it("all full source events are present in optimized version", () => {
for (const event of Object.keys(keptWebhooks)) {
const types = getSupportedEventTypes(event);
expect(types.length).toBeGreaterThan(0);
}
});
it("all full source actions are present in optimized version", () => {
for (const [event, actions] of Object.entries(keptWebhooks)) {
for (const action of Object.keys(actions)) {
const payload = getEventPayload(event, action);
expect(payload).toBeDefined();
}
}
});
});
@@ -1,7 +1,8 @@
import {data, DescriptionDictionary} from "@actions/expressions";
import webhookObjects from "./objects.min.json";
import webhooks from "./webhooks.min.json";
import webhooksData from "./webhooks.min.json";
import objectsData from "./webhooks.objects.min.json";
import stringsData from "./webhooks.strings.min.json";
import schedule from "./schedule.min.json";
import workflow_call from "./workflow_call.min.json";
@@ -49,9 +50,22 @@ type Param = {
};
/**
* A full {@link Param} or an index into the objects array for deduplicated parameters
* Compact format for params (written by update-webhooks.ts).
*
* Names can be interned (number = index into string table) or literal strings.
* Type-based dispatch:
* - number - interned name only (index into string table)
* - "name" - literal name only (singleton, not interned)
* - [name, desc] - name + description (name is number or string, desc is string)
* - [name, [...children]] - name + children (arr[1] is array)
* - [name, desc, [...children]] - name + description + children
*/
type DeduplicatedParam = Param | number;
type InternedName = number | string;
type CompactParam =
| InternedName
| [InternedName, string]
| [InternedName, CompactParam[]]
| [InternedName, string, CompactParam[]];
type WebhookPayload = {
descriptionHtml: string;
@@ -65,17 +79,33 @@ type Webhooks = {
};
};
type DeduplicatedWebhooks = {
[name: string]: {
[action: string]: WebhookPayload & {
bodyParameters: DeduplicatedParam[];
};
};
/**
* Webhooks data format after optimization:
* {
* [event]: { [action]: { p: CompactParam[] } }
* }
*
* String table and objects are loaded from separate files.
*/
type WebhooksData = {
[key: string]: {[action: string]: {p: CompactParam[]}};
};
/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any */
const dedupedWebhookPayloads: DeduplicatedWebhooks = webhooks as any;
const objects: Param[] = webhookObjects as any;
const webhooksJson: WebhooksData = webhooksData as any;
const objectsJson: CompactParam[] = objectsData as any;
// String table and objects are in separate files
const stringTable: string[] = stringsData;
const objects: CompactParam[] = objectsJson;
// Build event payloads map (skip "//" comment key)
const dedupedWebhookPayloads: {[event: string]: {[action: string]: {p: CompactParam[]}}} = {};
for (const [key, value] of Object.entries(webhooksJson)) {
if (key !== "//" && typeof value === "object" && value !== null) {
dedupedWebhookPayloads[key] = value as {[action: string]: {p: CompactParam[]}};
}
}
/* eslint-enable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any */
// Hydrated webhook payloads
@@ -169,10 +199,14 @@ function getWebhookPayload(event: string, action: string): WebhookPayload | unde
return undefined;
}
// Get params from 'p' (compact format)
const dedupedParams = deduplicatedPayload.p || [];
// Recreate the full payload and store it for reuse
const params = deduplicatedPayload.bodyParameters.map(p => fullParam(p));
const params = dedupedParams.map(p => fullParam(p));
const payload = {
...deduplicatedPayload,
descriptionHtml: "",
summaryHtml: "",
bodyParameters: params
};
webhookPayloads[event] ||= {};
@@ -180,13 +214,65 @@ function getWebhookPayload(event: string, action: string): WebhookPayload | unde
return payload;
}
function fullParam(dedupedParam: DeduplicatedParam): Param {
if (typeof dedupedParam === "number") {
if (dedupedParam >= objects.length) {
throw new Error(`Unknown object ${dedupedParam}`);
/**
* Resolve an interned name (non-negative number -> string table lookup) or return literal string
*/
function resolveName(name: InternedName): string {
if (typeof name === "number") {
if (name < 0 || name >= stringTable.length) {
throw new Error(`Unknown interned name index ${name}`);
}
return objects[dedupedParam];
return stringTable[name];
}
return name;
}
/**
* Convert a deduplicated param to a full Param.
*
* Compact format (type-based dispatch):
* - negative number - object index: -(n + 1) -> objects[-n - 1]
* - non-negative number - interned string index -> stringTable[n]
* - "name" - literal name only (singleton, not interned)
* - [name, desc] - name + description (name can be number or string)
* - [name, [...children]] - name + children (arr[1] is array)
* - [name, desc, [...children]] - name + description + children
*/
function fullParam(dedupedParam: CompactParam): Param {
// Negative number -> object index
if (typeof dedupedParam === "number" && dedupedParam < 0) {
const objectIndex = -(dedupedParam + 1);
if (objectIndex >= objects.length) {
throw new Error(`Unknown object index ${objectIndex} (from ${dedupedParam})`);
}
return fullParam(objects[objectIndex]);
}
return dedupedParam;
// Non-negative number or literal string -> name only
if (typeof dedupedParam === "number" || typeof dedupedParam === "string") {
return {
name: resolveName(dedupedParam),
description: ""
} as Param;
}
// Compact array format -> convert to Param object
if (Array.isArray(dedupedParam)) {
const arr = dedupedParam;
const name = resolveName(arr[0]);
// Type-based dispatch: if arr[1] is string -> description, if array -> children
const description = typeof arr[1] === "string" ? arr[1] : "";
// arr[1] is children if it's an array, otherwise arr[2] is children (if it exists and is an array)
const childrenArr = Array.isArray(arr[1]) ? arr[1] : Array.isArray(arr[2]) ? arr[2] : undefined;
const childParamsGroups = childrenArr ? childrenArr.map(c => fullParam(c)) : undefined;
return {
name,
description,
childParamsGroups
} as Param;
}
throw new Error(`Unexpected param format: ${JSON.stringify(dedupedParam)}`);
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,416 @@
[
"url",
"id",
"node_id",
"html_url",
"name",
"events_url",
"avatar_url",
"login",
"repos_url",
"type",
"gravatar_id",
"followers_url",
"following_url",
"gists_url",
"starred_url",
"subscriptions_url",
"organizations_url",
"received_events_url",
"site_admin",
"email",
"deleted",
"created_at",
"description",
"updated_at",
"href",
"labels_url",
"comments_url",
"number",
"user",
"statuses_url",
"owner",
"commits_url",
"state",
"open_issues",
"sha",
"title",
"ref",
"body",
"hooks_url",
"issues_url",
"full_name",
"private",
"fork",
"archive_url",
"assignees_url",
"blobs_url",
"branches_url",
"collaborators_url",
"compare_url",
"contents_url",
"contributors_url",
"deployments_url",
"downloads_url",
"forks_url",
"git_commits_url",
"git_refs_url",
"git_tags_url",
"issue_comment_url",
"issue_events_url",
"keys_url",
"languages_url",
"merges_url",
"milestones_url",
"notifications_url",
"pulls_url",
"releases_url",
"stargazers_url",
"subscribers_url",
"subscription_url",
"tags_url",
"teams_url",
"trees_url",
"slug",
"size",
"repo",
"closed_at",
"permissions",
"archived",
"comments",
"homepage",
"license",
"git_url",
"ssh_url",
"clone_url",
"mirror_url",
"svn_url",
"language",
"forks_count",
"stargazers_count",
"watchers_count",
"default_branch",
"open_issues_count",
"is_template",
"topics",
"has_issues",
"has_projects",
"has_wiki",
"has_pages",
"has_downloads",
"disabled",
"visibility",
"pushed_at",
"forks",
"delete_branch_on_merge",
"allow_forking",
"watchers",
"allow_rebase_merge",
"allow_squash_merge",
"allow_auto_merge",
"allow_update_branch",
"allow_merge_commit",
"from",
"web_commit_signoff_required",
"author_association",
"label",
"labels",
"key",
"admin",
"pull",
"triage",
"push",
"maintain",
"draft",
"public",
"commits",
"organization",
"spdx_id",
"master_branch",
"active_lock_reason",
"locked",
"review_comments",
"use_squash_pr_title_as_default",
"squash_merge_commit_title",
"squash_merge_commit_message",
"merge_commit_title",
"merge_commit_message",
"role_name",
"creator",
"stargazers",
"milestone",
"members_url",
"statuses",
"assignee",
"assignees",
"permission",
"privacy",
"repositories_url",
"base",
"status",
"head",
"reactions",
"+1",
"-1",
"confused",
"eyes",
"heart",
"hooray",
"laugh",
"rocket",
"total_count",
"diff_url",
"merged_at",
"patch_url",
"changes",
"due_on",
"repository_url",
"color",
"issue",
"closed_issues",
"pull_request",
"conclusion",
"default",
"_links",
"html",
"author",
"self",
"pull_requests",
"issue_url",
"has_discussions",
"metadata",
"date",
"review_comment",
"auto_merge",
"merge_commit_sha",
"requested_reviewers",
"requested_teams",
"review_comment_url",
"review_comments_url",
"starred_at",
"external_url",
"issues",
"checks",
"contents",
"deployments",
"events",
"head_sha",
"username",
"performed_via_github_app",
"timeline_url",
"additions",
"changed_files",
"deletions",
"maintainer_can_modify",
"mergeable",
"mergeable_state",
"merged",
"merged_by",
"rebaseable",
"head_branch",
"completed_at",
"started_at",
"parent",
"commit_message",
"commit_title",
"enabled_by",
"merge_method",
"pages",
"path",
"actions",
"administration",
"content_references",
"discussions",
"emails",
"environments",
"keys",
"members",
"organization_administration",
"organization_hooks",
"organization_packages",
"organization_plan",
"organization_projects",
"organization_secrets",
"organization_self_hosted_runners",
"organization_user_blocking",
"packages",
"repository_hooks",
"repository_projects",
"secret_scanning_alerts",
"secrets",
"security_events",
"security_scanning_alert",
"single_file",
"team_discussions",
"vulnerability_alerts",
"workflows",
"run_attempt",
"committer",
"message",
"prerelease",
"tag_name",
"target_commitish",
"repository",
"repository_id",
"head_commit",
"timestamp",
"tree_id",
"to",
"content_type",
"published_at",
"category",
"emoji",
"is_answerable",
"state_reason",
"affected_package_name",
"affected_range",
"external_identifier",
"external_reference",
"fixed_in",
"ghsa_id",
"severity",
"check_run_url",
"run_id",
"run_url",
"runner_group_id",
"runner_group_name",
"runner_id",
"runner_name",
"workflow_name",
"steps",
"installations_count",
"client_id",
"client_secret",
"webhook_secret",
"pem",
"after",
"before",
"answer_chosen_at",
"answer_chosen_by",
"answer_html_url",
"release",
"assets",
"assets_url",
"tarball_url",
"upload_url",
"zipball_url",
"app",
"environment",
"summary",
"actor",
"artifacts_url",
"cancel_url",
"check_suite_id",
"check_suite_node_id",
"check_suite_url",
"event",
"head_repository",
"jobs_url",
"logs_url",
"previous_attempt_url",
"referenced_workflows",
"rerun_url",
"run_number",
"run_started_at",
"triggering_actor",
"workflow_id",
"workflow_url",
"note",
"after_id",
"column_id",
"project_url",
"dismiss_reason",
"dismissed_at",
"dismisser",
"fix_reason",
"fixed_at",
"commit",
"workflow_job",
"action",
"temp_clone_token",
"subscribers_count",
"network_count",
"check_suite",
"deployment",
"task",
"original_environment",
"transient_environment",
"production_environment",
"payload",
"workflow_run",
"discussion",
"comment",
"child_comment_count",
"discussion_id",
"parent_id",
"column_url",
"commit_id",
"pull_request_url",
"browser_download_url",
"download_count",
"uploader",
"discussion_url",
"alert",
"manifest",
"installation_command",
"version",
"admin_enforced",
"authorized_actor_names",
"authorized_actors_only",
"authorized_dismissal_actors_only",
"linear_history_requirement_enforcement_level",
"required_status_checks",
"required_status_checks_enforcement_level",
"check_run",
"details_url",
"external_id",
"actions_meta",
"check_runs_url",
"latest_check_runs_count",
"rerequestable",
"runs_rerequestable",
"workflow",
"display_title",
"project_card",
"content_url",
"short_description",
"archived_at",
"reason",
"diff_hunk",
"in_reply_to_id",
"line",
"original_commit_id",
"original_line",
"original_position",
"original_start_line",
"position",
"pull_request_review_id",
"side",
"start_line",
"start_side",
"ref_type",
"target_url",
"new_repository",
"pusher",
"added",
"distinct",
"modified",
"removed",
"registry_package",
"ecosystem",
"namespace",
"package_type",
"package_version",
"body_html",
"docker_metadata",
"package_files",
"download_url",
"md5",
"sha1",
"sha256",
"package_url",
"rubygems_metadata",
"target_oid",
"registry",
"dismiss_comment"
]
+98
View File
@@ -0,0 +1,98 @@
# Workflow Schema Optimization Plan
## Current State (Commit 7660f61)
### What's Implemented
1. **Original schema preserved**: `workflow-v1.0.json` remains source of truth with 291 definitions
2. **Optimization script**: `script/optimize-workflow-schema.js` prunes unused definitions
3. **Generated files** (gitignored):
- `workflow-v1.0.optimized.json` - 281 definitions (pruned)
- `workflow-v1.0.optimized.min.json` - minified version loaded at runtime
4. **Build pipeline**: `npm run minify-json` chains optimize → minify
5. **Tests**: `workflow-schema.test.ts` validates schema integrity
### 10 Pruned Definitions
These are unreachable from `workflow-root-strict` entry point:
- `workflow-root` (non-strict variant)
- `on` (non-strict variant)
- `on-mapping` (non-strict variant)
- `job-if-result`
- `step-if-result`
- `boolean-needs-context`
- `number-needs-context`
- `string-needs-context`
- `boolean-steps-context`
- `number-steps-context`
### Size Savings
| Metric | Original | Optimized | Savings |
|--------|----------|-----------|---------|
| Definitions | 291 | 281 | 10 removed |
| Minified | 71,061 B | 69,022 B | 2.9% |
| Gzipped | 12,318 B | 12,172 B | 1.2% |
## Optimization Strategies Evaluated
### ✅ Pruning Unused Definitions (IMPLEMENTED)
- Removes definitions not reachable from entry point
- 1.2% gzip savings
- Low complexity, no runtime overhead
### ❌ Key Shortening
- Replace long keys with short codes (e.g., `description``d`)
- 1.5% gzip savings
- NOT WORTH IT: Adds complexity, minimal benefit after gzip
### ❌ String Interning
- Deduplicate repeated strings into lookup table
- Makes gzip WORSE (-0.5%)
- NOT WORTH IT: Gzip already handles repetition
### ❌ Compact Format (like webhooks)
- Restructure to array-based format
- Makes gzip WORSE (-0.4%)
- NOT WORTH IT: Schema structure doesn't benefit
### ❌ Split Descriptions
- Separate file for descriptions
- Adds 791 bytes when both files gzipped
- NOT WORTH IT: Worse total size
## Future Work
### Update from Server
The current schema may be missing some definitions from the latest server version. A future PR should:
1. Fetch latest `workflow-v1.0.json` from dotcom server
2. Update the source file
3. Verify tests still pass
4. Note: Server version had issues last checked (e.g., `coerce-raw`, missing `branches` on merge-group)
### Entry Point
- Entry point: `workflow-root-strict` (defined in `workflow-constants.ts`)
- Non-strict `workflow-root` exists but is unused in this codebase
## Files
```
workflow-parser/
├── src/
│ ├── workflow-v1.0.json # Source of truth (tracked, 291 defs)
│ ├── workflow-v1.0.optimized.json # Pruned (gitignored, 281 defs)
│ ├── workflow-v1.0.optimized.min.json # Minified (gitignored)
│ └── workflows/
│ ├── workflow-schema.ts # Loader (imports optimized.min.json)
│ └── workflow-schema.test.ts # Schema integrity tests
└── script/
└── optimize-workflow-schema.js # Pruning script
```
## Test Coverage
- `workflow-schema.test.ts`:
1. Schema loads from workflow-root-strict
2. All referenced definitions are reachable
3. Critical definitions exist (jobs, steps, runs-on, etc.)
+3 -2
View File
@@ -38,7 +38,8 @@
"format-check": "prettier --check '**/*.ts'",
"lint": "eslint 'src/**/*.ts'",
"lint-fix": "eslint --fix 'src/**/*.ts'",
"minify-json": "node ../script/minify-json.js src/workflow-v1.0.json",
"optimize-schema": "node script/optimize-workflow-schema.js",
"minify-json": "npm run optimize-schema && node ../script/minify-json.js src/workflow-v1.0.optimized.json",
"prebuild": "npm run minify-json",
"prepublishOnly": "npm run build && npm run test",
"pretest": "npm run minify-json",
@@ -53,7 +54,7 @@
"yaml": "^2.0.0-8"
},
"engines": {
"node": ">= 16.15"
"node": ">= 18"
},
"files": [
"dist/**/*"
@@ -0,0 +1,114 @@
#!/usr/bin/env node
/**
* Optimizes workflow-v1.0.json by pruning unused definitions.
*
* Removes definitions not reachable from the entry point (workflow-root-strict).
* Output is then minified by minify-json.js to produce the final .min.json file.
*
* Usage: node script/optimize-workflow-schema.js
*/
import {promises as fs} from "fs";
import path from "path";
import {fileURLToPath} from "url";
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const ENTRY_POINT = "workflow-root-strict";
const inputPath = path.join(__dirname, "..", "src", "workflow-v1.0.json");
const outputPath = path.join(__dirname, "..", "src", "workflow-v1.0.optimized.json");
/**
* Find all type references in a definition.
*/
function findRefs(obj) {
const refs = [];
function visit(node) {
if (!node || typeof node !== "object") return;
if (Array.isArray(node)) {
for (const item of node) {
if (typeof item === "string") refs.push(item);
else visit(item);
}
return;
}
for (const [key, value] of Object.entries(node)) {
if (["type", "item-type", "loose-key-type", "loose-value-type"].includes(key)) {
if (typeof value === "string") refs.push(value);
} else if (key === "one-of") {
visit(value);
} else if (key === "properties") {
for (const propValue of Object.values(value)) {
if (typeof propValue === "string") refs.push(propValue);
else if (propValue && typeof propValue === "object") visit(propValue);
}
} else if (["mapping", "sequence", "string"].includes(key)) {
visit(value);
}
}
}
visit(obj);
return refs;
}
/**
* Find all definitions reachable from entry point.
*/
function findReachable(schema, entryPoint) {
const reachable = new Set();
const queue = [entryPoint];
while (queue.length > 0) {
const name = queue.shift();
if (reachable.has(name) || !schema.definitions[name]) continue;
reachable.add(name);
const refs = findRefs(schema.definitions[name]);
for (const ref of refs) {
if (!reachable.has(ref) && schema.definitions[ref]) {
queue.push(ref);
}
}
}
return reachable;
}
async function main() {
const content = await fs.readFile(inputPath, "utf8");
const schema = JSON.parse(content);
const reachable = findReachable(schema, ENTRY_POINT);
const allDefs = Object.keys(schema.definitions);
const unused = allDefs.filter((name) => !reachable.has(name));
console.log(`Entry point: ${ENTRY_POINT}`);
console.log(`Definitions: ${allDefs.length} -> ${reachable.size} (${unused.length} pruned)`);
if (unused.length > 0) {
console.log("\nPruned:");
unused.forEach((name) => console.log(` - ${name}`));
}
// Create pruned schema preserving definition order
const pruned = {version: schema.version, definitions: {}};
for (const name of allDefs) {
if (reachable.has(name)) {
pruned.definitions[name] = schema.definitions[name];
}
}
// Write output (will be minified by minify-json.js)
await fs.writeFile(outputPath, JSON.stringify(pruned));
console.log(`\nOutput: ${outputPath}`);
}
main().catch((err) => {
console.error(err);
process.exit(1);
});
@@ -0,0 +1,85 @@
import {getWorkflowSchema} from "./workflow-schema";
describe("workflow-schema", () => {
it("loads successfully from workflow-root-strict entry point", () => {
const schema = getWorkflowSchema();
expect(schema).toBeDefined();
// Verify entry point exists
const rootDef = schema.getDefinition("workflow-root-strict");
expect(rootDef).toBeDefined();
});
it("has all referenced definitions reachable from workflow-root-strict", () => {
const schema = getWorkflowSchema();
const definitions = schema.definitions;
// Collect all type references from all definitions
const referencedTypes = new Set<string>();
const definedTypes = new Set<string>();
for (const name of Object.keys(definitions)) {
definedTypes.add(name);
collectReferences(definitions[name], referencedTypes);
}
// Every referenced type should be defined
const missingDefinitions: string[] = [];
for (const ref of referencedTypes) {
// Skip built-in types
if (isBuiltInType(ref)) continue;
if (!definedTypes.has(ref)) {
missingDefinitions.push(ref);
}
}
expect(missingDefinitions).toEqual([]);
});
it("can resolve key workflow definitions", () => {
const schema = getWorkflowSchema();
// These are critical definitions that must exist
const criticalDefinitions = [
"workflow-root-strict",
"jobs",
"steps",
"runs-on",
"step-uses",
"job-env",
"step-env",
];
for (const defName of criticalDefinitions) {
const def = schema.getDefinition(defName);
expect(def).toBeDefined();
}
});
});
function collectReferences(obj: unknown, refs: Set<string>): void {
if (!obj || typeof obj !== "object") return;
if (Array.isArray(obj)) {
for (const item of obj) {
if (typeof item === "string") refs.add(item);
else collectReferences(item, refs);
}
return;
}
const record = obj as Record<string, unknown>;
for (const [key, value] of Object.entries(record)) {
if (["type", "item-type", "loose-key-type", "loose-value-type"].includes(key)) {
if (typeof value === "string") refs.add(value);
} else if (key === "one-of" || key === "properties" || key === "mapping" || key === "sequence") {
collectReferences(value, refs);
}
}
}
function isBuiltInType(typeName: string): boolean {
const builtIns = ["null", "boolean", "number", "string", "sequence", "mapping", "any"];
return builtIns.includes(typeName);
}
@@ -1,6 +1,6 @@
import {JSONObjectReader} from "../templates/json-object-reader";
import {TemplateSchema} from "../templates/schema";
import WorkflowSchema from "../workflow-v1.0.min.json";
import WorkflowSchema from "../workflow-v1.0.optimized.min.json";
let schema: TemplateSchema;