Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 eric sciple
|
38ffd53f3b | ||
|
eric sciple
|
7660f61777 | ||
|
eric sciple
|
c04c1b26f4 |
@@ -64,3 +64,33 @@ jobs:
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate-webhooks:
|
||||
runs-on: ubuntu-latest
|
||||
name: Validate webhook optimization
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Use Node.js 22.x
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22.x
|
||||
cache: 'npm'
|
||||
registry-url: 'https://npm.pkg.github.com'
|
||||
- run: npm ci
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Build workspaces
|
||||
run: npm run build -ws
|
||||
- name: Generate full webhooks file
|
||||
run: cd languageservice && npm run update-webhooks
|
||||
- name: Run optimization validation tests
|
||||
run: cd languageservice && npm test -- --testPathPattern=eventPayloads
|
||||
- name: Verify validation tests ran
|
||||
run: |
|
||||
if [ ! -f languageservice/src/context-providers/events/webhooks.full.validation-complete ]; then
|
||||
echo "ERROR: Validation tests did not run!"
|
||||
echo "The webhooks.full.validation-complete marker file was not created."
|
||||
exit 1
|
||||
fi
|
||||
echo "Validation tests completed at: $(cat languageservice/src/context-providers/events/webhooks.full.validation-complete)"
|
||||
|
||||
+8
-4
@@ -7,7 +7,11 @@ node_modules
|
||||
# Minified JSON (generated at build time)
|
||||
*.min.json
|
||||
|
||||
# Intermediate JSON for size comparison (generated by update-webhooks --all)
|
||||
*.all.json
|
||||
*.drop.json
|
||||
*.strip.json
|
||||
# Optimized workflow schema (generated by optimize-workflow-schema.js)
|
||||
*.optimized.json
|
||||
|
||||
# Full webhooks source (generated by update-webhooks, used for validation tests)
|
||||
*.full.json
|
||||
|
||||
# Validation marker (generated by tests)
|
||||
*.validation-complete
|
||||
|
||||
+99
-26
@@ -6,8 +6,9 @@ This document describes the JSON data files used by the language service package
|
||||
|
||||
The language service uses several JSON files containing schema definitions, webhook payloads, and other metadata. To reduce bundle size, these files are:
|
||||
|
||||
1. **Optimized at generation time** — unused events are dropped, unused fields are stripped
|
||||
2. **Minified at build time** — whitespace is removed to produce `.min.json` files
|
||||
1. **Optimized at generation time** — unused events are dropped, unused fields are stripped, shared objects are deduplicated, property names are interned
|
||||
2. **Compacted using a space-efficient format** — params use type-based dispatch arrays instead of objects
|
||||
3. **Minified at build time** — whitespace is removed to produce `.min.json` files
|
||||
|
||||
The source `.json` files are human-readable and checked into the repository. The `.min.json` files are generated during build and gitignored.
|
||||
|
||||
@@ -18,7 +19,8 @@ The source `.json` files are human-readable and checked into the repository. The
|
||||
| File | Description |
|
||||
|------|-------------|
|
||||
| `src/context-providers/events/webhooks.json` | Webhook event payload schemas for autocompletion |
|
||||
| `src/context-providers/events/objects.json` | Deduplicated shared object definitions referenced by webhooks |
|
||||
| `src/context-providers/events/webhooks.objects.json` | Deduplicated shared object definitions referenced by webhooks |
|
||||
| `src/context-providers/events/webhooks.strings.json` | Interned property names shared by webhooks and objects |
|
||||
| `src/context-providers/events/schedule.json` | Schedule event context data |
|
||||
| `src/context-providers/events/workflow_call.json` | Reusable workflow call context data |
|
||||
| `src/context-providers/descriptions.json` | Context variable descriptions for hover |
|
||||
@@ -33,7 +35,7 @@ The source `.json` files are human-readable and checked into the repository. The
|
||||
|
||||
### Webhooks and Objects
|
||||
|
||||
The `webhooks.json` and `objects.json` files are generated from the [GitHub REST API description](https://github.com/github/rest-api-description):
|
||||
The `webhooks.json`, `webhooks.objects.json`, and `webhooks.strings.json` files are generated from the [GitHub REST API description](https://github.com/github/rest-api-description):
|
||||
|
||||
```bash
|
||||
cd languageservice
|
||||
@@ -44,9 +46,10 @@ This script:
|
||||
1. Fetches webhook schemas from the GitHub API description
|
||||
2. **Validates** all events are categorized (fails if new events are found)
|
||||
3. **Drops** events that aren't valid workflow triggers (see [Dropped Events](#dropped-events))
|
||||
4. **Strips** unused fields like `description` and `summary` (see [Stripped Fields](#stripped-fields))
|
||||
5. **Deduplicates** shared object definitions into `objects.json`
|
||||
6. Writes the optimized, pretty-printed JSON files
|
||||
4. **Compacts** params into a space-efficient array format, keeping only `name`, `description`, and `childParamsGroups` (see [Compact Format](#compact-format))
|
||||
5. **Deduplicates** shared object definitions into `webhooks.objects.json`
|
||||
6. **Interns** duplicate property names into `webhooks.strings.json` (see [String Interning](#string-interning))
|
||||
7. Writes the optimized, pretty-printed JSON files
|
||||
|
||||
### Handling New Webhook Events
|
||||
|
||||
@@ -67,9 +70,9 @@ Action required:
|
||||
|
||||
1. Check [Events that trigger workflows](https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows)
|
||||
|
||||
2. Edit `languageservice/script/webhooks/index.ts`:
|
||||
- Add to `KEPT_EVENTS` if it's a valid workflow trigger
|
||||
- Add to `DROPPED_EVENTS` if it's GitHub App or API-only
|
||||
2. Edit `languageservice/src/context-providers/events/event-filters.json`:
|
||||
- Add to `kept` array if it's a valid workflow trigger
|
||||
- Add to `dropped` array if it's GitHub App or API-only
|
||||
|
||||
3. Run `npm run update-webhooks` and commit the changes
|
||||
|
||||
@@ -101,13 +104,15 @@ The code imports the minified versions:
|
||||
|
||||
```ts
|
||||
import webhooks from "./events/webhooks.min.json"
|
||||
import objects from "./events/webhooks.objects.min.json"
|
||||
import strings from "./events/webhooks.strings.min.json"
|
||||
```
|
||||
|
||||
## CI Verification
|
||||
|
||||
CI verifies that generated source files are up-to-date:
|
||||
|
||||
1. Runs `npm run update-webhooks` to regenerate webhooks.json and objects.json
|
||||
1. Runs `npm run update-webhooks` to regenerate webhooks.json, webhooks.objects.json, and webhooks.strings.json
|
||||
2. Checks for uncommitted changes with `git diff --exit-code`
|
||||
|
||||
The `.min.json` files are generated at build time and are not committed to the repository.
|
||||
@@ -118,33 +123,101 @@ If the build fails, run `cd languageservice && npm run update-webhooks` locally
|
||||
|
||||
Webhook events that aren't valid workflow `on:` triggers are dropped (e.g., `installation`, `ping`, `member`, etc.). These are GitHub App or API-only events.
|
||||
|
||||
See `DROPPED_EVENTS` in `script/webhooks/index.ts` for the full list.
|
||||
See `dropped` array in `src/context-providers/events/event-filters.json` for the full list.
|
||||
|
||||
## Stripped Fields
|
||||
## Compact Format
|
||||
|
||||
Unused fields are stripped to reduce bundle size. For example:
|
||||
Params are converted from verbose objects into compact arrays, keeping only the fields needed for autocompletion and hover docs (`name`, `description`, `childParamsGroups`). Unused fields like `type`, `in`, `isRequired`, `enum`, and `default` are discarded.
|
||||
|
||||
| Format | Meaning |
|
||||
|--------|---------|
|
||||
| `"name"` | Name only (no description, no children) |
|
||||
| `[name, desc]` | Name + description (arr[1] is a string) |
|
||||
| `[name, children]` | Name + children (arr[1] is an array) |
|
||||
| `[name, desc, children]` | Name + description + children |
|
||||
|
||||
The reader uses `typeof arr[1]` to determine the format: if it's a string, it's a description; if it's an array, it's children.
|
||||
|
||||
**Example:**
|
||||
|
||||
```json
|
||||
// Before (from webhooks.all.json)
|
||||
// Before (object format)
|
||||
{
|
||||
"type": "object",
|
||||
"name": "issue",
|
||||
"in": "body",
|
||||
"description": "The issue itself.",
|
||||
"isRequired": true,
|
||||
"childParamsGroups": [...]
|
||||
"childParamsGroups": [
|
||||
{ "name": "id" },
|
||||
{ "name": "title", "description": "Issue title" }
|
||||
]
|
||||
}
|
||||
|
||||
// After (webhooks.json)
|
||||
// After (compact format)
|
||||
["issue", "The issue itself.", [
|
||||
"id",
|
||||
["title", "Issue title"]
|
||||
]]
|
||||
```
|
||||
|
||||
## String Interning
|
||||
|
||||
Property names that appear 2+ times are "interned" into a shared string table (`webhooks.strings.json`). In the compact arrays, these names are replaced with non-negative numeric indices:
|
||||
|
||||
```json
|
||||
// webhooks.strings.json
|
||||
["url", "id", "name", ...] // Index 0 = "url", 1 = "id", 2 = "name"
|
||||
|
||||
// webhooks.json - uses indices instead of strings
|
||||
{
|
||||
"name": "issue",
|
||||
"description": "The issue itself.",
|
||||
"childParamsGroups": [...]
|
||||
"push": {
|
||||
"default": {
|
||||
"p": [
|
||||
[0, "The URL..."], // 0 = "url" from string table
|
||||
[1, "Unique ID"], // 1 = "id"
|
||||
2 // 2 = "name" (name-only, no description)
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Only `name`, `description`, and `childParamsGroups` are kept — these are used for autocompletion and hover docs.
|
||||
**How to distinguish indices from other values:**
|
||||
|
||||
To compare all fields vs stripped, run `npm run update-webhooks -- --all` and diff the `.all.json` files against the regular ones.
|
||||
- **Negative numbers** → Object indices: `-1` = object 0, `-2` = object 1, etc. (formula: `-(index + 1)`)
|
||||
- **Non-negative numbers** → String indices (references into `webhooks.strings.json`)
|
||||
- **Literal strings** → Singletons (names appearing only once, not interned)
|
||||
|
||||
See `EVENT_ACTION_FIELDS` and `BODY_PARAM_FIELDS` in `script/webhooks/index.ts` to modify what gets stripped.
|
||||
Singletons are kept as literal strings for readability and to avoid the overhead of adding rarely-used names to the string table.
|
||||
|
||||
## Deduplication
|
||||
|
||||
Shared object definitions are extracted into `webhooks.objects.json` and referenced by negative index:
|
||||
|
||||
```json
|
||||
// webhooks.objects.json
|
||||
[
|
||||
["url", "The URL"], // Index 0 (referenced as -1)
|
||||
["id", "Unique identifier"], // Index 1 (referenced as -2)
|
||||
[...]
|
||||
]
|
||||
|
||||
// webhooks.json - negative numbers reference objects
|
||||
{
|
||||
"push": {
|
||||
"default": {
|
||||
"p": [-1, -2, ["ref", "The git ref"]] // -1 = object 0, -2 = object 1
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This reduces duplication when the same object structure appears in multiple events (e.g., `repository`, `sender`, `organization`).
|
||||
|
||||
## Size Reduction
|
||||
|
||||
The optimizations achieve approximately 99% file size reduction:
|
||||
|
||||
| Stage | Minified | Gzip |
|
||||
|-------|----------|------|
|
||||
| Original (webhooks.full.json) | 15.8 MB | 968 KB |
|
||||
| After optimization (combined) | 152 KB | 15.6 KB |
|
||||
| **Reduction** | **99%** | **98%** |
|
||||
|
||||
@@ -37,13 +37,13 @@
|
||||
"format-check": "prettier --check '**/*.ts'",
|
||||
"lint": "eslint 'src/**/*.ts'",
|
||||
"lint-fix": "eslint --fix 'src/**/*.ts'",
|
||||
"minify-json": "node ../script/minify-json.js src/context-providers/descriptions.json src/context-providers/events/webhooks.json src/context-providers/events/objects.json src/context-providers/events/schedule.json src/context-providers/events/workflow_call.json",
|
||||
"minify-json": "node ../script/minify-json.js src/context-providers/descriptions.json src/context-providers/events/webhooks.json src/context-providers/events/webhooks.objects.json src/context-providers/events/webhooks.strings.json src/context-providers/events/schedule.json src/context-providers/events/workflow_call.json",
|
||||
"prebuild": "npm run minify-json",
|
||||
"prepublishOnly": "npm run build && npm run test",
|
||||
"pretest": "npm run minify-json",
|
||||
"test": "NODE_OPTIONS=\"--experimental-vm-modules\" jest",
|
||||
"test-watch": "NODE_OPTIONS=\"--experimental-vm-modules\" jest --watch",
|
||||
"update-webhooks": "npx tsx script/webhooks/index.ts",
|
||||
"update-webhooks": "npx tsx script/webhooks/update-webhooks.ts",
|
||||
"watch": "tsc --build tsconfig.build.json --watch"
|
||||
},
|
||||
"dependencies": {
|
||||
|
||||
@@ -1,5 +1,38 @@
|
||||
import Webhook from "./webhook";
|
||||
|
||||
/**
|
||||
* Get the name from a param.
|
||||
* Formats: "name" (string), or [name, ...] (array)
|
||||
*/
|
||||
function getParamName(param: any): string {
|
||||
if (typeof param === "string") {
|
||||
return param;
|
||||
}
|
||||
if (Array.isArray(param)) {
|
||||
return param[0];
|
||||
}
|
||||
return param.name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get params from a webhook action.
|
||||
* Uses 'p' (short key) if present, falls back to 'bodyParameters'
|
||||
*/
|
||||
function getParams(webhook: any): any[] {
|
||||
return webhook.p || webhook.bodyParameters || [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Set params on a webhook action using the short key 'p'
|
||||
*/
|
||||
function setParams(webhook: any, params: any[]): void {
|
||||
if (webhook.p !== undefined) {
|
||||
webhook.p = params;
|
||||
} else {
|
||||
webhook.bodyParameters = params;
|
||||
}
|
||||
}
|
||||
|
||||
// Store any repeated body parameters in an array
|
||||
// and replace them in the webhook with an index in the array
|
||||
export function deduplicateWebhooks(webhooks: Record<string, Record<string, Webhook>>): any[] {
|
||||
@@ -10,10 +43,11 @@ export function deduplicateWebhooks(webhooks: Record<string, Record<string, Webh
|
||||
const objectCount: Record<string, number> = {};
|
||||
|
||||
for (const webhook of iterateWebhooks(webhooks)) {
|
||||
for (const param of webhook.bodyParameters) {
|
||||
objectsByName[param.name] ||= [];
|
||||
const index = findOrAdd(param, objectsByName[param.name]);
|
||||
const key = `${param.name}:${index}`;
|
||||
for (const param of getParams(webhook)) {
|
||||
const name = getParamName(param);
|
||||
objectsByName[name] ||= [];
|
||||
const index = findOrAdd(param, objectsByName[name]);
|
||||
const key = `${name}:${index}`;
|
||||
objectCount[key] ||= 0;
|
||||
objectCount[key]++;
|
||||
}
|
||||
@@ -27,18 +61,19 @@ export function deduplicateWebhooks(webhooks: Record<string, Record<string, Webh
|
||||
|
||||
for (const webhook of iterateWebhooks(webhooks)) {
|
||||
const newParams: any[] = [];
|
||||
for (const param of webhook.bodyParameters) {
|
||||
const index = find(param, objectsByName[param.name]);
|
||||
const key = `${param.name}:${index}`;
|
||||
for (const param of getParams(webhook)) {
|
||||
const name = getParamName(param);
|
||||
const index = find(param, objectsByName[name]);
|
||||
const key = `${name}:${index}`;
|
||||
if (objectCount[key] > 1) {
|
||||
newParams.push(indexForParam(param, index, bodyParamIndexMap, duplicatedBodyParams));
|
||||
newParams.push(indexForParam(param, name, index, bodyParamIndexMap, duplicatedBodyParams));
|
||||
} else {
|
||||
// If an object is only used once, keep it inline
|
||||
newParams.push(param);
|
||||
}
|
||||
}
|
||||
|
||||
webhook.bodyParameters = newParams;
|
||||
setParams(webhook, newParams);
|
||||
}
|
||||
|
||||
return duplicatedBodyParams;
|
||||
@@ -74,11 +109,12 @@ function find(param: any, objects: any[]): number {
|
||||
|
||||
function indexForParam(
|
||||
param: any,
|
||||
paramName: string,
|
||||
paramNameIndex: number,
|
||||
objectIndexMap: Record<string, number>,
|
||||
duplicatedBodyParams: any[]
|
||||
): number {
|
||||
const key = `${param.name}:${paramNameIndex}`;
|
||||
const key = `${paramName}:${paramNameIndex}`;
|
||||
|
||||
const existingIndex = objectIndexMap[key];
|
||||
if (existingIndex !== undefined) {
|
||||
|
||||
@@ -1,310 +0,0 @@
|
||||
import {promises as fs} from "fs";
|
||||
import Webhook from "./webhook.js";
|
||||
|
||||
import schemaImport from "rest-api-description/descriptions/api.github.com/dereferenced/api.github.com.deref.json";
|
||||
import {deduplicateWebhooks} from "./deduplicate.js";
|
||||
const schema = schemaImport as any;
|
||||
|
||||
const OUTPUT_PATH = "./src/context-providers/events/webhooks.json";
|
||||
const OBJECTS_PATH = "./src/context-providers/events/objects.json";
|
||||
const ALL_OUTPUT_PATH = "./src/context-providers/events/webhooks.all.json";
|
||||
const ALL_OBJECTS_PATH = "./src/context-providers/events/objects.all.json";
|
||||
const DROP_OUTPUT_PATH = "./src/context-providers/events/webhooks.drop.json";
|
||||
const DROP_OBJECTS_PATH = "./src/context-providers/events/objects.drop.json";
|
||||
const STRIP_OUTPUT_PATH = "./src/context-providers/events/webhooks.strip.json";
|
||||
const STRIP_OBJECTS_PATH = "./src/context-providers/events/objects.strip.json";
|
||||
|
||||
// Parse --all flag
|
||||
const generateAll = process.argv.includes("--all");
|
||||
|
||||
// Events to drop - not valid workflow triggers (GitHub App or API-only events)
|
||||
// See: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows
|
||||
const DROPPED_EVENTS = new Set([
|
||||
"branch_protection_configuration",
|
||||
"code_scanning_alert",
|
||||
"commit_comment",
|
||||
"custom_property",
|
||||
"custom_property_values",
|
||||
"dependabot_alert",
|
||||
"deploy_key",
|
||||
"github_app_authorization",
|
||||
"installation",
|
||||
"installation_repositories",
|
||||
"installation_target",
|
||||
"marketplace_purchase",
|
||||
"member",
|
||||
"membership",
|
||||
"merge_group",
|
||||
"meta",
|
||||
"org_block",
|
||||
"organization",
|
||||
"package",
|
||||
"personal_access_token_request",
|
||||
"ping",
|
||||
"repository",
|
||||
"repository_advisory",
|
||||
"repository_ruleset",
|
||||
"secret_scanning_alert",
|
||||
"secret_scanning_alert_location",
|
||||
"security_advisory",
|
||||
"security_and_analysis",
|
||||
"sponsorship",
|
||||
"star",
|
||||
"team",
|
||||
"team_add"
|
||||
]);
|
||||
|
||||
// Events to keep - valid workflow triggers
|
||||
// See: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows
|
||||
const KEPT_EVENTS = new Set([
|
||||
"branch_protection_rule",
|
||||
"check_run",
|
||||
"check_suite",
|
||||
"create",
|
||||
"delete",
|
||||
"deployment",
|
||||
"deployment_status",
|
||||
"discussion",
|
||||
"discussion_comment",
|
||||
"fork",
|
||||
"gollum",
|
||||
"issue_comment",
|
||||
"issues",
|
||||
"label",
|
||||
"milestone",
|
||||
"page_build",
|
||||
"project",
|
||||
"project_card",
|
||||
"project_column",
|
||||
"projects_v2",
|
||||
"projects_v2_item",
|
||||
"public",
|
||||
"pull_request",
|
||||
"pull_request_review",
|
||||
"pull_request_review_comment",
|
||||
"pull_request_review_thread",
|
||||
"push",
|
||||
"registry_package",
|
||||
"release",
|
||||
"repository_dispatch",
|
||||
"repository_import",
|
||||
"repository_vulnerability_alert",
|
||||
"status",
|
||||
"watch",
|
||||
"workflow_dispatch",
|
||||
"workflow_job",
|
||||
"workflow_run"
|
||||
]);
|
||||
|
||||
/**
|
||||
* Fields to strip from the JSON data.
|
||||
*
|
||||
* EVENT_ACTION_FIELDS: stripped from each event action object (top level only)
|
||||
* Example event action object before stripping:
|
||||
* {
|
||||
* "description": "This event is triggered when...", // <-- stripped
|
||||
* "summary": "A brief summary", // <-- stripped
|
||||
* "availability": ["repository"], // <-- stripped
|
||||
* "category": "issues", // <-- stripped
|
||||
* "action": "opened", // kept
|
||||
* "bodyParameters": [...] // kept
|
||||
* }
|
||||
*
|
||||
* BODY_PARAM_FIELDS: stripped from every bodyParameters object, recursively through childParamsGroups
|
||||
* Example bodyParameter object before stripping:
|
||||
* {
|
||||
* "type": "object", // <-- stripped
|
||||
* "name": "changes", // kept (used for property names)
|
||||
* "in": "body", // <-- stripped
|
||||
* "description": "The changes that were made.", // kept (used for hover docs)
|
||||
* "isRequired": true, // <-- stripped
|
||||
* "enum": ["a", "b"], // <-- stripped
|
||||
* "default": "a", // <-- stripped
|
||||
* "childParamsGroups": [ // kept (used for nested properties)
|
||||
* {
|
||||
* "type": "string", // <-- stripped (recursive)
|
||||
* "name": "from", // kept
|
||||
* "isRequired": true // <-- stripped (recursive)
|
||||
* }
|
||||
* ]
|
||||
* }
|
||||
*/
|
||||
const EVENT_ACTION_FIELDS = ["description", "summary", "availability", "category"];
|
||||
const BODY_PARAM_FIELDS = ["type", "in", "isRequired", "enum", "default"];
|
||||
|
||||
/**
|
||||
* Strip fields from a bodyParameter object and recursively from childParamsGroups.
|
||||
*/
|
||||
function stripBodyParam(param: any): any {
|
||||
if (typeof param !== "object" || param === null) {
|
||||
return param;
|
||||
}
|
||||
|
||||
const result: any = {};
|
||||
for (const [key, value] of Object.entries(param)) {
|
||||
if (BODY_PARAM_FIELDS.includes(key)) {
|
||||
continue; // Strip this field
|
||||
}
|
||||
if (key === "childParamsGroups" && Array.isArray(value)) {
|
||||
result[key] = value.map(stripBodyParam);
|
||||
} else {
|
||||
result[key] = value;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip unused fields from event action data.
|
||||
*/
|
||||
function stripEventActionFields(action: any): any {
|
||||
const result: any = {};
|
||||
for (const [key, value] of Object.entries(action)) {
|
||||
if (EVENT_ACTION_FIELDS.includes(key)) {
|
||||
continue; // Strip this field
|
||||
}
|
||||
if (key === "bodyParameters" && Array.isArray(value)) {
|
||||
result[key] = value.map((p: any) => (typeof p === "number" ? p : stripBodyParam(p)));
|
||||
} else {
|
||||
result[key] = value;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip unused fields from all webhooks.
|
||||
* Structure: { eventName: { actionName: { ...fields } } }
|
||||
*/
|
||||
function stripFields(webhooks: Record<string, Record<string, any>>): Record<string, Record<string, any>> {
|
||||
const result: Record<string, Record<string, any>> = {};
|
||||
for (const [eventName, actions] of Object.entries(webhooks)) {
|
||||
result[eventName] = {};
|
||||
for (const [actionName, actionData] of Object.entries(actions)) {
|
||||
result[eventName][actionName] = stripEventActionFields(actionData);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
const rawWebhooks = Object.values(schema.webhooks || schema["x-webhooks"]) as any[];
|
||||
if (!rawWebhooks) {
|
||||
throw new Error("No webhooks found in schema");
|
||||
}
|
||||
|
||||
const webhooks: Webhook[] = [];
|
||||
for (const webhook of Object.values(rawWebhooks)) {
|
||||
webhooks.push(new Webhook(webhook.post));
|
||||
}
|
||||
|
||||
await Promise.all(webhooks.map(webhook => webhook.process()));
|
||||
|
||||
// Check for unknown events (not in DROPPED_EVENTS or KEPT_EVENTS)
|
||||
const unknownEvents: string[] = [];
|
||||
for (const webhook of webhooks) {
|
||||
if (!DROPPED_EVENTS.has(webhook.category) && !KEPT_EVENTS.has(webhook.category)) {
|
||||
if (!unknownEvents.includes(webhook.category)) {
|
||||
unknownEvents.push(webhook.category);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (unknownEvents.length > 0) {
|
||||
console.error("");
|
||||
console.error("══════════════════════════════════════════════════════════════════");
|
||||
console.error("ERROR: New webhook event(s) detected!");
|
||||
console.error("══════════════════════════════════════════════════════════════════");
|
||||
console.error("");
|
||||
console.error("The following events are not categorized:");
|
||||
for (const event of unknownEvents.sort()) {
|
||||
console.error(` - ${event}`);
|
||||
}
|
||||
console.error("");
|
||||
console.error("Action required:");
|
||||
console.error(" 1. Check if the event is a valid workflow trigger:");
|
||||
console.error(
|
||||
" https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows"
|
||||
);
|
||||
console.error("");
|
||||
console.error(" 2. Add the event to DROPPED_EVENTS or KEPT_EVENTS in:");
|
||||
console.error(" languageservice/script/webhooks/index.ts");
|
||||
console.error("");
|
||||
console.error(" 3. See docs/json-data-files.md for more details.");
|
||||
console.error("");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// The category is the name of the webhook
|
||||
const categorizedWebhooks: Record<string, Record<string, Webhook>> = {};
|
||||
for (const webhook of webhooks) {
|
||||
if (!webhook.action) webhook.action = "default";
|
||||
|
||||
// Drop unused events
|
||||
if (DROPPED_EVENTS.has(webhook.category)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (categorizedWebhooks[webhook.category]) {
|
||||
categorizedWebhooks[webhook.category][webhook.action] = webhook;
|
||||
} else {
|
||||
categorizedWebhooks[webhook.category] = {};
|
||||
categorizedWebhooks[webhook.category][webhook.action] = webhook;
|
||||
}
|
||||
}
|
||||
|
||||
// Strip fields before deduplication
|
||||
const strippedWebhooks = stripFields(categorizedWebhooks);
|
||||
|
||||
// Deduplicate after dropping and stripping
|
||||
const objectsArray = deduplicateWebhooks(strippedWebhooks);
|
||||
|
||||
// Write optimized output
|
||||
await fs.writeFile(OBJECTS_PATH, JSON.stringify(objectsArray, null, 2));
|
||||
await fs.writeFile(OUTPUT_PATH, JSON.stringify(strippedWebhooks, null, 2));
|
||||
|
||||
console.log(`Wrote ${OUTPUT_PATH} (${Object.keys(strippedWebhooks).length} events)`);
|
||||
console.log(`Wrote ${OBJECTS_PATH} (${objectsArray.length} objects)`);
|
||||
|
||||
// Optionally generate intermediate versions for size comparison
|
||||
if (generateAll) {
|
||||
// Helper to deep clone
|
||||
function clone<T>(obj: T): T {
|
||||
return JSON.parse(JSON.stringify(obj));
|
||||
}
|
||||
|
||||
// Build full webhooks (no drop, no strip) from fresh data
|
||||
const fullWebhooks: Record<string, Record<string, any>> = {};
|
||||
for (const webhook of webhooks) {
|
||||
const w = clone(webhook);
|
||||
if (!w.action) w.action = "default";
|
||||
fullWebhooks[w.category] ||= {};
|
||||
fullWebhooks[w.category][w.action] = w;
|
||||
}
|
||||
|
||||
// Generate all version (no drop, no strip)
|
||||
const allWebhooks = clone(fullWebhooks);
|
||||
const allObjects = deduplicateWebhooks(allWebhooks);
|
||||
await fs.writeFile(ALL_OUTPUT_PATH, JSON.stringify(allWebhooks, null, 2));
|
||||
await fs.writeFile(ALL_OBJECTS_PATH, JSON.stringify(allObjects, null, 2));
|
||||
console.log(`Wrote ${ALL_OUTPUT_PATH} (${Object.keys(allWebhooks).length} events)`);
|
||||
console.log(`Wrote ${ALL_OBJECTS_PATH} (${allObjects.length} objects)`);
|
||||
|
||||
// Generate drop-only version (drop events, no strip)
|
||||
const dropWebhooks = clone(fullWebhooks);
|
||||
for (const event of DROPPED_EVENTS) {
|
||||
delete dropWebhooks[event];
|
||||
}
|
||||
const dropObjects = deduplicateWebhooks(dropWebhooks);
|
||||
await fs.writeFile(DROP_OUTPUT_PATH, JSON.stringify(dropWebhooks, null, 2));
|
||||
await fs.writeFile(DROP_OBJECTS_PATH, JSON.stringify(dropObjects, null, 2));
|
||||
console.log(`Wrote ${DROP_OUTPUT_PATH} (${Object.keys(dropWebhooks).length} events)`);
|
||||
console.log(`Wrote ${DROP_OBJECTS_PATH} (${dropObjects.length} objects)`);
|
||||
|
||||
// Generate strip-only version (strip fields, no drop)
|
||||
const stripWebhooks = stripFields(clone(fullWebhooks));
|
||||
const stripObjects = deduplicateWebhooks(stripWebhooks);
|
||||
await fs.writeFile(STRIP_OUTPUT_PATH, JSON.stringify(stripWebhooks, null, 2));
|
||||
await fs.writeFile(STRIP_OBJECTS_PATH, JSON.stringify(stripObjects, null, 2));
|
||||
console.log(`Wrote ${STRIP_OUTPUT_PATH} (${Object.keys(stripWebhooks).length} events)`);
|
||||
console.log(`Wrote ${STRIP_OBJECTS_PATH} (${stripObjects.length} objects)`);
|
||||
}
|
||||
+291
@@ -0,0 +1,291 @@
|
||||
import {promises as fs} from "fs";
|
||||
import Webhook from "./webhook.js";
|
||||
|
||||
import schemaImport from "rest-api-description/descriptions/api.github.com/dereferenced/api.github.com.deref.json";
|
||||
import {deduplicateWebhooks} from "./deduplicate.js";
|
||||
import eventFilters from "../../src/context-providers/events/event-filters.json";
|
||||
const schema = schemaImport as any;
|
||||
|
||||
const DROPPED_EVENTS = new Set(eventFilters.dropped);
|
||||
const KEPT_EVENTS = new Set(eventFilters.kept);
|
||||
|
||||
const OUTPUT_PATH = "./src/context-providers/events/webhooks.json";
|
||||
const OBJECTS_PATH = "./src/context-providers/events/webhooks.objects.json";
|
||||
const STRINGS_PATH = "./src/context-providers/events/webhooks.strings.json";
|
||||
const FULL_OUTPUT_PATH = "./src/context-providers/events/webhooks.full.json";
|
||||
|
||||
/**
|
||||
* Fields discarded from each event action object (top level only).
|
||||
* Body parameters are compacted to only keep name, description, and childParamsGroups.
|
||||
*/
|
||||
const EVENT_ACTION_FIELDS = ["description", "summary", "availability", "category", "action"];
|
||||
|
||||
/**
|
||||
* Convert a bodyParameter object to compact array format.
|
||||
*
|
||||
* Format (type-based dispatch):
|
||||
* - "name" - name only (just a string)
|
||||
* - [name, desc] - name + description (desc is string)
|
||||
* - [name, [...children]] - name + children (arr[1] is array)
|
||||
* - [name, desc, [...children]] - name + description + children
|
||||
*
|
||||
* The reader uses typeof to determine the meaning:
|
||||
* - string -> name only
|
||||
* - array with string arr[1] -> name + description
|
||||
* - array with array arr[1] -> name + children
|
||||
*/
|
||||
function compactParam(param: any): any {
|
||||
if (typeof param !== "object" || param === null) {
|
||||
return param;
|
||||
}
|
||||
|
||||
const name: string = param.name;
|
||||
const desc: string | undefined = param.description;
|
||||
const children: any[] | undefined = param.childParamsGroups;
|
||||
|
||||
const hasDesc = desc && desc.length > 0;
|
||||
const hasChildren = children && children.length > 0;
|
||||
|
||||
if (hasDesc && hasChildren) {
|
||||
return [name, desc, children.map(compactParam)];
|
||||
} else if (hasChildren) {
|
||||
return [name, children.map(compactParam)];
|
||||
} else if (hasDesc) {
|
||||
return [name, desc];
|
||||
} else {
|
||||
return name; // Just the string, not wrapped in array
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert event action data to compact format.
|
||||
*/
|
||||
function compactEventAction(action: any): any {
|
||||
const result: any = {};
|
||||
for (const [key, value] of Object.entries(action)) {
|
||||
if (EVENT_ACTION_FIELDS.includes(key)) {
|
||||
continue; // Discard this field
|
||||
}
|
||||
if (key === "bodyParameters" && Array.isArray(value)) {
|
||||
// Use short key 'p' for params
|
||||
result["p"] = value.map((p: any) => (typeof p === "number" ? p : compactParam(p)));
|
||||
} else {
|
||||
result[key] = value;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert all webhooks to compact format.
|
||||
* Structure: { eventName: { actionName: { ...fields } } }
|
||||
*/
|
||||
function compactWebhooks(webhooks: Record<string, Record<string, any>>): Record<string, Record<string, any>> {
|
||||
const result: Record<string, Record<string, any>> = {};
|
||||
for (const [eventName, actions] of Object.entries(webhooks)) {
|
||||
result[eventName] = {};
|
||||
for (const [actionName, actionData] of Object.entries(actions)) {
|
||||
result[eventName][actionName] = compactEventAction(actionData);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
const rawWebhooks = Object.values(schema.webhooks || schema["x-webhooks"]) as any[];
|
||||
if (!rawWebhooks) {
|
||||
throw new Error("No webhooks found in schema");
|
||||
}
|
||||
|
||||
const webhooks: Webhook[] = [];
|
||||
for (const webhook of Object.values(rawWebhooks)) {
|
||||
webhooks.push(new Webhook(webhook.post));
|
||||
}
|
||||
|
||||
await Promise.all(webhooks.map(webhook => webhook.process()));
|
||||
|
||||
// Check for unknown events (not in DROPPED_EVENTS or KEPT_EVENTS)
|
||||
const unknownEvents: string[] = [];
|
||||
for (const webhook of webhooks) {
|
||||
if (!DROPPED_EVENTS.has(webhook.category) && !KEPT_EVENTS.has(webhook.category)) {
|
||||
if (!unknownEvents.includes(webhook.category)) {
|
||||
unknownEvents.push(webhook.category);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (unknownEvents.length > 0) {
|
||||
console.error("");
|
||||
console.error("══════════════════════════════════════════════════════════════════");
|
||||
console.error("ERROR: New webhook event(s) detected!");
|
||||
console.error("══════════════════════════════════════════════════════════════════");
|
||||
console.error("");
|
||||
console.error("The following events are not categorized:");
|
||||
for (const event of unknownEvents.sort()) {
|
||||
console.error(` - ${event}`);
|
||||
}
|
||||
console.error("");
|
||||
console.error("Action required:");
|
||||
console.error(" 1. Check if the event is a valid workflow trigger:");
|
||||
console.error(
|
||||
" https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows"
|
||||
);
|
||||
console.error("");
|
||||
console.error(" 2. Add the event to 'dropped' or 'kept' array in:");
|
||||
console.error(" languageservice/src/context-providers/events/event-filters.json");
|
||||
console.error("");
|
||||
console.error(" 3. See docs/json-data-files.md for more details.");
|
||||
console.error("");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Build full webhooks (all events, no transformations) for validation tests
|
||||
const fullWebhooks: Record<string, Record<string, Webhook>> = {};
|
||||
for (const webhook of webhooks) {
|
||||
if (!webhook.action) webhook.action = "default";
|
||||
|
||||
if (fullWebhooks[webhook.category]) {
|
||||
fullWebhooks[webhook.category][webhook.action] = webhook;
|
||||
} else {
|
||||
fullWebhooks[webhook.category] = {};
|
||||
fullWebhooks[webhook.category][webhook.action] = webhook;
|
||||
}
|
||||
}
|
||||
|
||||
// Write full version (before any optimizations)
|
||||
await fs.writeFile(FULL_OUTPUT_PATH, JSON.stringify(fullWebhooks, null, 2));
|
||||
console.log(`Wrote ${FULL_OUTPUT_PATH} (${Object.keys(fullWebhooks).length} events, unoptimized)`);
|
||||
|
||||
// The category is the name of the webhook
|
||||
const categorizedWebhooks: Record<string, Record<string, Webhook>> = {};
|
||||
for (const webhook of webhooks) {
|
||||
if (!webhook.action) webhook.action = "default";
|
||||
|
||||
// Drop unused events
|
||||
if (DROPPED_EVENTS.has(webhook.category)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (categorizedWebhooks[webhook.category]) {
|
||||
categorizedWebhooks[webhook.category][webhook.action] = webhook;
|
||||
} else {
|
||||
categorizedWebhooks[webhook.category] = {};
|
||||
categorizedWebhooks[webhook.category][webhook.action] = webhook;
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to compact format before deduplication
|
||||
const compactedWebhooks = compactWebhooks(categorizedWebhooks);
|
||||
|
||||
// Deduplicate after compacting
|
||||
const objectsArray = deduplicateWebhooks(compactedWebhooks);
|
||||
|
||||
// ============================================================================
|
||||
// String Interning (Phase 3)
|
||||
// ============================================================================
|
||||
// Intern duplicate property names to reduce file size.
|
||||
// Names appearing 2+ times are stored in a string table and referenced by index.
|
||||
// Singleton names stay as literal strings for readability.
|
||||
|
||||
/**
|
||||
* Collect all property names from params (for frequency counting)
|
||||
*/
|
||||
function collectNames(param: any, counts: Map<string, number>): void {
|
||||
if (typeof param === "number") return;
|
||||
if (typeof param === "string") {
|
||||
counts.set(param, (counts.get(param) || 0) + 1);
|
||||
return;
|
||||
}
|
||||
if (Array.isArray(param)) {
|
||||
const name = param[0] as string;
|
||||
counts.set(name, (counts.get(name) || 0) + 1);
|
||||
const children = Array.isArray(param[1]) ? param[1] : param[2];
|
||||
if (children) children.forEach((c: any) => collectNames(c, counts));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace duplicate names with indices into the string table.
|
||||
* Object references use negative indices: objectIndex -> -(objectIndex + 1)
|
||||
* String references use non-negative indices: stringIndex -> stringIndex
|
||||
*
|
||||
* @param param - The param to process
|
||||
* @param nameToIndex - Map from name to string table index
|
||||
*/
|
||||
function internNames(param: any, nameToIndex: Map<string, number>): any {
|
||||
// Object reference (already a number from deduplication) -> make negative
|
||||
if (typeof param === "number") return -(param + 1);
|
||||
|
||||
// String -> intern if in table, otherwise keep as literal
|
||||
if (typeof param === "string") {
|
||||
const idx = nameToIndex.get(param);
|
||||
return idx !== undefined ? idx : param;
|
||||
}
|
||||
|
||||
if (Array.isArray(param)) {
|
||||
const name = param[0] as string;
|
||||
const idx = nameToIndex.get(name);
|
||||
const internedName = idx !== undefined ? idx : name;
|
||||
|
||||
// Handle different array formats
|
||||
if (typeof param[1] === "string" && !Array.isArray(param[1])) {
|
||||
// [name, desc] or [name, desc, children]
|
||||
if (param.length === 2) {
|
||||
return [internedName, param[1]];
|
||||
} else {
|
||||
return [internedName, param[1], (param[2] as any[]).map((c: any) => internNames(c, nameToIndex))];
|
||||
}
|
||||
} else if (Array.isArray(param[1])) {
|
||||
// [name, children]
|
||||
return [internedName, param[1].map((c: any) => internNames(c, nameToIndex))];
|
||||
}
|
||||
// Shouldn't happen, but fallback
|
||||
return [internedName, ...param.slice(1)];
|
||||
}
|
||||
return param;
|
||||
}
|
||||
|
||||
// Pass 1: Count all names
|
||||
const nameCounts = new Map<string, number>();
|
||||
objectsArray.forEach((obj: any) => collectNames(obj, nameCounts));
|
||||
for (const event of Object.values(compactedWebhooks)) {
|
||||
for (const action of Object.values(event as Record<string, any>)) {
|
||||
if (action.p) action.p.forEach((p: any) => collectNames(p, nameCounts));
|
||||
}
|
||||
}
|
||||
|
||||
// Build string table from duplicates, sorted by frequency (most common first = smaller indices)
|
||||
const sortedNames = [...nameCounts.entries()].filter(([, count]) => count >= 2).sort((a, b) => b[1] - a[1]);
|
||||
const stringTable = sortedNames.map(([name]) => name);
|
||||
const nameToIndex = new Map(stringTable.map((name, i) => [name, i]));
|
||||
|
||||
console.log(
|
||||
`String table: ${stringTable.length} interned names (${nameCounts.size - stringTable.length} singletons kept inline)`
|
||||
);
|
||||
|
||||
// Pass 2: Intern names in objects and webhooks
|
||||
// Objects use negative indices, strings use non-negative indices
|
||||
const internedObjects = objectsArray.map((obj: any) => internNames(obj, nameToIndex));
|
||||
const internedWebhooks: Record<string, Record<string, any>> = {};
|
||||
for (const [eventName, actions] of Object.entries(compactedWebhooks)) {
|
||||
internedWebhooks[eventName] = {};
|
||||
for (const [actionName, actionData] of Object.entries(actions as Record<string, any>)) {
|
||||
internedWebhooks[eventName][actionName] = {
|
||||
p: actionData.p.map((p: any) => internNames(p, nameToIndex))
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Write optimized output with separate string table
|
||||
// Format: webhooks.strings.json has string table, webhooks.json/webhooks.objects.json reference by index
|
||||
const finalOutput = {
|
||||
"//": "Generated file - refer to docs/json-data-files.md for format documentation",
|
||||
...internedWebhooks
|
||||
};
|
||||
|
||||
await fs.writeFile(STRINGS_PATH, JSON.stringify(stringTable, null, 2));
|
||||
await fs.writeFile(OBJECTS_PATH, JSON.stringify(internedObjects, null, 2));
|
||||
await fs.writeFile(OUTPUT_PATH, JSON.stringify(finalOutput, null, 2));
|
||||
|
||||
console.log(`Wrote ${STRINGS_PATH} (${stringTable.length} interned strings)`);
|
||||
console.log(`Wrote ${OUTPUT_PATH} (${Object.keys(compactedWebhooks).length} events)`);
|
||||
console.log(`Wrote ${OBJECTS_PATH} (${internedObjects.length} objects)`);
|
||||
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"dropped": [
|
||||
"branch_protection_configuration",
|
||||
"code_scanning_alert",
|
||||
"commit_comment",
|
||||
"custom_property",
|
||||
"custom_property_values",
|
||||
"dependabot_alert",
|
||||
"deploy_key",
|
||||
"github_app_authorization",
|
||||
"installation",
|
||||
"installation_repositories",
|
||||
"installation_target",
|
||||
"marketplace_purchase",
|
||||
"member",
|
||||
"membership",
|
||||
"merge_group",
|
||||
"meta",
|
||||
"org_block",
|
||||
"organization",
|
||||
"package",
|
||||
"personal_access_token_request",
|
||||
"ping",
|
||||
"repository",
|
||||
"repository_advisory",
|
||||
"repository_ruleset",
|
||||
"secret_scanning_alert",
|
||||
"secret_scanning_alert_location",
|
||||
"security_advisory",
|
||||
"security_and_analysis",
|
||||
"sponsorship",
|
||||
"star",
|
||||
"team",
|
||||
"team_add"
|
||||
],
|
||||
"kept": [
|
||||
"branch_protection_rule",
|
||||
"check_run",
|
||||
"check_suite",
|
||||
"create",
|
||||
"delete",
|
||||
"deployment",
|
||||
"deployment_status",
|
||||
"discussion",
|
||||
"discussion_comment",
|
||||
"fork",
|
||||
"gollum",
|
||||
"issue_comment",
|
||||
"issues",
|
||||
"label",
|
||||
"milestone",
|
||||
"page_build",
|
||||
"project",
|
||||
"project_card",
|
||||
"project_column",
|
||||
"projects_v2",
|
||||
"projects_v2_item",
|
||||
"public",
|
||||
"pull_request",
|
||||
"pull_request_review",
|
||||
"pull_request_review_comment",
|
||||
"pull_request_review_thread",
|
||||
"push",
|
||||
"registry_package",
|
||||
"release",
|
||||
"repository_dispatch",
|
||||
"repository_import",
|
||||
"repository_vulnerability_alert",
|
||||
"status",
|
||||
"watch",
|
||||
"workflow_dispatch",
|
||||
"workflow_job",
|
||||
"workflow_run"
|
||||
]
|
||||
}
|
||||
@@ -1,5 +1,34 @@
|
||||
import {DescriptionDictionary} from "@actions/expressions";
|
||||
import {existsSync} from "fs";
|
||||
import {fileURLToPath} from "url";
|
||||
import {dirname, join} from "path";
|
||||
import {data, DescriptionDictionary, DescriptionPair, isDescriptionDictionary} from "@actions/expressions";
|
||||
import {getEventPayload, getSupportedEventTypes} from "./eventPayloads";
|
||||
import eventFilters from "./event-filters.json";
|
||||
|
||||
const DROPPED_EVENTS = new Set(eventFilters.dropped);
|
||||
|
||||
// Check if full webhooks file exists (generated by npm run update-webhooks)
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const fullWebhooksPath = join(__dirname, "webhooks.full.json");
|
||||
const hasFullWebhooks = existsSync(fullWebhooksPath);
|
||||
|
||||
type Param = {
|
||||
name: string;
|
||||
description?: string;
|
||||
childParamsGroups?: Param[];
|
||||
};
|
||||
|
||||
type FullAction = {
|
||||
action?: string;
|
||||
bodyParameters?: Param[];
|
||||
};
|
||||
|
||||
type FullWebhooks = {
|
||||
[event: string]: {
|
||||
[action: string]: FullAction;
|
||||
};
|
||||
};
|
||||
|
||||
describe("eventPayloads", () => {
|
||||
describe("getSupportedEventTypes", () => {
|
||||
@@ -100,3 +129,149 @@ describe("eventPayloads", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Optimization validation tests - only run if webhooks.full.json exists
|
||||
// This file is generated by: npm run update-webhooks
|
||||
// In CI, a separate job runs these tests after generating the file
|
||||
const describeIfFullExists = hasFullWebhooks ? describe : describe.skip;
|
||||
|
||||
// Marker file path - written after validation tests complete
|
||||
const fullValidationMarkerPath = join(__dirname, "webhooks.full.validation-complete");
|
||||
|
||||
describeIfFullExists("optimization validation", () => {
|
||||
let keptWebhooks: FullWebhooks;
|
||||
|
||||
beforeAll(async () => {
|
||||
// Dynamically import the full webhooks file using fs to avoid TS module resolution
|
||||
const {readFileSync} = await import("fs");
|
||||
const fullWebhooks = JSON.parse(readFileSync(fullWebhooksPath, "utf-8")) as FullWebhooks;
|
||||
|
||||
// Filter to only kept events
|
||||
keptWebhooks = {};
|
||||
for (const [event, actions] of Object.entries(fullWebhooks)) {
|
||||
if (!DROPPED_EVENTS.has(event)) {
|
||||
keptWebhooks[event] = actions;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
// Write marker file to prove validation tests ran
|
||||
const {writeFileSync} = await import("fs");
|
||||
writeFileSync(fullValidationMarkerPath, new Date().toISOString());
|
||||
});
|
||||
|
||||
/**
|
||||
* Build a DescriptionDictionary from raw params (same logic as eventPayloads.ts)
|
||||
*/
|
||||
function buildFromParams(params: Param[]): DescriptionDictionary {
|
||||
const d = new DescriptionDictionary();
|
||||
for (const param of params) {
|
||||
if (param.childParamsGroups && param.childParamsGroups.length > 0) {
|
||||
const child = buildFromParams(param.childParamsGroups);
|
||||
d.add(param.name, child, param.description);
|
||||
} else {
|
||||
// Match the behavior in eventPayloads.ts - don't overwrite existing
|
||||
if (!d.get(param.name)) {
|
||||
d.add(param.name, new data.Null(), param.description);
|
||||
}
|
||||
}
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two DescriptionDictionary structures recursively
|
||||
*/
|
||||
function compareStructures(full: DescriptionDictionary, optimized: DescriptionDictionary, path: string): string[] {
|
||||
const errors: string[] = [];
|
||||
|
||||
const fullPairs = full.pairs();
|
||||
const optimizedPairs = optimized.pairs();
|
||||
|
||||
const fullKeys = new Set(fullPairs.map((p: DescriptionPair) => p.key));
|
||||
const optimizedKeys = new Set(optimizedPairs.map((p: DescriptionPair) => p.key));
|
||||
|
||||
// Check for missing keys in optimized
|
||||
for (const key of fullKeys) {
|
||||
if (!optimizedKeys.has(key)) {
|
||||
errors.push(`Missing key in optimized: ${path}.${key}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Check for extra keys in optimized
|
||||
for (const key of optimizedKeys) {
|
||||
if (!fullKeys.has(key)) {
|
||||
errors.push(`Extra key in optimized: ${path}.${key}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Compare descriptions and recurse into nested structures
|
||||
for (const fullPair of fullPairs) {
|
||||
const optimizedValue = optimized.get(fullPair.key);
|
||||
if (optimizedValue === undefined) continue;
|
||||
|
||||
// Compare descriptions
|
||||
const fullDesc = full.getDescription(fullPair.key) ?? "";
|
||||
const optimizedDesc = optimized.getDescription(fullPair.key) ?? "";
|
||||
if (fullDesc !== optimizedDesc) {
|
||||
errors.push(
|
||||
`Description mismatch at ${path}.${fullPair.key}: ` + `full="${fullDesc}" vs optimized="${optimizedDesc}"`
|
||||
);
|
||||
}
|
||||
|
||||
// Recurse into nested dictionaries
|
||||
if (isDescriptionDictionary(fullPair.value) && isDescriptionDictionary(optimizedValue)) {
|
||||
errors.push(...compareStructures(fullPair.value, optimizedValue, `${path}.${fullPair.key}`));
|
||||
}
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
it("optimized webhooks match full source for all events and actions", () => {
|
||||
const allErrors: string[] = [];
|
||||
|
||||
for (const [event, actions] of Object.entries(keptWebhooks)) {
|
||||
for (const [action, actionData] of Object.entries(actions)) {
|
||||
// Build from full source (use bodyParameters, may be undefined)
|
||||
const params = actionData.bodyParameters || [];
|
||||
const fullPayload = buildFromParams(params);
|
||||
|
||||
// Get from optimized (deduplicated) source
|
||||
const optimizedPayload = getEventPayload(event, action);
|
||||
|
||||
if (!optimizedPayload) {
|
||||
allErrors.push(`Missing optimized payload for ${event}.${action}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const errors = compareStructures(fullPayload, optimizedPayload, `${event}.${action}`);
|
||||
allErrors.push(...errors);
|
||||
}
|
||||
}
|
||||
|
||||
if (allErrors.length > 0) {
|
||||
fail(
|
||||
`Optimization validation failed:\n${allErrors.slice(0, 20).join("\n")}` +
|
||||
(allErrors.length > 20 ? `\n... and ${allErrors.length - 20} more errors` : "")
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("all full source events are present in optimized version", () => {
|
||||
for (const event of Object.keys(keptWebhooks)) {
|
||||
const types = getSupportedEventTypes(event);
|
||||
expect(types.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it("all full source actions are present in optimized version", () => {
|
||||
for (const [event, actions] of Object.entries(keptWebhooks)) {
|
||||
for (const action of Object.keys(actions)) {
|
||||
const payload = getEventPayload(event, action);
|
||||
expect(payload).toBeDefined();
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import {data, DescriptionDictionary} from "@actions/expressions";
|
||||
|
||||
import webhookObjects from "./objects.min.json";
|
||||
import webhooks from "./webhooks.min.json";
|
||||
import webhooksData from "./webhooks.min.json";
|
||||
import objectsData from "./webhooks.objects.min.json";
|
||||
import stringsData from "./webhooks.strings.min.json";
|
||||
|
||||
import schedule from "./schedule.min.json";
|
||||
import workflow_call from "./workflow_call.min.json";
|
||||
@@ -49,9 +50,22 @@ type Param = {
|
||||
};
|
||||
|
||||
/**
|
||||
* A full {@link Param} or an index into the objects array for deduplicated parameters
|
||||
* Compact format for params (written by update-webhooks.ts).
|
||||
*
|
||||
* Names can be interned (number = index into string table) or literal strings.
|
||||
* Type-based dispatch:
|
||||
* - number - interned name only (index into string table)
|
||||
* - "name" - literal name only (singleton, not interned)
|
||||
* - [name, desc] - name + description (name is number or string, desc is string)
|
||||
* - [name, [...children]] - name + children (arr[1] is array)
|
||||
* - [name, desc, [...children]] - name + description + children
|
||||
*/
|
||||
type DeduplicatedParam = Param | number;
|
||||
type InternedName = number | string;
|
||||
type CompactParam =
|
||||
| InternedName
|
||||
| [InternedName, string]
|
||||
| [InternedName, CompactParam[]]
|
||||
| [InternedName, string, CompactParam[]];
|
||||
|
||||
type WebhookPayload = {
|
||||
descriptionHtml: string;
|
||||
@@ -65,17 +79,33 @@ type Webhooks = {
|
||||
};
|
||||
};
|
||||
|
||||
type DeduplicatedWebhooks = {
|
||||
[name: string]: {
|
||||
[action: string]: WebhookPayload & {
|
||||
bodyParameters: DeduplicatedParam[];
|
||||
};
|
||||
};
|
||||
/**
|
||||
* Webhooks data format after optimization:
|
||||
* {
|
||||
* [event]: { [action]: { p: CompactParam[] } }
|
||||
* }
|
||||
*
|
||||
* String table and objects are loaded from separate files.
|
||||
*/
|
||||
type WebhooksData = {
|
||||
[key: string]: {[action: string]: {p: CompactParam[]}};
|
||||
};
|
||||
|
||||
/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any */
|
||||
const dedupedWebhookPayloads: DeduplicatedWebhooks = webhooks as any;
|
||||
const objects: Param[] = webhookObjects as any;
|
||||
const webhooksJson: WebhooksData = webhooksData as any;
|
||||
const objectsJson: CompactParam[] = objectsData as any;
|
||||
|
||||
// String table and objects are in separate files
|
||||
const stringTable: string[] = stringsData;
|
||||
const objects: CompactParam[] = objectsJson;
|
||||
|
||||
// Build event payloads map (skip "//" comment key)
|
||||
const dedupedWebhookPayloads: {[event: string]: {[action: string]: {p: CompactParam[]}}} = {};
|
||||
for (const [key, value] of Object.entries(webhooksJson)) {
|
||||
if (key !== "//" && typeof value === "object" && value !== null) {
|
||||
dedupedWebhookPayloads[key] = value as {[action: string]: {p: CompactParam[]}};
|
||||
}
|
||||
}
|
||||
/* eslint-enable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any */
|
||||
|
||||
// Hydrated webhook payloads
|
||||
@@ -169,10 +199,14 @@ function getWebhookPayload(event: string, action: string): WebhookPayload | unde
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Get params from 'p' (compact format)
|
||||
const dedupedParams = deduplicatedPayload.p || [];
|
||||
|
||||
// Recreate the full payload and store it for reuse
|
||||
const params = deduplicatedPayload.bodyParameters.map(p => fullParam(p));
|
||||
const params = dedupedParams.map(p => fullParam(p));
|
||||
const payload = {
|
||||
...deduplicatedPayload,
|
||||
descriptionHtml: "",
|
||||
summaryHtml: "",
|
||||
bodyParameters: params
|
||||
};
|
||||
webhookPayloads[event] ||= {};
|
||||
@@ -180,13 +214,65 @@ function getWebhookPayload(event: string, action: string): WebhookPayload | unde
|
||||
return payload;
|
||||
}
|
||||
|
||||
function fullParam(dedupedParam: DeduplicatedParam): Param {
|
||||
if (typeof dedupedParam === "number") {
|
||||
if (dedupedParam >= objects.length) {
|
||||
throw new Error(`Unknown object ${dedupedParam}`);
|
||||
/**
|
||||
* Resolve an interned name (non-negative number -> string table lookup) or return literal string
|
||||
*/
|
||||
function resolveName(name: InternedName): string {
|
||||
if (typeof name === "number") {
|
||||
if (name < 0 || name >= stringTable.length) {
|
||||
throw new Error(`Unknown interned name index ${name}`);
|
||||
}
|
||||
return objects[dedupedParam];
|
||||
return stringTable[name];
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a deduplicated param to a full Param.
|
||||
*
|
||||
* Compact format (type-based dispatch):
|
||||
* - negative number - object index: -(n + 1) -> objects[-n - 1]
|
||||
* - non-negative number - interned string index -> stringTable[n]
|
||||
* - "name" - literal name only (singleton, not interned)
|
||||
* - [name, desc] - name + description (name can be number or string)
|
||||
* - [name, [...children]] - name + children (arr[1] is array)
|
||||
* - [name, desc, [...children]] - name + description + children
|
||||
*/
|
||||
function fullParam(dedupedParam: CompactParam): Param {
|
||||
// Negative number -> object index
|
||||
if (typeof dedupedParam === "number" && dedupedParam < 0) {
|
||||
const objectIndex = -(dedupedParam + 1);
|
||||
if (objectIndex >= objects.length) {
|
||||
throw new Error(`Unknown object index ${objectIndex} (from ${dedupedParam})`);
|
||||
}
|
||||
return fullParam(objects[objectIndex]);
|
||||
}
|
||||
|
||||
return dedupedParam;
|
||||
// Non-negative number or literal string -> name only
|
||||
if (typeof dedupedParam === "number" || typeof dedupedParam === "string") {
|
||||
return {
|
||||
name: resolveName(dedupedParam),
|
||||
description: ""
|
||||
} as Param;
|
||||
}
|
||||
|
||||
// Compact array format -> convert to Param object
|
||||
if (Array.isArray(dedupedParam)) {
|
||||
const arr = dedupedParam;
|
||||
const name = resolveName(arr[0]);
|
||||
|
||||
// Type-based dispatch: if arr[1] is string -> description, if array -> children
|
||||
const description = typeof arr[1] === "string" ? arr[1] : "";
|
||||
// arr[1] is children if it's an array, otherwise arr[2] is children (if it exists and is an array)
|
||||
const childrenArr = Array.isArray(arr[1]) ? arr[1] : Array.isArray(arr[2]) ? arr[2] : undefined;
|
||||
const childParamsGroups = childrenArr ? childrenArr.map(c => fullParam(c)) : undefined;
|
||||
|
||||
return {
|
||||
name,
|
||||
description,
|
||||
childParamsGroups
|
||||
} as Param;
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected param format: ${JSON.stringify(dedupedParam)}`);
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,416 @@
|
||||
[
|
||||
"url",
|
||||
"id",
|
||||
"node_id",
|
||||
"html_url",
|
||||
"name",
|
||||
"events_url",
|
||||
"avatar_url",
|
||||
"login",
|
||||
"repos_url",
|
||||
"type",
|
||||
"gravatar_id",
|
||||
"followers_url",
|
||||
"following_url",
|
||||
"gists_url",
|
||||
"starred_url",
|
||||
"subscriptions_url",
|
||||
"organizations_url",
|
||||
"received_events_url",
|
||||
"site_admin",
|
||||
"email",
|
||||
"deleted",
|
||||
"created_at",
|
||||
"description",
|
||||
"updated_at",
|
||||
"href",
|
||||
"labels_url",
|
||||
"comments_url",
|
||||
"number",
|
||||
"user",
|
||||
"statuses_url",
|
||||
"owner",
|
||||
"commits_url",
|
||||
"state",
|
||||
"open_issues",
|
||||
"sha",
|
||||
"title",
|
||||
"ref",
|
||||
"body",
|
||||
"hooks_url",
|
||||
"issues_url",
|
||||
"full_name",
|
||||
"private",
|
||||
"fork",
|
||||
"archive_url",
|
||||
"assignees_url",
|
||||
"blobs_url",
|
||||
"branches_url",
|
||||
"collaborators_url",
|
||||
"compare_url",
|
||||
"contents_url",
|
||||
"contributors_url",
|
||||
"deployments_url",
|
||||
"downloads_url",
|
||||
"forks_url",
|
||||
"git_commits_url",
|
||||
"git_refs_url",
|
||||
"git_tags_url",
|
||||
"issue_comment_url",
|
||||
"issue_events_url",
|
||||
"keys_url",
|
||||
"languages_url",
|
||||
"merges_url",
|
||||
"milestones_url",
|
||||
"notifications_url",
|
||||
"pulls_url",
|
||||
"releases_url",
|
||||
"stargazers_url",
|
||||
"subscribers_url",
|
||||
"subscription_url",
|
||||
"tags_url",
|
||||
"teams_url",
|
||||
"trees_url",
|
||||
"slug",
|
||||
"size",
|
||||
"repo",
|
||||
"closed_at",
|
||||
"permissions",
|
||||
"archived",
|
||||
"comments",
|
||||
"homepage",
|
||||
"license",
|
||||
"git_url",
|
||||
"ssh_url",
|
||||
"clone_url",
|
||||
"mirror_url",
|
||||
"svn_url",
|
||||
"language",
|
||||
"forks_count",
|
||||
"stargazers_count",
|
||||
"watchers_count",
|
||||
"default_branch",
|
||||
"open_issues_count",
|
||||
"is_template",
|
||||
"topics",
|
||||
"has_issues",
|
||||
"has_projects",
|
||||
"has_wiki",
|
||||
"has_pages",
|
||||
"has_downloads",
|
||||
"disabled",
|
||||
"visibility",
|
||||
"pushed_at",
|
||||
"forks",
|
||||
"delete_branch_on_merge",
|
||||
"allow_forking",
|
||||
"watchers",
|
||||
"allow_rebase_merge",
|
||||
"allow_squash_merge",
|
||||
"allow_auto_merge",
|
||||
"allow_update_branch",
|
||||
"allow_merge_commit",
|
||||
"from",
|
||||
"web_commit_signoff_required",
|
||||
"author_association",
|
||||
"label",
|
||||
"labels",
|
||||
"key",
|
||||
"admin",
|
||||
"pull",
|
||||
"triage",
|
||||
"push",
|
||||
"maintain",
|
||||
"draft",
|
||||
"public",
|
||||
"commits",
|
||||
"organization",
|
||||
"spdx_id",
|
||||
"master_branch",
|
||||
"active_lock_reason",
|
||||
"locked",
|
||||
"review_comments",
|
||||
"use_squash_pr_title_as_default",
|
||||
"squash_merge_commit_title",
|
||||
"squash_merge_commit_message",
|
||||
"merge_commit_title",
|
||||
"merge_commit_message",
|
||||
"role_name",
|
||||
"creator",
|
||||
"stargazers",
|
||||
"milestone",
|
||||
"members_url",
|
||||
"statuses",
|
||||
"assignee",
|
||||
"assignees",
|
||||
"permission",
|
||||
"privacy",
|
||||
"repositories_url",
|
||||
"base",
|
||||
"status",
|
||||
"head",
|
||||
"reactions",
|
||||
"+1",
|
||||
"-1",
|
||||
"confused",
|
||||
"eyes",
|
||||
"heart",
|
||||
"hooray",
|
||||
"laugh",
|
||||
"rocket",
|
||||
"total_count",
|
||||
"diff_url",
|
||||
"merged_at",
|
||||
"patch_url",
|
||||
"changes",
|
||||
"due_on",
|
||||
"repository_url",
|
||||
"color",
|
||||
"issue",
|
||||
"closed_issues",
|
||||
"pull_request",
|
||||
"conclusion",
|
||||
"default",
|
||||
"_links",
|
||||
"html",
|
||||
"author",
|
||||
"self",
|
||||
"pull_requests",
|
||||
"issue_url",
|
||||
"has_discussions",
|
||||
"metadata",
|
||||
"date",
|
||||
"review_comment",
|
||||
"auto_merge",
|
||||
"merge_commit_sha",
|
||||
"requested_reviewers",
|
||||
"requested_teams",
|
||||
"review_comment_url",
|
||||
"review_comments_url",
|
||||
"starred_at",
|
||||
"external_url",
|
||||
"issues",
|
||||
"checks",
|
||||
"contents",
|
||||
"deployments",
|
||||
"events",
|
||||
"head_sha",
|
||||
"username",
|
||||
"performed_via_github_app",
|
||||
"timeline_url",
|
||||
"additions",
|
||||
"changed_files",
|
||||
"deletions",
|
||||
"maintainer_can_modify",
|
||||
"mergeable",
|
||||
"mergeable_state",
|
||||
"merged",
|
||||
"merged_by",
|
||||
"rebaseable",
|
||||
"head_branch",
|
||||
"completed_at",
|
||||
"started_at",
|
||||
"parent",
|
||||
"commit_message",
|
||||
"commit_title",
|
||||
"enabled_by",
|
||||
"merge_method",
|
||||
"pages",
|
||||
"path",
|
||||
"actions",
|
||||
"administration",
|
||||
"content_references",
|
||||
"discussions",
|
||||
"emails",
|
||||
"environments",
|
||||
"keys",
|
||||
"members",
|
||||
"organization_administration",
|
||||
"organization_hooks",
|
||||
"organization_packages",
|
||||
"organization_plan",
|
||||
"organization_projects",
|
||||
"organization_secrets",
|
||||
"organization_self_hosted_runners",
|
||||
"organization_user_blocking",
|
||||
"packages",
|
||||
"repository_hooks",
|
||||
"repository_projects",
|
||||
"secret_scanning_alerts",
|
||||
"secrets",
|
||||
"security_events",
|
||||
"security_scanning_alert",
|
||||
"single_file",
|
||||
"team_discussions",
|
||||
"vulnerability_alerts",
|
||||
"workflows",
|
||||
"run_attempt",
|
||||
"committer",
|
||||
"message",
|
||||
"prerelease",
|
||||
"tag_name",
|
||||
"target_commitish",
|
||||
"repository",
|
||||
"repository_id",
|
||||
"head_commit",
|
||||
"timestamp",
|
||||
"tree_id",
|
||||
"to",
|
||||
"content_type",
|
||||
"published_at",
|
||||
"category",
|
||||
"emoji",
|
||||
"is_answerable",
|
||||
"state_reason",
|
||||
"affected_package_name",
|
||||
"affected_range",
|
||||
"external_identifier",
|
||||
"external_reference",
|
||||
"fixed_in",
|
||||
"ghsa_id",
|
||||
"severity",
|
||||
"check_run_url",
|
||||
"run_id",
|
||||
"run_url",
|
||||
"runner_group_id",
|
||||
"runner_group_name",
|
||||
"runner_id",
|
||||
"runner_name",
|
||||
"workflow_name",
|
||||
"steps",
|
||||
"installations_count",
|
||||
"client_id",
|
||||
"client_secret",
|
||||
"webhook_secret",
|
||||
"pem",
|
||||
"after",
|
||||
"before",
|
||||
"answer_chosen_at",
|
||||
"answer_chosen_by",
|
||||
"answer_html_url",
|
||||
"release",
|
||||
"assets",
|
||||
"assets_url",
|
||||
"tarball_url",
|
||||
"upload_url",
|
||||
"zipball_url",
|
||||
"app",
|
||||
"environment",
|
||||
"summary",
|
||||
"actor",
|
||||
"artifacts_url",
|
||||
"cancel_url",
|
||||
"check_suite_id",
|
||||
"check_suite_node_id",
|
||||
"check_suite_url",
|
||||
"event",
|
||||
"head_repository",
|
||||
"jobs_url",
|
||||
"logs_url",
|
||||
"previous_attempt_url",
|
||||
"referenced_workflows",
|
||||
"rerun_url",
|
||||
"run_number",
|
||||
"run_started_at",
|
||||
"triggering_actor",
|
||||
"workflow_id",
|
||||
"workflow_url",
|
||||
"note",
|
||||
"after_id",
|
||||
"column_id",
|
||||
"project_url",
|
||||
"dismiss_reason",
|
||||
"dismissed_at",
|
||||
"dismisser",
|
||||
"fix_reason",
|
||||
"fixed_at",
|
||||
"commit",
|
||||
"workflow_job",
|
||||
"action",
|
||||
"temp_clone_token",
|
||||
"subscribers_count",
|
||||
"network_count",
|
||||
"check_suite",
|
||||
"deployment",
|
||||
"task",
|
||||
"original_environment",
|
||||
"transient_environment",
|
||||
"production_environment",
|
||||
"payload",
|
||||
"workflow_run",
|
||||
"discussion",
|
||||
"comment",
|
||||
"child_comment_count",
|
||||
"discussion_id",
|
||||
"parent_id",
|
||||
"column_url",
|
||||
"commit_id",
|
||||
"pull_request_url",
|
||||
"browser_download_url",
|
||||
"download_count",
|
||||
"uploader",
|
||||
"discussion_url",
|
||||
"alert",
|
||||
"manifest",
|
||||
"installation_command",
|
||||
"version",
|
||||
"admin_enforced",
|
||||
"authorized_actor_names",
|
||||
"authorized_actors_only",
|
||||
"authorized_dismissal_actors_only",
|
||||
"linear_history_requirement_enforcement_level",
|
||||
"required_status_checks",
|
||||
"required_status_checks_enforcement_level",
|
||||
"check_run",
|
||||
"details_url",
|
||||
"external_id",
|
||||
"actions_meta",
|
||||
"check_runs_url",
|
||||
"latest_check_runs_count",
|
||||
"rerequestable",
|
||||
"runs_rerequestable",
|
||||
"workflow",
|
||||
"display_title",
|
||||
"project_card",
|
||||
"content_url",
|
||||
"short_description",
|
||||
"archived_at",
|
||||
"reason",
|
||||
"diff_hunk",
|
||||
"in_reply_to_id",
|
||||
"line",
|
||||
"original_commit_id",
|
||||
"original_line",
|
||||
"original_position",
|
||||
"original_start_line",
|
||||
"position",
|
||||
"pull_request_review_id",
|
||||
"side",
|
||||
"start_line",
|
||||
"start_side",
|
||||
"ref_type",
|
||||
"target_url",
|
||||
"new_repository",
|
||||
"pusher",
|
||||
"added",
|
||||
"distinct",
|
||||
"modified",
|
||||
"removed",
|
||||
"registry_package",
|
||||
"ecosystem",
|
||||
"namespace",
|
||||
"package_type",
|
||||
"package_version",
|
||||
"body_html",
|
||||
"docker_metadata",
|
||||
"package_files",
|
||||
"download_url",
|
||||
"md5",
|
||||
"sha1",
|
||||
"sha256",
|
||||
"package_url",
|
||||
"rubygems_metadata",
|
||||
"target_oid",
|
||||
"registry",
|
||||
"dismiss_comment"
|
||||
]
|
||||
@@ -0,0 +1,98 @@
|
||||
# Workflow Schema Optimization Plan
|
||||
|
||||
## Current State (Commit 7660f61)
|
||||
|
||||
### What's Implemented
|
||||
|
||||
1. **Original schema preserved**: `workflow-v1.0.json` remains source of truth with 291 definitions
|
||||
2. **Optimization script**: `script/optimize-workflow-schema.js` prunes unused definitions
|
||||
3. **Generated files** (gitignored):
|
||||
- `workflow-v1.0.optimized.json` - 281 definitions (pruned)
|
||||
- `workflow-v1.0.optimized.min.json` - minified version loaded at runtime
|
||||
4. **Build pipeline**: `npm run minify-json` chains optimize → minify
|
||||
5. **Tests**: `workflow-schema.test.ts` validates schema integrity
|
||||
|
||||
### 10 Pruned Definitions
|
||||
|
||||
These are unreachable from `workflow-root-strict` entry point:
|
||||
|
||||
- `workflow-root` (non-strict variant)
|
||||
- `on` (non-strict variant)
|
||||
- `on-mapping` (non-strict variant)
|
||||
- `job-if-result`
|
||||
- `step-if-result`
|
||||
- `boolean-needs-context`
|
||||
- `number-needs-context`
|
||||
- `string-needs-context`
|
||||
- `boolean-steps-context`
|
||||
- `number-steps-context`
|
||||
|
||||
### Size Savings
|
||||
|
||||
| Metric | Original | Optimized | Savings |
|
||||
|--------|----------|-----------|---------|
|
||||
| Definitions | 291 | 281 | 10 removed |
|
||||
| Minified | 71,061 B | 69,022 B | 2.9% |
|
||||
| Gzipped | 12,318 B | 12,172 B | 1.2% |
|
||||
|
||||
## Optimization Strategies Evaluated
|
||||
|
||||
### ✅ Pruning Unused Definitions (IMPLEMENTED)
|
||||
- Removes definitions not reachable from entry point
|
||||
- 1.2% gzip savings
|
||||
- Low complexity, no runtime overhead
|
||||
|
||||
### ❌ Key Shortening
|
||||
- Replace long keys with short codes (e.g., `description` → `d`)
|
||||
- 1.5% gzip savings
|
||||
- NOT WORTH IT: Adds complexity, minimal benefit after gzip
|
||||
|
||||
### ❌ String Interning
|
||||
- Deduplicate repeated strings into lookup table
|
||||
- Makes gzip WORSE (-0.5%)
|
||||
- NOT WORTH IT: Gzip already handles repetition
|
||||
|
||||
### ❌ Compact Format (like webhooks)
|
||||
- Restructure to array-based format
|
||||
- Makes gzip WORSE (-0.4%)
|
||||
- NOT WORTH IT: Schema structure doesn't benefit
|
||||
|
||||
### ❌ Split Descriptions
|
||||
- Separate file for descriptions
|
||||
- Adds 791 bytes when both files gzipped
|
||||
- NOT WORTH IT: Worse total size
|
||||
|
||||
## Future Work
|
||||
|
||||
### Update from Server
|
||||
The current schema may be missing some definitions from the latest server version. A future PR should:
|
||||
1. Fetch latest `workflow-v1.0.json` from dotcom server
|
||||
2. Update the source file
|
||||
3. Verify tests still pass
|
||||
4. Note: Server version had issues last checked (e.g., `coerce-raw`, missing `branches` on merge-group)
|
||||
|
||||
### Entry Point
|
||||
- Entry point: `workflow-root-strict` (defined in `workflow-constants.ts`)
|
||||
- Non-strict `workflow-root` exists but is unused in this codebase
|
||||
|
||||
## Files
|
||||
|
||||
```
|
||||
workflow-parser/
|
||||
├── src/
|
||||
│ ├── workflow-v1.0.json # Source of truth (tracked, 291 defs)
|
||||
│ ├── workflow-v1.0.optimized.json # Pruned (gitignored, 281 defs)
|
||||
│ ├── workflow-v1.0.optimized.min.json # Minified (gitignored)
|
||||
│ └── workflows/
|
||||
│ ├── workflow-schema.ts # Loader (imports optimized.min.json)
|
||||
│ └── workflow-schema.test.ts # Schema integrity tests
|
||||
└── script/
|
||||
└── optimize-workflow-schema.js # Pruning script
|
||||
```
|
||||
|
||||
## Test Coverage
|
||||
|
||||
- `workflow-schema.test.ts`:
|
||||
1. Schema loads from workflow-root-strict
|
||||
2. All referenced definitions are reachable
|
||||
3. Critical definitions exist (jobs, steps, runs-on, etc.)
|
||||
@@ -38,7 +38,8 @@
|
||||
"format-check": "prettier --check '**/*.ts'",
|
||||
"lint": "eslint 'src/**/*.ts'",
|
||||
"lint-fix": "eslint --fix 'src/**/*.ts'",
|
||||
"minify-json": "node ../script/minify-json.js src/workflow-v1.0.json",
|
||||
"optimize-schema": "node script/optimize-workflow-schema.js",
|
||||
"minify-json": "npm run optimize-schema && node ../script/minify-json.js src/workflow-v1.0.optimized.json",
|
||||
"prebuild": "npm run minify-json",
|
||||
"prepublishOnly": "npm run build && npm run test",
|
||||
"pretest": "npm run minify-json",
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Optimizes workflow-v1.0.json by pruning unused definitions.
|
||||
*
|
||||
* Removes definitions not reachable from the entry point (workflow-root-strict).
|
||||
* Output is then minified by minify-json.js to produce the final .min.json file.
|
||||
*
|
||||
* Usage: node script/optimize-workflow-schema.js
|
||||
*/
|
||||
|
||||
import {promises as fs} from "fs";
|
||||
import path from "path";
|
||||
import {fileURLToPath} from "url";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ENTRY_POINT = "workflow-root-strict";
|
||||
|
||||
const inputPath = path.join(__dirname, "..", "src", "workflow-v1.0.json");
|
||||
const outputPath = path.join(__dirname, "..", "src", "workflow-v1.0.optimized.json");
|
||||
|
||||
/**
|
||||
* Find all type references in a definition.
|
||||
*/
|
||||
function findRefs(obj) {
|
||||
const refs = [];
|
||||
|
||||
function visit(node) {
|
||||
if (!node || typeof node !== "object") return;
|
||||
|
||||
if (Array.isArray(node)) {
|
||||
for (const item of node) {
|
||||
if (typeof item === "string") refs.push(item);
|
||||
else visit(item);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for (const [key, value] of Object.entries(node)) {
|
||||
if (["type", "item-type", "loose-key-type", "loose-value-type"].includes(key)) {
|
||||
if (typeof value === "string") refs.push(value);
|
||||
} else if (key === "one-of") {
|
||||
visit(value);
|
||||
} else if (key === "properties") {
|
||||
for (const propValue of Object.values(value)) {
|
||||
if (typeof propValue === "string") refs.push(propValue);
|
||||
else if (propValue && typeof propValue === "object") visit(propValue);
|
||||
}
|
||||
} else if (["mapping", "sequence", "string"].includes(key)) {
|
||||
visit(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
visit(obj);
|
||||
return refs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find all definitions reachable from entry point.
|
||||
*/
|
||||
function findReachable(schema, entryPoint) {
|
||||
const reachable = new Set();
|
||||
const queue = [entryPoint];
|
||||
|
||||
while (queue.length > 0) {
|
||||
const name = queue.shift();
|
||||
if (reachable.has(name) || !schema.definitions[name]) continue;
|
||||
reachable.add(name);
|
||||
|
||||
const refs = findRefs(schema.definitions[name]);
|
||||
for (const ref of refs) {
|
||||
if (!reachable.has(ref) && schema.definitions[ref]) {
|
||||
queue.push(ref);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return reachable;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const content = await fs.readFile(inputPath, "utf8");
|
||||
const schema = JSON.parse(content);
|
||||
|
||||
const reachable = findReachable(schema, ENTRY_POINT);
|
||||
const allDefs = Object.keys(schema.definitions);
|
||||
const unused = allDefs.filter((name) => !reachable.has(name));
|
||||
|
||||
console.log(`Entry point: ${ENTRY_POINT}`);
|
||||
console.log(`Definitions: ${allDefs.length} -> ${reachable.size} (${unused.length} pruned)`);
|
||||
|
||||
if (unused.length > 0) {
|
||||
console.log("\nPruned:");
|
||||
unused.forEach((name) => console.log(` - ${name}`));
|
||||
}
|
||||
|
||||
// Create pruned schema preserving definition order
|
||||
const pruned = {version: schema.version, definitions: {}};
|
||||
for (const name of allDefs) {
|
||||
if (reachable.has(name)) {
|
||||
pruned.definitions[name] = schema.definitions[name];
|
||||
}
|
||||
}
|
||||
|
||||
// Write output (will be minified by minify-json.js)
|
||||
await fs.writeFile(outputPath, JSON.stringify(pruned));
|
||||
|
||||
console.log(`\nOutput: ${outputPath}`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,85 @@
|
||||
import {getWorkflowSchema} from "./workflow-schema";
|
||||
|
||||
describe("workflow-schema", () => {
|
||||
it("loads successfully from workflow-root-strict entry point", () => {
|
||||
const schema = getWorkflowSchema();
|
||||
expect(schema).toBeDefined();
|
||||
|
||||
// Verify entry point exists
|
||||
const rootDef = schema.getDefinition("workflow-root-strict");
|
||||
expect(rootDef).toBeDefined();
|
||||
});
|
||||
|
||||
it("has all referenced definitions reachable from workflow-root-strict", () => {
|
||||
const schema = getWorkflowSchema();
|
||||
const definitions = schema.definitions;
|
||||
|
||||
// Collect all type references from all definitions
|
||||
const referencedTypes = new Set<string>();
|
||||
const definedTypes = new Set<string>();
|
||||
|
||||
for (const name of Object.keys(definitions)) {
|
||||
definedTypes.add(name);
|
||||
collectReferences(definitions[name], referencedTypes);
|
||||
}
|
||||
|
||||
// Every referenced type should be defined
|
||||
const missingDefinitions: string[] = [];
|
||||
for (const ref of referencedTypes) {
|
||||
// Skip built-in types
|
||||
if (isBuiltInType(ref)) continue;
|
||||
|
||||
if (!definedTypes.has(ref)) {
|
||||
missingDefinitions.push(ref);
|
||||
}
|
||||
}
|
||||
|
||||
expect(missingDefinitions).toEqual([]);
|
||||
});
|
||||
|
||||
it("can resolve key workflow definitions", () => {
|
||||
const schema = getWorkflowSchema();
|
||||
|
||||
// These are critical definitions that must exist
|
||||
const criticalDefinitions = [
|
||||
"workflow-root-strict",
|
||||
"jobs",
|
||||
"steps",
|
||||
"runs-on",
|
||||
"step-uses",
|
||||
"job-env",
|
||||
"step-env",
|
||||
];
|
||||
|
||||
for (const defName of criticalDefinitions) {
|
||||
const def = schema.getDefinition(defName);
|
||||
expect(def).toBeDefined();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
function collectReferences(obj: unknown, refs: Set<string>): void {
|
||||
if (!obj || typeof obj !== "object") return;
|
||||
|
||||
if (Array.isArray(obj)) {
|
||||
for (const item of obj) {
|
||||
if (typeof item === "string") refs.add(item);
|
||||
else collectReferences(item, refs);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const record = obj as Record<string, unknown>;
|
||||
for (const [key, value] of Object.entries(record)) {
|
||||
if (["type", "item-type", "loose-key-type", "loose-value-type"].includes(key)) {
|
||||
if (typeof value === "string") refs.add(value);
|
||||
} else if (key === "one-of" || key === "properties" || key === "mapping" || key === "sequence") {
|
||||
collectReferences(value, refs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isBuiltInType(typeName: string): boolean {
|
||||
const builtIns = ["null", "boolean", "number", "string", "sequence", "mapping", "any"];
|
||||
return builtIns.includes(typeName);
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import {JSONObjectReader} from "../templates/json-object-reader";
|
||||
import {TemplateSchema} from "../templates/schema";
|
||||
import WorkflowSchema from "../workflow-v1.0.min.json";
|
||||
import WorkflowSchema from "../workflow-v1.0.optimized.min.json";
|
||||
|
||||
let schema: TemplateSchema;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user