Optimize webhooks JSON with compact format and string interning
- Convert params to compact array format (type-based dispatch) - Intern duplicate property names into string table - Use negative indices for object references to distinguish from string indices - Rename objects.json → webhooks.objects.json, add webhooks.strings.json - Move event filters to JSON for maintainability - Add CI validation job to verify optimization correctness Reduces combined minified size by ~67% (453 KB → 148 KB), ~27% gzipped (23 KB → 17 KB).
This commit is contained in:
eric sciple
@@ -1,5 +1,38 @@
|
||||
import Webhook from "./webhook";
|
||||
|
||||
/**
|
||||
* Get the name from a param.
|
||||
* Formats: "name" (string), or [name, ...] (array)
|
||||
*/
|
||||
function getParamName(param: any): string {
|
||||
if (typeof param === "string") {
|
||||
return param;
|
||||
}
|
||||
if (Array.isArray(param)) {
|
||||
return param[0];
|
||||
}
|
||||
return param.name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get params from a webhook action.
|
||||
* Uses 'p' (short key) if present, falls back to 'bodyParameters'
|
||||
*/
|
||||
function getParams(webhook: any): any[] {
|
||||
return webhook.p || webhook.bodyParameters || [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Set params on a webhook action using the short key 'p'
|
||||
*/
|
||||
function setParams(webhook: any, params: any[]): void {
|
||||
if (webhook.p !== undefined) {
|
||||
webhook.p = params;
|
||||
} else {
|
||||
webhook.bodyParameters = params;
|
||||
}
|
||||
}
|
||||
|
||||
// Store any repeated body parameters in an array
|
||||
// and replace them in the webhook with an index in the array
|
||||
export function deduplicateWebhooks(webhooks: Record<string, Record<string, Webhook>>): any[] {
|
||||
@@ -10,10 +43,11 @@ export function deduplicateWebhooks(webhooks: Record<string, Record<string, Webh
|
||||
const objectCount: Record<string, number> = {};
|
||||
|
||||
for (const webhook of iterateWebhooks(webhooks)) {
|
||||
for (const param of webhook.bodyParameters) {
|
||||
objectsByName[param.name] ||= [];
|
||||
const index = findOrAdd(param, objectsByName[param.name]);
|
||||
const key = `${param.name}:${index}`;
|
||||
for (const param of getParams(webhook)) {
|
||||
const name = getParamName(param);
|
||||
objectsByName[name] ||= [];
|
||||
const index = findOrAdd(param, objectsByName[name]);
|
||||
const key = `${name}:${index}`;
|
||||
objectCount[key] ||= 0;
|
||||
objectCount[key]++;
|
||||
}
|
||||
@@ -27,18 +61,19 @@ export function deduplicateWebhooks(webhooks: Record<string, Record<string, Webh
|
||||
|
||||
for (const webhook of iterateWebhooks(webhooks)) {
|
||||
const newParams: any[] = [];
|
||||
for (const param of webhook.bodyParameters) {
|
||||
const index = find(param, objectsByName[param.name]);
|
||||
const key = `${param.name}:${index}`;
|
||||
for (const param of getParams(webhook)) {
|
||||
const name = getParamName(param);
|
||||
const index = find(param, objectsByName[name]);
|
||||
const key = `${name}:${index}`;
|
||||
if (objectCount[key] > 1) {
|
||||
newParams.push(indexForParam(param, index, bodyParamIndexMap, duplicatedBodyParams));
|
||||
newParams.push(indexForParam(param, name, index, bodyParamIndexMap, duplicatedBodyParams));
|
||||
} else {
|
||||
// If an object is only used once, keep it inline
|
||||
newParams.push(param);
|
||||
}
|
||||
}
|
||||
|
||||
webhook.bodyParameters = newParams;
|
||||
setParams(webhook, newParams);
|
||||
}
|
||||
|
||||
return duplicatedBodyParams;
|
||||
@@ -74,11 +109,12 @@ function find(param: any, objects: any[]): number {
|
||||
|
||||
function indexForParam(
|
||||
param: any,
|
||||
paramName: string,
|
||||
paramNameIndex: number,
|
||||
objectIndexMap: Record<string, number>,
|
||||
duplicatedBodyParams: any[]
|
||||
): number {
|
||||
const key = `${param.name}:${paramNameIndex}`;
|
||||
const key = `${paramName}:${paramNameIndex}`;
|
||||
|
||||
const existingIndex = objectIndexMap[key];
|
||||
if (existingIndex !== undefined) {
|
||||
|
||||
@@ -1,310 +0,0 @@
|
||||
import {promises as fs} from "fs";
|
||||
import Webhook from "./webhook.js";
|
||||
|
||||
import schemaImport from "rest-api-description/descriptions/api.github.com/dereferenced/api.github.com.deref.json";
|
||||
import {deduplicateWebhooks} from "./deduplicate.js";
|
||||
const schema = schemaImport as any;
|
||||
|
||||
const OUTPUT_PATH = "./src/context-providers/events/webhooks.json";
|
||||
const OBJECTS_PATH = "./src/context-providers/events/objects.json";
|
||||
const ALL_OUTPUT_PATH = "./src/context-providers/events/webhooks.all.json";
|
||||
const ALL_OBJECTS_PATH = "./src/context-providers/events/objects.all.json";
|
||||
const DROP_OUTPUT_PATH = "./src/context-providers/events/webhooks.drop.json";
|
||||
const DROP_OBJECTS_PATH = "./src/context-providers/events/objects.drop.json";
|
||||
const STRIP_OUTPUT_PATH = "./src/context-providers/events/webhooks.strip.json";
|
||||
const STRIP_OBJECTS_PATH = "./src/context-providers/events/objects.strip.json";
|
||||
|
||||
// Parse --all flag
|
||||
const generateAll = process.argv.includes("--all");
|
||||
|
||||
// Events to drop - not valid workflow triggers (GitHub App or API-only events)
|
||||
// See: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows
|
||||
const DROPPED_EVENTS = new Set([
|
||||
"branch_protection_configuration",
|
||||
"code_scanning_alert",
|
||||
"commit_comment",
|
||||
"custom_property",
|
||||
"custom_property_values",
|
||||
"dependabot_alert",
|
||||
"deploy_key",
|
||||
"github_app_authorization",
|
||||
"installation",
|
||||
"installation_repositories",
|
||||
"installation_target",
|
||||
"marketplace_purchase",
|
||||
"member",
|
||||
"membership",
|
||||
"merge_group",
|
||||
"meta",
|
||||
"org_block",
|
||||
"organization",
|
||||
"package",
|
||||
"personal_access_token_request",
|
||||
"ping",
|
||||
"repository",
|
||||
"repository_advisory",
|
||||
"repository_ruleset",
|
||||
"secret_scanning_alert",
|
||||
"secret_scanning_alert_location",
|
||||
"security_advisory",
|
||||
"security_and_analysis",
|
||||
"sponsorship",
|
||||
"star",
|
||||
"team",
|
||||
"team_add"
|
||||
]);
|
||||
|
||||
// Events to keep - valid workflow triggers
|
||||
// See: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows
|
||||
const KEPT_EVENTS = new Set([
|
||||
"branch_protection_rule",
|
||||
"check_run",
|
||||
"check_suite",
|
||||
"create",
|
||||
"delete",
|
||||
"deployment",
|
||||
"deployment_status",
|
||||
"discussion",
|
||||
"discussion_comment",
|
||||
"fork",
|
||||
"gollum",
|
||||
"issue_comment",
|
||||
"issues",
|
||||
"label",
|
||||
"milestone",
|
||||
"page_build",
|
||||
"project",
|
||||
"project_card",
|
||||
"project_column",
|
||||
"projects_v2",
|
||||
"projects_v2_item",
|
||||
"public",
|
||||
"pull_request",
|
||||
"pull_request_review",
|
||||
"pull_request_review_comment",
|
||||
"pull_request_review_thread",
|
||||
"push",
|
||||
"registry_package",
|
||||
"release",
|
||||
"repository_dispatch",
|
||||
"repository_import",
|
||||
"repository_vulnerability_alert",
|
||||
"status",
|
||||
"watch",
|
||||
"workflow_dispatch",
|
||||
"workflow_job",
|
||||
"workflow_run"
|
||||
]);
|
||||
|
||||
/**
|
||||
* Fields to strip from the JSON data.
|
||||
*
|
||||
* EVENT_ACTION_FIELDS: stripped from each event action object (top level only)
|
||||
* Example event action object before stripping:
|
||||
* {
|
||||
* "description": "This event is triggered when...", // <-- stripped
|
||||
* "summary": "A brief summary", // <-- stripped
|
||||
* "availability": ["repository"], // <-- stripped
|
||||
* "category": "issues", // <-- stripped
|
||||
* "action": "opened", // kept
|
||||
* "bodyParameters": [...] // kept
|
||||
* }
|
||||
*
|
||||
* BODY_PARAM_FIELDS: stripped from every bodyParameters object, recursively through childParamsGroups
|
||||
* Example bodyParameter object before stripping:
|
||||
* {
|
||||
* "type": "object", // <-- stripped
|
||||
* "name": "changes", // kept (used for property names)
|
||||
* "in": "body", // <-- stripped
|
||||
* "description": "The changes that were made.", // kept (used for hover docs)
|
||||
* "isRequired": true, // <-- stripped
|
||||
* "enum": ["a", "b"], // <-- stripped
|
||||
* "default": "a", // <-- stripped
|
||||
* "childParamsGroups": [ // kept (used for nested properties)
|
||||
* {
|
||||
* "type": "string", // <-- stripped (recursive)
|
||||
* "name": "from", // kept
|
||||
* "isRequired": true // <-- stripped (recursive)
|
||||
* }
|
||||
* ]
|
||||
* }
|
||||
*/
|
||||
const EVENT_ACTION_FIELDS = ["description", "summary", "availability", "category"];
|
||||
const BODY_PARAM_FIELDS = ["type", "in", "isRequired", "enum", "default"];
|
||||
|
||||
/**
|
||||
* Strip fields from a bodyParameter object and recursively from childParamsGroups.
|
||||
*/
|
||||
function stripBodyParam(param: any): any {
|
||||
if (typeof param !== "object" || param === null) {
|
||||
return param;
|
||||
}
|
||||
|
||||
const result: any = {};
|
||||
for (const [key, value] of Object.entries(param)) {
|
||||
if (BODY_PARAM_FIELDS.includes(key)) {
|
||||
continue; // Strip this field
|
||||
}
|
||||
if (key === "childParamsGroups" && Array.isArray(value)) {
|
||||
result[key] = value.map(stripBodyParam);
|
||||
} else {
|
||||
result[key] = value;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip unused fields from event action data.
|
||||
*/
|
||||
function stripEventActionFields(action: any): any {
|
||||
const result: any = {};
|
||||
for (const [key, value] of Object.entries(action)) {
|
||||
if (EVENT_ACTION_FIELDS.includes(key)) {
|
||||
continue; // Strip this field
|
||||
}
|
||||
if (key === "bodyParameters" && Array.isArray(value)) {
|
||||
result[key] = value.map((p: any) => (typeof p === "number" ? p : stripBodyParam(p)));
|
||||
} else {
|
||||
result[key] = value;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip unused fields from all webhooks.
|
||||
* Structure: { eventName: { actionName: { ...fields } } }
|
||||
*/
|
||||
function stripFields(webhooks: Record<string, Record<string, any>>): Record<string, Record<string, any>> {
|
||||
const result: Record<string, Record<string, any>> = {};
|
||||
for (const [eventName, actions] of Object.entries(webhooks)) {
|
||||
result[eventName] = {};
|
||||
for (const [actionName, actionData] of Object.entries(actions)) {
|
||||
result[eventName][actionName] = stripEventActionFields(actionData);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
const rawWebhooks = Object.values(schema.webhooks || schema["x-webhooks"]) as any[];
|
||||
if (!rawWebhooks) {
|
||||
throw new Error("No webhooks found in schema");
|
||||
}
|
||||
|
||||
const webhooks: Webhook[] = [];
|
||||
for (const webhook of Object.values(rawWebhooks)) {
|
||||
webhooks.push(new Webhook(webhook.post));
|
||||
}
|
||||
|
||||
await Promise.all(webhooks.map(webhook => webhook.process()));
|
||||
|
||||
// Check for unknown events (not in DROPPED_EVENTS or KEPT_EVENTS)
|
||||
const unknownEvents: string[] = [];
|
||||
for (const webhook of webhooks) {
|
||||
if (!DROPPED_EVENTS.has(webhook.category) && !KEPT_EVENTS.has(webhook.category)) {
|
||||
if (!unknownEvents.includes(webhook.category)) {
|
||||
unknownEvents.push(webhook.category);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (unknownEvents.length > 0) {
|
||||
console.error("");
|
||||
console.error("══════════════════════════════════════════════════════════════════");
|
||||
console.error("ERROR: New webhook event(s) detected!");
|
||||
console.error("══════════════════════════════════════════════════════════════════");
|
||||
console.error("");
|
||||
console.error("The following events are not categorized:");
|
||||
for (const event of unknownEvents.sort()) {
|
||||
console.error(` - ${event}`);
|
||||
}
|
||||
console.error("");
|
||||
console.error("Action required:");
|
||||
console.error(" 1. Check if the event is a valid workflow trigger:");
|
||||
console.error(
|
||||
" https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows"
|
||||
);
|
||||
console.error("");
|
||||
console.error(" 2. Add the event to DROPPED_EVENTS or KEPT_EVENTS in:");
|
||||
console.error(" languageservice/script/webhooks/index.ts");
|
||||
console.error("");
|
||||
console.error(" 3. See docs/json-data-files.md for more details.");
|
||||
console.error("");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// The category is the name of the webhook
|
||||
const categorizedWebhooks: Record<string, Record<string, Webhook>> = {};
|
||||
for (const webhook of webhooks) {
|
||||
if (!webhook.action) webhook.action = "default";
|
||||
|
||||
// Drop unused events
|
||||
if (DROPPED_EVENTS.has(webhook.category)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (categorizedWebhooks[webhook.category]) {
|
||||
categorizedWebhooks[webhook.category][webhook.action] = webhook;
|
||||
} else {
|
||||
categorizedWebhooks[webhook.category] = {};
|
||||
categorizedWebhooks[webhook.category][webhook.action] = webhook;
|
||||
}
|
||||
}
|
||||
|
||||
// Strip fields before deduplication
|
||||
const strippedWebhooks = stripFields(categorizedWebhooks);
|
||||
|
||||
// Deduplicate after dropping and stripping
|
||||
const objectsArray = deduplicateWebhooks(strippedWebhooks);
|
||||
|
||||
// Write optimized output
|
||||
await fs.writeFile(OBJECTS_PATH, JSON.stringify(objectsArray, null, 2));
|
||||
await fs.writeFile(OUTPUT_PATH, JSON.stringify(strippedWebhooks, null, 2));
|
||||
|
||||
console.log(`Wrote ${OUTPUT_PATH} (${Object.keys(strippedWebhooks).length} events)`);
|
||||
console.log(`Wrote ${OBJECTS_PATH} (${objectsArray.length} objects)`);
|
||||
|
||||
// Optionally generate intermediate versions for size comparison
|
||||
if (generateAll) {
|
||||
// Helper to deep clone
|
||||
function clone<T>(obj: T): T {
|
||||
return JSON.parse(JSON.stringify(obj));
|
||||
}
|
||||
|
||||
// Build full webhooks (no drop, no strip) from fresh data
|
||||
const fullWebhooks: Record<string, Record<string, any>> = {};
|
||||
for (const webhook of webhooks) {
|
||||
const w = clone(webhook);
|
||||
if (!w.action) w.action = "default";
|
||||
fullWebhooks[w.category] ||= {};
|
||||
fullWebhooks[w.category][w.action] = w;
|
||||
}
|
||||
|
||||
// Generate all version (no drop, no strip)
|
||||
const allWebhooks = clone(fullWebhooks);
|
||||
const allObjects = deduplicateWebhooks(allWebhooks);
|
||||
await fs.writeFile(ALL_OUTPUT_PATH, JSON.stringify(allWebhooks, null, 2));
|
||||
await fs.writeFile(ALL_OBJECTS_PATH, JSON.stringify(allObjects, null, 2));
|
||||
console.log(`Wrote ${ALL_OUTPUT_PATH} (${Object.keys(allWebhooks).length} events)`);
|
||||
console.log(`Wrote ${ALL_OBJECTS_PATH} (${allObjects.length} objects)`);
|
||||
|
||||
// Generate drop-only version (drop events, no strip)
|
||||
const dropWebhooks = clone(fullWebhooks);
|
||||
for (const event of DROPPED_EVENTS) {
|
||||
delete dropWebhooks[event];
|
||||
}
|
||||
const dropObjects = deduplicateWebhooks(dropWebhooks);
|
||||
await fs.writeFile(DROP_OUTPUT_PATH, JSON.stringify(dropWebhooks, null, 2));
|
||||
await fs.writeFile(DROP_OBJECTS_PATH, JSON.stringify(dropObjects, null, 2));
|
||||
console.log(`Wrote ${DROP_OUTPUT_PATH} (${Object.keys(dropWebhooks).length} events)`);
|
||||
console.log(`Wrote ${DROP_OBJECTS_PATH} (${dropObjects.length} objects)`);
|
||||
|
||||
// Generate strip-only version (strip fields, no drop)
|
||||
const stripWebhooks = stripFields(clone(fullWebhooks));
|
||||
const stripObjects = deduplicateWebhooks(stripWebhooks);
|
||||
await fs.writeFile(STRIP_OUTPUT_PATH, JSON.stringify(stripWebhooks, null, 2));
|
||||
await fs.writeFile(STRIP_OBJECTS_PATH, JSON.stringify(stripObjects, null, 2));
|
||||
console.log(`Wrote ${STRIP_OUTPUT_PATH} (${Object.keys(stripWebhooks).length} events)`);
|
||||
console.log(`Wrote ${STRIP_OBJECTS_PATH} (${stripObjects.length} objects)`);
|
||||
}
|
||||
+291
@@ -0,0 +1,291 @@
|
||||
import {promises as fs} from "fs";
|
||||
import Webhook from "./webhook.js";
|
||||
|
||||
import schemaImport from "rest-api-description/descriptions/api.github.com/dereferenced/api.github.com.deref.json";
|
||||
import {deduplicateWebhooks} from "./deduplicate.js";
|
||||
import eventFilters from "../../src/context-providers/events/event-filters.json";
|
||||
const schema = schemaImport as any;
|
||||
|
||||
const DROPPED_EVENTS = new Set(eventFilters.dropped);
|
||||
const KEPT_EVENTS = new Set(eventFilters.kept);
|
||||
|
||||
const OUTPUT_PATH = "./src/context-providers/events/webhooks.json";
|
||||
const OBJECTS_PATH = "./src/context-providers/events/webhooks.objects.json";
|
||||
const STRINGS_PATH = "./src/context-providers/events/webhooks.strings.json";
|
||||
const FULL_OUTPUT_PATH = "./src/context-providers/events/webhooks.full.json";
|
||||
|
||||
/**
|
||||
* Fields discarded from each event action object (top level only).
|
||||
* Body parameters are compacted to only keep name, description, and childParamsGroups.
|
||||
*/
|
||||
const EVENT_ACTION_FIELDS = ["description", "summary", "availability", "category", "action"];
|
||||
|
||||
/**
|
||||
* Convert a bodyParameter object to compact array format.
|
||||
*
|
||||
* Format (type-based dispatch):
|
||||
* - "name" - name only (just a string)
|
||||
* - [name, desc] - name + description (desc is string)
|
||||
* - [name, [...children]] - name + children (arr[1] is array)
|
||||
* - [name, desc, [...children]] - name + description + children
|
||||
*
|
||||
* The reader uses typeof to determine the meaning:
|
||||
* - string -> name only
|
||||
* - array with string arr[1] -> name + description
|
||||
* - array with array arr[1] -> name + children
|
||||
*/
|
||||
function compactParam(param: any): any {
|
||||
if (typeof param !== "object" || param === null) {
|
||||
return param;
|
||||
}
|
||||
|
||||
const name: string = param.name;
|
||||
const desc: string | undefined = param.description;
|
||||
const children: any[] | undefined = param.childParamsGroups;
|
||||
|
||||
const hasDesc = desc && desc.length > 0;
|
||||
const hasChildren = children && children.length > 0;
|
||||
|
||||
if (hasDesc && hasChildren) {
|
||||
return [name, desc, children.map(compactParam)];
|
||||
} else if (hasChildren) {
|
||||
return [name, children.map(compactParam)];
|
||||
} else if (hasDesc) {
|
||||
return [name, desc];
|
||||
} else {
|
||||
return name; // Just the string, not wrapped in array
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert event action data to compact format.
|
||||
*/
|
||||
function compactEventAction(action: any): any {
|
||||
const result: any = {};
|
||||
for (const [key, value] of Object.entries(action)) {
|
||||
if (EVENT_ACTION_FIELDS.includes(key)) {
|
||||
continue; // Discard this field
|
||||
}
|
||||
if (key === "bodyParameters" && Array.isArray(value)) {
|
||||
// Use short key 'p' for params
|
||||
result["p"] = value.map((p: any) => (typeof p === "number" ? p : compactParam(p)));
|
||||
} else {
|
||||
result[key] = value;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert all webhooks to compact format.
|
||||
* Structure: { eventName: { actionName: { ...fields } } }
|
||||
*/
|
||||
function compactWebhooks(webhooks: Record<string, Record<string, any>>): Record<string, Record<string, any>> {
|
||||
const result: Record<string, Record<string, any>> = {};
|
||||
for (const [eventName, actions] of Object.entries(webhooks)) {
|
||||
result[eventName] = {};
|
||||
for (const [actionName, actionData] of Object.entries(actions)) {
|
||||
result[eventName][actionName] = compactEventAction(actionData);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
const rawWebhooks = Object.values(schema.webhooks || schema["x-webhooks"]) as any[];
|
||||
if (!rawWebhooks) {
|
||||
throw new Error("No webhooks found in schema");
|
||||
}
|
||||
|
||||
const webhooks: Webhook[] = [];
|
||||
for (const webhook of Object.values(rawWebhooks)) {
|
||||
webhooks.push(new Webhook(webhook.post));
|
||||
}
|
||||
|
||||
await Promise.all(webhooks.map(webhook => webhook.process()));
|
||||
|
||||
// Check for unknown events (not in DROPPED_EVENTS or KEPT_EVENTS)
|
||||
const unknownEvents: string[] = [];
|
||||
for (const webhook of webhooks) {
|
||||
if (!DROPPED_EVENTS.has(webhook.category) && !KEPT_EVENTS.has(webhook.category)) {
|
||||
if (!unknownEvents.includes(webhook.category)) {
|
||||
unknownEvents.push(webhook.category);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (unknownEvents.length > 0) {
|
||||
console.error("");
|
||||
console.error("══════════════════════════════════════════════════════════════════");
|
||||
console.error("ERROR: New webhook event(s) detected!");
|
||||
console.error("══════════════════════════════════════════════════════════════════");
|
||||
console.error("");
|
||||
console.error("The following events are not categorized:");
|
||||
for (const event of unknownEvents.sort()) {
|
||||
console.error(` - ${event}`);
|
||||
}
|
||||
console.error("");
|
||||
console.error("Action required:");
|
||||
console.error(" 1. Check if the event is a valid workflow trigger:");
|
||||
console.error(
|
||||
" https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows"
|
||||
);
|
||||
console.error("");
|
||||
console.error(" 2. Add the event to 'dropped' or 'kept' array in:");
|
||||
console.error(" languageservice/src/context-providers/events/event-filters.json");
|
||||
console.error("");
|
||||
console.error(" 3. See docs/json-data-files.md for more details.");
|
||||
console.error("");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Build full webhooks (all events, no transformations) for validation tests
|
||||
const fullWebhooks: Record<string, Record<string, Webhook>> = {};
|
||||
for (const webhook of webhooks) {
|
||||
if (!webhook.action) webhook.action = "default";
|
||||
|
||||
if (fullWebhooks[webhook.category]) {
|
||||
fullWebhooks[webhook.category][webhook.action] = webhook;
|
||||
} else {
|
||||
fullWebhooks[webhook.category] = {};
|
||||
fullWebhooks[webhook.category][webhook.action] = webhook;
|
||||
}
|
||||
}
|
||||
|
||||
// Write full version (before any optimizations)
|
||||
await fs.writeFile(FULL_OUTPUT_PATH, JSON.stringify(fullWebhooks, null, 2));
|
||||
console.log(`Wrote ${FULL_OUTPUT_PATH} (${Object.keys(fullWebhooks).length} events, unoptimized)`);
|
||||
|
||||
// The category is the name of the webhook
|
||||
const categorizedWebhooks: Record<string, Record<string, Webhook>> = {};
|
||||
for (const webhook of webhooks) {
|
||||
if (!webhook.action) webhook.action = "default";
|
||||
|
||||
// Drop unused events
|
||||
if (DROPPED_EVENTS.has(webhook.category)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (categorizedWebhooks[webhook.category]) {
|
||||
categorizedWebhooks[webhook.category][webhook.action] = webhook;
|
||||
} else {
|
||||
categorizedWebhooks[webhook.category] = {};
|
||||
categorizedWebhooks[webhook.category][webhook.action] = webhook;
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to compact format before deduplication
|
||||
const compactedWebhooks = compactWebhooks(categorizedWebhooks);
|
||||
|
||||
// Deduplicate after compacting
|
||||
const objectsArray = deduplicateWebhooks(compactedWebhooks);
|
||||
|
||||
// ============================================================================
|
||||
// String Interning (Phase 3)
|
||||
// ============================================================================
|
||||
// Intern duplicate property names to reduce file size.
|
||||
// Names appearing 2+ times are stored in a string table and referenced by index.
|
||||
// Singleton names stay as literal strings for readability.
|
||||
|
||||
/**
|
||||
* Collect all property names from params (for frequency counting)
|
||||
*/
|
||||
function collectNames(param: any, counts: Map<string, number>): void {
|
||||
if (typeof param === "number") return;
|
||||
if (typeof param === "string") {
|
||||
counts.set(param, (counts.get(param) || 0) + 1);
|
||||
return;
|
||||
}
|
||||
if (Array.isArray(param)) {
|
||||
const name = param[0] as string;
|
||||
counts.set(name, (counts.get(name) || 0) + 1);
|
||||
const children = Array.isArray(param[1]) ? param[1] : param[2];
|
||||
if (children) children.forEach((c: any) => collectNames(c, counts));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace duplicate names with indices into the string table.
|
||||
* Object references use negative indices: objectIndex -> -(objectIndex + 1)
|
||||
* String references use non-negative indices: stringIndex -> stringIndex
|
||||
*
|
||||
* @param param - The param to process
|
||||
* @param nameToIndex - Map from name to string table index
|
||||
*/
|
||||
function internNames(param: any, nameToIndex: Map<string, number>): any {
|
||||
// Object reference (already a number from deduplication) -> make negative
|
||||
if (typeof param === "number") return -(param + 1);
|
||||
|
||||
// String -> intern if in table, otherwise keep as literal
|
||||
if (typeof param === "string") {
|
||||
const idx = nameToIndex.get(param);
|
||||
return idx !== undefined ? idx : param;
|
||||
}
|
||||
|
||||
if (Array.isArray(param)) {
|
||||
const name = param[0] as string;
|
||||
const idx = nameToIndex.get(name);
|
||||
const internedName = idx !== undefined ? idx : name;
|
||||
|
||||
// Handle different array formats
|
||||
if (typeof param[1] === "string" && !Array.isArray(param[1])) {
|
||||
// [name, desc] or [name, desc, children]
|
||||
if (param.length === 2) {
|
||||
return [internedName, param[1]];
|
||||
} else {
|
||||
return [internedName, param[1], (param[2] as any[]).map((c: any) => internNames(c, nameToIndex))];
|
||||
}
|
||||
} else if (Array.isArray(param[1])) {
|
||||
// [name, children]
|
||||
return [internedName, param[1].map((c: any) => internNames(c, nameToIndex))];
|
||||
}
|
||||
// Shouldn't happen, but fallback
|
||||
return [internedName, ...param.slice(1)];
|
||||
}
|
||||
return param;
|
||||
}
|
||||
|
||||
// Pass 1: Count all names
|
||||
const nameCounts = new Map<string, number>();
|
||||
objectsArray.forEach((obj: any) => collectNames(obj, nameCounts));
|
||||
for (const event of Object.values(compactedWebhooks)) {
|
||||
for (const action of Object.values(event as Record<string, any>)) {
|
||||
if (action.p) action.p.forEach((p: any) => collectNames(p, nameCounts));
|
||||
}
|
||||
}
|
||||
|
||||
// Build string table from duplicates, sorted by frequency (most common first = smaller indices)
|
||||
const sortedNames = [...nameCounts.entries()].filter(([, count]) => count >= 2).sort((a, b) => b[1] - a[1]);
|
||||
const stringTable = sortedNames.map(([name]) => name);
|
||||
const nameToIndex = new Map(stringTable.map((name, i) => [name, i]));
|
||||
|
||||
console.log(
|
||||
`String table: ${stringTable.length} interned names (${nameCounts.size - stringTable.length} singletons kept inline)`
|
||||
);
|
||||
|
||||
// Pass 2: Intern names in objects and webhooks
|
||||
// Objects use negative indices, strings use non-negative indices
|
||||
const internedObjects = objectsArray.map((obj: any) => internNames(obj, nameToIndex));
|
||||
const internedWebhooks: Record<string, Record<string, any>> = {};
|
||||
for (const [eventName, actions] of Object.entries(compactedWebhooks)) {
|
||||
internedWebhooks[eventName] = {};
|
||||
for (const [actionName, actionData] of Object.entries(actions as Record<string, any>)) {
|
||||
internedWebhooks[eventName][actionName] = {
|
||||
p: actionData.p.map((p: any) => internNames(p, nameToIndex))
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Write optimized output with separate string table
|
||||
// Format: webhooks.strings.json has string table, webhooks.json/webhooks.objects.json reference by index
|
||||
const finalOutput = {
|
||||
"//": "Generated file - refer to docs/json-data-files.md for format documentation",
|
||||
...internedWebhooks
|
||||
};
|
||||
|
||||
await fs.writeFile(STRINGS_PATH, JSON.stringify(stringTable, null, 2));
|
||||
await fs.writeFile(OBJECTS_PATH, JSON.stringify(internedObjects, null, 2));
|
||||
await fs.writeFile(OUTPUT_PATH, JSON.stringify(finalOutput, null, 2));
|
||||
|
||||
console.log(`Wrote ${STRINGS_PATH} (${stringTable.length} interned strings)`);
|
||||
console.log(`Wrote ${OUTPUT_PATH} (${Object.keys(compactedWebhooks).length} events)`);
|
||||
console.log(`Wrote ${OBJECTS_PATH} (${internedObjects.length} objects)`);
|
||||
Reference in New Issue
Block a user