From 3db2cfa8d0be776a38c333e982043530845b4ccd Mon Sep 17 00:00:00 2001 From: Aiqiao Yan <55104035+aiqiaoy@users.noreply.github.com> Date: Mon, 13 Apr 2026 16:24:07 -0400 Subject: [PATCH] Add ability to cache only latest N versions and cache github/gh-aw-actions (#136) * add ability to cache only latest N versions * fix test --- config/actions/github_gh-aw-actions.json | 26 +++++++ script/generated/github_gh-aw-actions.sh | 9 +++ script/internal/action-config.js | 94 +++++++++++++++++++++++- script/internal/add-action.js | 31 ++++++-- script/internal/filter-tags.js | 87 ++++++++++++++++++++++ script/internal/generate-scripts.sh | 15 ++++ script/internal/update-action.js | 26 ++++++- script/test.sh | 4 + 8 files changed, 279 insertions(+), 13 deletions(-) create mode 100644 config/actions/github_gh-aw-actions.json create mode 100644 script/generated/github_gh-aw-actions.sh create mode 100644 script/internal/filter-tags.js diff --git a/config/actions/github_gh-aw-actions.json b/config/actions/github_gh-aw-actions.json new file mode 100644 index 0000000..a7631e8 --- /dev/null +++ b/config/actions/github_gh-aw-actions.json @@ -0,0 +1,26 @@ +{ + "owner": "github", + "repo": "gh-aw-actions", + "patterns": [ + "+^master$", + "+^v[0-9]+(\\.[0-9]+){0,2}$" + ], + "branches": {}, + "defaultBranch": "master", + "latestMajorVersions": 1, + "latestVersionsPerMajor": 3, + "tags": { + "v0.67.4": { + "commit": "2b3c275b3652caa01c2ebe31cbab50ec2df0f927", + "tag": "9d6ae06250fc0ec536a0e5f35de313b35bad7246" + }, + "v0.68.0": { + "commit": "6715c81fe97e4bcbfa0734c3422491672ebda34f", + "tag": "0acfb4a691fe207cd8bc982ea5cb9d750d57a702" + }, + "v0.68.1": { + "commit": "ea222e359276c0702a5f5203547ff9d88d0ddd76", + "tag": "2fe53acc038ba01c3bbdc767d4b25df31ca5bdfc" + } + } +} \ No newline at end of file diff --git a/script/generated/github_gh-aw-actions.sh b/script/generated/github_gh-aw-actions.sh new file mode 100644 index 0000000..f9115b9 --- /dev/null +++ b/script/generated/github_gh-aw-actions.sh @@ -0,0 +1,9 @@ +mkdir github_gh-aw-actions +pushd github_gh-aw-actions +curl -s -S -L -o '2b3c275b3652caa01c2ebe31cbab50ec2df0f927.tar.gz' 'https://api.github.com/repos/github/gh-aw-actions/tarball/2b3c275b3652caa01c2ebe31cbab50ec2df0f927' +curl -s -S -L -o '2b3c275b3652caa01c2ebe31cbab50ec2df0f927.zip' 'https://api.github.com/repos/github/gh-aw-actions/zipball/2b3c275b3652caa01c2ebe31cbab50ec2df0f927' +curl -s -S -L -o '6715c81fe97e4bcbfa0734c3422491672ebda34f.tar.gz' 'https://api.github.com/repos/github/gh-aw-actions/tarball/6715c81fe97e4bcbfa0734c3422491672ebda34f' +curl -s -S -L -o '6715c81fe97e4bcbfa0734c3422491672ebda34f.zip' 'https://api.github.com/repos/github/gh-aw-actions/zipball/6715c81fe97e4bcbfa0734c3422491672ebda34f' +curl -s -S -L -o 'ea222e359276c0702a5f5203547ff9d88d0ddd76.tar.gz' 'https://api.github.com/repos/github/gh-aw-actions/tarball/ea222e359276c0702a5f5203547ff9d88d0ddd76' +curl -s -S -L -o 'ea222e359276c0702a5f5203547ff9d88d0ddd76.zip' 'https://api.github.com/repos/github/gh-aw-actions/zipball/ea222e359276c0702a5f5203547ff9d88d0ddd76' +popd diff --git a/script/internal/action-config.js b/script/internal/action-config.js index e01a7ac..41fcf08 100644 --- a/script/internal/action-config.js +++ b/script/internal/action-config.js @@ -44,6 +44,18 @@ class ActionConfig { */ defaultBranch = 'master' + /** + * Maximum number of latest major versions to include (default to unlimited) + * @type {number|undefined} + */ + latestMajorVersions = undefined + + /** + * Maximum number of latest version tags per major version (default to unlimited) + * @type {number|undefined} + */ + latestVersionsPerMajor = undefined + /** * Tag versions * @type {{[ref: string]: TagVersion}} @@ -73,9 +85,11 @@ exports.TagVersion = TagVersion * @param {string[]} patternStrings * @param {string} defaultBranch * @param {string[]|undefined} ignoreTags + * @param {number|undefined} latestMajorVersions + * @param {number|undefined} latestVersionsPerMajor * @returns {Promise} */ -async function add(owner, repo, patternStrings, defaultBranch, ignoreTags) { +async function add(owner, repo, patternStrings, defaultBranch, ignoreTags, latestMajorVersions, latestVersionsPerMajor) { assert.ok(owner, "Arg 'owner' must not be empty") assert.ok(repo, "Arg 'repo' must not be empty") assert.ok(patternStrings, "Arg 'patternStrings' must not be null") @@ -94,6 +108,12 @@ async function add(owner, repo, patternStrings, defaultBranch, ignoreTags) { if (ignoreTags && ignoreTags.length > 0) { config.ignoreTags = ignoreTags } + if (latestMajorVersions && latestMajorVersions > 0) { + config.latestMajorVersions = latestMajorVersions + } + if (latestVersionsPerMajor && latestVersionsPerMajor > 0) { + config.latestVersionsPerMajor = latestVersionsPerMajor + } config.defaultBranch = defaultBranch const tempDir = path.join(paths.temp, `${owner}_${repo}`) @@ -130,6 +150,9 @@ async function add(owner, repo, patternStrings, defaultBranch, ignoreTags) { config.tags[tag] = tagVersion } + // Prune old tags based on version limits + pruneOldTags(config) + // Write config await exec.exec('mkdir', ['-p', path.dirname(file)]) await fs.promises.writeFile(file, JSON.stringify(config, null, ' ')) @@ -141,6 +164,75 @@ async function add(owner, repo, patternStrings, defaultBranch, ignoreTags) { } exports.add = add +/** + * Prunes old tags from the config based on latestMajorVersions and latestVersionsPerMajor. + * Modifies config.tags in place. + * @param {ActionConfig} config + */ +function pruneOldTags(config) { + const maxMajors = config.latestMajorVersions || 0 + const maxPerMajor = config.latestVersionsPerMajor || 0 + + if (!maxMajors && !maxPerMajor) { + return + } + + const tagNames = Object.keys(config.tags) + const versionTags = [] + const keepTags = new Set() + + for (const tag of tagNames) { + const match = tag.match(/^v(\d+)(?:\.(\d+))?(?:\.(\d+))?$/) + if (!match) { + // Always keep non-version tags + keepTags.add(tag) + continue + } + + const major = parseInt(match[1], 10) + const minor = match[2] !== undefined ? parseInt(match[2], 10) : -1 + const patch = match[3] !== undefined ? parseInt(match[3], 10) : -1 + versionTags.push({ tag, major, minor, patch, isMajorOnly: minor === -1 }) + } + + // Distinct major versions sorted descending (newest first) + const majorVersions = [...new Set(versionTags.map(v => v.major))].sort((a, b) => b - a) + const allowedMajors = new Set( + maxMajors > 0 ? majorVersions.slice(0, maxMajors) : majorVersions + ) + + for (const major of allowedMajors) { + const tagsForMajor = versionTags.filter(v => v.major === major) + + // Always keep major-only pointers (e.g. "v4") + for (const v of tagsForMajor.filter(v => v.isMajorOnly)) { + keepTags.add(v.tag) + } + + // Sort non-major-only tags by version descending (latest first) + const sorted = tagsForMajor + .filter(v => !v.isMajorOnly) + .sort((a, b) => { + if (a.minor !== b.minor) return b.minor - a.minor + return b.patch - a.patch + }) + + const kept = maxPerMajor > 0 ? sorted.slice(0, maxPerMajor) : sorted + for (const v of kept) { + keepTags.add(v.tag) + } + } + + // Remove pruned tags + for (const tag of tagNames) { + if (!keepTags.has(tag)) { + console.log(`Pruning tag '${tag}' from config (version limit)`) + delete config.tags[tag] + } + } +} +exports.pruneOldTags = pruneOldTags + /** * Returns the action config file path * @param {string} owner diff --git a/script/internal/add-action.js b/script/internal/add-action.js index 024aabe..b250e8a 100644 --- a/script/internal/add-action.js +++ b/script/internal/add-action.js @@ -15,6 +15,8 @@ async function main() { const patterns = args.patterns const defaultBranch = args.defaultBranch || 'master' const ignoreTags = args.ignoreTags + const latestMajorVersions = args.latestMajorVersions + const latestVersionsPerMajor = args.latestVersionsPerMajor // File exists? const file = actionConfig.getFilePath(owner, repo) @@ -24,7 +26,7 @@ async function main() { await fsHelper.reinitTemp() // Add the config - await actionConfig.add(owner, repo, patterns, defaultBranch, ignoreTags) + await actionConfig.add(owner, repo, patterns, defaultBranch, ignoreTags, latestMajorVersions, latestVersionsPerMajor) } catch (err) { // Help @@ -60,7 +62,7 @@ class Args { */ function getArgs() { // Parse - const parsedArgs = argHelper.parse([], ['default-branch', 'ignore-tags']) + const parsedArgs = argHelper.parse([], ['default-branch', 'ignore-tags', 'latest-major-versions', 'latest-versions-per-major']) if (parsedArgs.arguments.length < 1) { argHelper.throwError('Expected at least one arg') } @@ -101,16 +103,29 @@ function getArgs() { repo: splitNwo[1], patterns: patterns, defaultBranch: parsedArgs.options['default-branch'], - ignoreTags: ignoreTags + ignoreTags: ignoreTags, + latestMajorVersions: parseNonNegativeInt(parsedArgs.options['latest-major-versions'], 'latest-major-versions'), + latestVersionsPerMajor: parseNonNegativeInt(parsedArgs.options['latest-versions-per-major'], 'latest-versions-per-major') } } +function parseNonNegativeInt(value, name) { + if (!value) return 0 + const n = Number(value) + if (!Number.isInteger(n) || n < 0) { + argHelper.throwError(`--${name} must be a non-negative integer, got '${value}'`) + } + return n +} + function printUsage() { - console.error('USAGE: add-action.sh [--default-branch branch] [--ignore-tags versions] nwo [(+|-)regexp [...]]') - console.error(` --default-branch Default branch name. For example: master`) - console.error(` --ignore-tags Comma-separated version prefixes to ignore. For example: v1,v2`) - console.error(` nwo Name with owner. For example: actions/checkout`) - console.error(` regexp Refs to include or exclude. Default: ${actionConfig.defaultPatterns.join(' ')}`) + console.error('USAGE: add-action.sh [--default-branch branch] [--ignore-tags versions] [--latest-major-versions N] [--latest-versions-per-major N] nwo [(+|-)regexp [...]]') + console.error(` --default-branch Default branch name. For example: master`) + console.error(` --ignore-tags Comma-separated version prefixes to ignore. For example: v1,v2`) + console.error(` --latest-major-versions Only cache the latest N major versions. For example: 3`) + console.error(` --latest-versions-per-major Only cache the latest N version tags per major version. For example: 5`) + console.error(` nwo Name with owner. For example: actions/checkout`) + console.error(` regexp Refs to include or exclude. Default: ${actionConfig.defaultPatterns.join(' ')}`) } main() \ No newline at end of file diff --git a/script/internal/filter-tags.js b/script/internal/filter-tags.js new file mode 100644 index 0000000..2ad5243 --- /dev/null +++ b/script/internal/filter-tags.js @@ -0,0 +1,87 @@ +// Filters tags from an action config based on latestMajorVersions and latestVersionsPerMajor. +// Reads JSON config from stdin, outputs allowed tag names (one per line). + +async function main() { + let input = '' + for await (const chunk of process.stdin) { + input += chunk + } + + const config = JSON.parse(input) + const tags = Object.keys(config.tags || {}) + const latestMajorVersions = config.latestMajorVersions || 0 // 0 = unlimited + const latestVersionsPerMajor = config.latestVersionsPerMajor || 0 // 0 = unlimited + + if (!latestMajorVersions && !latestVersionsPerMajor) { + // No filtering configured, output all tags + for (const tag of tags) { + console.log(tag) + } + return + } + + // Parse version info from tag names + const versionTags = [] + const nonVersionTags = [] + + for (const tag of tags) { + const match = tag.match(/^v(\d+)(?:\.(\d+))?(?:\.(\d+))?$/) + if (!match) { + nonVersionTags.push(tag) + continue + } + + const major = parseInt(match[1], 10) + const minor = match[2] !== undefined ? parseInt(match[2], 10) : -1 + const patch = match[3] !== undefined ? parseInt(match[3], 10) : -1 + const isMajorOnly = minor === -1 + + versionTags.push({ tag, major, minor, patch, isMajorOnly }) + } + + // Find distinct major versions sorted descending (newest first) + const majorVersions = [...new Set(versionTags.map(v => v.major))].sort((a, b) => b - a) + + // Apply latestMajorVersions filter + const allowedMajors = new Set( + latestMajorVersions > 0 ? majorVersions.slice(0, latestMajorVersions) : majorVersions + ) + + // Always include non-version tags + const result = [...nonVersionTags] + + for (const major of allowedMajors) { + const tagsForMajor = versionTags.filter(v => v.major === major) + + // Always include major-only pointers (e.g., "v4") + for (const v of tagsForMajor.filter(v => v.isMajorOnly)) { + result.push(v.tag) + } + + // Sort non-major-only tags by version descending (latest first) + const sortedVersions = tagsForMajor + .filter(v => !v.isMajorOnly) + .sort((a, b) => { + if (a.minor !== b.minor) return b.minor - a.minor + return b.patch - a.patch + }) + + // Apply latestVersionsPerMajor filter + const kept = latestVersionsPerMajor > 0 + ? sortedVersions.slice(0, latestVersionsPerMajor) + : sortedVersions + + for (const v of kept) { + result.push(v.tag) + } + } + + for (const tag of result) { + console.log(tag) + } +} + +main().catch(err => { + console.error(err.message) + process.exitCode = 1 +}) diff --git a/script/internal/generate-scripts.sh b/script/internal/generate-scripts.sh index 6f3b64a..a771163 100755 --- a/script/internal/generate-scripts.sh +++ b/script/internal/generate-scripts.sh @@ -44,6 +44,15 @@ for json_file in $script_dir/../../config/actions/*.json; do ignore_patterns=() IFS=$'\n' read -r -d '' -a ignore_patterns < <( echo "$json" | jq --raw-output '.ignoreTags // [] | .[]' && printf '\0' ) + # Get version-filtered tags (applies latestMajorVersions and latestVersionsPerMajor) + filtered_tags=() + IFS=$'\n' read -r -d '' -a filtered_tags < <( echo "$json" | node "$script_dir/filter-tags.js" && printf '\0' ) + unset filtered_tag_set + declare -A filtered_tag_set + for t in "${filtered_tags[@]}"; do + filtered_tag_set[$t]=1 + done + # Get an array of tag info. Each item contains " " tag_info=() IFS=$'\n' read -r -d '' -a tag_info < <( echo "$json" | jq --raw-output '.tags | to_entries | .[] | .key + " " + .value.commit' && printf '\0' ) @@ -67,6 +76,12 @@ for json_file in $script_dir/../../config/actions/*.json; do continue fi + # Check if the tag passes version filter + if [ -z "${filtered_tag_set[$tag]+x}" ]; then + echo "Skipping tag '$tag' (filtered by version limits)" + continue + fi + # Append curl download command curl_download_commands+=("curl -s -S -L -o '$sha.tar.gz' 'https://api.github.com/repos/$owner/$repo/tarball/$sha'") curl_download_commands+=("curl -s -S -L -o '$sha.zip' 'https://api.github.com/repos/$owner/$repo/zipball/$sha'") diff --git a/script/internal/update-action.js b/script/internal/update-action.js index b43a328..20dd0b3 100644 --- a/script/internal/update-action.js +++ b/script/internal/update-action.js @@ -23,8 +23,10 @@ async function main() { const patterns = config.patterns const defaultBranch = config.defaultBranch const ignoreTags = config.ignoreTags + const latestMajorVersions = args.latestMajorVersions || config.latestMajorVersions + const latestVersionsPerMajor = args.latestVersionsPerMajor || config.latestVersionsPerMajor assert.ok(patterns && patterns.length, 'Existing patterns must not be empty') - await actionConfig.add(owner, repo, patterns, defaultBranch, ignoreTags) + await actionConfig.add(owner, repo, patterns, defaultBranch, ignoreTags, latestMajorVersions, latestVersionsPerMajor) } } catch (err) { @@ -51,6 +53,8 @@ class Args { all = false owner = '' repo = '' + latestMajorVersions = 0 + latestVersionsPerMajor = 0 } /** @@ -58,9 +62,11 @@ class Args { * @returns {Args} */ function getArgs() { - const parsedArgs = argHelper.parse(['all']) + const parsedArgs = argHelper.parse(['all'], ['latest-major-versions', 'latest-versions-per-major']) const result = new Args() result.all = !!parsedArgs.flags['all'] + result.latestMajorVersions = parseNonNegativeInt(parsedArgs.options['latest-major-versions'], 'latest-major-versions') + result.latestVersionsPerMajor = parseNonNegativeInt(parsedArgs.options['latest-versions-per-major'], 'latest-versions-per-major') // All if (result.all) { @@ -88,9 +94,21 @@ function getArgs() { return result } +function parseNonNegativeInt(value, name) { + if (!value) return 0 + const n = Number(value) + if (!Number.isInteger(n) || n < 0) { + argHelper.throwError(`--${name} must be a non-negative integer, got '${value}'`) + } + return n +} + function printUsage() { - console.error('USAGE: update-action.sh nwo') - console.error(` nwo Name with owner. For example: actions/checkout`) + console.error('USAGE: update-action.sh [--all] [--latest-major-versions N] [--latest-versions-per-major N] [nwo]') + console.error(` --all Update all configured actions`) + console.error(` --latest-major-versions Update to only keep the latest N major versions`) + console.error(` --latest-versions-per-major Update to only keep the latest N version tags per major`) + console.error(` nwo Name with owner. For example: actions/checkout`) } main() \ No newline at end of file diff --git a/script/test.sh b/script/test.sh index da49789..66f9c4e 100755 --- a/script/test.sh +++ b/script/test.sh @@ -46,6 +46,8 @@ function test_tar_gz () echo "Find action.yml under $sha_archive_full_path" elif [[ -f "$first_dir/action.yaml" ]]; then echo "Find action.yaml under $sha_archive_full_path" + elif find "$first_dir" -name 'action.yml' -o -name 'action.yaml' | grep -q .; then + echo "Find action.yml in subdirectory under $sha_archive_full_path" else echo "$sha_archive_full_path doesn't contain an action.yml or action.yaml" exit 1 @@ -83,6 +85,8 @@ function test_zip () echo "Find action.yml under $sha_archive_full_path" elif [[ -f "$first_dir/action.yaml" ]]; then echo "Find action.yaml under $sha_archive_full_path" + elif find "$first_dir" -name 'action.yml' -o -name 'action.yaml' | grep -q .; then + echo "Find action.yml in subdirectory under $sha_archive_full_path" else echo "$sha_archive_full_path doesn't contain an action.yml or action.yaml" exit 1