From 508f6970a5f85097117972847749c79e0a276b26 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Wed, 22 Dec 2021 09:59:18 +0000 Subject: [PATCH] bypass rate limit --- dist/index.js | 49 +++++++++++++++++++++-------------- src/delete.ts | 28 +++++++++++++------- src/version/delete-version.ts | 10 ++++++- src/version/get-versions.ts | 2 +- 4 files changed, 58 insertions(+), 31 deletions(-) diff --git a/dist/index.js b/dist/index.js index 1eb34df..52ec2ab 100644 --- a/dist/index.js +++ b/dist/index.js @@ -27,7 +27,7 @@ exports.deleteVersions = exports.finalIds = exports.getVersionIds = void 0; const rxjs_1 = __nccwpck_require__(5805); const version_1 = __nccwpck_require__(4428); const operators_1 = __nccwpck_require__(7801); -const RATE_LIMIT = 99; +//const RATE_LIMIT = 99 let totalCount; function getVersionIds(owner, repo, packageName, numVersions, cursor, token) { return version_1.getOldestVersions(owner, repo, packageName, numVersions, cursor, token).pipe(operators_1.expand(value => value.paginate @@ -41,40 +41,44 @@ function finalIds(input) { } if (input.hasOldestVersionQueryInfo()) { if (input.minVersionsToKeep < 0) { + /* input.numOldVersionsToDelete = - input.numOldVersionsToDelete < RATE_LIMIT - ? input.numOldVersionsToDelete - : RATE_LIMIT; + input.numOldVersionsToDelete < RATE_LIMIT + ? input.numOldVersionsToDelete + : RATE_LIMIT*/ + console.log(`input.numOldVersionsToDelete: ${input.numOldVersionsToDelete}`); return getVersionIds(input.owner, input.repo, input.packageName, input.numOldVersionsToDelete, '', input.token).pipe(operators_1.map(value => { + value = value.filter(info => !input.ignoreVersions.test(info.version)); const temp = input.numOldVersionsToDelete; input.numOldVersionsToDelete = input.numOldVersionsToDelete - value.length <= 0 ? 0 : input.numOldVersionsToDelete - value.length; - input.numDeleted += value.filter(info => !input.ignoreVersions.test(info.version)).length; - return value - .filter(info => !input.ignoreVersions.test(info.version)) - .map(info => info.id) - .slice(0, temp); + console.log(`temp: ${temp}, numOldeVersions: ${input.numOldVersionsToDelete}`); + return value.map(info => info.id).slice(0, temp); })); } else { - return getVersionIds(input.owner, input.repo, input.packageName, RATE_LIMIT, '', input.token).pipe(operators_1.map(value => { + return getVersionIds(input.owner, input.repo, input.packageName, 100, '', input.token).pipe(operators_1.map(value => { totalCount = totalCount - value.filter(info => input.ignoreVersions.test(info.version)).length; value = value.filter(info => !input.ignoreVersions.test(info.version)); let toDelete = totalCount - input.minVersionsToKeep - input.numDeleted; toDelete = toDelete > value.length ? value.length : toDelete; + /* if (toDelete > 0 && input.numDeleted < RATE_LIMIT) { - // using input.numDeleted to keep track of deleted and remaining packages - if (input.numDeleted + toDelete > 99) { - toDelete = RATE_LIMIT - input.numDeleted; - input.numDeleted = RATE_LIMIT; - } - else { - input.numDeleted = input.numDeleted + toDelete; - } + // using input.numDeleted to keep track of deleted and remaining packages + if (input.numDeleted + toDelete > 99) { + toDelete = RATE_LIMIT - input.numDeleted + input.numDeleted = RATE_LIMIT + } else { + input.numDeleted = input.numDeleted + toDelete + } + return value.map(info => info.id).slice(0, toDelete) + } else return []*/ + if (toDelete > 0) { + input.numDeleted += toDelete; return value.map(info => info.id).slice(0, toDelete); } else @@ -177,6 +181,7 @@ exports.deletePackageVersions = exports.deletePackageVersion = void 0; const rxjs_1 = __nccwpck_require__(5805); const operators_1 = __nccwpck_require__(7801); const graphql_1 = __nccwpck_require__(6320); +let deleted = 0; const mutation = ` mutation deletePackageVersion($packageVersionId: String!) { deletePackageVersion(input: {packageVersionId: $packageVersionId}) { @@ -184,6 +189,11 @@ const mutation = ` } }`; function deletePackageVersion(packageVersionId, token) { + if (deleted === 99) { + console.log(`reaching rate limit`); + operators_1.delay(5000); + } + deleted += 1; return rxjs_1.from(graphql_1.graphql(token, mutation, { packageVersionId, headers: { @@ -202,6 +212,7 @@ function deletePackageVersions(packageVersionIds, token) { return rxjs_1.of(true); } const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe(operators_1.tap(result => { + console.log(`versions Deleted 0 : ${deleted}`); if (result) { console.log(`version with id: ${id}, deleted`); } @@ -282,7 +293,7 @@ function queryForOldestVersions(owner, repo, packageName, numVersions, startCurs owner, repo, package: packageName, - last: numVersions, + last: numVersions > 100 ? 100 : numVersions, headers: { Accept: 'application/vnd.github.packages-preview+json' } diff --git a/src/delete.ts b/src/delete.ts index 330e2b6..0466fbf 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -3,7 +3,7 @@ import {EMPTY, Observable, of, throwError} from 'rxjs' import {deletePackageVersions, getOldestVersions, VersionInfo} from './version' import {concatMap, map, expand, tap} from 'rxjs/operators' -const RATE_LIMIT = 99 +//const RATE_LIMIT = 99 let totalCount: number export function getVersionIds( @@ -45,10 +45,14 @@ export function finalIds(input: Input): Observable { } if (input.hasOldestVersionQueryInfo()) { if (input.minVersionsToKeep < 0) { + /* input.numOldVersionsToDelete = input.numOldVersionsToDelete < RATE_LIMIT ? input.numOldVersionsToDelete - : RATE_LIMIT + : RATE_LIMIT*/ + console.log( + `input.numOldVersionsToDelete: ${input.numOldVersionsToDelete}` + ) return getVersionIds( input.owner, input.repo, @@ -58,18 +62,16 @@ export function finalIds(input: Input): Observable { input.token ).pipe( map(value => { + value = value.filter(info => !input.ignoreVersions.test(info.version)) const temp = input.numOldVersionsToDelete input.numOldVersionsToDelete = input.numOldVersionsToDelete - value.length <= 0 ? 0 : input.numOldVersionsToDelete - value.length - input.numDeleted += value.filter( - info => !input.ignoreVersions.test(info.version) - ).length - return value - .filter(info => !input.ignoreVersions.test(info.version)) - .map(info => info.id) - .slice(0, temp) + console.log( + `temp: ${temp}, numOldeVersions: ${input.numOldVersionsToDelete}` + ) + return value.map(info => info.id).slice(0, temp) }) ) } else { @@ -77,7 +79,7 @@ export function finalIds(input: Input): Observable { input.owner, input.repo, input.packageName, - RATE_LIMIT, + 100, '', input.token ).pipe( @@ -88,6 +90,7 @@ export function finalIds(input: Input): Observable { value = value.filter(info => !input.ignoreVersions.test(info.version)) let toDelete = totalCount - input.minVersionsToKeep - input.numDeleted toDelete = toDelete > value.length ? value.length : toDelete + /* if (toDelete > 0 && input.numDeleted < RATE_LIMIT) { // using input.numDeleted to keep track of deleted and remaining packages if (input.numDeleted + toDelete > 99) { @@ -97,6 +100,11 @@ export function finalIds(input: Input): Observable { input.numDeleted = input.numDeleted + toDelete } return value.map(info => info.id).slice(0, toDelete) + } else return []*/ + + if (toDelete > 0) { + input.numDeleted += toDelete + return value.map(info => info.id).slice(0, toDelete) } else return [] }) ) diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index 40a7269..1e6c079 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -1,8 +1,10 @@ import {from, Observable, merge, throwError, of} from 'rxjs' -import {catchError, map, tap} from 'rxjs/operators' +import {catchError, delay, map, tap} from 'rxjs/operators' import {GraphQlQueryResponse} from '@octokit/graphql/dist-types/types' import {graphql} from './graphql' +let deleted = 0 + export interface DeletePackageVersionMutationResponse { deletePackageVersion: { success: boolean @@ -20,6 +22,11 @@ export function deletePackageVersion( packageVersionId: string, token: string ): Observable { + if (deleted === 99) { + console.log(`reaching rate limit`) + delay(5000) + } + deleted += 1 return from( graphql(token, mutation, { packageVersionId, @@ -51,6 +58,7 @@ export function deletePackageVersions( const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe( tap(result => { + console.log(`versions Deleted 0 : ${deleted}`) if (result) { console.log(`version with id: ${id}, deleted`) } else { diff --git a/src/version/get-versions.ts b/src/version/get-versions.ts index dad6909..5d91e6d 100644 --- a/src/version/get-versions.ts +++ b/src/version/get-versions.ts @@ -101,7 +101,7 @@ export function queryForOldestVersions( owner, repo, package: packageName, - last: numVersions, + last: numVersions > 100 ? 100 : numVersions, headers: { Accept: 'application/vnd.github.packages-preview+json' }