bypass rate limit

2021-12-22 09:59:18 +00:00
parent e6a9d9c802
commit 508f6970a5
4 changed files with 58 additions and 31 deletions
@@ -27,7 +27,7 @@ exports.deleteVersions = exports.finalIds = exports.getVersionIds = void 0;
 const rxjs_1 = __nccwpck_require__(5805);
 const version_1 = __nccwpck_require__(4428);
 const operators_1 = __nccwpck_require__(7801);
-const RATE_LIMIT = 99;
+//const RATE_LIMIT = 99
 let totalCount;
 function getVersionIds(owner, repo, packageName, numVersions, cursor, token) {
    return version_1.getOldestVersions(owner, repo, packageName, numVersions, cursor, token).pipe(operators_1.expand(value => value.paginate
@@ -41,40 +41,44 @@ function finalIds(input) {
    }
    if (input.hasOldestVersionQueryInfo()) {
        if (input.minVersionsToKeep < 0) {
+            /*
            input.numOldVersionsToDelete =
-                input.numOldVersionsToDelete < RATE_LIMIT
-                    ? input.numOldVersionsToDelete
-                    : RATE_LIMIT;
+              input.numOldVersionsToDelete < RATE_LIMIT
+                ? input.numOldVersionsToDelete
+                : RATE_LIMIT*/
+            console.log(`input.numOldVersionsToDelete: ${input.numOldVersionsToDelete}`);
            return getVersionIds(input.owner, input.repo, input.packageName, input.numOldVersionsToDelete, '', input.token).pipe(operators_1.map(value => {
+                value = value.filter(info => !input.ignoreVersions.test(info.version));
                const temp = input.numOldVersionsToDelete;
                input.numOldVersionsToDelete =
                    input.numOldVersionsToDelete - value.length <= 0
                        ? 0
                        : input.numOldVersionsToDelete - value.length;
-                input.numDeleted += value.filter(info => !input.ignoreVersions.test(info.version)).length;
-                return value
-                    .filter(info => !input.ignoreVersions.test(info.version))
-                    .map(info => info.id)
-                    .slice(0, temp);
+                console.log(`temp: ${temp}, numOldeVersions: ${input.numOldVersionsToDelete}`);
+                return value.map(info => info.id).slice(0, temp);
            }));
        }
        else {
-            return getVersionIds(input.owner, input.repo, input.packageName, RATE_LIMIT, '', input.token).pipe(operators_1.map(value => {
+            return getVersionIds(input.owner, input.repo, input.packageName, 100, '', input.token).pipe(operators_1.map(value => {
                totalCount =
                    totalCount -
                        value.filter(info => input.ignoreVersions.test(info.version)).length;
                value = value.filter(info => !input.ignoreVersions.test(info.version));
                let toDelete = totalCount - input.minVersionsToKeep - input.numDeleted;
                toDelete = toDelete > value.length ? value.length : toDelete;
+                /*
                if (toDelete > 0 && input.numDeleted < RATE_LIMIT) {
-                    // using input.numDeleted to keep track of deleted and remaining packages
-                    if (input.numDeleted + toDelete > 99) {
-                        toDelete = RATE_LIMIT - input.numDeleted;
-                        input.numDeleted = RATE_LIMIT;
-                    }
-                    else {
-                        input.numDeleted = input.numDeleted + toDelete;
-                    }
+                  // using input.numDeleted to keep track of deleted and remaining packages
+                  if (input.numDeleted + toDelete > 99) {
+                    toDelete = RATE_LIMIT - input.numDeleted
+                    input.numDeleted = RATE_LIMIT
+                  } else {
+                    input.numDeleted = input.numDeleted + toDelete
+                  }
+                  return value.map(info => info.id).slice(0, toDelete)
+                } else return []*/
+                if (toDelete > 0) {
+                    input.numDeleted += toDelete;
                    return value.map(info => info.id).slice(0, toDelete);
                }
                else
@@ -177,6 +181,7 @@ exports.deletePackageVersions = exports.deletePackageVersion = void 0;
 const rxjs_1 = __nccwpck_require__(5805);
 const operators_1 = __nccwpck_require__(7801);
 const graphql_1 = __nccwpck_require__(6320);
+let deleted = 0;
 const mutation = `
  mutation deletePackageVersion($packageVersionId: String!) {
      deletePackageVersion(input: {packageVersionId: $packageVersionId}) {
@@ -184,6 +189,11 @@ const mutation = `
      }
  }`;
 function deletePackageVersion(packageVersionId, token) {
+    if (deleted === 99) {
+        console.log(`reaching rate limit`);
+        operators_1.delay(5000);
+    }
+    deleted += 1;
    return rxjs_1.from(graphql_1.graphql(token, mutation, {
        packageVersionId,
        headers: {
@@ -202,6 +212,7 @@ function deletePackageVersions(packageVersionIds, token) {
        return rxjs_1.of(true);
    }
    const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe(operators_1.tap(result => {
+        console.log(`versions Deleted 0 : ${deleted}`);
        if (result) {
            console.log(`version with id: ${id}, deleted`);
        }
@@ -282,7 +293,7 @@ function queryForOldestVersions(owner, repo, packageName, numVersions, startCurs
            owner,
            repo,
            package: packageName,
-            last: numVersions,
+            last: numVersions > 100 ? 100 : numVersions,
            headers: {
                Accept: 'application/vnd.github.packages-preview+json'
            }
@@ -3,7 +3,7 @@ import {EMPTY, Observable, of, throwError} from 'rxjs'
 import {deletePackageVersions, getOldestVersions, VersionInfo} from './version'
 import {concatMap, map, expand, tap} from 'rxjs/operators'

-const RATE_LIMIT = 99
+//const RATE_LIMIT = 99
 let totalCount: number

 export function getVersionIds(
@@ -45,10 +45,14 @@ export function finalIds(input: Input): Observable<string[]> {
  }
  if (input.hasOldestVersionQueryInfo()) {
    if (input.minVersionsToKeep < 0) {
+      /*
      input.numOldVersionsToDelete =
        input.numOldVersionsToDelete < RATE_LIMIT
          ? input.numOldVersionsToDelete
-          : RATE_LIMIT
+          : RATE_LIMIT*/
+      console.log(
+        `input.numOldVersionsToDelete: ${input.numOldVersionsToDelete}`
+      )
      return getVersionIds(
        input.owner,
        input.repo,
@@ -58,18 +62,16 @@ export function finalIds(input: Input): Observable<string[]> {
        input.token
      ).pipe(
        map(value => {
+          value = value.filter(info => !input.ignoreVersions.test(info.version))
          const temp = input.numOldVersionsToDelete
          input.numOldVersionsToDelete =
            input.numOldVersionsToDelete - value.length <= 0
              ? 0
              : input.numOldVersionsToDelete - value.length
-          input.numDeleted += value.filter(
-            info => !input.ignoreVersions.test(info.version)
-          ).length
-          return value
-            .filter(info => !input.ignoreVersions.test(info.version))
-            .map(info => info.id)
-            .slice(0, temp)
+          console.log(
+            `temp: ${temp}, numOldeVersions: ${input.numOldVersionsToDelete}`
+          )
+          return value.map(info => info.id).slice(0, temp)
        })
      )
    } else {
@@ -77,7 +79,7 @@ export function finalIds(input: Input): Observable<string[]> {
        input.owner,
        input.repo,
        input.packageName,
-        RATE_LIMIT,
+        100,
        '',
        input.token
      ).pipe(
@@ -88,6 +90,7 @@ export function finalIds(input: Input): Observable<string[]> {
          value = value.filter(info => !input.ignoreVersions.test(info.version))
          let toDelete = totalCount - input.minVersionsToKeep - input.numDeleted
          toDelete = toDelete > value.length ? value.length : toDelete
+          /*
          if (toDelete > 0 && input.numDeleted < RATE_LIMIT) {
            // using input.numDeleted to keep track of deleted and remaining packages
            if (input.numDeleted + toDelete > 99) {
@@ -97,6 +100,11 @@ export function finalIds(input: Input): Observable<string[]> {
              input.numDeleted = input.numDeleted + toDelete
            }
            return value.map(info => info.id).slice(0, toDelete)
+          } else return []*/
+
+          if (toDelete > 0) {
+            input.numDeleted += toDelete
+            return value.map(info => info.id).slice(0, toDelete)
          } else return []
        })
      )
@@ -1,8 +1,10 @@
 import {from, Observable, merge, throwError, of} from 'rxjs'
-import {catchError, map, tap} from 'rxjs/operators'
+import {catchError, delay, map, tap} from 'rxjs/operators'
 import {GraphQlQueryResponse} from '@octokit/graphql/dist-types/types'
 import {graphql} from './graphql'

+let deleted = 0
+
 export interface DeletePackageVersionMutationResponse {
  deletePackageVersion: {
    success: boolean
@@ -20,6 +22,11 @@ export function deletePackageVersion(
  packageVersionId: string,
  token: string
 ): Observable<boolean> {
+  if (deleted === 99) {
+    console.log(`reaching rate limit`)
+    delay(5000)
+  }
+  deleted += 1
  return from(
    graphql(token, mutation, {
      packageVersionId,
@@ -51,6 +58,7 @@ export function deletePackageVersions(
  const deletes = packageVersionIds.map(id =>
    deletePackageVersion(id, token).pipe(
      tap(result => {
+        console.log(`versions Deleted 0 : ${deleted}`)
        if (result) {
          console.log(`version with id: ${id}, deleted`)
        } else {
@@ -101,7 +101,7 @@ export function queryForOldestVersions(
        owner,
        repo,
        package: packageName,
-        last: numVersions,
+        last: numVersions > 100 ? 100 : numVersions,
        headers: {
          Accept: 'application/vnd.github.packages-preview+json'
        }