batch processing w/ exponential backoff (#79)

Signed-off-by: Brian DeHamer <bdehamer@github.com>

src/index.ts

@@ -5,7 +5,6 @@ import * as core from '@actions/core'
 import { run, RunInputs } from './main'
 
 const DEFAULT_BATCH_SIZE = 50
-const DEFAULT_BATCH_DELAY = 5000
 
 const inputs: RunInputs = {
   subjectPath: core.getInput('subject-path'),
@@ -21,8 +20,7 @@ const inputs: RunInputs = {
     core.getInput('private-signing')
   ),
   // internal only
-  batchSize: DEFAULT_BATCH_SIZE,
-  batchDelay: DEFAULT_BATCH_DELAY
+  batchSize: DEFAULT_BATCH_SIZE
 }
 
 // eslint-disable-next-line @typescript-eslint/no-floating-promises

src/main.ts

@@ -10,6 +10,8 @@ import * as style from './style'
 import { SubjectInputs, subjectFromInputs } from './subject'
 
 const ATTESTATION_FILE_NAME = 'attestation.jsonl'
+const DELAY_INTERVAL_MS = 75
+const DELAY_MAX_MS = 1200
 
 export type RunInputs = SubjectInputs &
   PredicateInputs & {
@@ -17,7 +19,6 @@ export type RunInputs = SubjectInputs &
     githubToken: string
     privateSigning: boolean
     batchSize: number
-    batchDelay: number
   }
 
 /* istanbul ignore next */
@@ -62,22 +63,22 @@ export async function run(inputs: RunInputs): Promise<void> {
     core.setOutput('bundle-path', outputPath)
 
     const subjectChunks = chunkArray(subjects, inputs.batchSize)
-    let chunkCount = 0
 
     // Generate attestations for each subject serially, working in batches
-    for (const subjectChunk of subjectChunks) {
-      // Delay between batches (only when chunkCount > 0)
-      if (chunkCount++) {
-        await new Promise(resolve => setTimeout(resolve, inputs.batchDelay))
-      }
-
+    for (let i = 0; i < subjectChunks.length; i++) {
       if (subjectChunks.length > 1) {
-        core.info(
-          `Processing subject batch ${chunkCount}/${subjectChunks.length}`
-        )
+        core.info(`Processing subject batch ${i + 1}/${subjectChunks.length}`)
       }
 
-      for (const subject of subjectChunk) {
+      // Calculate the delay time for this batch
+      const delayTime = delay(i)
+
+      for (const subject of subjectChunks[i]) {
+        // Delay between attestations (only when chunk size > 1)
+        if (i > 0) {
+          await new Promise(resolve => setTimeout(resolve, delayTime))
+        }
+
         const att = await createAttestation(subject, predicate, {
           sigstoreInstance,
           pushToRegistry: inputs.pushToRegistry,
@@ -197,5 +198,9 @@ const chunkArray = <T>(array: T[], chunkSize: number): T[][] => {
   )
 }
 
+// Calculate the delay time for a given iteration
+const delay = (iteration: number): number =>
+  Math.min(DELAY_INTERVAL_MS * 2 ** iteration, DELAY_MAX_MS)
+
 const attestationURL = (id: string): string =>
   `${github.context.serverUrl}/${github.context.repo.owner}/${github.context.repo.repo}/attestations/${id}`

src/subject.ts

@@ -6,6 +6,7 @@ import path from 'path'
 
 import type { Subject } from '@actions/attest'
 
+const MAX_SUBJECT_COUNT = 2500
 const DIGEST_ALGORITHM = 'sha256'
 
 export type SubjectInputs = {
@@ -54,34 +55,41 @@ const getSubjectFromPath = async (
   subjectPath: string,
   subjectName?: string
 ): Promise<Subject[]> => {
-  const subjects: Subject[] = []
+  const digestedSubjects: Subject[] = []
+  const files: string[] = []
 
   // Parse the list of subject paths
   const subjectPaths = parseList(subjectPath)
 
+  // Expand the globbed paths to a list of files
   for (const subPath of subjectPaths) {
-    // Expand the globbed path to a list of files
     /* eslint-disable-next-line github/no-then */
-    const files = await glob.create(subPath).then(async g => g.glob())
-
-    for (const file of files) {
-      // Skip anything that is NOT a file
-      if (!fs.statSync(file).isFile()) {
-        continue
-      }
-
-      const name = subjectName || path.parse(file).base
-      const digest = await digestFile(DIGEST_ALGORITHM, file)
-
-      subjects.push({ name, digest: { [DIGEST_ALGORITHM]: digest } })
-    }
+    files.push(...(await glob.create(subPath).then(async g => g.glob())))
   }
 
-  if (subjects.length === 0) {
+  if (files.length > MAX_SUBJECT_COUNT) {
+    throw new Error(
+      `Too many subjects specified. The maximum number of subjects is ${MAX_SUBJECT_COUNT}.`
+    )
+  }
+
+  for (const file of files) {
+    // Skip anything that is NOT a file
+    if (!fs.statSync(file).isFile()) {
+      continue
+    }
+
+    const name = subjectName || path.parse(file).base
+    const digest = await digestFile(DIGEST_ALGORITHM, file)
+
+    digestedSubjects.push({ name, digest: { [DIGEST_ALGORITHM]: digest } })
+  }
+
+  if (digestedSubjects.length === 0) {
     throw new Error(`Could not find subject at path ${subjectPath}`)
   }
 
-  return Promise.all(subjects)
+  return digestedSubjects
 }
 
 // Returns the subject specified by the digest of a file. The digest is returned