Merge pull request #2165 from austenstone/max-list-artifact-2k

fix: artifact pagination bugs and configurable artifact count limits

packages/artifact/__tests__/config.test.ts

@@ -105,3 +105,45 @@ describe('uploadConcurrencyEnv', () => {
     }).toThrow()
   })
 })
+
+describe('getMaxArtifactListCount', () => {
+  beforeEach(() => {
+    delete process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT
+  })
+
+  it('should return default 1000 when no env set', () => {
+    expect(config.getMaxArtifactListCount()).toBe(1000)
+  })
+
+  it('should return value set in ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT', () => {
+    process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT = '2000'
+    expect(config.getMaxArtifactListCount()).toBe(2000)
+  })
+
+  it('should throw if value set in ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT is invalid', () => {
+    process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT = 'abc'
+    expect(() => {
+      config.getMaxArtifactListCount()
+    }).toThrow(
+      'Invalid value set for ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT env variable'
+    )
+  })
+
+  it('should throw if ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT is < 1', () => {
+    process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT = '0'
+    expect(() => {
+      config.getMaxArtifactListCount()
+    }).toThrow(
+      'Invalid value set for ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT env variable'
+    )
+  })
+
+  it('should throw if ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT is negative', () => {
+    process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT = '-100'
+    expect(() => {
+      config.getMaxArtifactListCount()
+    }).toThrow(
+      'Invalid value set for ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT env variable'
+    )
+  })
+})

packages/artifact/__tests__/list-artifacts.test.ts

@@ -170,6 +170,126 @@ describe('list-artifact', () => {
         )
       ).rejects.toThrow('boom')
     })
+
+    it('should handle pagination correctly when fetching multiple pages', async () => {
+      const mockRequest = github.getOctokit(fixtures.token)
+        .request as MockedRequest
+
+      const manyArtifacts = Array.from({length: 150}, (_, i) => ({
+        id: i + 1,
+        name: `artifact-${i + 1}`,
+        size: 100,
+        createdAt: new Date('2023-12-01')
+      }))
+
+      mockRequest
+        .mockResolvedValueOnce({
+          status: 200,
+          headers: {},
+          url: '',
+          data: {
+            ...artifactsToListResponse(manyArtifacts.slice(0, 100)),
+            total_count: 150
+          }
+        })
+        .mockResolvedValueOnce({
+          status: 200,
+          headers: {},
+          url: '',
+          data: {
+            ...artifactsToListResponse(manyArtifacts.slice(100, 150)),
+            total_count: 150
+          }
+        })
+
+      const response = await listArtifactsPublic(
+        fixtures.runId,
+        fixtures.owner,
+        fixtures.repo,
+        fixtures.token,
+        false
+      )
+
+      // Verify that both API calls were made
+      expect(mockRequest).toHaveBeenCalledTimes(2)
+
+      // Should return all 150 artifacts across both pages
+      expect(response.artifacts).toHaveLength(150)
+
+      // Verify we got artifacts from both pages
+      expect(response.artifacts[0].name).toBe('artifact-1')
+      expect(response.artifacts[99].name).toBe('artifact-100')
+      expect(response.artifacts[100].name).toBe('artifact-101')
+      expect(response.artifacts[149].name).toBe('artifact-150')
+    })
+
+    it('should respect ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT environment variable', async () => {
+      const originalEnv = process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT
+      process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT = '150'
+
+      jest.resetModules()
+
+      try {
+        const {listArtifactsPublic: listArtifactsPublicReloaded} = await import(
+          '../src/internal/find/list-artifacts'
+        )
+        const githubReloaded = await import('@actions/github')
+
+        const mockRequest = (githubReloaded.getOctokit as jest.Mock)(
+          fixtures.token
+        ).request as MockedRequest
+
+        const manyArtifacts = Array.from({length: 200}, (_, i) => ({
+          id: i + 1,
+          name: `artifact-${i + 1}`,
+          size: 100,
+          createdAt: new Date('2023-12-01')
+        }))
+
+        mockRequest
+          .mockResolvedValueOnce({
+            status: 200,
+            headers: {},
+            url: '',
+            data: {
+              ...artifactsToListResponse(manyArtifacts.slice(0, 100)),
+              total_count: 200
+            }
+          })
+          .mockResolvedValueOnce({
+            status: 200,
+            headers: {},
+            url: '',
+            data: {
+              ...artifactsToListResponse(manyArtifacts.slice(100, 150)),
+              total_count: 200
+            }
+          })
+
+        const response = await listArtifactsPublicReloaded(
+          fixtures.runId,
+          fixtures.owner,
+          fixtures.repo,
+          fixtures.token,
+          false
+        )
+
+        // Should only return 150 artifacts due to the limit
+        expect(response.artifacts).toHaveLength(150)
+        expect(response.artifacts[0].name).toBe('artifact-1')
+        expect(response.artifacts[149].name).toBe('artifact-150')
+      } finally {
+        // Restore original environment variable
+        if (originalEnv !== undefined) {
+          process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT = originalEnv
+        } else {
+          delete process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT
+        }
+
+        // Reset modules again to restore original state
+        jest.resetModules()
+      }
+    })
   })
 
   describe('internal', () => {

packages/artifact/src/internal/find/list-artifacts.ts

@@ -9,12 +9,12 @@ import {retry} from '@octokit/plugin-retry'
 import {OctokitOptions} from '@octokit/core/dist-types/types'
 import {internalArtifactTwirpClient} from '../shared/artifact-twirp-client'
 import {getBackendIdsFromToken} from '../shared/util'
+import {getMaxArtifactListCount} from '../shared/config'
 import {ListArtifactsRequest, Timestamp} from '../../generated'
 
-// Limiting to 1000 for perf reasons
-const maximumArtifactCount = 1000
+const maximumArtifactCount = getMaxArtifactListCount()
 const paginationCount = 100
-const maxNumberOfPages = maximumArtifactCount / paginationCount
+const maxNumberOfPages = Math.ceil(maximumArtifactCount / paginationCount)
 
 export async function listArtifactsPublic(
   workflowRunId: number,
@@ -59,7 +59,7 @@ export async function listArtifactsPublic(
   const totalArtifactCount = listArtifactResponse.total_count
   if (totalArtifactCount > maximumArtifactCount) {
     warning(
-      `Workflow run ${workflowRunId} has more than 1000 artifacts. Results will be incomplete as only the first ${maximumArtifactCount} artifacts will be returned`
+      `Workflow run ${workflowRunId} has ${totalArtifactCount} artifacts, exceeding the limit of ${maximumArtifactCount}. Results will be incomplete as only the first ${maximumArtifactCount} artifacts will be returned`
     )
     numberOfPages = maxNumberOfPages
   }
@@ -81,7 +81,7 @@ export async function listArtifactsPublic(
   // Iterate over any remaining pages
   for (
     currentPageNumber;
-    currentPageNumber < numberOfPages;
+    currentPageNumber <= numberOfPages;
     currentPageNumber++
   ) {
     debug(`Fetching page ${currentPageNumber} of artifact list`)

packages/artifact/src/internal/shared/config.ts

@@ -97,3 +97,19 @@ export function getUploadChunkTimeout(): number {
 
   return timeout
 }
+
+// This value can be changed with ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT variable.
+// Defaults to 1000 as a safeguard for rate limiting.
+export function getMaxArtifactListCount(): number {
+  const maxCountVar =
+    process.env['ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT'] || '1000'
+
+  const maxCount = parseInt(maxCountVar)
+  if (isNaN(maxCount) || maxCount < 1) {
+    throw new Error(
+      'Invalid value set for ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT env variable'
+    )
+  }
+
+  return maxCount
+}