fix: resolve critical pagination bugs and add comprehensive testing
- Fix off-by-one error in pagination loop (< to <=) that prevented fetching last page - Add Math.ceil() to maxNumberOfPages calculation for proper limit handling - Replace hardcoded 2000 limit with configurable getMaxArtifactListCount() - Add pagination test for multi-page artifact listing - Add environment variable test for ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT - Add comprehensive test coverage for getMaxArtifactListCount() function Fixes compound bug where pagination and limit logic capped results at 900 artifacts instead of intended 1000.
This commit is contained in:
@@ -105,3 +105,39 @@ describe('uploadConcurrencyEnv', () => {
|
|||||||
}).toThrow()
|
}).toThrow()
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('getMaxArtifactListCount', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
delete process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return default 1000 when no env set', () => {
|
||||||
|
expect(config.getMaxArtifactListCount()).toBe(1000)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return value set in ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT', () => {
|
||||||
|
process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT = '2000'
|
||||||
|
expect(config.getMaxArtifactListCount()).toBe(2000)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should throw if value set in ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT is invalid', () => {
|
||||||
|
process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT = 'abc'
|
||||||
|
expect(() => {
|
||||||
|
config.getMaxArtifactListCount()
|
||||||
|
}).toThrow('Invalid value set for ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT env variable')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should throw if ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT is < 1', () => {
|
||||||
|
process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT = '0'
|
||||||
|
expect(() => {
|
||||||
|
config.getMaxArtifactListCount()
|
||||||
|
}).toThrow('Invalid value set for ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT env variable')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should throw if ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT is negative', () => {
|
||||||
|
process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT = '-100'
|
||||||
|
expect(() => {
|
||||||
|
config.getMaxArtifactListCount()
|
||||||
|
}).toThrow('Invalid value set for ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT env variable')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|||||||
@@ -170,6 +170,126 @@ describe('list-artifact', () => {
|
|||||||
)
|
)
|
||||||
).rejects.toThrow('boom')
|
).rejects.toThrow('boom')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('should handle pagination correctly when fetching multiple pages', async () => {
|
||||||
|
const mockRequest = github.getOctokit(fixtures.token)
|
||||||
|
.request as MockedRequest
|
||||||
|
|
||||||
|
const manyArtifacts = Array.from({length: 150}, (_, i) => ({
|
||||||
|
id: i + 1,
|
||||||
|
name: `artifact-${i + 1}`,
|
||||||
|
size: 100,
|
||||||
|
createdAt: new Date('2023-12-01')
|
||||||
|
}))
|
||||||
|
|
||||||
|
mockRequest
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
status: 200,
|
||||||
|
headers: {},
|
||||||
|
url: '',
|
||||||
|
data: {
|
||||||
|
...artifactsToListResponse(manyArtifacts.slice(0, 100)),
|
||||||
|
total_count: 150
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
status: 200,
|
||||||
|
headers: {},
|
||||||
|
url: '',
|
||||||
|
data: {
|
||||||
|
...artifactsToListResponse(manyArtifacts.slice(100, 150)),
|
||||||
|
total_count: 150
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
const response = await listArtifactsPublic(
|
||||||
|
fixtures.runId,
|
||||||
|
fixtures.owner,
|
||||||
|
fixtures.repo,
|
||||||
|
fixtures.token,
|
||||||
|
false
|
||||||
|
)
|
||||||
|
|
||||||
|
// Verify that both API calls were made
|
||||||
|
expect(mockRequest).toHaveBeenCalledTimes(2)
|
||||||
|
|
||||||
|
// Should return all 150 artifacts across both pages
|
||||||
|
expect(response.artifacts).toHaveLength(150)
|
||||||
|
|
||||||
|
// Verify we got artifacts from both pages
|
||||||
|
expect(response.artifacts[0].name).toBe('artifact-1')
|
||||||
|
expect(response.artifacts[99].name).toBe('artifact-100')
|
||||||
|
expect(response.artifacts[100].name).toBe('artifact-101')
|
||||||
|
expect(response.artifacts[149].name).toBe('artifact-150')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should respect ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT environment variable', async () => {
|
||||||
|
const originalEnv = process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT
|
||||||
|
process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT = '150'
|
||||||
|
|
||||||
|
jest.resetModules()
|
||||||
|
|
||||||
|
try {
|
||||||
|
const {listArtifactsPublic: listArtifactsPublicReloaded} = await import(
|
||||||
|
'../src/internal/find/list-artifacts'
|
||||||
|
)
|
||||||
|
const githubReloaded = await import('@actions/github')
|
||||||
|
|
||||||
|
const mockRequest = (githubReloaded.getOctokit as jest.Mock)(
|
||||||
|
fixtures.token
|
||||||
|
).request as MockedRequest
|
||||||
|
|
||||||
|
const manyArtifacts = Array.from({length: 200}, (_, i) => ({
|
||||||
|
id: i + 1,
|
||||||
|
name: `artifact-${i + 1}`,
|
||||||
|
size: 100,
|
||||||
|
createdAt: new Date('2023-12-01')
|
||||||
|
}))
|
||||||
|
|
||||||
|
mockRequest
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
status: 200,
|
||||||
|
headers: {},
|
||||||
|
url: '',
|
||||||
|
data: {
|
||||||
|
...artifactsToListResponse(manyArtifacts.slice(0, 100)),
|
||||||
|
total_count: 200
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
status: 200,
|
||||||
|
headers: {},
|
||||||
|
url: '',
|
||||||
|
data: {
|
||||||
|
...artifactsToListResponse(manyArtifacts.slice(100, 150)),
|
||||||
|
total_count: 200
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
const response = await listArtifactsPublicReloaded(
|
||||||
|
fixtures.runId,
|
||||||
|
fixtures.owner,
|
||||||
|
fixtures.repo,
|
||||||
|
fixtures.token,
|
||||||
|
false
|
||||||
|
)
|
||||||
|
|
||||||
|
// Should only return 150 artifacts due to the limit
|
||||||
|
expect(response.artifacts).toHaveLength(150)
|
||||||
|
expect(response.artifacts[0].name).toBe('artifact-1')
|
||||||
|
expect(response.artifacts[149].name).toBe('artifact-150')
|
||||||
|
} finally {
|
||||||
|
// Restore original environment variable
|
||||||
|
if (originalEnv !== undefined) {
|
||||||
|
process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT = originalEnv
|
||||||
|
} else {
|
||||||
|
delete process.env.ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset modules again to restore original state
|
||||||
|
jest.resetModules()
|
||||||
|
}
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('internal', () => {
|
describe('internal', () => {
|
||||||
|
|||||||
@@ -9,12 +9,12 @@ import {retry} from '@octokit/plugin-retry'
|
|||||||
import {OctokitOptions} from '@octokit/core/dist-types/types'
|
import {OctokitOptions} from '@octokit/core/dist-types/types'
|
||||||
import {internalArtifactTwirpClient} from '../shared/artifact-twirp-client'
|
import {internalArtifactTwirpClient} from '../shared/artifact-twirp-client'
|
||||||
import {getBackendIdsFromToken} from '../shared/util'
|
import {getBackendIdsFromToken} from '../shared/util'
|
||||||
|
import {getMaxArtifactListCount} from '../shared/config'
|
||||||
import {ListArtifactsRequest, Timestamp} from '../../generated'
|
import {ListArtifactsRequest, Timestamp} from '../../generated'
|
||||||
|
|
||||||
// Limiting to 2000 for perf reasons
|
const maximumArtifactCount = getMaxArtifactListCount()
|
||||||
const maximumArtifactCount = 2000
|
|
||||||
const paginationCount = 100
|
const paginationCount = 100
|
||||||
const maxNumberOfPages = maximumArtifactCount / paginationCount
|
const maxNumberOfPages = Math.ceil(maximumArtifactCount / paginationCount)
|
||||||
|
|
||||||
export async function listArtifactsPublic(
|
export async function listArtifactsPublic(
|
||||||
workflowRunId: number,
|
workflowRunId: number,
|
||||||
@@ -81,7 +81,7 @@ export async function listArtifactsPublic(
|
|||||||
// Iterate over any remaining pages
|
// Iterate over any remaining pages
|
||||||
for (
|
for (
|
||||||
currentPageNumber;
|
currentPageNumber;
|
||||||
currentPageNumber < numberOfPages;
|
currentPageNumber <= numberOfPages;
|
||||||
currentPageNumber++
|
currentPageNumber++
|
||||||
) {
|
) {
|
||||||
debug(`Fetching page ${currentPageNumber} of artifact list`)
|
debug(`Fetching page ${currentPageNumber} of artifact list`)
|
||||||
|
|||||||
@@ -97,3 +97,22 @@ export function getUploadChunkTimeout(): number {
|
|||||||
|
|
||||||
return timeout
|
return timeout
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// This value can be changed with ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT variable.
|
||||||
|
// Defaults to 1000 as a safeguard for rate limiting.
|
||||||
|
export function getMaxArtifactListCount(): number {
|
||||||
|
const maxCountVar = process.env['ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT']
|
||||||
|
if (!maxCountVar) {
|
||||||
|
return 1000
|
||||||
|
}
|
||||||
|
|
||||||
|
const maxCount = parseInt(maxCountVar)
|
||||||
|
if (isNaN(maxCount) || maxCount < 1) {
|
||||||
|
throw new Error(
|
||||||
|
'Invalid value set for ACTIONS_ARTIFACT_MAX_ARTIFACT_COUNT env variable'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return maxCount
|
||||||
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user