Compare commits

..

8 Commits

Author SHA1 Message Date
Vallie Joseph 123bccb3d9 adding network tests 2023-12-19 06:40:01 +00:00
Rob Herley 68f22927e7 Merge pull request #1608 from actions/robherley/artifact-client-import
Update artifact module quick start
2023-12-14 15:46:14 -05:00
Rob Herley 11a2dd3117 update artifact module quick start 2023-12-14 15:38:49 -05:00
Rob Herley 43c63eef65 Merge pull request #1607 from actions/robherley/update-artifact-tests
Update artifact workflow tests
2023-12-13 12:47:12 -05:00
Rob Herley 6a9034d692 update artifact workflow tests 2023-12-13 12:19:14 -05:00
Rob Herley eff198be5b Merge pull request #1605 from actions/robherley/usage-message
Better error message for artifact usage limits
2023-12-12 09:49:55 -05:00
Rob Herley 16b786a545 better error message for usage limits 2023-12-11 22:01:08 -05:00
Rob Herley 18ce228b82 Merge pull request #1603 from actions/robherley/network-errors
Add specific messages for network-specific node error codes
2023-12-11 17:34:24 -05:00
10 changed files with 313 additions and 65 deletions
+69 -33
View File
@@ -1,5 +1,3 @@
# Temporarily disabled while v2.0.0 of @actions/artifact is under development
name: artifact-unit-tests
on:
push:
@@ -12,8 +10,8 @@ on:
- '**.md'
jobs:
build:
name: Build
upload:
name: Upload
strategy:
matrix:
@@ -42,19 +40,13 @@ jobs:
npm run tsc
working-directory: packages/artifact
- name: Set artifact file contents
shell: bash
run: |
echo "file1=hello from file 1" >> $GITHUB_ENV
echo "file2=hello from file 2" >> $GITHUB_ENV
- name: Create files that will be uploaded
run: |
mkdir artifact-path
echo '${{ env.file1 }}' > artifact-path/first.txt
echo '${{ env.file2 }}' > artifact-path/second.txt
echo -n 'hello from file 1' > artifact-path/first.txt
echo -n 'hello from file 2' > artifact-path/second.txt
- name: Upload Artifacts using actions/github-script@v7
- name: Upload Artifacts
uses: actions/github-script@v7
with:
script: |
@@ -73,9 +65,16 @@ jobs:
console.log(`Successfully uploaded artifact ${id}`)
try {
await artifact.uploadArtifact(artifactName, fileContents, './')
throw new Error('should have failed second upload')
} catch (err) {
console.log('Successfully blocked second artifact upload')
}
verify:
name: Verify
runs-on: ubuntu-latest
needs: [build]
needs: [upload]
steps:
- name: Checkout
uses: actions/checkout@v4
@@ -96,35 +95,72 @@ jobs:
npm run tsc
working-directory: packages/artifact
- name: List artifacts using actions/github-script@v7
- name: List and Download Artifacts
uses: actions/github-script@v7
with:
script: |
const {default: artifact} = require('./packages/artifact/lib/artifact')
const {default: artifactClient} = require('./packages/artifact/lib/artifact')
const workflowRunId = process.env.GITHUB_RUN_ID
const repository = process.env.GITHUB_REPOSITORY
const repositoryOwner = repository.split('/')[0]
const repositoryName = repository.split('/')[1]
const {readFile} = require('fs/promises')
const path = require('path')
const listResult = await artifact.listArtifacts(workflowRunId, repositoryOwner, repositoryName, '${{ secrets.GITHUB_TOKEN }}')
const findBy = {
repositoryOwner: process.env.GITHUB_REPOSITORY.split('/')[0],
repositoryName: process.env.GITHUB_REPOSITORY.split('/')[1],
token: '${{ secrets.GITHUB_TOKEN }}',
workflowRunId: process.env.GITHUB_RUN_ID
}
const listResult = await artifactClient.listArtifacts({latest: true, findBy})
console.log(listResult)
const artifacts = listResult.artifacts
const expected = [
'my-artifact-ubuntu-latest',
'my-artifact-windows-latest',
'my-artifact-macos-latest'
]
if (artifacts.length !== 3) {
throw new Error('Expected 3 artifacts but only found ' + artifacts.length + ' artifacts')
}
const foundArtifacts = artifacts.filter(artifact =>
expected.includes(artifact.name)
)
const artifactNames = artifacts.map(artifact => artifact.name)
if (!artifactNames.includes('my-artifact-ubuntu-latest')){
throw new Error("Expected artifact list to contain an artifact named my-artifact-ubuntu-latest but it's missing")
}
if (!artifactNames.includes('my-artifact-windows-latest')){
throw new Error("Expected artifact list to contain an artifact named my-artifact-windows-latest but it's missing")
}
if (!artifactNames.includes('my-artifact-macos-latest')){
throw new Error("Expected artifact list to contain an artifact named my-artifact-macos-latest but it's missing")
if (foundArtifacts.length !== 3) {
console.log('Unexpected length of found artifacts', foundArtifacts)
throw new Error(
`Expected 3 artifacts but found ${foundArtifacts.length} artifacts.`
)
}
console.log('Successfully listed artifacts that were uploaded')
const files = [
{name: 'artifact-path/first.txt', content: 'hello from file 1'},
{name: 'artifact-path/second.txt', content: 'hello from file 2'}
]
for (const artifact of foundArtifacts) {
const {downloadPath} = await artifactClient.downloadArtifact(artifact.id, {
path: artifact.name,
findBy
})
console.log('Downloaded artifact to:', downloadPath)
for (const file of files) {
const filepath = path.join(
process.env.GITHUB_WORKSPACE,
downloadPath,
file.name
)
console.log('Checking file:', filepath)
const content = await readFile(filepath, 'utf8')
if (content.trim() !== file.content.trim()) {
throw new Error(
`Expected file '${file.name}' to contain '${file.content}' but found '${content}'`
)
}
}
}
+8 -2
View File
@@ -63,10 +63,16 @@ Import the module:
```js
// ES6 module
import artifact from '@actions/artifact'
import {DefaultArtifactClient} from '@actions/artifact'
// CommonJS
const {default: artifact} = require('@actions/artifact')
const {DefaultArtifactClient} = require('@actions/artifact')
```
Then instantiate:
```js
const artifact = new DefaultArtifactClient()
```
️ For a comprehensive list of classes, interfaces, functions and more, see the [generated documentation](./docs/generated/README.md).
@@ -4,6 +4,7 @@ import {HttpClient} from '@actions/http-client'
import * as config from '../src/internal/shared/config'
import {internalArtifactTwirpClient} from '../src/internal/shared/artifact-twirp-client'
import {noopLogs} from './common'
import {NetworkError, UsageError} from '../src/internal/shared/errors'
jest.mock('@actions/http-client')
@@ -257,9 +258,42 @@ describe('artifact-http-client', () => {
name: 'artifact',
version: 4
})
}).rejects.toThrowError(
'Failed to CreateArtifact: Unable to make request: ENOTFOUND\nIf you are using self-hosted runners, please make sure your runner has access to all GitHub endpoints: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#communication-between-self-hosted-runners-and-github'
)
}).rejects.toThrowError(new NetworkError('ENOTFOUND').message)
expect(mockHttpClient).toHaveBeenCalledTimes(1)
expect(mockPost).toHaveBeenCalledTimes(1)
})
it('should properly describe a usage error', async () => {
const mockPost = jest.fn(() => {
const msgFailed = new http.IncomingMessage(new net.Socket())
msgFailed.statusCode = 403
msgFailed.statusMessage = 'Forbidden'
return {
message: msgFailed,
readBody: async () => {
return Promise.resolve(
`{"msg": "insufficient usage to create artifact"}`
)
}
}
})
const mockHttpClient = (
HttpClient as unknown as jest.Mock
).mockImplementation(() => {
return {
post: mockPost
}
})
const client = internalArtifactTwirpClient()
await expect(async () => {
await client.CreateArtifact({
workflowRunBackendId: '1234',
workflowJobRunBackendId: '5678',
name: 'artifact',
version: 4
})
}).rejects.toThrowError(new UsageError().message)
expect(mockHttpClient).toHaveBeenCalledTimes(1)
expect(mockPost).toHaveBeenCalledTimes(1)
})
+81 -14
View File
@@ -21,6 +21,7 @@
"archiver": "^5.3.1",
"crypto": "^1.0.1",
"jwt-decode": "^3.1.2",
"nock": "^13.4.0",
"twirp-ts": "^2.5.0",
"unzip-stream": "^0.3.1"
},
@@ -108,6 +109,25 @@
"node": ">=14.0.0"
}
},
"node_modules/@azure/core-http/node_modules/node-fetch": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
"dependencies": {
"whatwg-url": "^5.0.0"
},
"engines": {
"node": "4.x || >=6.0.0"
},
"peerDependencies": {
"encoding": "^0.1.0"
},
"peerDependenciesMeta": {
"encoding": {
"optional": true
}
}
},
"node_modules/@azure/core-lro": {
"version": "2.5.4",
"resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.5.4.tgz",
@@ -332,6 +352,25 @@
"once": "^1.4.0"
}
},
"node_modules/@octokit/request/node_modules/node-fetch": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
"dependencies": {
"whatwg-url": "^5.0.0"
},
"engines": {
"node": "4.x || >=6.0.0"
},
"peerDependencies": {
"encoding": "^0.1.0"
},
"peerDependenciesMeta": {
"encoding": {
"optional": true
}
}
},
"node_modules/@octokit/types": {
"version": "6.41.0",
"resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.41.0.tgz",
@@ -756,6 +795,22 @@
"integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==",
"deprecated": "This package is no longer supported. It's now a built-in Node module. If you've depended on crypto, you should switch to the one that's built-in."
},
"node_modules/debug": {
"version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
"dependencies": {
"ms": "2.1.2"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@@ -911,6 +966,11 @@
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
},
"node_modules/json-stringify-safe": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
"integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA=="
},
"node_modules/jsonc-parser": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz",
@@ -1065,6 +1125,11 @@
"mkdirp": "bin/cmd.js"
}
},
"node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/neo-async": {
"version": "2.6.2",
"resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz",
@@ -1080,23 +1145,17 @@
"tslib": "^2.0.3"
}
},
"node_modules/node-fetch": {
"version": "2.6.12",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz",
"integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==",
"node_modules/nock": {
"version": "13.4.0",
"resolved": "https://registry.npmjs.org/nock/-/nock-13.4.0.tgz",
"integrity": "sha512-W8NVHjO/LCTNA64yxAPHV/K47LpGYcVzgKd3Q0n6owhwvD0Dgoterc25R4rnZbckJEb6Loxz1f5QMuJpJnbSyQ==",
"dependencies": {
"whatwg-url": "^5.0.0"
"debug": "^4.1.0",
"json-stringify-safe": "^5.0.1",
"propagate": "^2.0.0"
},
"engines": {
"node": "4.x || >=6.0.0"
},
"peerDependencies": {
"encoding": "^0.1.0"
},
"peerDependenciesMeta": {
"encoding": {
"optional": true
}
"node": ">= 10.13"
}
},
"node_modules/normalize-path": {
@@ -1164,6 +1223,14 @@
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"node_modules/propagate": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz",
"integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==",
"engines": {
"node": ">= 8"
}
},
"node_modules/readable-stream": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
+1
View File
@@ -52,6 +52,7 @@
"archiver": "^5.3.1",
"crypto": "^1.0.1",
"jwt-decode": "^3.1.2",
"nock": "^13.4.0",
"twirp-ts": "^2.5.0",
"unzip-stream": "^0.3.1"
},
@@ -4,7 +4,7 @@ import {info, debug} from '@actions/core'
import {ArtifactServiceClientJSON} from '../../generated'
import {getResultsServiceUrl, getRuntimeToken} from './config'
import {getUserAgentString} from './user-agent'
import {NetworkError} from './errors'
import {NetworkError, UsageError} from './errors'
// The twirp http client must implement this interface
interface Rpc {
@@ -64,7 +64,7 @@ class ArtifactHttpClient implements Rpc {
this.httpClient.post(url, JSON.stringify(data), headers)
)
return JSON.parse(body)
return body
} catch (error) {
throw new Error(`Failed to ${method}: ${error.message}`)
}
@@ -72,34 +72,49 @@ class ArtifactHttpClient implements Rpc {
async retryableRequest(
operation: () => Promise<HttpClientResponse>
): Promise<{response: HttpClientResponse; body: string}> {
): Promise<{response: HttpClientResponse; body: object}> {
let attempt = 0
let errorMessage = ''
let rawBody = ''
while (attempt < this.maxAttempts) {
let isRetryable = false
try {
const response = await operation()
const statusCode = response.message.statusCode
const body = await response.readBody()
rawBody = await response.readBody()
debug(`[Response] - ${response.message.statusCode}`)
debug(`Headers: ${JSON.stringify(response.message.headers, null, 2)}`)
debug(`Body: ${body}`)
const body = JSON.parse(rawBody)
debug(`Body: ${JSON.stringify(body, null, 2)}`)
if (this.isSuccessStatusCode(statusCode)) {
return {response, body}
}
isRetryable = this.isRetryableHttpStatusCode(statusCode)
errorMessage = `Failed request: (${statusCode}) ${response.message.statusMessage}`
const responseMessage = JSON.parse(body).msg
if (responseMessage) {
errorMessage = `${errorMessage}: ${responseMessage}`
if (body.msg) {
if (UsageError.isUsageErrorMessage(body.msg)) {
throw new UsageError()
}
errorMessage = `${errorMessage}: ${body.msg}`
}
} catch (error) {
isRetryable = true
errorMessage = error.message
if (error instanceof SyntaxError) {
debug(`Raw Body: ${rawBody}`)
throw error
}
if (error instanceof UsageError) {
throw error
}
if (NetworkError.isNetworkErrorCode(error?.code)) {
throw new NetworkError(error?.code)
}
isRetryable = true
errorMessage = error.message
}
if (!isRetryable) {
@@ -57,3 +57,16 @@ export class NetworkError extends Error {
].includes(code)
}
}
export class UsageError extends Error {
constructor() {
const message = `Artifact storage quota has been hit. Unable to upload any new artifacts. Usage is recalculated every 6-12 hours.\nMore info on storage limits: https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#calculating-minute-and-storage-spending`
super(message)
this.name = 'UsageError'
}
static isUsageErrorMessage = (msg?: string): boolean => {
if (!msg) return false
return msg.includes('insufficient usage')
}
}
@@ -45,6 +45,21 @@ export interface UploadArtifactOptions {
* For large files that are not easily compressed, a value of 0 is recommended for significantly faster uploads.
*/
compressionLevel?: number
/**
* The simulated network error we'll temporarily use to test the azure blob
* client behavior.
* The value can range from 0 to 6
* 0 - fetchError
* 1 - abortError
* 2 - networkError
* 3 - securityError
* 4 - notAllowedError
* 5 - quotaExceededError
* 6 - random
* 7 - none
*
*/
simulateError?: number
}
/**
@@ -5,8 +5,17 @@ import {getUploadChunkSize, getConcurrency} from '../shared/config'
import * as core from '@actions/core'
import * as crypto from 'crypto'
import * as stream from 'stream'
import nock from 'nock'
import {NetworkError} from '../shared/errors'
export const DEFAULT_ERROR_NUMBER = 7
export const ERROR_TYPES = [
'fetchError',
'abortError',
'securityError',
'notAllowedError',
'quotaExceededError'
]
export interface BlobUploadResponse {
/**
* The total reported upload size in bytes. Empty if the upload failed
@@ -18,10 +27,58 @@ export interface BlobUploadResponse {
*/
sha256Hash?: string
}
export async function sendSimulatedError(
simulatedError: number,
authenticatedUploadURL: string
): Promise<void> {
switch (simulatedError) {
case 0: {
nock(authenticatedUploadURL).get('/').replyWithError({
code: 'ECONNRESET',
message: 'socket hang up'
})
break
}
case 1: {
const controller = new AbortController()
controller.abort()
break
}
case 2: {
nock(authenticatedUploadURL).get('/').replyWithError({
code: 'ETIMEDOUT'
})
break
}
case 3: {
nock(authenticatedUploadURL).get('/').reply(403)
break
}
case 4: {
nock(authenticatedUploadURL).get('/').reply(405)
break
}
case 5: {
nock(authenticatedUploadURL).get('/').reply(429)
break
}
case 6: {
const rand = Math.floor(Math.random() * ERROR_TYPES.length)
sendSimulatedError(rand, authenticatedUploadURL)
break
}
case 7: {
core.info('no error selected')
break
}
default:
core.error('something went wrong')
}
}
export async function uploadZipToBlobStorage(
authenticatedUploadURL: string,
zipUploadStream: ZipUploadStream
zipUploadStream: ZipUploadStream,
simulatedError: number = DEFAULT_ERROR_NUMBER
): Promise<BlobUploadResponse> {
let uploadByteCount = 0
@@ -36,6 +93,9 @@ export async function uploadZipToBlobStorage(
const uploadCallback = (progress: TransferProgressEvent): void => {
core.info(`Uploaded bytes ${progress.loadedBytes}`)
if (progress.loadedBytes > 1) {
sendSimulatedError(simulatedError, authenticatedUploadURL)
}
uploadByteCount = progress.loadedBytes
}
@@ -76,7 +76,8 @@ export async function uploadArtifact(
// Upload zip to blob storage
const uploadResult = await uploadZipToBlobStorage(
createArtifactResp.signedUploadUrl,
zipUploadStream
zipUploadStream,
options?.simulateError
)
// finalize the artifact