Compare commits

..
Author SHA1 Message Date
Jonathan Tamsut 7789d4b6cd install type 2023-12-08 13:25:14 -08:00
Jonathan Tamsut 6aad3fe814 replace unzipper with unzip stream 2023-12-08 13:09:36 -08:00
12 changed files with 82 additions and 293 deletions
+33 -69
View File
@@ -1,3 +1,5 @@
# Temporarily disabled while v2.0.0 of @actions/artifact is under development
name: artifact-unit-tests name: artifact-unit-tests
on: on:
push: push:
@@ -10,8 +12,8 @@ on:
- '**.md' - '**.md'
jobs: jobs:
upload: build:
name: Upload name: Build
strategy: strategy:
matrix: matrix:
@@ -40,13 +42,19 @@ jobs:
npm run tsc npm run tsc
working-directory: packages/artifact working-directory: packages/artifact
- name: Set artifact file contents
shell: bash
run: |
echo "file1=hello from file 1" >> $GITHUB_ENV
echo "file2=hello from file 2" >> $GITHUB_ENV
- name: Create files that will be uploaded - name: Create files that will be uploaded
run: | run: |
mkdir artifact-path mkdir artifact-path
echo -n 'hello from file 1' > artifact-path/first.txt echo '${{ env.file1 }}' > artifact-path/first.txt
echo -n 'hello from file 2' > artifact-path/second.txt echo '${{ env.file2 }}' > artifact-path/second.txt
- name: Upload Artifacts - name: Upload Artifacts using actions/github-script@v7
uses: actions/github-script@v7 uses: actions/github-script@v7
with: with:
script: | script: |
@@ -65,16 +73,9 @@ jobs:
console.log(`Successfully uploaded artifact ${id}`) console.log(`Successfully uploaded artifact ${id}`)
try {
await artifact.uploadArtifact(artifactName, fileContents, './')
throw new Error('should have failed second upload')
} catch (err) {
console.log('Successfully blocked second artifact upload')
}
verify: verify:
name: Verify
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [upload] needs: [build]
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
@@ -95,72 +96,35 @@ jobs:
npm run tsc npm run tsc
working-directory: packages/artifact working-directory: packages/artifact
- name: List and Download Artifacts - name: List artifacts using actions/github-script@v7
uses: actions/github-script@v7 uses: actions/github-script@v7
with: with:
script: | script: |
const {default: artifactClient} = require('./packages/artifact/lib/artifact') const {default: artifact} = require('./packages/artifact/lib/artifact')
const {readFile} = require('fs/promises') const workflowRunId = process.env.GITHUB_RUN_ID
const path = require('path') const repository = process.env.GITHUB_REPOSITORY
const repositoryOwner = repository.split('/')[0]
const repositoryName = repository.split('/')[1]
const findBy = { const listResult = await artifact.listArtifacts(workflowRunId, repositoryOwner, repositoryName, '${{ secrets.GITHUB_TOKEN }}')
repositoryOwner: process.env.GITHUB_REPOSITORY.split('/')[0],
repositoryName: process.env.GITHUB_REPOSITORY.split('/')[1],
token: '${{ secrets.GITHUB_TOKEN }}',
workflowRunId: process.env.GITHUB_RUN_ID
}
const listResult = await artifactClient.listArtifacts({latest: true, findBy})
console.log(listResult) console.log(listResult)
const artifacts = listResult.artifacts const artifacts = listResult.artifacts
const expected = [
'my-artifact-ubuntu-latest',
'my-artifact-windows-latest',
'my-artifact-macos-latest'
]
const foundArtifacts = artifacts.filter(artifact => if (artifacts.length !== 3) {
expected.includes(artifact.name) throw new Error('Expected 3 artifacts but only found ' + artifacts.length + ' artifacts')
) }
if (foundArtifacts.length !== 3) { const artifactNames = artifacts.map(artifact => artifact.name)
console.log('Unexpected length of found artifacts', foundArtifacts) if (!artifactNames.includes('my-artifact-ubuntu-latest')){
throw new Error( throw new Error("Expected artifact list to contain an artifact named my-artifact-ubuntu-latest but it's missing")
`Expected 3 artifacts but found ${foundArtifacts.length} artifacts.` }
) if (!artifactNames.includes('my-artifact-windows-latest')){
throw new Error("Expected artifact list to contain an artifact named my-artifact-windows-latest but it's missing")
}
if (!artifactNames.includes('my-artifact-macos-latest')){
throw new Error("Expected artifact list to contain an artifact named my-artifact-macos-latest but it's missing")
} }
console.log('Successfully listed artifacts that were uploaded') console.log('Successfully listed artifacts that were uploaded')
const files = [
{name: 'artifact-path/first.txt', content: 'hello from file 1'},
{name: 'artifact-path/second.txt', content: 'hello from file 2'}
]
for (const artifact of foundArtifacts) {
const {downloadPath} = await artifactClient.downloadArtifact(artifact.id, {
path: artifact.name,
findBy
})
console.log('Downloaded artifact to:', downloadPath)
for (const file of files) {
const filepath = path.join(
process.env.GITHUB_WORKSPACE,
downloadPath,
file.name
)
console.log('Checking file:', filepath)
const content = await readFile(filepath, 'utf8')
if (content.trim() !== file.content.trim()) {
throw new Error(
`Expected file '${file.name}' to contain '${file.content}' but found '${content}'`
)
}
}
}
+10
View File
@@ -9,6 +9,7 @@
"@types/jest": "^29.5.4", "@types/jest": "^29.5.4",
"@types/node": "^20.5.7", "@types/node": "^20.5.7",
"@types/signale": "^1.4.1", "@types/signale": "^1.4.1",
"@types/unzip-stream": "^0.3.4",
"concurrently": "^6.1.0", "concurrently": "^6.1.0",
"eslint": "^8.0.1", "eslint": "^8.0.1",
"eslint-config-prettier": "^8.9.0", "eslint-config-prettier": "^8.9.0",
@@ -2587,6 +2588,15 @@
"integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==", "integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==",
"dev": true "dev": true
}, },
"node_modules/@types/unzip-stream": {
"version": "0.3.4",
"resolved": "https://registry.npmjs.org/@types/unzip-stream/-/unzip-stream-0.3.4.tgz",
"integrity": "sha512-ud0vtsNRF+joUCyvNMyo0j5DKX2Lh/im+xVgRzBEsfHhQYZ+i4fKTveova9XxLzt6Jl6G0e/0mM4aC0gqZYSnA==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/yargs": { "node_modules/@types/yargs": {
"version": "17.0.24", "version": "17.0.24",
"resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.24.tgz", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.24.tgz",
+2 -1
View File
@@ -19,6 +19,7 @@
"@types/jest": "^29.5.4", "@types/jest": "^29.5.4",
"@types/node": "^20.5.7", "@types/node": "^20.5.7",
"@types/signale": "^1.4.1", "@types/signale": "^1.4.1",
"@types/unzip-stream": "^0.3.4",
"concurrently": "^6.1.0", "concurrently": "^6.1.0",
"eslint": "^8.0.1", "eslint": "^8.0.1",
"eslint-config-prettier": "^8.9.0", "eslint-config-prettier": "^8.9.0",
@@ -33,4 +34,4 @@
"ts-jest": "^29.1.1", "ts-jest": "^29.1.1",
"typescript": "^5.2.2" "typescript": "^5.2.2"
} }
} }
+2 -8
View File
@@ -63,16 +63,10 @@ Import the module:
```js ```js
// ES6 module // ES6 module
import {DefaultArtifactClient} from '@actions/artifact' import artifact from '@actions/artifact'
// CommonJS // CommonJS
const {DefaultArtifactClient} = require('@actions/artifact') const {default: artifact} = require('@actions/artifact')
```
Then instantiate:
```js
const artifact = new DefaultArtifactClient()
``` ```
️ For a comprehensive list of classes, interfaces, functions and more, see the [generated documentation](./docs/generated/README.md). ️ For a comprehensive list of classes, interfaces, functions and more, see the [generated documentation](./docs/generated/README.md).
@@ -4,16 +4,9 @@ import {HttpClient} from '@actions/http-client'
import * as config from '../src/internal/shared/config' import * as config from '../src/internal/shared/config'
import {internalArtifactTwirpClient} from '../src/internal/shared/artifact-twirp-client' import {internalArtifactTwirpClient} from '../src/internal/shared/artifact-twirp-client'
import {noopLogs} from './common' import {noopLogs} from './common'
import {NetworkError, UsageError} from '../src/internal/shared/errors'
jest.mock('@actions/http-client') jest.mock('@actions/http-client')
const clientOptions = {
maxAttempts: 5,
retryIntervalMs: 1,
retryMultiplier: 1.5
}
describe('artifact-http-client', () => { describe('artifact-http-client', () => {
beforeAll(() => { beforeAll(() => {
noopLogs() noopLogs()
@@ -101,7 +94,11 @@ describe('artifact-http-client', () => {
} }
}) })
const client = internalArtifactTwirpClient(clientOptions) const client = internalArtifactTwirpClient({
maxAttempts: 5,
retryIntervalMs: 1,
retryMultiplier: 1.5
})
const artifact = await client.CreateArtifact({ const artifact = await client.CreateArtifact({
workflowRunBackendId: '1234', workflowRunBackendId: '1234',
workflowJobRunBackendId: '5678', workflowJobRunBackendId: '5678',
@@ -136,7 +133,11 @@ describe('artifact-http-client', () => {
post: mockPost post: mockPost
} }
}) })
const client = internalArtifactTwirpClient(clientOptions) const client = internalArtifactTwirpClient({
maxAttempts: 5,
retryIntervalMs: 1,
retryMultiplier: 1.5
})
await expect(async () => { await expect(async () => {
await client.CreateArtifact({ await client.CreateArtifact({
workflowRunBackendId: '1234', workflowRunBackendId: '1234',
@@ -171,7 +172,11 @@ describe('artifact-http-client', () => {
post: mockPost post: mockPost
} }
}) })
const client = internalArtifactTwirpClient(clientOptions) const client = internalArtifactTwirpClient({
maxAttempts: 5,
retryIntervalMs: 1,
retryMultiplier: 1.5
})
await expect(async () => { await expect(async () => {
await client.CreateArtifact({ await client.CreateArtifact({
workflowRunBackendId: '1234', workflowRunBackendId: '1234',
@@ -185,116 +190,4 @@ describe('artifact-http-client', () => {
expect(mockHttpClient).toHaveBeenCalledTimes(1) expect(mockHttpClient).toHaveBeenCalledTimes(1)
expect(mockPost).toHaveBeenCalledTimes(1) expect(mockPost).toHaveBeenCalledTimes(1)
}) })
it('should fail with a descriptive error', async () => {
// 409 duplicate error
const mockPost = jest.fn(() => {
const msgFailed = new http.IncomingMessage(new net.Socket())
msgFailed.statusCode = 409
msgFailed.statusMessage = 'Conflict'
return {
message: msgFailed,
readBody: async () => {
return Promise.resolve(
`{"msg": "an artifact with this name already exists on the workflow run"}`
)
}
}
})
const mockHttpClient = (
HttpClient as unknown as jest.Mock
).mockImplementation(() => {
return {
post: mockPost
}
})
const client = internalArtifactTwirpClient(clientOptions)
await expect(async () => {
await client.CreateArtifact({
workflowRunBackendId: '1234',
workflowJobRunBackendId: '5678',
name: 'artifact',
version: 4
})
await client.CreateArtifact({
workflowRunBackendId: '1234',
workflowJobRunBackendId: '5678',
name: 'artifact',
version: 4
})
}).rejects.toThrowError(
'Failed to CreateArtifact: Received non-retryable error: Failed request: (409) Conflict: an artifact with this name already exists on the workflow run'
)
expect(mockHttpClient).toHaveBeenCalledTimes(1)
expect(mockPost).toHaveBeenCalledTimes(1)
})
it('should properly describe a network failure', async () => {
class FakeNodeError extends Error {
code: string
constructor(code: string) {
super()
this.code = code
}
}
const mockPost = jest.fn(() => {
throw new FakeNodeError('ENOTFOUND')
})
const mockHttpClient = (
HttpClient as unknown as jest.Mock
).mockImplementation(() => {
return {
post: mockPost
}
})
const client = internalArtifactTwirpClient()
await expect(async () => {
await client.CreateArtifact({
workflowRunBackendId: '1234',
workflowJobRunBackendId: '5678',
name: 'artifact',
version: 4
})
}).rejects.toThrowError(new NetworkError('ENOTFOUND').message)
expect(mockHttpClient).toHaveBeenCalledTimes(1)
expect(mockPost).toHaveBeenCalledTimes(1)
})
it('should properly describe a usage error', async () => {
const mockPost = jest.fn(() => {
const msgFailed = new http.IncomingMessage(new net.Socket())
msgFailed.statusCode = 403
msgFailed.statusMessage = 'Forbidden'
return {
message: msgFailed,
readBody: async () => {
return Promise.resolve(
`{"msg": "insufficient usage to create artifact"}`
)
}
}
})
const mockHttpClient = (
HttpClient as unknown as jest.Mock
).mockImplementation(() => {
return {
post: mockPost
}
})
const client = internalArtifactTwirpClient()
await expect(async () => {
await client.CreateArtifact({
workflowRunBackendId: '1234',
workflowJobRunBackendId: '5678',
name: 'artifact',
version: 4
})
}).rejects.toThrowError(new UsageError().message)
expect(mockHttpClient).toHaveBeenCalledTimes(1)
expect(mockPost).toHaveBeenCalledTimes(1)
})
}) })
@@ -104,7 +104,6 @@ const mockGetArtifactSuccess = jest.fn(() => {
const message = new http.IncomingMessage(new net.Socket()) const message = new http.IncomingMessage(new net.Socket())
message.statusCode = 200 message.statusCode = 200
message.push(fs.readFileSync(fixtures.exampleArtifact.path)) message.push(fs.readFileSync(fixtures.exampleArtifact.path))
message.push(null)
return { return {
message message
} }
@@ -114,7 +113,6 @@ const mockGetArtifactFailure = jest.fn(() => {
const message = new http.IncomingMessage(new net.Socket()) const message = new http.IncomingMessage(new net.Socket())
message.statusCode = 500 message.statusCode = 500
message.push('Internal Server Error') message.push('Internal Server Error')
message.push(null)
return { return {
message message
} }
+5 -6
View File
@@ -18,6 +18,7 @@
"@octokit/plugin-retry": "^3.0.9", "@octokit/plugin-retry": "^3.0.9",
"@octokit/request-error": "^5.0.0", "@octokit/request-error": "^5.0.0",
"@protobuf-ts/plugin": "^2.2.3-alpha.1", "@protobuf-ts/plugin": "^2.2.3-alpha.1",
"@types/unzipper": "^0.10.6",
"archiver": "^5.3.1", "archiver": "^5.3.1",
"crypto": "^1.0.1", "crypto": "^1.0.1",
"jwt-decode": "^3.1.2", "jwt-decode": "^3.1.2",
@@ -26,7 +27,6 @@
}, },
"devDependencies": { "devDependencies": {
"@types/archiver": "^5.3.2", "@types/archiver": "^5.3.2",
"@types/unzip-stream": "^0.3.4",
"typedoc": "^0.25.4", "typedoc": "^0.25.4",
"typedoc-plugin-markdown": "^3.17.1", "typedoc-plugin-markdown": "^3.17.1",
"typescript": "^5.2.2" "typescript": "^5.2.2"
@@ -471,11 +471,10 @@
"@types/node": "*" "@types/node": "*"
} }
}, },
"node_modules/@types/unzip-stream": { "node_modules/@types/unzipper": {
"version": "0.3.4", "version": "0.10.6",
"resolved": "https://registry.npmjs.org/@types/unzip-stream/-/unzip-stream-0.3.4.tgz", "resolved": "https://registry.npmjs.org/@types/unzipper/-/unzipper-0.10.6.tgz",
"integrity": "sha512-ud0vtsNRF+joUCyvNMyo0j5DKX2Lh/im+xVgRzBEsfHhQYZ+i4fKTveova9XxLzt6Jl6G0e/0mM4aC0gqZYSnA==", "integrity": "sha512-zcBj329AHgKLQyz209N/S9R0GZqXSkUQO4tJSYE3x02qg4JuDFpgKMj50r82Erk1natCWQDIvSccDddt7jPzjA==",
"dev": true,
"dependencies": { "dependencies": {
"@types/node": "*" "@types/node": "*"
} }
+1 -1
View File
@@ -49,6 +49,7 @@
"@octokit/plugin-retry": "^3.0.9", "@octokit/plugin-retry": "^3.0.9",
"@octokit/request-error": "^5.0.0", "@octokit/request-error": "^5.0.0",
"@protobuf-ts/plugin": "^2.2.3-alpha.1", "@protobuf-ts/plugin": "^2.2.3-alpha.1",
"@types/unzipper": "^0.10.6",
"archiver": "^5.3.1", "archiver": "^5.3.1",
"crypto": "^1.0.1", "crypto": "^1.0.1",
"jwt-decode": "^3.1.2", "jwt-decode": "^3.1.2",
@@ -57,7 +58,6 @@
}, },
"devDependencies": { "devDependencies": {
"@types/archiver": "^5.3.2", "@types/archiver": "^5.3.2",
"@types/unzip-stream": "^0.3.4",
"typedoc": "^0.25.4", "typedoc": "^0.25.4",
"typedoc-plugin-markdown": "^3.17.1", "typedoc-plugin-markdown": "^3.17.1",
"typescript": "^5.2.2" "typescript": "^5.2.2"
@@ -2,7 +2,7 @@ import fs from 'fs/promises'
import * as github from '@actions/github' import * as github from '@actions/github'
import * as core from '@actions/core' import * as core from '@actions/core'
import * as httpClient from '@actions/http-client' import * as httpClient from '@actions/http-client'
import unzip from 'unzip-stream' import unzipper from 'unzip-stream'
import { import {
DownloadArtifactOptions, DownloadArtifactOptions,
DownloadArtifactResponse DownloadArtifactResponse
@@ -47,12 +47,7 @@ async function streamExtract(url: string, directory: string): Promise<void> {
) )
} }
return new Promise((resolve, reject) => { return response.message.pipe(unzipper.Extract({path: directory})).promise()
response.message
.pipe(unzip.Extract({path: directory}))
.on('close', resolve)
.on('error', reject)
})
} }
export async function downloadArtifactPublic( export async function downloadArtifactPublic(
@@ -4,7 +4,6 @@ import {info, debug} from '@actions/core'
import {ArtifactServiceClientJSON} from '../../generated' import {ArtifactServiceClientJSON} from '../../generated'
import {getResultsServiceUrl, getRuntimeToken} from './config' import {getResultsServiceUrl, getRuntimeToken} from './config'
import {getUserAgentString} from './user-agent' import {getUserAgentString} from './user-agent'
import {NetworkError, UsageError} from './errors'
// The twirp http client must implement this interface // The twirp http client must implement this interface
interface Rpc { interface Rpc {
@@ -64,7 +63,7 @@ class ArtifactHttpClient implements Rpc {
this.httpClient.post(url, JSON.stringify(data), headers) this.httpClient.post(url, JSON.stringify(data), headers)
) )
return body return JSON.parse(body)
} catch (error) { } catch (error) {
throw new Error(`Failed to ${method}: ${error.message}`) throw new Error(`Failed to ${method}: ${error.message}`)
} }
@@ -72,47 +71,27 @@ class ArtifactHttpClient implements Rpc {
async retryableRequest( async retryableRequest(
operation: () => Promise<HttpClientResponse> operation: () => Promise<HttpClientResponse>
): Promise<{response: HttpClientResponse; body: object}> { ): Promise<{response: HttpClientResponse; body: string}> {
let attempt = 0 let attempt = 0
let errorMessage = '' let errorMessage = ''
let rawBody = ''
while (attempt < this.maxAttempts) { while (attempt < this.maxAttempts) {
let isRetryable = false let isRetryable = false
try { try {
const response = await operation() const response = await operation()
const statusCode = response.message.statusCode const statusCode = response.message.statusCode
rawBody = await response.readBody() const body = await response.readBody()
debug(`[Response] - ${response.message.statusCode}`) debug(`[Response] - ${response.message.statusCode}`)
debug(`Headers: ${JSON.stringify(response.message.headers, null, 2)}`) debug(`Headers: ${JSON.stringify(response.message.headers, null, 2)}`)
const body = JSON.parse(rawBody) debug(`Body: ${body}`)
debug(`Body: ${JSON.stringify(body, null, 2)}`)
if (this.isSuccessStatusCode(statusCode)) { if (this.isSuccessStatusCode(statusCode)) {
return {response, body} return {response, body}
} }
isRetryable = this.isRetryableHttpStatusCode(statusCode) isRetryable = this.isRetryableHttpStatusCode(statusCode)
errorMessage = `Failed request: (${statusCode}) ${response.message.statusMessage}` errorMessage = `Failed request: (${statusCode}) ${response.message.statusMessage}`
if (body.msg) {
if (UsageError.isUsageErrorMessage(body.msg)) {
throw new UsageError()
}
errorMessage = `${errorMessage}: ${body.msg}`
}
} catch (error) { } catch (error) {
if (error instanceof SyntaxError) {
debug(`Raw Body: ${rawBody}`)
throw error
}
if (error instanceof UsageError) {
throw error
}
if (NetworkError.isNetworkErrorCode(error?.code)) {
throw new NetworkError(error?.code)
}
isRetryable = true isRetryable = true
errorMessage = error.message errorMessage = error.message
} }
@@ -35,38 +35,3 @@ export class GHESNotSupportedError extends Error {
this.name = 'GHESNotSupportedError' this.name = 'GHESNotSupportedError'
} }
} }
export class NetworkError extends Error {
code: string
constructor(code: string) {
const message = `Unable to make request: ${code}\nIf you are using self-hosted runners, please make sure your runner has access to all GitHub endpoints: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#communication-between-self-hosted-runners-and-github`
super(message)
this.code = code
this.name = 'NetworkError'
}
static isNetworkErrorCode = (code?: string): boolean => {
if (!code) return false
return [
'ECONNRESET',
'ENOTFOUND',
'ETIMEDOUT',
'ECONNREFUSED',
'EHOSTUNREACH'
].includes(code)
}
}
export class UsageError extends Error {
constructor() {
const message = `Artifact storage quota has been hit. Unable to upload any new artifacts. Usage is recalculated every 6-12 hours.\nMore info on storage limits: https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#calculating-minute-and-storage-spending`
super(message)
this.name = 'UsageError'
}
static isUsageErrorMessage = (msg?: string): boolean => {
if (!msg) return false
return msg.includes('insufficient usage')
}
}
@@ -5,7 +5,6 @@ import {getUploadChunkSize, getConcurrency} from '../shared/config'
import * as core from '@actions/core' import * as core from '@actions/core'
import * as crypto from 'crypto' import * as crypto from 'crypto'
import * as stream from 'stream' import * as stream from 'stream'
import {NetworkError} from '../shared/errors'
export interface BlobUploadResponse { export interface BlobUploadResponse {
/** /**
@@ -53,20 +52,12 @@ export async function uploadZipToBlobStorage(
core.info('Beginning upload of artifact content to blob storage') core.info('Beginning upload of artifact content to blob storage')
try { await blockBlobClient.uploadStream(
await blockBlobClient.uploadStream( uploadStream,
uploadStream, bufferSize,
bufferSize, maxConcurrency,
maxConcurrency, options
options )
)
} catch (error) {
if (NetworkError.isNetworkErrorCode(error?.code)) {
throw new NetworkError(error?.code)
}
throw error
}
core.info('Finished uploading artifact content to blob storage!') core.info('Finished uploading artifact content to blob storage!')