support http proxy for artifact upload

.github/workflows/artifact-tests.yml

@@ -1,3 +1,5 @@
+# Temporarily disabled while v2.0.0 of @actions/artifact is under development
+
 name: artifact-unit-tests
 on:
   push:
@@ -10,8 +12,8 @@ on:
       - '**.md'
 
 jobs:
-  upload:
+  build:
-    name: Upload
+    name: Build
 
     strategy:
       matrix:
@@ -40,13 +42,19 @@ jobs:
         npm run tsc
       working-directory: packages/artifact
 
+    - name: Set artifact file contents
+      shell: bash
+      run: |
+        echo "file1=hello from file 1" >> $GITHUB_ENV
+        echo "file2=hello from file 2" >> $GITHUB_ENV
+
     - name: Create files that will be uploaded
       run: |
         mkdir artifact-path
-        echo -n 'hello from file 1' > artifact-path/first.txt
+        echo '${{ env.file1 }}' > artifact-path/first.txt
-        echo -n 'hello from file 2' > artifact-path/second.txt
+        echo '${{ env.file2 }}' > artifact-path/second.txt
 
-    - name: Upload Artifacts
+    - name: Upload Artifacts using actions/github-script@v7
       uses: actions/github-script@v7
       with:
         script: |
@@ -65,16 +73,9 @@ jobs:
 
           console.log(`Successfully uploaded artifact ${id}`)
 
-          try {
-            await artifact.uploadArtifact(artifactName, fileContents, './')
-            throw new Error('should have failed second upload')
-          } catch (err) {
-            console.log('Successfully blocked second artifact upload')
-          }
   verify:
-    name: Verify
     runs-on: ubuntu-latest
-    needs: [upload]
+    needs: [build]
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -95,72 +96,35 @@ jobs:
         npm run tsc
       working-directory: packages/artifact
 
-    - name: List and Download Artifacts
+    - name: List artifacts using actions/github-script@v7
       uses: actions/github-script@v7
       with:
         script: |
-          const {default: artifactClient} = require('./packages/artifact/lib/artifact')
+          const {default: artifact} = require('./packages/artifact/lib/artifact')
 
-          const {readFile} = require('fs/promises')
+          const workflowRunId = process.env.GITHUB_RUN_ID
-          const path = require('path')
+          const repository = process.env.GITHUB_REPOSITORY
+          const repositoryOwner = repository.split('/')[0]
+          const repositoryName = repository.split('/')[1]
 
-          const findBy = {
+          const listResult = await artifact.listArtifacts(workflowRunId, repositoryOwner, repositoryName, '${{ secrets.GITHUB_TOKEN }}')
-            repositoryOwner: process.env.GITHUB_REPOSITORY.split('/')[0],
-            repositoryName: process.env.GITHUB_REPOSITORY.split('/')[1],
-            token: '${{ secrets.GITHUB_TOKEN }}',
-            workflowRunId: process.env.GITHUB_RUN_ID
-          }
-
-          const listResult = await artifactClient.listArtifacts({latest: true, findBy})
           console.log(listResult)
 
           const artifacts = listResult.artifacts
-          const expected = [
-            'my-artifact-ubuntu-latest',
-            'my-artifact-windows-latest',
-            'my-artifact-macos-latest'
-          ]
 
-          const foundArtifacts = artifacts.filter(artifact =>
+          if (artifacts.length !== 3) {
-            expected.includes(artifact.name)
+            throw new Error('Expected 3 artifacts but only found ' + artifacts.length + ' artifacts')
-          )
+          }
 
-          if (foundArtifacts.length !== 3) {
+          const artifactNames = artifacts.map(artifact => artifact.name)
-            console.log('Unexpected length of found artifacts', foundArtifacts)
+          if (!artifactNames.includes('my-artifact-ubuntu-latest')){
-            throw new Error(
+            throw new Error("Expected artifact list to contain an artifact named my-artifact-ubuntu-latest but it's missing")
-              `Expected 3 artifacts but found ${foundArtifacts.length} artifacts.`
+          }
-            )
+          if (!artifactNames.includes('my-artifact-windows-latest')){
+            throw new Error("Expected artifact list to contain an artifact named my-artifact-windows-latest but it's missing")
+          }
+          if (!artifactNames.includes('my-artifact-macos-latest')){
+            throw new Error("Expected artifact list to contain an artifact named my-artifact-macos-latest but it's missing")
           }
 
           console.log('Successfully listed artifacts that were uploaded')
-
-          const files = [
-            {name: 'artifact-path/first.txt', content: 'hello from file 1'},
-            {name: 'artifact-path/second.txt', content: 'hello from file 2'}
-          ]
-
-          for (const artifact of foundArtifacts) {
-            const {downloadPath} = await artifactClient.downloadArtifact(artifact.id, {
-              path: artifact.name,
-              findBy
-            })
-
-            console.log('Downloaded artifact to:', downloadPath)
-
-            for (const file of files) {
-              const filepath = path.join(
-                process.env.GITHUB_WORKSPACE,
-                downloadPath,
-                file.name
-              )
-
-              console.log('Checking file:', filepath)
-
-              const content = await readFile(filepath, 'utf8')
-              if (content.trim() !== file.content.trim()) {
-                throw new Error(
-                  `Expected file '${file.name}' to contain '${file.content}' but found '${content}'`
-                )
-              }
-            }
-          }

packages/artifact/README.md

@@ -63,16 +63,10 @@ Import the module:
 
 ```js
 // ES6 module
-import {DefaultArtifactClient} from '@actions/artifact'
+import artifact from '@actions/artifact'
 
 // CommonJS
-const {DefaultArtifactClient} = require('@actions/artifact')
+const {default: artifact} = require('@actions/artifact')
-```
-
-Then instantiate:
-
-```js
-const artifact = new DefaultArtifactClient()
 ```
 
 ℹ️ For a comprehensive list of classes, interfaces, functions and more, see the [generated documentation](./docs/generated/README.md).

packages/artifact/__tests__/artifact-http-client.test.ts

@@ -4,7 +4,6 @@ import {HttpClient} from '@actions/http-client'
 import * as config from '../src/internal/shared/config'
 import {internalArtifactTwirpClient} from '../src/internal/shared/artifact-twirp-client'
 import {noopLogs} from './common'
-import {NetworkError, UsageError} from '../src/internal/shared/errors'
 
 jest.mock('@actions/http-client')
 
@@ -258,42 +257,9 @@ describe('artifact-http-client', () => {
         name: 'artifact',
         version: 4
       })
-    }).rejects.toThrowError(new NetworkError('ENOTFOUND').message)
+    }).rejects.toThrowError(
-    expect(mockHttpClient).toHaveBeenCalledTimes(1)
+      'Failed to CreateArtifact: Unable to make request: ENOTFOUND\nIf you are using self-hosted runners, please make sure your runner has access to all GitHub endpoints: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#communication-between-self-hosted-runners-and-github'
-    expect(mockPost).toHaveBeenCalledTimes(1)
+    )
-  })
-
-  it('should properly describe a usage error', async () => {
-    const mockPost = jest.fn(() => {
-      const msgFailed = new http.IncomingMessage(new net.Socket())
-      msgFailed.statusCode = 403
-      msgFailed.statusMessage = 'Forbidden'
-      return {
-        message: msgFailed,
-        readBody: async () => {
-          return Promise.resolve(
-            `{"msg": "insufficient usage to create artifact"}`
-          )
-        }
-      }
-    })
-
-    const mockHttpClient = (
-      HttpClient as unknown as jest.Mock
-    ).mockImplementation(() => {
-      return {
-        post: mockPost
-      }
-    })
-    const client = internalArtifactTwirpClient()
-    await expect(async () => {
-      await client.CreateArtifact({
-        workflowRunBackendId: '1234',
-        workflowJobRunBackendId: '5678',
-        name: 'artifact',
-        version: 4
-      })
-    }).rejects.toThrowError(new UsageError().message)
     expect(mockHttpClient).toHaveBeenCalledTimes(1)
     expect(mockPost).toHaveBeenCalledTimes(1)
   })

packages/artifact/__tests__/upload-artifact.test.ts

@@ -352,3 +352,46 @@ describe('upload-artifact', () => {
     expect(uploadResp).rejects.toThrow()
   })
 })
+
+describe('getBlobClientOptions', () => {
+  afterEach(() => {
+    delete process.env['HTTPS_PROXY']
+    delete process.env['HTTP_PROXY']
+    delete process.env['NO_PROXY']
+    jest.restoreAllMocks()
+  })
+
+  it('should not use proxy settings if not specified', () => {
+    const opts = blobUpload.getBlobClientOptions('https://blob-storage.local')
+    expect(opts.proxyOptions).toBeUndefined()
+  })
+
+  it('should use https proxy settings from environment', () => {
+    process.env['HTTPS_PROXY'] = 'https://foo:[email protected]'
+    const opts = blobUpload.getBlobClientOptions('https://blob-storage.local')
+    expect(opts.proxyOptions).toEqual({
+      host: 'my-proxy.local',
+      port: 443,
+      username: 'foo',
+      password: 'bar'
+    })
+  })
+
+  it('should use http proxy settings from environment', () => {
+    process.env['HTTP_PROXY'] = 'http://foo:[email protected]:1234'
+    const opts = blobUpload.getBlobClientOptions('http://blob-storage.local')
+    expect(opts.proxyOptions).toEqual({
+      host: 'my-proxy.local',
+      port: 1234,
+      username: 'foo',
+      password: 'bar'
+    })
+  })
+
+  it('should respect NO_PROXY', () => {
+    process.env['HTTPS_PROXY'] = 'https://foo:[email protected]'
+    process.env['NO_PROXY'] = 'no-proxy-me.local'
+    const opts = blobUpload.getBlobClientOptions('https://no-proxy-me.local')
+    expect(opts.proxyOptions).toBeUndefined()
+  })
+})

packages/artifact/src/internal/shared/artifact-twirp-client.ts

@@ -4,7 +4,7 @@ import {info, debug} from '@actions/core'
 import {ArtifactServiceClientJSON} from '../../generated'
 import {getResultsServiceUrl, getRuntimeToken} from './config'
 import {getUserAgentString} from './user-agent'
-import {NetworkError, UsageError} from './errors'
+import {NetworkError} from './errors'
 
 // The twirp http client must implement this interface
 interface Rpc {
@@ -64,7 +64,7 @@ class ArtifactHttpClient implements Rpc {
         this.httpClient.post(url, JSON.stringify(data), headers)
       )
 
-      return body
+      return JSON.parse(body)
     } catch (error) {
       throw new Error(`Failed to ${method}: ${error.message}`)
     }
@@ -72,49 +72,34 @@ class ArtifactHttpClient implements Rpc {
 
   async retryableRequest(
     operation: () => Promise<HttpClientResponse>
-  ): Promise<{response: HttpClientResponse; body: object}> {
+  ): Promise<{response: HttpClientResponse; body: string}> {
     let attempt = 0
     let errorMessage = ''
-    let rawBody = ''
     while (attempt < this.maxAttempts) {
       let isRetryable = false
 
       try {
         const response = await operation()
         const statusCode = response.message.statusCode
-        rawBody = await response.readBody()
+        const body = await response.readBody()
         debug(`[Response] - ${response.message.statusCode}`)
         debug(`Headers: ${JSON.stringify(response.message.headers, null, 2)}`)
-        const body = JSON.parse(rawBody)
+        debug(`Body: ${body}`)
-        debug(`Body: ${JSON.stringify(body, null, 2)}`)
         if (this.isSuccessStatusCode(statusCode)) {
           return {response, body}
         }
         isRetryable = this.isRetryableHttpStatusCode(statusCode)
         errorMessage = `Failed request: (${statusCode}) ${response.message.statusMessage}`
-        if (body.msg) {
+        const responseMessage = JSON.parse(body).msg
-          if (UsageError.isUsageErrorMessage(body.msg)) {
+        if (responseMessage) {
-            throw new UsageError()
+          errorMessage = `${errorMessage}: ${responseMessage}`
-          }
-
-          errorMessage = `${errorMessage}: ${body.msg}`
         }
       } catch (error) {
-        if (error instanceof SyntaxError) {
+        isRetryable = true
-          debug(`Raw Body: ${rawBody}`)
+        errorMessage = error.message
-          throw error
-        }
-
-        if (error instanceof UsageError) {
-          throw error
-        }
-
         if (NetworkError.isNetworkErrorCode(error?.code)) {
           throw new NetworkError(error?.code)
         }
-
-        isRetryable = true
-        errorMessage = error.message
       }
 
       if (!isRetryable) {

packages/artifact/src/internal/shared/errors.ts

@@ -57,16 +57,3 @@ export class NetworkError extends Error {
     ].includes(code)
   }
 }
-
-export class UsageError extends Error {
-  constructor() {
-    const message = `Artifact storage quota has been hit. Unable to upload any new artifacts. Usage is recalculated every 6-12 hours.\nMore info on storage limits: https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#calculating-minute-and-storage-spending`
-    super(message)
-    this.name = 'UsageError'
-  }
-
-  static isUsageErrorMessage = (msg?: string): boolean => {
-    if (!msg) return false
-    return msg.includes('insufficient usage')
-  }
-}

packages/artifact/src/internal/upload/blob-upload.ts

@@ -1,11 +1,18 @@
-import {BlobClient, BlockBlobUploadStreamOptions} from '@azure/storage-blob'
+import {
+  AnonymousCredential,
+  BlobClient,
+  BlockBlobUploadStreamOptions,
+  StoragePipelineOptions
+} from '@azure/storage-blob'
 import {TransferProgressEvent} from '@azure/core-http'
 import {ZipUploadStream} from './zip'
 import {getUploadChunkSize, getConcurrency} from '../shared/config'
+import {getProxyUrl} from '@actions/http-client'
 import * as core from '@actions/core'
 import * as crypto from 'crypto'
 import * as stream from 'stream'
 import {NetworkError} from '../shared/errors'
+import {getUserAgentString} from '../shared/user-agent'
 
 export interface BlobUploadResponse {
   /**
@@ -27,7 +34,12 @@ export async function uploadZipToBlobStorage(
 
   const maxConcurrency = getConcurrency()
   const bufferSize = getUploadChunkSize()
-  const blobClient = new BlobClient(authenticatedUploadURL)
+
+  const blobClient = new BlobClient(
+    authenticatedUploadURL,
+    new AnonymousCredential(),
+    getBlobClientOptions(authenticatedUploadURL)
+  )
   const blockBlobClient = blobClient.getBlockBlobClient()
 
   core.debug(
@@ -85,3 +97,37 @@ export async function uploadZipToBlobStorage(
     sha256Hash
   }
 }
+
+export function getBlobClientOptions(sasURL: string): StoragePipelineOptions {
+  const options: StoragePipelineOptions = {
+    userAgentOptions: {
+      userAgentPrefix: getUserAgentString()
+    }
+  }
+
+  const proxyUrl = getProxyUrl(sasURL)
+  if (proxyUrl !== '') {
+    const {
+      port: portString,
+      hostname: host,
+      username,
+      password,
+      protocol
+    } = new URL(proxyUrl)
+    core.debug(`Using proxy server for blob storage upload, host: ${host}`)
+
+    let port = protocol === 'https:' ? 443 : 80
+    if (portString !== '') {
+      port = parseInt(portString)
+    }
+
+    options.proxyOptions = {
+      host,
+      port,
+      username,
+      password
+    }
+  }
+
+  return options
+}