Compare commits

..
Author SHA1 Message Date
konradpabjan d0c9131261 Run npm format 2021-12-06 15:54:15 -05:00
konradpabjan cfba9377f8 More detailed information for chunked uploads 2021-12-06 15:44:12 -05:00
16 changed files with 6450 additions and 321 deletions
+4 -11
View File
@@ -44,27 +44,24 @@ jobs:
npm ci
npm run tsc
working-directory: packages/artifact
- name: Set artifact file contents
shell: bash
run: |
echo "non-gzip-artifact-content=hello" >> $GITHUB_ENV
echo "gzip-artifact-content=Some large amount of text that has a compression ratio that is greater than 100%. If greater than 100%, gzip is used to upload the file" >> $GITHUB_ENV
echo "empty-artifact-content=_EMPTY_" >> $GITHUB_ENV
- name: Create files that will be uploaded
run: |
mkdir artifact-path
mkdir artifact-path
echo ${{ env.non-gzip-artifact-content }} > artifact-path/world.txt
echo ${{ env.gzip-artifact-content }} > artifact-path/gzip.txt
touch artifact-path/empty.txt
# We're using node -e to call the functions directly available in the @actions/artifact package
- name: Upload artifacts using uploadArtifact()
run: |
node -e "Promise.resolve(require('./packages/artifact/lib/artifact-client').create().uploadArtifact('my-artifact-1',['artifact-path/world.txt'], '${{ github.workspace }}'))"
node -e "Promise.resolve(require('./packages/artifact/lib/artifact-client').create().uploadArtifact('my-artifact-2',['artifact-path/gzip.txt'], '${{ github.workspace }}'))"
node -e "Promise.resolve(require('./packages/artifact/lib/artifact-client').create().uploadArtifact('my-artifact-3',['artifact-path/empty.txt'], '${{ github.workspace }}'))"
- name: Download artifacts using downloadArtifact()
run: |
@@ -72,15 +69,12 @@ jobs:
node -e "Promise.resolve(require('./packages/artifact/lib/artifact-client').create().downloadArtifact('my-artifact-1','artifact-1-directory'))"
mkdir artifact-2-directory
node -e "Promise.resolve(require('./packages/artifact/lib/artifact-client').create().downloadArtifact('my-artifact-2','artifact-2-directory'))"
mkdir artifact-3-directory
node -e "Promise.resolve(require('./packages/artifact/lib/artifact-client').create().downloadArtifact('my-artifact-3','artifact-3-directory'))"
- name: Verify downloadArtifact()
shell: bash
run: |
packages/artifact/__tests__/test-artifact-file.sh "artifact-1-directory/artifact-path/world.txt" "${{ env.non-gzip-artifact-content }}"
packages/artifact/__tests__/test-artifact-file.sh "artifact-2-directory/artifact-path/gzip.txt" "${{ env.gzip-artifact-content }}"
packages/artifact/__tests__/test-artifact-file.sh "artifact-3-directory/artifact-path/empty.txt" "${{ env.empty-artifact-content }}"
- name: Download artifacts using downloadAllArtifacts()
run: |
@@ -91,5 +85,4 @@ jobs:
shell: bash
run: |
packages/artifact/__tests__/test-artifact-file.sh "multi-artifact-directory/my-artifact-1/artifact-path/world.txt" "${{ env.non-gzip-artifact-content }}"
packages/artifact/__tests__/test-artifact-file.sh "multi-artifact-directory/my-artifact-2/artifact-path/gzip.txt" "${{ env.gzip-artifact-content }}"
packages/artifact/__tests__/test-artifact-file.sh "multi-artifact-directory/my-artifact-3/artifact-path/empty.txt" "${{ env.empty-artifact-content }}"
packages/artifact/__tests__/test-artifact-file.sh "multi-artifact-directory/my-artifact-2/artifact-path/gzip.txt" "${{ env.gzip-artifact-content }}"
+2 -3
View File
@@ -31,10 +31,9 @@ jobs:
- name: Bootstrap
run: npm run bootstrap
# commenting out until https://github.com/lerna/lerna/issues/3010 is fixed
#- name: audit tools
- name: audit tools
# `|| npm audit` to pretty-print the output if vulnerabilies are found after filtering.
# run: npm audit --audit-level=moderate --json | scripts/audit-allow-list || npm audit --audit-level=moderate
run: npm audit --audit-level=moderate --json | scripts/audit-allow-list || npm audit --audit-level=moderate
- name: audit packages
run: npm run audit-all
-5
View File
@@ -18,11 +18,6 @@ jobs:
- name: verify package exists
run: ls packages/${{ github.event.inputs.package }}
- name: Set Node.js 12.x
uses: actions/setup-node@v1
with:
node-version: 12.x
- name: npm install
run: npm install
+2 -2
View File
@@ -1,4 +1,4 @@
* @actions/actions-runtime
/packages/artifact/ @actions/artifacts-actions
/packages/cache/ @actions/actions-cache
/packages/artifact/ @actions/actions-service
/packages/cache/ @actions/actions-service
+6428 -171
View File
File diff suppressed because it is too large Load Diff
+1 -14
View File
@@ -60,17 +60,4 @@
### 0.5.2
- Add HTTP 500 as a retryable status code for artifact upload and download.
### 0.6.0
- Support upload from named pipes [#748](https://github.com/actions/toolkit/pull/748)
- Fixes to percentage values being greater than 100% when downloading all artifacts [#889](https://github.com/actions/toolkit/pull/889)
- Improved logging and output during artifact upload [#949](https://github.com/actions/toolkit/pull/949)
- Improvements to client-side validation for certain invalid characters not allowed during upload: [#951](https://github.com/actions/toolkit/pull/951)
- Faster upload speeds for certain types of large files by exempting gzip compression [#956](https://github.com/actions/toolkit/pull/956)
- More detailed logging when dealing with chunked uploads [#957](https://github.com/actions/toolkit/pull/957)
### 0.6.1
- Fix for failing 0 byte file uploads on Windows [#962](https://github.com/actions/toolkit/pull/962)
- Add HTTP 500 as a retryable status code for artifact upload and download.
@@ -18,10 +18,8 @@ if [ ! -f "$path" ]; then
exit 1
fi
actualContent=$(cat "$path")
if [ "$expectedContent" == "_EMPTY_" ] && [ ! -s "$path" ]; then
exit 0
elif [ "$actualContent" != "$expectedContent" ]; then
actualContent=$(cat $path)
if [ "$actualContent" != "$expectedContent" ];then
echo "File contents are not correct, expected $expectedContent, received $actualContent"
exit 1
fi
@@ -1,79 +0,0 @@
import * as core from '@actions/core'
import * as tmp from 'tmp-promise'
import * as path from 'path'
import * as io from '../../io/src/io'
import {promises as fs} from 'fs'
import {createGZipFileOnDisk} from '../src/internal/upload-gzip'
const root = path.join(__dirname, '_temp', 'upload-gzip')
const tempGzipFilePath = path.join(root, 'file1.gzip')
const tempZipFilePath = path.join(root, 'file2.zip')
const tempTarlzFilePath = path.join(root, 'file3.tar.lz')
const tempGzFilePath = path.join(root, 'file4.tar.gz')
const tempBz2FilePath = path.join(root, 'file5.tar.bz2')
const temp7zFilePath = path.join(root, 'file6.7z')
const tempNormalFilePath = path.join(root, 'file6.txt')
jest.mock('../src/internal/config-variables')
beforeAll(async () => {
// mock all output so that there is less noise when running tests
jest.spyOn(console, 'log').mockImplementation(() => {})
jest.spyOn(core, 'debug').mockImplementation(() => {})
jest.spyOn(core, 'info').mockImplementation(() => {})
jest.spyOn(core, 'warning').mockImplementation(() => {})
jest.spyOn(core, 'error').mockImplementation(() => {})
// clear temp directory and create files that will be "uploaded"
await io.rmRF(root)
await fs.mkdir(path.join(root))
await fs.writeFile(tempGzipFilePath, 'a file with a .gzip file extension')
await fs.writeFile(tempZipFilePath, 'a file with a .zip file extension')
await fs.writeFile(tempTarlzFilePath, 'a file with a tar.lz file extension')
await fs.writeFile(tempGzFilePath, 'a file with a gz file file extension')
await fs.writeFile(tempBz2FilePath, 'a file with a .bz2 file extension')
await fs.writeFile(temp7zFilePath, 'a file with a .7z file extension')
await fs.writeFile(tempNormalFilePath, 'a file with a .txt file extension')
})
test('Number.MAX_SAFE_INTEGER is returned when an existing compressed file is used', async () => {
// create temporary file
const tempFile = await tmp.file()
expect(await createGZipFileOnDisk(tempGzipFilePath, tempFile.path)).toEqual(
Number.MAX_SAFE_INTEGER
)
expect(await createGZipFileOnDisk(tempZipFilePath, tempFile.path)).toEqual(
Number.MAX_SAFE_INTEGER
)
expect(await createGZipFileOnDisk(tempTarlzFilePath, tempFile.path)).toEqual(
Number.MAX_SAFE_INTEGER
)
expect(await createGZipFileOnDisk(tempGzFilePath, tempFile.path)).toEqual(
Number.MAX_SAFE_INTEGER
)
expect(await createGZipFileOnDisk(tempBz2FilePath, tempFile.path)).toEqual(
Number.MAX_SAFE_INTEGER
)
expect(await createGZipFileOnDisk(temp7zFilePath, tempFile.path)).toEqual(
Number.MAX_SAFE_INTEGER
)
expect(
await createGZipFileOnDisk(tempNormalFilePath, tempFile.path)
).not.toEqual(Number.MAX_SAFE_INTEGER)
})
test('gzip file on disk gets successfully created', async () => {
// create temporary file
const tempFile = await tmp.file()
const gzipFileSize = await createGZipFileOnDisk(
tempNormalFilePath,
tempFile.path
)
const fileStat = await fs.stat(tempNormalFilePath)
const totalFileSize = fileStat.size
// original file and gzip file should not be equal in size
expect(gzipFileSize).not.toEqual(totalFileSize)
})
+1 -4
View File
@@ -181,10 +181,7 @@ describe('Upload Tests', () => {
function hasMkfifo(): boolean {
try {
// make sure we drain the stdout
return (
process.platform !== 'win32' &&
execSync('which mkfifo').toString().length > 0
)
return execSync('which mkfifo').toString().length > 0
} catch (e) {
return false
}
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@actions/artifact",
"version": "0.6.1",
"version": "0.5.2",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@actions/artifact",
"version": "0.6.1",
"version": "0.5.2",
"preview": true,
"description": "Actions artifact lib",
"keywords": [
@@ -240,9 +240,6 @@ Note: The size of downloaded zips can differ significantly from the reported siz
while (downloadedArtifacts < artifacts.count) {
const currentArtifactToDownload = artifacts.value[downloadedArtifacts]
downloadedArtifacts += 1
core.info(
`starting download of artifact ${currentArtifactToDownload.name} : ${downloadedArtifacts}/${artifacts.count}`
)
// Get container entries for the specific artifact
const items = await downloadHttpClient.getContainerItems(
@@ -228,6 +228,9 @@ export class DownloadHttpClient {
let response: IHttpClientResponse
try {
response = await makeDownloadRequest()
if (core.isDebug()) {
displayHttpDiagnostics(response)
}
} catch (error) {
// if an error is caught, it is usually indicative of a timeout so retry the download
core.info('An error occurred while attempting to download a file')
@@ -57,7 +57,7 @@ export class StatusReporter {
`Uploaded ${fileName} (${percentage.slice(
0,
percentage.indexOf('.') + 2
)}%) bytes ${chunkStartIndex}:${chunkEndIndex}`
)}%) chunks ${chunkStartIndex}:${chunkEndIndex}`
)
}
@@ -3,20 +3,6 @@ import * as zlib from 'zlib'
import {promisify} from 'util'
const stat = promisify(fs.stat)
/**
* GZipping certain files that are already compressed will likely not yield further size reductions. Creating large temporary gzip
* files then will just waste a lot of time before ultimately being discarded (especially for very large files).
* If any of these types of files are encountered then on-disk gzip creation will be skipped and the original file will be uploaded as-is
*/
const gzipExemptFileExtensions = [
'.gzip',
'.zip',
'.tar.lz',
'.tar.gz',
'.tar.bz2',
'.7z'
]
/**
* Creates a Gzip compressed file of an original file at the provided temporary filepath location
* @param {string} originalFilePath filepath of whatever will be compressed. The original file will be unmodified
@@ -27,13 +13,6 @@ export async function createGZipFileOnDisk(
originalFilePath: string,
tempFilePath: string
): Promise<number> {
for (const gzipExemptExtension of gzipExemptFileExtensions) {
if (originalFilePath.endsWith(gzipExemptExtension)) {
// return a really large number so that the original file gets uploaded
return Number.MAX_SAFE_INTEGER
}
}
return new Promise((resolve, reject) => {
const inputStream = fs.createReadStream(originalFilePath)
const gzip = zlib.createGzip()
@@ -221,7 +221,10 @@ export class UploadHttpClient {
): Promise<UploadFileResult> {
const fileStat: fs.Stats = await stat(parameters.file)
const totalFileSize = fileStat.size
const isFIFO = fileStat.isFIFO()
// on Windows with mkfifo from MSYS2 stats.isFIFO returns false, so we check if running on Windows node and
// if the file has size of 0 to compensate
const isFIFO =
fileStat.isFIFO() || (process.platform === 'win32' && totalFileSize === 0)
let offset = 0
let isUploadSuccessful = true
let failedChunkSizes = 0