Compare commits

..

65 Commits

Author SHA1 Message Date
Sampark Sharma 2d3c79e6fe Address review comments 2022-12-12 12:18:07 +00:00
Sampark Sharma e7e19845ca Update packages/cache/src/internal/cacheHttpClient.ts
Co-authored-by: Bishal Prasad <bishal-pdmsft@github.com>
2022-12-12 17:44:34 +05:30
Sampark Sharma cad074ceef Add better comments 2022-12-12 10:58:58 +00:00
Sampark Sharma d7ae8cd1ef Fix tests 2022-12-12 07:51:54 +00:00
Sampark Sharma 6bc5dc5a23 Fix test 2022-12-12 07:25:36 +00:00
Sampark Sharma bbf5659dfa Fix test 2022-12-12 07:04:15 +00:00
Sampark Sharma d175a181a0 Add end to end test for cache using bsd on windows
and address review comments
2022-12-12 06:53:11 +00:00
Sampark Sharma 0690c10515 Fix test 2022-12-09 11:47:00 +00:00
Sampark Sharma 0c23c38c68 Add debug logging for gzip fall back 2022-12-09 11:10:28 +00:00
Sampark Sharma 7a532d03f4 Reconfigure catch block 2022-12-09 11:10:18 +00:00
Sampark Sharma d31c2dd88d Fix issues 2022-12-09 11:09:54 +00:00
Sampark Sharma cb3dc49fec Merge pull request #1249 from actions/phantsure/gzip-fallback
Add fallback to gzip compression if cache not found
2022-12-08 10:59:17 +05:30
Sampark Sharma 27f9a7d461 Revert Address review comments 2022-12-07 08:41:56 +00:00
Sampark Sharma d79a09bc0e Address review comments 2022-12-07 08:27:08 +00:00
Sampark Sharma d1094e1523 Add test 2022-12-07 07:39:41 +00:00
Sampark Sharma c0085d7739 Fix test 2022-12-06 11:52:46 +00:00
Sampark Sharma 1dae855746 Add fallback to gzip compression if cache not found 2022-12-06 11:38:17 +00:00
Sampark Sharma de8fda6976 Merge pull request #1244 from actions/phantsure/release-beta-1
Update for beta release of cache package
2022-12-05 13:01:30 +05:30
Sampark Sharma 61e630822a Fix audit issues 2022-12-05 06:09:05 +00:00
Sampark Sharma c207fbd5db Update for beta release 2022-12-01 10:08:48 +00:00
Sampark Sharma 785599dd2e Merge pull request #1237 from actions/pdotl/win-bsdtar-zstd
Bsdtar + zstd fallback to prevent zstd hanging issue on windows
2022-12-01 12:14:55 +05:30
Sampark Sharma 8595831452 Fix test 2022-11-30 10:37:32 +00:00
Sampark Sharma afbc5c0a9e Add await to async function calls 2022-11-30 06:16:19 +00:00
Sampark Sharma 424ae62ee7 Fix tar test 2022-11-30 06:05:34 +00:00
Sampark Sharma 2e5a517460 Fix test 2022-11-29 12:16:17 +00:00
Sampark Sharma 34f0143be2 Address review comments 2022-11-29 10:35:22 +00:00
Sampark Sharma 0fd856d0a0 Refactor code 2022-11-28 10:24:40 +00:00
Sampark Sharma 0822441ee0 Fix lint test 2022-11-23 11:51:46 +00:00
Sampark Sharma 187781e273 Fix tests 2022-11-23 11:40:11 +00:00
Sampark Sharma 1f3371766a Add new tests 2022-11-23 11:20:24 +00:00
Sampark Sharma 39b7a86b23 Fix old tests 2022-11-23 07:58:46 +00:00
Sampark Sharma 2f73afa843 Separate args 2022-11-23 07:39:15 +00:00
Sampark Sharma ffde3e4bd5 Fix test 2022-11-22 07:33:01 +00:00
Sampark Sharma 32f538163d Fix windows test 2022-11-21 12:51:07 +00:00
Sampark Sharma 32b95825ba Add windows bsdtar test 2022-11-21 12:19:44 +00:00
Sampark Sharma 54eb9b8055 Address some comments and correct compression commands 2022-11-21 12:05:03 +00:00
Lovepreet Singh f9dfb05bd2 Temporarily remove thhe condition that prevents zstd usage on windows unless with GNUtar 2022-11-17 12:13:49 +00:00
Lovepreet Singh b3bd482c0f Fix windows gnutar test case 2022-11-17 09:46:01 +00:00
Lovepreet Singh 4fa5b7d133 Fix lint issues 2022-11-17 09:12:53 +00:00
Lovepreet Singh ea9856079f Fix tar tests 2022-11-17 08:53:46 +00:00
Sampark Sharma caadeee090 Merge pull request #1239 from actions/phantsure/gnutar_windows
Add GNUtar as default on windows
2022-11-17 13:12:02 +05:30
Sampark Sharma 964682b5d4 Fix test 2022-11-17 07:34:07 +00:00
Sampark Sharma 6f7397feb6 Add GNUtar as default on windows 2022-11-17 07:26:52 +00:00
Sampark Sharma 819157bf87 Merge pull request #1238 from actions/revert-1232-phantsure/gnutar_windows
Revert "Add GNUtar as default in windows"
2022-11-17 12:42:10 +05:30
Sampark Sharma 86102e88e9 Revert "Add GNUtar as default in windows" 2022-11-17 12:26:27 +05:30
Lovepreet Singh 5f89653f1b Fix order of args for tar 2022-11-17 06:22:13 +00:00
Lovepreet Singh e9e146bbf8 Add -v option for testing 2022-11-16 21:04:25 +00:00
Lovepreet Singh 27bf8304ea Fix tar operations 2022-11-16 20:50:04 +00:00
Lovepreet Singh db3517fe3b bsd + zstd fallback implementation 2022-11-16 20:21:40 +00:00
Lovepreet Singh 6349c3ca3a bsd + zstd fallback implementation 2022-11-16 20:18:31 +00:00
Sampark Sharma e4c071ba19 Merge pull request #1232 from actions/phantsure/gnutar_windows
Add GNUtar as default in windows
2022-11-16 16:55:59 +05:30
Sampark Sharma 9443e26349 Address review comments 2022-11-15 10:18:46 +00:00
Sampark Sharma 5a0405df4e Fix tests 2022-11-15 08:48:29 +00:00
Sampark Sharma 0e707aeabc Address comments 2022-11-15 08:41:04 +00:00
Sampark Sharma 7441dc5e59 Fix tests 2022-11-14 10:25:17 +00:00
Sampark Sharma 1ddac5e02f Fix tests 2022-11-14 10:11:46 +00:00
Sampark Sharma 55484166d8 Address review comments 2022-11-14 08:12:15 +00:00
Sampark Sharma 7181b913f5 Remove export 2022-11-14 06:34:18 +00:00
Sampark Sharma 81cd5a5c2e Revert "Add logs for cache version on miss"
This reverts commit 4b348086a9.
2022-11-14 06:33:53 +00:00
Sampark Sharma 4b348086a9 Add logs for cache version on miss 2022-11-14 05:24:09 +00:00
Sampark Sharma 436cf8d6ea Fix tests 2022-11-10 15:50:06 +00:00
Sampark Sharma e0aadb573c Fix GNUtar path 2022-11-08 08:29:31 +00:00
Sampark Sharma 62a66a8ce9 Fix tests 2022-11-08 06:43:28 +00:00
Sampark Sharma 0c58e4113e Fix gnutar check on windows 2022-11-07 13:43:54 +00:00
Sampark Sharma 9366237c90 Add gnuTar as default in windows 2022-11-07 13:39:07 +00:00
30 changed files with 2244 additions and 3060 deletions
+9 -16
View File
@@ -22,12 +22,12 @@ jobs:
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v2
- name: Set Node.js 16.x - name: Set Node.js 12.x
uses: actions/setup-node@v3 uses: actions/setup-node@v1
with: with:
node-version: 16.x node-version: 12.x
# In order to upload & download artifacts from a shell script, certain env variables need to be set that are only available in the # In order to upload & download artifacts from a shell script, certain env variables need to be set that are only available in the
# node context. This runs a local action that gets and sets the necessary env variables that are needed # node context. This runs a local action that gets and sets the necessary env variables that are needed
@@ -55,16 +55,16 @@ jobs:
- name: Create files that will be uploaded - name: Create files that will be uploaded
run: | run: |
mkdir artifact-path mkdir artifact-path
echo '${{ env.non-gzip-artifact-content }}' > artifact-path/world.txt echo ${{ env.non-gzip-artifact-content }} > artifact-path/world.txt
echo '${{ env.gzip-artifact-content }}' > artifact-path/gzip.txt echo ${{ env.gzip-artifact-content }} > artifact-path/gzip.txt
touch artifact-path/empty.txt touch artifact-path/empty.txt
# We're using node -e to call the functions directly available in the @actions/artifact package # We're using node -e to call the functions directly available in the @actions/artifact package
- name: Upload artifacts using uploadArtifact() - name: Upload artifacts using uploadArtifact()
run: | run: |
node -e "Promise.resolve(require('./packages/artifact/lib/artifact-client').create().uploadArtifact('my-artifact-1',['artifact-path/world.txt'], process.argv[1]))" "${{ github.workspace }}" node -e "Promise.resolve(require('./packages/artifact/lib/artifact-client').create().uploadArtifact('my-artifact-1',['artifact-path/world.txt'], '${{ github.workspace }}'))"
node -e "Promise.resolve(require('./packages/artifact/lib/artifact-client').create().uploadArtifact('my-artifact-2',['artifact-path/gzip.txt'], process.argv[1]))" "${{ github.workspace }}" node -e "Promise.resolve(require('./packages/artifact/lib/artifact-client').create().uploadArtifact('my-artifact-2',['artifact-path/gzip.txt'], '${{ github.workspace }}'))"
node -e "Promise.resolve(require('./packages/artifact/lib/artifact-client').create().uploadArtifact('my-artifact-3',['artifact-path/empty.txt'], process.argv[1]))" "${{ github.workspace }}" node -e "Promise.resolve(require('./packages/artifact/lib/artifact-client').create().uploadArtifact('my-artifact-3',['artifact-path/empty.txt'], '${{ github.workspace }}'))"
- name: Download artifacts using downloadArtifact() - name: Download artifacts using downloadArtifact()
run: | run: |
@@ -75,13 +75,6 @@ jobs:
mkdir artifact-3-directory mkdir artifact-3-directory
node -e "Promise.resolve(require('./packages/artifact/lib/artifact-client').create().downloadArtifact('my-artifact-3','artifact-3-directory'))" node -e "Promise.resolve(require('./packages/artifact/lib/artifact-client').create().downloadArtifact('my-artifact-3','artifact-3-directory'))"
- name: Add additional logging
shell: bash
run: |
cat ./artifact-1-directory/artifact-path/world.txt
cat ./artifact-2-directory/artifact-path/gzip.txt
cat ./artifact-3-directory/artifact-path/empty.txt
- name: Verify downloadArtifact() - name: Verify downloadArtifact()
shell: bash shell: bash
run: | run: |
+4 -4
View File
@@ -18,12 +18,12 @@ jobs:
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v2
- name: Set Node.js 16.x - name: Set Node.js 12.x
uses: actions/setup-node@v3 uses: actions/setup-node@v1
with: with:
node-version: 16.x node-version: 12.x
- name: npm install - name: npm install
run: npm install run: npm install
+4 -4
View File
@@ -22,12 +22,12 @@ jobs:
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v2
- name: Set Node.js 16.x - name: Set Node.js 12.x
uses: actions/setup-node@v3 uses: actions/setup-node@v1
with: with:
node-version: 16.x node-version: 12.x
# In order to save & restore cache from a shell script, certain env variables need to be set that are only available in the # In order to save & restore cache from a shell script, certain env variables need to be set that are only available in the
# node context. This runs a local action that gets and sets the necessary env variables that are needed # node context. This runs a local action that gets and sets the necessary env variables that are needed
+91
View File
@@ -0,0 +1,91 @@
name: cache-windows-bsd-unit-tests
on:
push:
branches:
- main
paths-ignore:
- '**.md'
pull_request:
paths-ignore:
- '**.md'
jobs:
build:
name: Build
runs-on: windows-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- shell: bash
run: |
rm "C:\Program Files\Git\usr\bin\tar.exe"
- name: Set Node.js 12.x
uses: actions/setup-node@v1
with:
node-version: 12.x
# In order to save & restore cache from a shell script, certain env variables need to be set that are only available in the
# node context. This runs a local action that gets and sets the necessary env variables that are needed
- name: Set env variables
uses: ./packages/cache/__tests__/__fixtures__/
# Need root node_modules because certain npm packages like jest are configured for the entire repository and it won't be possible
# without these to just compile the cache package
- name: Install root npm packages
run: npm ci
- name: Compile cache package
run: |
npm ci
npm run tsc
working-directory: packages/cache
- name: Generate files in working directory
shell: bash
run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} test-cache
- name: Generate files outside working directory
shell: bash
run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} ~/test-cache
# We're using node -e to call the functions directly available in the @actions/cache package
- name: Save cache using saveCache()
run: |
node -e "Promise.resolve(require('./packages/cache/lib/cache').saveCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))"
- name: Delete cache folders before restoring
shell: bash
run: |
rm -rf test-cache
rm -rf ~/test-cache
- name: Restore cache using restoreCache() with http-client
run: |
node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}',[],{useAzureSdk: false}))"
- name: Verify cache restored with http-client
shell: bash
run: |
packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} test-cache
packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache
- name: Delete cache folders before restoring
shell: bash
run: |
rm -rf test-cache
rm -rf ~/test-cache
rm -f cache.tar
- name: Restore cache using restoreCache() with Azure SDK
run: |
node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))"
- name: Verify cache restored with Azure SDK
shell: bash
run: |
packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} test-cache
packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache
+1 -1
View File
@@ -20,7 +20,7 @@ jobs:
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v3 uses: actions/checkout@v2
# Initializes the CodeQL tools for scanning. # Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL - name: Initialize CodeQL
+4 -4
View File
@@ -13,15 +13,15 @@ jobs:
steps: steps:
- name: setup repo - name: setup repo
uses: actions/checkout@v3 uses: actions/checkout@v2
- name: verify package exists - name: verify package exists
run: ls packages/${{ github.event.inputs.package }} run: ls packages/${{ github.event.inputs.package }}
- name: Set Node.js 16.x - name: Set Node.js 12.x
uses: actions/setup-node@v3 uses: actions/setup-node@v1
with: with:
node-version: 16.x node-version: 12.x
- name: npm install - name: npm install
run: npm install run: npm install
+4 -5
View File
@@ -23,13 +23,12 @@ jobs:
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v2
- name: Set Node.js 16.x - name: Set Node.js 12.x
uses: actions/setup-node@v3 uses: actions/setup-node@v1
with: with:
node-version: 16.x node-version: 12.x
- run: npm -v
- name: npm install - name: npm install
run: npm install run: npm install
+1 -1
View File
@@ -9,7 +9,7 @@ jobs:
if: ${{ github.repository_owner == 'actions' }} if: ${{ github.repository_owner == 'actions' }}
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v3 uses: actions/checkout@v2
- name: Update Octokit - name: Update Octokit
working-directory: packages/github working-directory: packages/github
run: | run: |
+1 -1
View File
@@ -32,7 +32,7 @@ jobs:
os: [ubuntu-16.04, windows-2019] os: [ubuntu-16.04, windows-2019]
runs-on: ${{matrix.os}} runs-on: ${{matrix.os}}
actions: actions:
- uses: actions/setup-node@v3 - uses: actions/setup-node@v1
with: with:
version: ${{matrix.node}} version: ${{matrix.node}}
- run: | - run: |
+1 -1
View File
@@ -18,7 +18,7 @@ e.g. To use https://github.com/actions/setup-node, users will author:
```yaml ```yaml
steps: steps:
using: actions/setup-node@v3 using: actions/setup-node@v1
``` ```
# Define Metadata # Define Metadata
+1477 -2657
View File
File diff suppressed because it is too large Load Diff
@@ -1,5 +1,5 @@
name: 'Set env variables' name: 'Set env variables'
description: 'Sets certain env variables so that e2e artifact upload and download can be tested in a shell' description: 'Sets certain env variables so that e2e artifact upload and download can be tested in a shell'
runs: runs:
using: 'node16' using: 'node12'
main: 'index.js' main: 'index.js'
+18 -25
View File
@@ -4,17 +4,17 @@ import * as io from '../../io/src/io'
import * as net from 'net' import * as net from 'net'
import * as path from 'path' import * as path from 'path'
import * as configVariables from '../src/internal/config-variables' import * as configVariables from '../src/internal/config-variables'
import { promises as fs } from 'fs' import {promises as fs} from 'fs'
import { DownloadItem } from '../src/internal/download-specification' import {DownloadItem} from '../src/internal/download-specification'
import { HttpClient, HttpClientResponse } from '@actions/http-client' import {HttpClient, HttpClientResponse} from '@actions/http-client'
import { DownloadHttpClient } from '../src/internal/download-http-client' import {DownloadHttpClient} from '../src/internal/download-http-client'
import { import {
ListArtifactsResponse, ListArtifactsResponse,
QueryArtifactResponse QueryArtifactResponse
} from '../src/internal/contracts' } from '../src/internal/contracts'
import * as stream from 'stream' import * as stream from 'stream'
import { gzip } from 'zlib' import {gzip} from 'zlib'
import { promisify } from 'util' import {promisify} from 'util'
const root = path.join(__dirname, '_temp', 'artifact-download-tests') const root = path.join(__dirname, '_temp', 'artifact-download-tests')
const defaultEncoding = 'utf8' const defaultEncoding = 'utf8'
@@ -24,18 +24,17 @@ jest.mock('@actions/http-client')
describe('Download Tests', () => { describe('Download Tests', () => {
beforeAll(async () => { beforeAll(async () => {
jest.setTimeout(300000);
await io.rmRF(root) await io.rmRF(root)
await fs.mkdir(path.join(root), { await fs.mkdir(path.join(root), {
recursive: true recursive: true
}) })
// mock all output so that there is less noise when running tests // mock all output so that there is less noise when running tests
jest.spyOn(console, 'log').mockImplementation(() => { }) jest.spyOn(console, 'log').mockImplementation(() => {})
jest.spyOn(core, 'debug').mockImplementation(() => { }) jest.spyOn(core, 'debug').mockImplementation(() => {})
jest.spyOn(core, 'info').mockImplementation(() => { }) jest.spyOn(core, 'info').mockImplementation(() => {})
jest.spyOn(core, 'warning').mockImplementation(() => { }) jest.spyOn(core, 'warning').mockImplementation(() => {})
jest.spyOn(core, 'error').mockImplementation(() => { }) jest.spyOn(core, 'error').mockImplementation(() => {})
}) })
/** /**
@@ -164,20 +163,14 @@ describe('Download Tests', () => {
await checkDestinationFile(targetPath, fileContents) await checkDestinationFile(targetPath, fileContents)
}) })
it.each([ it('Test retryable status codes during artifact download', async () => {
429,
500,
502,
503,
504
])('Test retryable status code %p during artifact download', async (scode: number) => {
// The first http response should return a retryable status call while the subsequent call should return a 200 so // The first http response should return a retryable status call while the subsequent call should return a 200 so
// the download should successfully finish // the download should successfully finish
const retryableStatusCodes = [scode] const retryableStatusCodes = [429, 500, 502, 503, 504]
for (const statusCode of retryableStatusCodes) { for (const statusCode of retryableStatusCodes) {
const fileContents = Buffer.from('try, try again\n', defaultEncoding) const fileContents = Buffer.from('try, try again\n', defaultEncoding)
const targetPath = path.join(root, `AAAFileC-${statusCode}.txt`) const targetPath = path.join(root, `FileC-${statusCode}.txt`)
setupDownloadItemResponse(fileContents, false, statusCode, false, true) setupDownloadItemResponse(fileContents, false, statusCode, false, true)
const downloadHttpClient = new DownloadHttpClient() const downloadHttpClient = new DownloadHttpClient()
@@ -193,7 +186,7 @@ describe('Download Tests', () => {
await checkDestinationFile(targetPath, fileContents) await checkDestinationFile(targetPath, fileContents)
} }
}); })
it('Test retry on truncated response with gzip', async () => { it('Test retry on truncated response with gzip', async () => {
const fileContents = Buffer.from( const fileContents = Buffer.from(
@@ -283,7 +276,7 @@ describe('Download Tests', () => {
] ]
} }
const returnData: string = JSON.stringify(response, null, 2) const returnData: string = JSON.stringify(response, null, 2)
mockReadBody = async function (): Promise<string> { mockReadBody = async function(): Promise<string> {
return new Promise(resolve => { return new Promise(resolve => {
resolve(returnData) resolve(returnData)
}) })
@@ -454,7 +447,7 @@ describe('Download Tests', () => {
] ]
} }
const returnData: string = JSON.stringify(response, null, 2) const returnData: string = JSON.stringify(response, null, 2)
mockReadBody = async function (): Promise<string> { mockReadBody = async function(): Promise<string> {
return new Promise(resolve => { return new Promise(resolve => {
resolve(returnData) resolve(returnData)
}) })
+27 -27
View File
@@ -9,7 +9,7 @@
"version": "1.1.0", "version": "1.1.0",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@actions/core": "^1.10.0", "@actions/core": "^1.9.1",
"@actions/http-client": "^2.0.1", "@actions/http-client": "^2.0.1",
"tmp": "^0.2.1", "tmp": "^0.2.1",
"tmp-promise": "^3.0.2" "tmp-promise": "^3.0.2"
@@ -20,9 +20,9 @@
} }
}, },
"node_modules/@actions/core": { "node_modules/@actions/core": {
"version": "1.10.0", "version": "1.9.1",
"resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.0.tgz", "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.9.1.tgz",
"integrity": "sha512-2aZDDa3zrrZbP5ZYg159sNoLRb61nQ7awl5pSvIq5Qpj81vwDzdMRKzkWJGJuwVvWpvZKx7vspJALyvaaIQyug==", "integrity": "sha512-5ad+U2YGrmmiw6du20AQW5XuWo7UKN2052FjSV7MX+Wfjf8sCqcsZe62NfgHys4QI4/Y+vQvLKYL8jWtA1ZBTA==",
"dependencies": { "dependencies": {
"@actions/http-client": "^2.0.1", "@actions/http-client": "^2.0.1",
"uuid": "^8.3.2" "uuid": "^8.3.2"
@@ -59,22 +59,22 @@
"node_modules/concat-map": { "node_modules/concat-map": {
"version": "0.0.1", "version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
}, },
"node_modules/fs.realpath": { "node_modules/fs.realpath": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
}, },
"node_modules/glob": { "node_modules/glob": {
"version": "7.2.3", "version": "7.2.0",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz",
"integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==",
"dependencies": { "dependencies": {
"fs.realpath": "^1.0.0", "fs.realpath": "^1.0.0",
"inflight": "^1.0.4", "inflight": "^1.0.4",
"inherits": "2", "inherits": "2",
"minimatch": "^3.1.1", "minimatch": "^3.0.4",
"once": "^1.3.0", "once": "^1.3.0",
"path-is-absolute": "^1.0.0" "path-is-absolute": "^1.0.0"
}, },
@@ -88,7 +88,7 @@
"node_modules/inflight": { "node_modules/inflight": {
"version": "1.0.6", "version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
"integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
"dependencies": { "dependencies": {
"once": "^1.3.0", "once": "^1.3.0",
"wrappy": "1" "wrappy": "1"
@@ -113,7 +113,7 @@
"node_modules/once": { "node_modules/once": {
"version": "1.4.0", "version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"dependencies": { "dependencies": {
"wrappy": "1" "wrappy": "1"
} }
@@ -121,7 +121,7 @@
"node_modules/path-is-absolute": { "node_modules/path-is-absolute": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
"engines": { "engines": {
"node": ">=0.10.0" "node": ">=0.10.0"
} }
@@ -191,14 +191,14 @@
"node_modules/wrappy": { "node_modules/wrappy": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
} }
}, },
"dependencies": { "dependencies": {
"@actions/core": { "@actions/core": {
"version": "1.10.0", "version": "1.9.1",
"resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.0.tgz", "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.9.1.tgz",
"integrity": "sha512-2aZDDa3zrrZbP5ZYg159sNoLRb61nQ7awl5pSvIq5Qpj81vwDzdMRKzkWJGJuwVvWpvZKx7vspJALyvaaIQyug==", "integrity": "sha512-5ad+U2YGrmmiw6du20AQW5XuWo7UKN2052FjSV7MX+Wfjf8sCqcsZe62NfgHys4QI4/Y+vQvLKYL8jWtA1ZBTA==",
"requires": { "requires": {
"@actions/http-client": "^2.0.1", "@actions/http-client": "^2.0.1",
"uuid": "^8.3.2" "uuid": "^8.3.2"
@@ -235,22 +235,22 @@
"concat-map": { "concat-map": {
"version": "0.0.1", "version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
}, },
"fs.realpath": { "fs.realpath": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
}, },
"glob": { "glob": {
"version": "7.2.3", "version": "7.2.0",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz",
"integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==",
"requires": { "requires": {
"fs.realpath": "^1.0.0", "fs.realpath": "^1.0.0",
"inflight": "^1.0.4", "inflight": "^1.0.4",
"inherits": "2", "inherits": "2",
"minimatch": "^3.1.1", "minimatch": "^3.0.4",
"once": "^1.3.0", "once": "^1.3.0",
"path-is-absolute": "^1.0.0" "path-is-absolute": "^1.0.0"
} }
@@ -258,7 +258,7 @@
"inflight": { "inflight": {
"version": "1.0.6", "version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
"integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
"requires": { "requires": {
"once": "^1.3.0", "once": "^1.3.0",
"wrappy": "1" "wrappy": "1"
@@ -280,7 +280,7 @@
"once": { "once": {
"version": "1.4.0", "version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"requires": { "requires": {
"wrappy": "1" "wrappy": "1"
} }
@@ -288,7 +288,7 @@
"path-is-absolute": { "path-is-absolute": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
}, },
"rimraf": { "rimraf": {
"version": "3.0.2", "version": "3.0.2",
@@ -333,7 +333,7 @@
"wrappy": { "wrappy": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
} }
} }
} }
+1 -1
View File
@@ -37,7 +37,7 @@
"url": "https://github.com/actions/toolkit/issues" "url": "https://github.com/actions/toolkit/issues"
}, },
"dependencies": { "dependencies": {
"@actions/core": "^1.10.0", "@actions/core": "^1.9.1",
"@actions/http-client": "^2.0.1", "@actions/http-client": "^2.0.1",
"tmp": "^0.2.1", "tmp": "^0.2.1",
"tmp-promise": "^3.0.2" "tmp-promise": "^3.0.2"
@@ -21,7 +21,7 @@ export function getUploadSpecification(
): UploadSpecification[] { ): UploadSpecification[] {
// artifact name was checked earlier on, no need to check again // artifact name was checked earlier on, no need to check again
const specifications: UploadSpecification[] = [] const specifications: UploadSpecification[] = []
debug(rootDirectory)
if (!fs.existsSync(rootDirectory)) { if (!fs.existsSync(rootDirectory)) {
throw new Error(`Provided rootDirectory ${rootDirectory} does not exist`) throw new Error(`Provided rootDirectory ${rootDirectory} does not exist`)
} }
+3
View File
@@ -91,3 +91,6 @@
### 3.0.6 ### 3.0.6
- Added `@azure/abort-controller` to dependencies to fix compatibility issue with ESM [#1208](https://github.com/actions/toolkit/issues/1208) - Added `@azure/abort-controller` to dependencies to fix compatibility issue with ESM [#1208](https://github.com/actions/toolkit/issues/1208)
### 3.1.0-beta.1
- Update actions/cache on windows to use gnu tar and zstd by default and fallback to bsdtar and zstd if gnu tar is not available. ([issue](https://github.com/actions/cache/issues/984))
+1 -1
View File
@@ -1,5 +1,5 @@
name: 'Set env variables' name: 'Set env variables'
description: 'Sets certain env variables so that e2e restore and save cache can be tested in a shell' description: 'Sets certain env variables so that e2e restore and save cache can be tested in a shell'
runs: runs:
using: 'node16' using: 'node12'
main: 'index.js' main: 'index.js'
+75
View File
@@ -161,6 +161,81 @@ test('restore with gzip compressed cache found', async () => {
expect(getCompressionMock).toHaveBeenCalledTimes(1) expect(getCompressionMock).toHaveBeenCalledTimes(1)
}) })
test('restore with zstd as default but gzip compressed cache found on windows', async () => {
if (process.platform === 'win32') {
const paths = ['node_modules']
const key = 'node-test'
const cacheEntry: ArtifactCacheEntry = {
cacheKey: key,
scope: 'refs/heads/main',
archiveLocation: 'www.actionscache.test/download'
}
const getCacheMock = jest.spyOn(cacheHttpClient, 'getCacheEntry')
getCacheMock
.mockImplementationOnce(async () => {
return Promise.resolve(null)
})
.mockImplementationOnce(async () => {
return Promise.resolve(cacheEntry)
})
const tempPath = '/foo/bar'
const createTempDirectoryMock = jest.spyOn(
cacheUtils,
'createTempDirectory'
)
createTempDirectoryMock.mockImplementation(async () => {
return Promise.resolve(tempPath)
})
const archivePath = path.join(tempPath, CacheFilename.Gzip)
const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache')
const fileSize = 142
const getArchiveFileSizeInBytesMock = jest
.spyOn(cacheUtils, 'getArchiveFileSizeInBytes')
.mockReturnValue(fileSize)
const extractTarMock = jest.spyOn(tar, 'extractTar')
const unlinkFileMock = jest.spyOn(cacheUtils, 'unlinkFile')
const compression = CompressionMethod.Zstd
const getCompressionMock = jest
.spyOn(cacheUtils, 'getCompressionMethod')
.mockReturnValue(Promise.resolve(compression))
const cacheKey = await restoreCache(paths, key)
expect(cacheKey).toBe(key)
expect(getCacheMock).toHaveBeenNthCalledWith(1, [key], paths, {
compressionMethod: compression
})
expect(getCacheMock).toHaveBeenNthCalledWith(2, [key], paths, {
compressionMethod: CompressionMethod.Gzip
})
expect(createTempDirectoryMock).toHaveBeenCalledTimes(1)
expect(downloadCacheMock).toHaveBeenCalledWith(
cacheEntry.archiveLocation,
archivePath,
undefined
)
expect(getArchiveFileSizeInBytesMock).toHaveBeenCalledWith(archivePath)
expect(extractTarMock).toHaveBeenCalledTimes(1)
expect(extractTarMock).toHaveBeenCalledWith(
archivePath,
CompressionMethod.Gzip
)
expect(unlinkFileMock).toHaveBeenCalledTimes(1)
expect(unlinkFileMock).toHaveBeenCalledWith(archivePath)
expect(getCompressionMock).toHaveBeenCalledTimes(1)
}
})
test('restore with zstd compressed cache found', async () => { test('restore with zstd compressed cache found', async () => {
const paths = ['node_modules'] const paths = ['node_modules']
const key = 'node-test' const key = 'node-test'
+224 -49
View File
@@ -1,7 +1,14 @@
import * as exec from '@actions/exec' import * as exec from '@actions/exec'
import * as io from '@actions/io' import * as io from '@actions/io'
import * as path from 'path' import * as path from 'path'
import {CacheFilename, CompressionMethod} from '../src/internal/constants' import {
CacheFilename,
CompressionMethod,
GnuTarPathOnWindows,
ManifestFilename,
SystemTarPathOnWindows,
TarFilename
} from '../src/internal/constants'
import * as tar from '../src/internal/tar' import * as tar from '../src/internal/tar'
import * as utils from '../src/internal/cacheUtils' import * as utils from '../src/internal/cacheUtils'
// eslint-disable-next-line @typescript-eslint/no-require-imports // eslint-disable-next-line @typescript-eslint/no-require-imports
@@ -13,7 +20,7 @@ jest.mock('@actions/io')
const IS_WINDOWS = process.platform === 'win32' const IS_WINDOWS = process.platform === 'win32'
const IS_MAC = process.platform === 'darwin' const IS_MAC = process.platform === 'darwin'
const defaultTarPath = process.platform === 'darwin' ? 'gtar' : 'tar' const defaultTarPath = IS_MAC ? 'gtar' : 'tar'
function getTempDir(): string { function getTempDir(): string {
return path.join(__dirname, '_temp', 'tar') return path.join(__dirname, '_temp', 'tar')
@@ -28,6 +35,10 @@ beforeAll(async () => {
await jest.requireActual('@actions/io').rmRF(getTempDir()) await jest.requireActual('@actions/io').rmRF(getTempDir())
}) })
beforeEach(async () => {
jest.restoreAllMocks()
})
afterAll(async () => { afterAll(async () => {
delete process.env['GITHUB_WORKSPACE'] delete process.env['GITHUB_WORKSPACE']
await jest.requireActual('@actions/io').rmRF(getTempDir()) await jest.requireActual('@actions/io').rmRF(getTempDir())
@@ -41,16 +52,15 @@ test('zstd extract tar', async () => {
? `${process.env['windir']}\\fakepath\\cache.tar` ? `${process.env['windir']}\\fakepath\\cache.tar`
: 'cache.tar' : 'cache.tar'
const workspace = process.env['GITHUB_WORKSPACE'] const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
await tar.extractTar(archivePath, CompressionMethod.Zstd) await tar.extractTar(archivePath, CompressionMethod.Zstd)
expect(mkdirMock).toHaveBeenCalledWith(workspace) expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${defaultTarPath}"`,
[ [
'--use-compress-program', `"${tarPath}"`,
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
'-xf', '-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P', '-P',
@@ -58,11 +68,61 @@ test('zstd extract tar', async () => {
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
] ]
.concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []), .concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
])
.join(' '),
undefined,
{cwd: undefined} {cwd: undefined}
) )
}) })
test('zstd extract tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const mkdirMock = jest.spyOn(io, 'mkdirP')
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = SystemTarPathOnWindows
await tar.extractTar(archivePath, CompressionMethod.Zstd)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(2)
expect(execMock).toHaveBeenNthCalledWith(
1,
[
'zstd -d --long=30 -o',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
].join(' '),
undefined,
{cwd: undefined}
)
expect(execMock).toHaveBeenNthCalledWith(
2,
[
`"${tarPath}"`,
'-xf',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/')
].join(' '),
undefined,
{cwd: undefined}
)
}
})
test('gzip extract tar', async () => { test('gzip extract tar', async () => {
const mkdirMock = jest.spyOn(io, 'mkdirP') const mkdirMock = jest.spyOn(io, 'mkdirP')
const execMock = jest.spyOn(exec, 'exec') const execMock = jest.spyOn(exec, 'exec')
@@ -74,50 +134,51 @@ test('gzip extract tar', async () => {
await tar.extractTar(archivePath, CompressionMethod.Gzip) await tar.extractTar(archivePath, CompressionMethod.Gzip)
expect(mkdirMock).toHaveBeenCalledWith(workspace) expect(mkdirMock).toHaveBeenCalledWith(workspace)
const tarPath = IS_WINDOWS const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[ [
'-z', `"${tarPath}"`,
'-xf', '-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P', '-P',
'-C', '-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
].concat(IS_MAC ? ['--delay-directory-restore'] : []), ]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z'])
.join(' '),
undefined,
{cwd: undefined} {cwd: undefined}
) )
}) })
test('gzip extract GNU tar on windows', async () => { test('gzip extract GNU tar on windows with GNUtar in path', async () => {
if (IS_WINDOWS) { if (IS_WINDOWS) {
jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false) // GNU tar present in path but not at default location
jest
const isGnuMock = jest .spyOn(utils, 'getGnuTarPathOnWindows')
.spyOn(utils, 'isGnuTarInstalled') .mockReturnValue(Promise.resolve('tar'))
.mockReturnValue(Promise.resolve(true))
const execMock = jest.spyOn(exec, 'exec') const execMock = jest.spyOn(exec, 'exec')
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
const workspace = process.env['GITHUB_WORKSPACE'] const workspace = process.env['GITHUB_WORKSPACE']
await tar.extractTar(archivePath, CompressionMethod.Gzip) await tar.extractTar(archivePath, CompressionMethod.Gzip)
expect(isGnuMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"tar"`,
[ [
'-z', `"tar"`,
'-xf', '-xf',
archivePath.replace(/\\/g, '/'), archivePath.replace(/\\/g, '/'),
'-P', '-P',
'-C', '-C',
workspace?.replace(/\\/g, '/'), workspace?.replace(/\\/g, '/'),
'--force-local' '--force-local',
], '-z'
].join(' '),
undefined,
{cwd: undefined} {cwd: undefined}
) )
} }
@@ -134,13 +195,13 @@ test('zstd create tar', async () => {
await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd) await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${defaultTarPath}"`,
[ [
`"${tarPath}"`,
'--posix', '--posix',
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30',
'-cf', '-cf',
IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd,
'--exclude', '--exclude',
@@ -149,16 +210,81 @@ test('zstd create tar', async () => {
'-C', '-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from', '--files-from',
'manifest.txt' ManifestFilename
] ]
.concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []), .concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30'
])
.join(' '),
undefined, // args
{ {
cwd: archiveFolder cwd: archiveFolder
} }
) )
}) })
test('zstd create tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archiveFolder = getTempDir()
const workspace = process.env['GITHUB_WORKSPACE']
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
await fs.promises.mkdir(archiveFolder, {recursive: true})
await tar.createTar(
archiveFolder,
sourceDirectories,
CompressionMethod.Zstd
)
const tarPath = SystemTarPathOnWindows
expect(execMock).toHaveBeenCalledTimes(2)
expect(execMock).toHaveBeenNthCalledWith(
1,
[
`"${tarPath}"`,
'--posix',
'-cf',
TarFilename.replace(/\\/g, '/'),
'--exclude',
TarFilename.replace(/\\/g, '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/'),
'--files-from',
ManifestFilename
].join(' '),
undefined, // args
{
cwd: archiveFolder
}
)
expect(execMock).toHaveBeenNthCalledWith(
2,
[
'zstd -T0 --long=30 -o',
CacheFilename.Zstd.replace(/\\/g, '/'),
TarFilename.replace(/\\/g, '/')
].join(' '),
undefined, // args
{
cwd: archiveFolder
}
)
}
})
test('gzip create tar', async () => { test('gzip create tar', async () => {
const execMock = jest.spyOn(exec, 'exec') const execMock = jest.spyOn(exec, 'exec')
@@ -170,16 +296,13 @@ test('gzip create tar', async () => {
await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Gzip) await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Gzip)
const tarPath = IS_WINDOWS const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[ [
`"${tarPath}"`,
'--posix', '--posix',
'-z',
'-cf', '-cf',
IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip, IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip,
'--exclude', '--exclude',
@@ -188,8 +311,13 @@ test('gzip create tar', async () => {
'-C', '-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from', '--files-from',
'manifest.txt' ManifestFilename
].concat(IS_MAC ? ['--delay-directory-restore'] : []), ]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z'])
.join(' '),
undefined, // args
{ {
cwd: archiveFolder cwd: archiveFolder
} }
@@ -205,22 +333,65 @@ test('zstd list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.Zstd) await tar.listTar(archivePath, CompressionMethod.Zstd)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${defaultTarPath}"`,
[ [
'--use-compress-program', `"${tarPath}"`,
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
'-tf', '-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P' '-P'
] ]
.concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []), .concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
])
.join(' '),
undefined,
{cwd: undefined} {cwd: undefined}
) )
}) })
test('zstd list tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
await tar.listTar(archivePath, CompressionMethod.Zstd)
const tarPath = SystemTarPathOnWindows
expect(execMock).toHaveBeenCalledTimes(2)
expect(execMock).toHaveBeenNthCalledWith(
1,
[
'zstd -d --long=30 -o',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
].join(' '),
undefined,
{cwd: undefined}
)
expect(execMock).toHaveBeenNthCalledWith(
2,
[
`"${tarPath}"`,
'-tf',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
].join(' '),
undefined,
{cwd: undefined}
)
}
})
test('zstdWithoutLong list tar', async () => { test('zstdWithoutLong list tar', async () => {
const execMock = jest.spyOn(exec, 'exec') const execMock = jest.spyOn(exec, 'exec')
@@ -230,18 +401,20 @@ test('zstdWithoutLong list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.ZstdWithoutLong) await tar.listTar(archivePath, CompressionMethod.ZstdWithoutLong)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${defaultTarPath}"`,
[ [
'--use-compress-program', `"${tarPath}"`,
IS_WINDOWS ? 'zstd -d' : 'unzstd',
'-tf', '-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P' '-P'
] ]
.concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []), .concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd'])
.join(' '),
undefined,
{cwd: undefined} {cwd: undefined}
) )
}) })
@@ -254,18 +427,20 @@ test('gzip list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.Gzip) await tar.listTar(archivePath, CompressionMethod.Gzip)
const tarPath = IS_WINDOWS const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[ [
'-z', `"${tarPath}"`,
'-tf', '-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P' '-P'
].concat(IS_MAC ? ['--delay-directory-restore'] : []), ]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z'])
.join(' '),
undefined,
{cwd: undefined} {cwd: undefined}
) )
}) })
+2 -2
View File
@@ -1,12 +1,12 @@
{ {
"name": "@actions/cache", "name": "@actions/cache",
"version": "3.0.6", "version": "3.1.0-beta.1",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@actions/cache", "name": "@actions/cache",
"version": "3.0.6", "version": "3.1.0-beta.1",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@actions/core": "^1.10.0", "@actions/core": "^1.10.0",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@actions/cache", "name": "@actions/cache",
"version": "3.0.6", "version": "3.1.0-beta.1",
"preview": true, "preview": true,
"description": "Actions cache lib", "description": "Actions cache lib",
"keywords": [ "keywords": [
+25 -5
View File
@@ -4,6 +4,8 @@ import * as utils from './internal/cacheUtils'
import * as cacheHttpClient from './internal/cacheHttpClient' import * as cacheHttpClient from './internal/cacheHttpClient'
import {createTar, extractTar, listTar} from './internal/tar' import {createTar, extractTar, listTar} from './internal/tar'
import {DownloadOptions, UploadOptions} from './options' import {DownloadOptions, UploadOptions} from './options'
import {CompressionMethod} from './internal/constants'
import {ArtifactCacheEntry} from './internal/contracts'
export class ValidationError extends Error { export class ValidationError extends Error {
constructor(message: string) { constructor(message: string) {
@@ -85,17 +87,35 @@ export async function restoreCache(
checkKey(key) checkKey(key)
} }
const compressionMethod = await utils.getCompressionMethod() let cacheEntry: ArtifactCacheEntry | null
let compressionMethod = await utils.getCompressionMethod()
let archivePath = '' let archivePath = ''
try { try {
// path are needed to compute version // path are needed to compute version
const cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, { cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, {
compressionMethod compressionMethod
}) })
if (!cacheEntry?.archiveLocation) { if (!cacheEntry?.archiveLocation) {
// Cache not found // This is to support the old cache entry created by gzip on windows.
return undefined if (
process.platform === 'win32' &&
compressionMethod !== CompressionMethod.Gzip
) {
compressionMethod = CompressionMethod.Gzip
cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, {
compressionMethod
})
if (!cacheEntry?.archiveLocation) {
return undefined
}
core.debug(
"Couldn't find cache entry with zstd compression, falling back to gzip compression."
)
} else {
// Cache not found
return undefined
}
} }
archivePath = path.join( archivePath = path.join(
+2
View File
@@ -104,6 +104,7 @@ export async function getCacheEntry(
httpClient.getJson<ArtifactCacheEntry>(getCacheApiUrl(resource)) httpClient.getJson<ArtifactCacheEntry>(getCacheApiUrl(resource))
) )
if (response.statusCode === 204) { if (response.statusCode === 204) {
// Cache not found
return null return null
} }
if (!isSuccessStatusCode(response.statusCode)) { if (!isSuccessStatusCode(response.statusCode)) {
@@ -113,6 +114,7 @@ export async function getCacheEntry(
const cacheResult = response.result const cacheResult = response.result
const cacheDownloadUrl = cacheResult?.archiveLocation const cacheDownloadUrl = cacheResult?.archiveLocation
if (!cacheDownloadUrl) { if (!cacheDownloadUrl) {
// Cache achiveLocation not found. This should never happen, and hence bail out.
throw new Error('Cache not found.') throw new Error('Cache not found.')
} }
core.setSecret(cacheDownloadUrl) core.setSecret(cacheDownloadUrl)
+10 -8
View File
@@ -7,7 +7,11 @@ import * as path from 'path'
import * as semver from 'semver' import * as semver from 'semver'
import * as util from 'util' import * as util from 'util'
import {v4 as uuidV4} from 'uuid' import {v4 as uuidV4} from 'uuid'
import {CacheFilename, CompressionMethod} from './constants' import {
CacheFilename,
CompressionMethod,
GnuTarPathOnWindows
} from './constants'
// From https://github.com/actions/toolkit/blob/main/packages/tool-cache/src/tool-cache.ts#L23 // From https://github.com/actions/toolkit/blob/main/packages/tool-cache/src/tool-cache.ts#L23
export async function createTempDirectory(): Promise<string> { export async function createTempDirectory(): Promise<string> {
@@ -90,11 +94,6 @@ async function getVersion(app: string): Promise<string> {
// Use zstandard if possible to maximize cache performance // Use zstandard if possible to maximize cache performance
export async function getCompressionMethod(): Promise<CompressionMethod> { export async function getCompressionMethod(): Promise<CompressionMethod> {
if (process.platform === 'win32' && !(await isGnuTarInstalled())) {
// Disable zstd due to bug https://github.com/actions/cache/issues/301
return CompressionMethod.Gzip
}
const versionOutput = await getVersion('zstd') const versionOutput = await getVersion('zstd')
const version = semver.clean(versionOutput) const version = semver.clean(versionOutput)
@@ -116,9 +115,12 @@ export function getCacheFileName(compressionMethod: CompressionMethod): string {
: CacheFilename.Zstd : CacheFilename.Zstd
} }
export async function isGnuTarInstalled(): Promise<boolean> { export async function getGnuTarPathOnWindows(): Promise<string> {
if (fs.existsSync(GnuTarPathOnWindows)) {
return GnuTarPathOnWindows
}
const versionOutput = await getVersion('tar') const versionOutput = await getVersion('tar')
return versionOutput.toLowerCase().includes('gnu tar') return versionOutput.toLowerCase().includes('gnu tar') ? io.which('tar') : ''
} }
export function assertDefined<T>(name: string, value?: T): T { export function assertDefined<T>(name: string, value?: T): T {
+15
View File
@@ -11,6 +11,11 @@ export enum CompressionMethod {
Zstd = 'zstd' Zstd = 'zstd'
} }
export enum ArchiveToolType {
GNU = 'gnu',
BSD = 'bsd'
}
// The default number of retry attempts. // The default number of retry attempts.
export const DefaultRetryAttempts = 2 export const DefaultRetryAttempts = 2
@@ -21,3 +26,13 @@ export const DefaultRetryDelay = 5000
// over the socket during this period, the socket is destroyed and the download // over the socket during this period, the socket is destroyed and the download
// is aborted. // is aborted.
export const SocketTimeout = 5000 export const SocketTimeout = 5000
// The default path of GNUtar on hosted Windows runners
export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe`
// The default path of BSDtar on hosted Windows runners
export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe`
export const TarFilename = 'cache.tar'
export const ManifestFilename = 'manifest.txt'
+5
View File
@@ -31,3 +31,8 @@ export interface InternalCacheOptions {
compressionMethod?: CompressionMethod compressionMethod?: CompressionMethod
cacheSize?: number cacheSize?: number
} }
export interface ArchiveTool {
path: string
type: string
}
+223 -84
View File
@@ -3,25 +3,28 @@ import * as io from '@actions/io'
import {existsSync, writeFileSync} from 'fs' import {existsSync, writeFileSync} from 'fs'
import * as path from 'path' import * as path from 'path'
import * as utils from './cacheUtils' import * as utils from './cacheUtils'
import {CompressionMethod} from './constants' import {ArchiveTool} from './contracts'
import {
CompressionMethod,
SystemTarPathOnWindows,
ArchiveToolType,
TarFilename,
ManifestFilename
} from './constants'
const IS_WINDOWS = process.platform === 'win32' const IS_WINDOWS = process.platform === 'win32'
async function getTarPath( // Returns tar path and type: BSD or GNU
args: string[], async function getTarPath(): Promise<ArchiveTool> {
compressionMethod: CompressionMethod
): Promise<string> {
switch (process.platform) { switch (process.platform) {
case 'win32': { case 'win32': {
const systemTar = `${process.env['windir']}\\System32\\tar.exe` const gnuTar = await utils.getGnuTarPathOnWindows()
if (compressionMethod !== CompressionMethod.Gzip) { const systemTar = SystemTarPathOnWindows
// We only use zstandard compression on windows when gnu tar is installed due to if (gnuTar) {
// a bug with compressing large files with bsdtar + zstd // Use GNUtar as default on windows
args.push('--force-local') return <ArchiveTool>{path: gnuTar, type: ArchiveToolType.GNU}
} else if (existsSync(systemTar)) { } else if (existsSync(systemTar)) {
return systemTar return <ArchiveTool>{path: systemTar, type: ArchiveToolType.BSD}
} else if (await utils.isGnuTarInstalled()) {
args.push('--force-local')
} }
break break
} }
@@ -29,27 +32,133 @@ async function getTarPath(
const gnuTar = await io.which('gtar', false) const gnuTar = await io.which('gtar', false)
if (gnuTar) { if (gnuTar) {
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
args.push('--delay-directory-restore') return <ArchiveTool>{path: gnuTar, type: ArchiveToolType.GNU}
return gnuTar } else {
return <ArchiveTool>{
path: await io.which('tar', true),
type: ArchiveToolType.BSD
}
} }
break
} }
default: default:
break break
} }
return await io.which('tar', true) // Default assumption is GNU tar is present in path
return <ArchiveTool>{
path: await io.which('tar', true),
type: ArchiveToolType.GNU
}
} }
async function execTar( // Return arguments for tar as per tarPath, compressionMethod, method type and os
args: string[], async function getTarArgs(
tarPath: ArchiveTool,
compressionMethod: CompressionMethod, compressionMethod: CompressionMethod,
cwd?: string type: string,
): Promise<void> { archivePath = ''
try { ): Promise<string[]> {
await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd}) const args = [`"${tarPath.path}"`]
} catch (error) { const cacheFileName = utils.getCacheFileName(compressionMethod)
throw new Error(`Tar failed with error: ${error?.message}`) const tarFile = 'cache.tar'
const workingDirectory = getWorkingDirectory()
// Speficic args for BSD tar on windows for workaround
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
// Method specific args
switch (type) {
case 'create':
args.push(
'--posix',
'-cf',
BSD_TAR_ZSTD
? tarFile
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--exclude',
BSD_TAR_ZSTD
? tarFile
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--files-from',
ManifestFilename
)
break
case 'extract':
args.push(
'-xf',
BSD_TAR_ZSTD
? tarFile
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
)
break
case 'list':
args.push(
'-tf',
BSD_TAR_ZSTD
? tarFile
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
)
break
} }
// Platform specific args
if (tarPath.type === ArchiveToolType.GNU) {
switch (process.platform) {
case 'win32':
args.push('--force-local')
break
case 'darwin':
args.push('--delay-directory-restore')
break
}
}
return args
}
// Returns commands to run tar and compression program
async function getCommands(
compressionMethod: CompressionMethod,
type: string,
archivePath = ''
): Promise<string[]> {
let args
const tarPath = await getTarPath()
const tarArgs = await getTarArgs(
tarPath,
compressionMethod,
type,
archivePath
)
const compressionArgs =
type !== 'create'
? await getDecompressionProgram(tarPath, compressionMethod, archivePath)
: await getCompressionProgram(tarPath, compressionMethod)
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
if (BSD_TAR_ZSTD && type !== 'create') {
args = [[...compressionArgs].join(' '), [...tarArgs].join(' ')]
} else {
args = [[...tarArgs].join(' '), [...compressionArgs].join(' ')]
}
if (BSD_TAR_ZSTD) {
return args
}
return [args.join(' ')]
} }
function getWorkingDirectory(): string { function getWorkingDirectory(): string {
@@ -57,37 +166,107 @@ function getWorkingDirectory(): string {
} }
// Common function for extractTar and listTar to get the compression method // Common function for extractTar and listTar to get the compression method
function getCompressionProgram(compressionMethod: CompressionMethod): string[] { async function getDecompressionProgram(
tarPath: ArchiveTool,
compressionMethod: CompressionMethod,
archivePath: string
): Promise<string[]> {
// -d: Decompress. // -d: Decompress.
// unzstd is equivalent to 'zstd -d' // unzstd is equivalent to 'zstd -d'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners. // Using 30 here because we also support 32-bit self-hosted runners.
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
switch (compressionMethod) { switch (compressionMethod) {
case CompressionMethod.Zstd: case CompressionMethod.Zstd:
return [ return BSD_TAR_ZSTD
'--use-compress-program', ? [
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' 'zstd -d --long=30 -o',
] TarFilename,
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
]
: [
'--use-compress-program',
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
]
case CompressionMethod.ZstdWithoutLong: case CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] return BSD_TAR_ZSTD
? [
'zstd -d -o',
TarFilename,
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
]
: ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']
default: default:
return ['-z'] return ['-z']
} }
} }
// Used for creating the archive
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// zstdmt is equivalent to 'zstd -T0'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
async function getCompressionProgram(
tarPath: ArchiveTool,
compressionMethod: CompressionMethod
): Promise<string[]> {
const cacheFileName = utils.getCacheFileName(compressionMethod)
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
switch (compressionMethod) {
case CompressionMethod.Zstd:
return BSD_TAR_ZSTD
? [
'zstd -T0 --long=30 -o',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
TarFilename
]
: [
'--use-compress-program',
IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return BSD_TAR_ZSTD
? [
'zstd -T0 -o',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
TarFilename
]
: ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt']
default:
return ['-z']
}
}
// Executes all commands as separate processes
async function execCommands(commands: string[], cwd?: string): Promise<void> {
for (const command of commands) {
try {
await exec(command, undefined, {cwd})
} catch (error) {
throw new Error(
`${command.split(' ')[0]} failed with error: ${error?.message}`
)
}
}
}
// List the contents of a tar
export async function listTar( export async function listTar(
archivePath: string, archivePath: string,
compressionMethod: CompressionMethod compressionMethod: CompressionMethod
): Promise<void> { ): Promise<void> {
const args = [ const commands = await getCommands(compressionMethod, 'list', archivePath)
...getCompressionProgram(compressionMethod), await execCommands(commands)
'-tf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
]
await execTar(args, compressionMethod)
} }
// Extract a tar
export async function extractTar( export async function extractTar(
archivePath: string, archivePath: string,
compressionMethod: CompressionMethod compressionMethod: CompressionMethod
@@ -95,61 +274,21 @@ export async function extractTar(
// Create directory to extract tar into // Create directory to extract tar into
const workingDirectory = getWorkingDirectory() const workingDirectory = getWorkingDirectory()
await io.mkdirP(workingDirectory) await io.mkdirP(workingDirectory)
const args = [ const commands = await getCommands(compressionMethod, 'extract', archivePath)
...getCompressionProgram(compressionMethod), await execCommands(commands)
'-xf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
]
await execTar(args, compressionMethod)
} }
// Create a tar
export async function createTar( export async function createTar(
archiveFolder: string, archiveFolder: string,
sourceDirectories: string[], sourceDirectories: string[],
compressionMethod: CompressionMethod compressionMethod: CompressionMethod
): Promise<void> { ): Promise<void> {
// Write source directories to manifest.txt to avoid command length limits // Write source directories to manifest.txt to avoid command length limits
const manifestFilename = 'manifest.txt'
const cacheFileName = utils.getCacheFileName(compressionMethod)
writeFileSync( writeFileSync(
path.join(archiveFolder, manifestFilename), path.join(archiveFolder, ManifestFilename),
sourceDirectories.join('\n') sourceDirectories.join('\n')
) )
const workingDirectory = getWorkingDirectory() const commands = await getCommands(compressionMethod, 'create')
await execCommands(commands, archiveFolder)
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// zstdmt is equivalent to 'zstd -T0'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
function getCompressionProgram(): string[] {
switch (compressionMethod) {
case CompressionMethod.Zstd:
return [
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']
default:
return ['-z']
}
}
const args = [
'--posix',
...getCompressionProgram(),
'-cf',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--exclude',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--files-from',
manifestFilename
]
await execTar(args, compressionMethod, archiveFolder)
} }
+6 -6
View File
@@ -50,9 +50,9 @@
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
}, },
"node_modules/minimatch": { "node_modules/minimatch": {
"version": "3.1.2", "version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"dependencies": { "dependencies": {
"brace-expansion": "^1.1.7" "brace-expansion": "^1.1.7"
}, },
@@ -115,9 +115,9 @@
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
}, },
"minimatch": { "minimatch": {
"version": "3.1.2", "version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"requires": { "requires": {
"brace-expansion": "^1.1.7" "brace-expansion": "^1.1.7"
} }
+7 -155
View File
@@ -100,19 +100,6 @@
"node": "*" "node": "*"
} }
}, },
"node_modules/call-bind": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
"integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
"dev": true,
"dependencies": {
"function-bind": "^1.1.1",
"get-intrinsic": "^1.0.2"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/chai": { "node_modules/chai": {
"version": "4.2.0", "version": "4.2.0",
"resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz", "resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz",
@@ -167,12 +154,6 @@
"integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=", "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=",
"dev": true "dev": true
}, },
"node_modules/function-bind": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
"integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==",
"dev": true
},
"node_modules/get-func-name": { "node_modules/get-func-name": {
"version": "2.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz",
@@ -182,44 +163,6 @@
"node": "*" "node": "*"
} }
}, },
"node_modules/get-intrinsic": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz",
"integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==",
"dev": true,
"dependencies": {
"function-bind": "^1.1.1",
"has": "^1.0.3",
"has-symbols": "^1.0.3"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/has": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
"integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
"dev": true,
"dependencies": {
"function-bind": "^1.1.1"
},
"engines": {
"node": ">= 0.4.0"
}
},
"node_modules/has-symbols": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
"integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
"dev": true,
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/json-stringify-safe": { "node_modules/json-stringify-safe": {
"version": "5.0.1", "version": "5.0.1",
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
@@ -285,15 +228,6 @@
"semver": "bin/semver" "semver": "bin/semver"
} }
}, },
"node_modules/object-inspect": {
"version": "1.12.2",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz",
"integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==",
"dev": true,
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/pathval": { "node_modules/pathval": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz", "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz",
@@ -313,18 +247,12 @@
] ]
}, },
"node_modules/qs": { "node_modules/qs": {
"version": "6.11.0", "version": "6.7.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
"integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==",
"dev": true, "dev": true,
"dependencies": {
"side-channel": "^1.0.4"
},
"engines": { "engines": {
"node": ">=0.6" "node": ">=0.6"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
} }
}, },
"node_modules/semver": { "node_modules/semver": {
@@ -335,20 +263,6 @@
"semver": "bin/semver.js" "semver": "bin/semver.js"
} }
}, },
"node_modules/side-channel": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
"integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
"dev": true,
"dependencies": {
"call-bind": "^1.0.0",
"get-intrinsic": "^1.0.2",
"object-inspect": "^1.9.0"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/tunnel": { "node_modules/tunnel": {
"version": "0.0.6", "version": "0.0.6",
"resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
@@ -450,16 +364,6 @@
"integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==",
"dev": true "dev": true
}, },
"call-bind": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
"integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
"dev": true,
"requires": {
"function-bind": "^1.1.1",
"get-intrinsic": "^1.0.2"
}
},
"chai": { "chai": {
"version": "4.2.0", "version": "4.2.0",
"resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz", "resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz",
@@ -504,44 +408,12 @@
"integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=", "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=",
"dev": true "dev": true
}, },
"function-bind": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
"integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==",
"dev": true
},
"get-func-name": { "get-func-name": {
"version": "2.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz",
"integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=",
"dev": true "dev": true
}, },
"get-intrinsic": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz",
"integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==",
"dev": true,
"requires": {
"function-bind": "^1.1.1",
"has": "^1.0.3",
"has-symbols": "^1.0.3"
}
},
"has": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
"integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
"dev": true,
"requires": {
"function-bind": "^1.1.1"
}
},
"has-symbols": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
"integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
"dev": true
},
"json-stringify-safe": { "json-stringify-safe": {
"version": "5.0.1", "version": "5.0.1",
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
@@ -600,12 +472,6 @@
} }
} }
}, },
"object-inspect": {
"version": "1.12.2",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz",
"integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==",
"dev": true
},
"pathval": { "pathval": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz", "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz",
@@ -619,30 +485,16 @@
"dev": true "dev": true
}, },
"qs": { "qs": {
"version": "6.11.0", "version": "6.7.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
"integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==",
"dev": true, "dev": true
"requires": {
"side-channel": "^1.0.4"
}
}, },
"semver": { "semver": {
"version": "6.3.0", "version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
"integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
}, },
"side-channel": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
"integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
"dev": true,
"requires": {
"call-bind": "^1.0.0",
"get-intrinsic": "^1.0.2",
"object-inspect": "^1.9.0"
}
},
"tunnel": { "tunnel": {
"version": "0.0.6", "version": "0.0.6",
"resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",