Compare commits

..
Author SHA1 Message Date
Ferenc Hammerl 56146a6713 Bump actions to newer versions 2023-01-03 16:59:01 +01:00
Ferenc Hammerl 74f24b41d1 Use most recent setup-node 2023-01-03 16:43:09 +01:00
23 changed files with 185 additions and 681 deletions
+2 -2
View File
@@ -22,10 +22,10 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Set Node.js 16.x
uses: actions/setup-node@v1
uses: actions/setup-node@v3
with:
node-version: 16.x
+2 -2
View File
@@ -18,10 +18,10 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Set Node.js 16.x
uses: actions/setup-node@v1
uses: actions/setup-node@v3
with:
node-version: 16.x
+2 -2
View File
@@ -22,10 +22,10 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Set Node.js 16.x
uses: actions/setup-node@v1
uses: actions/setup-node@v3
with:
node-version: 16.x
-90
View File
@@ -1,90 +0,0 @@
name: cache-windows-bsd-unit-tests
on:
push:
branches:
- main
paths-ignore:
- '**.md'
pull_request:
paths-ignore:
- '**.md'
jobs:
build:
name: Build
runs-on: windows-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- shell: bash
run: |
rm "C:\Program Files\Git\usr\bin\tar.exe"
- name: Set Node.js 12.x
uses: actions/setup-node@v1
with:
node-version: 12.x
# In order to save & restore cache from a shell script, certain env variables need to be set that are only available in the
# node context. This runs a local action that gets and sets the necessary env variables that are needed
- name: Set env variables
uses: ./packages/cache/__tests__/__fixtures__/
# Need root node_modules because certain npm packages like jest are configured for the entire repository and it won't be possible
# without these to just compile the cache package
- name: Install root npm packages
run: npm ci
- name: Compile cache package
run: |
npm ci
npm run tsc
working-directory: packages/cache
- name: Generate files in working directory
shell: bash
run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} test-cache
- name: Generate files outside working directory
shell: bash
run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} ~/test-cache
# We're using node -e to call the functions directly available in the @actions/cache package
- name: Save cache using saveCache()
run: |
node -e "Promise.resolve(require('./packages/cache/lib/cache').saveCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))"
- name: Delete cache folders before restoring
shell: bash
run: |
rm -rf test-cache
rm -rf ~/test-cache
- name: Restore cache using restoreCache() with http-client
run: |
node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}',[],{useAzureSdk: false}))"
- name: Verify cache restored with http-client
shell: bash
run: |
packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} test-cache
packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache
- name: Delete cache folders before restoring
shell: bash
run: |
rm -rf test-cache
rm -rf ~/test-cache
- name: Restore cache using restoreCache() with Azure SDK
run: |
node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))"
- name: Verify cache restored with Azure SDK
shell: bash
run: |
packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} test-cache
packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache
+1 -1
View File
@@ -20,7 +20,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v2
uses: actions/checkout@v3
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
+4 -4
View File
@@ -13,13 +13,13 @@ jobs:
steps:
- name: setup repo
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: verify package exists
run: ls packages/${{ github.event.inputs.package }}
- name: Set Node.js 16.x
uses: actions/setup-node@v1
uses: actions/setup-node@v3
with:
node-version: 16.x
@@ -40,7 +40,7 @@ jobs:
working-directory: packages/${{ github.event.inputs.package }}
- name: upload artifact
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: ${{ github.event.inputs.package }}
path: packages/${{ github.event.inputs.package }}/*.tgz
@@ -52,7 +52,7 @@ jobs:
steps:
- name: download artifact
uses: actions/download-artifact@v2
uses: actions/download-artifact@v3
with:
name: ${{ github.event.inputs.package }}
+2 -2
View File
@@ -23,10 +23,10 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Set Node.js 16.x
uses: actions/setup-node@v1
uses: actions/setup-node@v3
with:
node-version: 16.x
+2 -2
View File
@@ -9,7 +9,7 @@ jobs:
if: ${{ github.repository_owner == 'actions' }}
steps:
- name: Checkout repository
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Update Octokit
working-directory: packages/github
run: |
@@ -30,7 +30,7 @@ jobs:
fi
- name: Create PR
if: ${{steps.status.outputs.createPR}}
uses: actions/github-script@v2
uses: actions/github-script@v6
with:
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
+1 -1
View File
@@ -32,7 +32,7 @@ jobs:
os: [ubuntu-16.04, windows-2019]
runs-on: ${{matrix.os}}
actions:
- uses: actions/setup-node@v1
- uses: actions/setup-node@v3
with:
version: ${{matrix.node}}
- run: |
+1 -1
View File
@@ -18,7 +18,7 @@ e.g. To use https://github.com/actions/setup-node, users will author:
```yaml
steps:
using: actions/setup-node@v1
using: actions/setup-node@v3
```
# Define Metadata
-21
View File
@@ -91,24 +91,3 @@
### 3.0.6
- Added `@azure/abort-controller` to dependencies to fix compatibility issue with ESM [#1208](https://github.com/actions/toolkit/issues/1208)
### 3.1.0-beta.1
- Update actions/cache on windows to use gnu tar and zstd by default and fallback to bsdtar and zstd if gnu tar is not available. ([issue](https://github.com/actions/cache/issues/984))
### 3.1.0-beta.2
- Added support for fallback to gzip to restore old caches on windows.
### 3.1.0-beta.3
- Bug Fixes for fallback to gzip to restore old caches on windows and bsdtar if gnutar is not available.
### 3.1.0
- Update actions/cache on windows to use gnu tar and zstd by default
- Update actions/cache on windows to fallback to bsdtar and zstd if gnu tar is not available.
- Added support for fallback to gzip to restore old caches on windows.
### 3.1.1
- Reverted changes in 3.1.0 to fix issue with symlink restoration on windows.
- Added support for verbose logging about cache version during cache miss.
### 3.1.2
- Fix issue with symlink restoration on windows.
+6 -17
View File
@@ -7,7 +7,7 @@ jest.mock('../src/internal/downloadUtils')
test('getCacheVersion with one path returns version', async () => {
const paths = ['node_modules']
const result = getCacheVersion(paths, undefined, true)
const result = getCacheVersion(paths)
expect(result).toEqual(
'b3e0c6cb5ecf32614eeb2997d905b9c297046d7cbf69062698f25b14b4cb0985'
)
@@ -15,7 +15,7 @@ test('getCacheVersion with one path returns version', async () => {
test('getCacheVersion with multiple paths returns version', async () => {
const paths = ['node_modules', 'dist']
const result = getCacheVersion(paths, undefined, true)
const result = getCacheVersion(paths)
expect(result).toEqual(
'165c3053bc646bf0d4fac17b1f5731caca6fe38e0e464715c0c3c6b6318bf436'
)
@@ -23,33 +23,22 @@ test('getCacheVersion with multiple paths returns version', async () => {
test('getCacheVersion with zstd compression returns version', async () => {
const paths = ['node_modules']
const result = getCacheVersion(paths, CompressionMethod.Zstd, true)
const result = getCacheVersion(paths, CompressionMethod.Zstd)
expect(result).toEqual(
'273877e14fd65d270b87a198edbfa2db5a43de567c9a548d2a2505b408befe24'
)
})
test('getCacheVersion with gzip compression returns version', async () => {
test('getCacheVersion with gzip compression does not change vesion', async () => {
const paths = ['node_modules']
const result = getCacheVersion(paths, CompressionMethod.Gzip, true)
const result = getCacheVersion(paths, CompressionMethod.Gzip)
expect(result).toEqual(
'470e252814dbffc9524891b17cf4e5749b26c1b5026e63dd3f00972db2393117'
'b3e0c6cb5ecf32614eeb2997d905b9c297046d7cbf69062698f25b14b4cb0985'
)
})
test('getCacheVersion with enableCrossOsArchive as false returns version on windows', async () => {
if (process.platform === 'win32') {
const paths = ['node_modules']
const result = getCacheVersion(paths)
expect(result).toEqual(
'2db19d6596dc34f51f0043120148827a264863f5c6ac857569c2af7119bad14e'
)
}
})
test('downloadCache uses http-client for non-Azure URLs', async () => {
const downloadCacheHttpClientMock = jest.spyOn(
downloadUtils,
+3 -6
View File
@@ -142,8 +142,7 @@ test('restore with gzip compressed cache found', async () => {
expect(cacheKey).toBe(key)
expect(getCacheMock).toHaveBeenCalledWith([key], paths, {
compressionMethod: compression,
enableCrossOsArchive: false
compressionMethod: compression
})
expect(createTempDirectoryMock).toHaveBeenCalledTimes(1)
expect(downloadCacheMock).toHaveBeenCalledWith(
@@ -202,8 +201,7 @@ test('restore with zstd compressed cache found', async () => {
expect(cacheKey).toBe(key)
expect(getCacheMock).toHaveBeenCalledWith([key], paths, {
compressionMethod: compression,
enableCrossOsArchive: false
compressionMethod: compression
})
expect(createTempDirectoryMock).toHaveBeenCalledTimes(1)
expect(downloadCacheMock).toHaveBeenCalledWith(
@@ -260,8 +258,7 @@ test('restore with cache found for restore key', async () => {
expect(cacheKey).toBe(restoreKey)
expect(getCacheMock).toHaveBeenCalledWith([key, restoreKey], paths, {
compressionMethod: compression,
enableCrossOsArchive: false
compressionMethod: compression
})
expect(createTempDirectoryMock).toHaveBeenCalledTimes(1)
expect(downloadCacheMock).toHaveBeenCalledWith(
+3 -9
View File
@@ -209,9 +209,7 @@ test('save with reserve cache failure should fail', async () => {
expect(reserveCacheMock).toHaveBeenCalledTimes(1)
expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey, paths, {
cacheSize: undefined,
compressionMethod: compression,
enableCrossOsArchive: false
compressionMethod: compression
})
expect(createTarMock).toHaveBeenCalledTimes(1)
expect(saveCacheMock).toHaveBeenCalledTimes(0)
@@ -255,9 +253,7 @@ test('save with server error should fail', async () => {
expect(reserveCacheMock).toHaveBeenCalledTimes(1)
expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey, [filePath], {
cacheSize: undefined,
compressionMethod: compression,
enableCrossOsArchive: false
compressionMethod: compression
})
const archiveFolder = '/foo/bar'
const archiveFile = path.join(archiveFolder, CacheFilename.Zstd)
@@ -300,9 +296,7 @@ test('save with valid inputs uploads a cache', async () => {
expect(reserveCacheMock).toHaveBeenCalledTimes(1)
expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey, [filePath], {
cacheSize: undefined,
compressionMethod: compression,
enableCrossOsArchive: false
compressionMethod: compression
})
const archiveFolder = '/foo/bar'
const archiveFile = path.join(archiveFolder, CacheFilename.Zstd)
+49 -227
View File
@@ -1,29 +1,19 @@
import * as exec from '@actions/exec'
import {exportVariable} from '@actions/core'
import * as io from '@actions/io'
import * as path from 'path'
import {
CacheFilename,
CompressionMethod,
GnuTarPathOnWindows,
ManifestFilename,
SystemTarPathOnWindows,
TarFilename
} from '../src/internal/constants'
import {CacheFilename, CompressionMethod} from '../src/internal/constants'
import * as tar from '../src/internal/tar'
import * as utils from '../src/internal/cacheUtils'
// eslint-disable-next-line @typescript-eslint/no-require-imports
import fs = require('fs')
exportVariable('MSYS', 'winsymlinks:nativestrict')
jest.mock('@actions/exec')
jest.mock('@actions/io')
const IS_WINDOWS = process.platform === 'win32'
const IS_MAC = process.platform === 'darwin'
const defaultTarPath = IS_MAC ? 'gtar' : 'tar'
const defaultTarPath = process.platform === 'darwin' ? 'gtar' : 'tar'
function getTempDir(): string {
return path.join(__dirname, '_temp', 'tar')
@@ -38,10 +28,6 @@ beforeAll(async () => {
await jest.requireActual('@actions/io').rmRF(getTempDir())
})
beforeEach(async () => {
jest.restoreAllMocks()
})
afterAll(async () => {
delete process.env['GITHUB_WORKSPACE']
await jest.requireActual('@actions/io').rmRF(getTempDir())
@@ -55,15 +41,16 @@ test('zstd extract tar', async () => {
? `${process.env['windir']}\\fakepath\\cache.tar`
: 'cache.tar'
const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
await tar.extractTar(archivePath, CompressionMethod.Zstd)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${defaultTarPath}"`,
[
`"${tarPath}"`,
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
'-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P',
@@ -71,61 +58,11 @@ test('zstd extract tar', async () => {
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
])
.join(' '),
undefined,
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
)
})
test('zstd extract tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const mkdirMock = jest.spyOn(io, 'mkdirP')
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = SystemTarPathOnWindows
await tar.extractTar(archivePath, CompressionMethod.Zstd)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(2)
expect(execMock).toHaveBeenNthCalledWith(
1,
[
'zstd -d --long=30 --force -o',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
].join(' '),
undefined,
{cwd: undefined}
)
expect(execMock).toHaveBeenNthCalledWith(
2,
[
`"${tarPath}"`,
'-xf',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/')
].join(' '),
undefined,
{cwd: undefined}
)
}
})
test('gzip extract tar', async () => {
const mkdirMock = jest.spyOn(io, 'mkdirP')
const execMock = jest.spyOn(exec, 'exec')
@@ -137,51 +74,50 @@ test('gzip extract tar', async () => {
await tar.extractTar(archivePath, CompressionMethod.Gzip)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[
`"${tarPath}"`,
'-z',
'-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P',
'-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z'])
.join(' '),
undefined,
].concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
)
})
test('gzip extract GNU tar on windows with GNUtar in path', async () => {
test('gzip extract GNU tar on windows', async () => {
if (IS_WINDOWS) {
// GNU tar present in path but not at default location
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve('tar'))
jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false)
const isGnuMock = jest
.spyOn(utils, 'isGnuTarInstalled')
.mockReturnValue(Promise.resolve(true))
const execMock = jest.spyOn(exec, 'exec')
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
const workspace = process.env['GITHUB_WORKSPACE']
await tar.extractTar(archivePath, CompressionMethod.Gzip)
expect(isGnuMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"tar"`,
[
`"tar"`,
'-z',
'-xf',
archivePath.replace(/\\/g, '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/'),
'--force-local',
'-z'
].join(' '),
undefined,
'--force-local'
],
{cwd: undefined}
)
}
@@ -198,13 +134,13 @@ test('zstd create tar', async () => {
await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${defaultTarPath}"`,
[
`"${tarPath}"`,
'--posix',
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30',
'-cf',
IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd,
'--exclude',
@@ -213,81 +149,16 @@ test('zstd create tar', async () => {
'-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from',
ManifestFilename
'manifest.txt'
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30'
])
.join(' '),
undefined, // args
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
{
cwd: archiveFolder
}
)
})
test('zstd create tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archiveFolder = getTempDir()
const workspace = process.env['GITHUB_WORKSPACE']
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
await fs.promises.mkdir(archiveFolder, {recursive: true})
await tar.createTar(
archiveFolder,
sourceDirectories,
CompressionMethod.Zstd
)
const tarPath = SystemTarPathOnWindows
expect(execMock).toHaveBeenCalledTimes(2)
expect(execMock).toHaveBeenNthCalledWith(
1,
[
`"${tarPath}"`,
'--posix',
'-cf',
TarFilename.replace(/\\/g, '/'),
'--exclude',
TarFilename.replace(/\\/g, '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/'),
'--files-from',
ManifestFilename
].join(' '),
undefined, // args
{
cwd: archiveFolder
}
)
expect(execMock).toHaveBeenNthCalledWith(
2,
[
'zstd -T0 --long=30 --force -o',
CacheFilename.Zstd.replace(/\\/g, '/'),
TarFilename.replace(/\\/g, '/')
].join(' '),
undefined, // args
{
cwd: archiveFolder
}
)
}
})
test('gzip create tar', async () => {
const execMock = jest.spyOn(exec, 'exec')
@@ -299,13 +170,16 @@ test('gzip create tar', async () => {
await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Gzip)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[
`"${tarPath}"`,
'--posix',
'-z',
'-cf',
IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip,
'--exclude',
@@ -314,13 +188,8 @@ test('gzip create tar', async () => {
'-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from',
ManifestFilename
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z'])
.join(' '),
undefined, // args
'manifest.txt'
].concat(IS_MAC ? ['--delay-directory-restore'] : []),
{
cwd: archiveFolder
}
@@ -336,65 +205,22 @@ test('zstd list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.Zstd)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${defaultTarPath}"`,
[
`"${tarPath}"`,
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
])
.join(' '),
undefined,
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
)
})
test('zstd list tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
await tar.listTar(archivePath, CompressionMethod.Zstd)
const tarPath = SystemTarPathOnWindows
expect(execMock).toHaveBeenCalledTimes(2)
expect(execMock).toHaveBeenNthCalledWith(
1,
[
'zstd -d --long=30 --force -o',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
].join(' '),
undefined,
{cwd: undefined}
)
expect(execMock).toHaveBeenNthCalledWith(
2,
[
`"${tarPath}"`,
'-tf',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
].join(' '),
undefined,
{cwd: undefined}
)
}
})
test('zstdWithoutLong list tar', async () => {
const execMock = jest.spyOn(exec, 'exec')
@@ -404,20 +230,18 @@ test('zstdWithoutLong list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.ZstdWithoutLong)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${defaultTarPath}"`,
[
`"${tarPath}"`,
'--use-compress-program',
IS_WINDOWS ? 'zstd -d' : 'unzstd',
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd'])
.join(' '),
undefined,
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
)
})
@@ -430,20 +254,18 @@ test('gzip list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.Gzip)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[
`"${tarPath}"`,
'-z',
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z'])
.join(' '),
undefined,
].concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
)
})
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@actions/cache",
"version": "3.1.2",
"version": "3.1.1",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@actions/cache",
"version": "3.1.2",
"version": "3.1.1",
"license": "MIT",
"dependencies": {
"@actions/core": "^1.10.0",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@actions/cache",
"version": "3.1.2",
"version": "3.1.1",
"preview": true,
"description": "Actions cache lib",
"keywords": [
+4 -9
View File
@@ -60,15 +60,13 @@ export function isFeatureAvailable(): boolean {
* @param primaryKey an explicit key for restoring the cache
* @param restoreKeys an optional ordered list of keys to use for restoring the cache if no cache hit occurred for key
* @param downloadOptions cache download options
* @param enableCrossOsArchive an optional boolean enabled to restore on windows any cache created on any platform
* @returns string returns the key for the cache hit, otherwise returns undefined
*/
export async function restoreCache(
paths: string[],
primaryKey: string,
restoreKeys?: string[],
options?: DownloadOptions,
enableCrossOsArchive = false
options?: DownloadOptions
): Promise<string | undefined> {
checkPaths(paths)
@@ -92,9 +90,9 @@ export async function restoreCache(
try {
// path are needed to compute version
const cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, {
compressionMethod,
enableCrossOsArchive
compressionMethod
})
if (!cacheEntry?.archiveLocation) {
// Cache not found
return undefined
@@ -153,15 +151,13 @@ export async function restoreCache(
*
* @param paths a list of file paths to be cached
* @param key an explicit key for restoring the cache
* @param enableCrossOsArchive an optional boolean enabled to save cache on windows which could be restored on any platform
* @param options cache upload options
* @returns number returns cacheId if the cache was saved successfully and throws an error if save fails
*/
export async function saveCache(
paths: string[],
key: string,
options?: UploadOptions,
enableCrossOsArchive = false
options?: UploadOptions
): Promise<number> {
checkPaths(paths)
checkKey(key)
@@ -211,7 +207,6 @@ export async function saveCache(
paths,
{
compressionMethod,
enableCrossOsArchive,
cacheSize: archiveFileSize
}
)
+8 -26
View File
@@ -73,21 +73,13 @@ function createHttpClient(): HttpClient {
export function getCacheVersion(
paths: string[],
compressionMethod?: CompressionMethod,
enableCrossOsArchive = false
compressionMethod?: CompressionMethod
): string {
const components = paths
// Add compression method to cache version to restore
// compressed cache as per compression method
if (compressionMethod) {
components.push(compressionMethod)
}
// Only check for windows platforms if enableCrossOsArchive is false
if (process.platform === 'win32' && !enableCrossOsArchive) {
components.push('windows-only')
}
const components = paths.concat(
!compressionMethod || compressionMethod === CompressionMethod.Gzip
? []
: [compressionMethod]
)
// Add salt to cache version to support breaking changes in cache entry
components.push(versionSalt)
@@ -104,11 +96,7 @@ export async function getCacheEntry(
options?: InternalCacheOptions
): Promise<ArtifactCacheEntry | null> {
const httpClient = createHttpClient()
const version = getCacheVersion(
paths,
options?.compressionMethod,
options?.enableCrossOsArchive
)
const version = getCacheVersion(paths, options?.compressionMethod)
const resource = `cache?keys=${encodeURIComponent(
keys.join(',')
)}&version=${version}`
@@ -116,7 +104,6 @@ export async function getCacheEntry(
const response = await retryTypedResponse('getCacheEntry', async () =>
httpClient.getJson<ArtifactCacheEntry>(getCacheApiUrl(resource))
)
// Cache not found
if (response.statusCode === 204) {
// List cache for primary key only if cache miss occurs
if (core.isDebug()) {
@@ -131,7 +118,6 @@ export async function getCacheEntry(
const cacheResult = response.result
const cacheDownloadUrl = cacheResult?.archiveLocation
if (!cacheDownloadUrl) {
// Cache achiveLocation not found. This should never happen, and hence bail out.
throw new Error('Cache not found.')
}
core.setSecret(cacheDownloadUrl)
@@ -193,11 +179,7 @@ export async function reserveCache(
options?: InternalCacheOptions
): Promise<ITypedResponseWithError<ReserveCacheResponse>> {
const httpClient = createHttpClient()
const version = getCacheVersion(
paths,
options?.compressionMethod,
options?.enableCrossOsArchive
)
const version = getCacheVersion(paths, options?.compressionMethod)
const reserveCacheRequest: ReserveCacheRequest = {
key,
+8 -10
View File
@@ -7,11 +7,7 @@ import * as path from 'path'
import * as semver from 'semver'
import * as util from 'util'
import {v4 as uuidV4} from 'uuid'
import {
CacheFilename,
CompressionMethod,
GnuTarPathOnWindows
} from './constants'
import {CacheFilename, CompressionMethod} from './constants'
// From https://github.com/actions/toolkit/blob/main/packages/tool-cache/src/tool-cache.ts#L23
export async function createTempDirectory(): Promise<string> {
@@ -94,6 +90,11 @@ async function getVersion(app: string): Promise<string> {
// Use zstandard if possible to maximize cache performance
export async function getCompressionMethod(): Promise<CompressionMethod> {
if (process.platform === 'win32' && !(await isGnuTarInstalled())) {
// Disable zstd due to bug https://github.com/actions/cache/issues/301
return CompressionMethod.Gzip
}
const versionOutput = await getVersion('zstd')
const version = semver.clean(versionOutput)
@@ -115,12 +116,9 @@ export function getCacheFileName(compressionMethod: CompressionMethod): string {
: CacheFilename.Zstd
}
export async function getGnuTarPathOnWindows(): Promise<string> {
if (fs.existsSync(GnuTarPathOnWindows)) {
return GnuTarPathOnWindows
}
export async function isGnuTarInstalled(): Promise<boolean> {
const versionOutput = await getVersion('tar')
return versionOutput.toLowerCase().includes('gnu tar') ? io.which('tar') : ''
return versionOutput.toLowerCase().includes('gnu tar')
}
export function assertDefined<T>(name: string, value?: T): T {
-15
View File
@@ -11,11 +11,6 @@ export enum CompressionMethod {
Zstd = 'zstd'
}
export enum ArchiveToolType {
GNU = 'gnu',
BSD = 'bsd'
}
// The default number of retry attempts.
export const DefaultRetryAttempts = 2
@@ -26,13 +21,3 @@ export const DefaultRetryDelay = 5000
// over the socket during this period, the socket is destroyed and the download
// is aborted.
export const SocketTimeout = 5000
// The default path of GNUtar on hosted Windows runners
export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe`
// The default path of BSDtar on hosted Windows runners
export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe`
export const TarFilename = 'cache.tar'
export const ManifestFilename = 'manifest.txt'
-6
View File
@@ -35,11 +35,5 @@ export interface ReserveCacheResponse {
export interface InternalCacheOptions {
compressionMethod?: CompressionMethod
enableCrossOsArchive?: boolean
cacheSize?: number
}
export interface ArchiveTool {
path: string
type: string
}
+84 -225
View File
@@ -1,32 +1,27 @@
import {exec} from '@actions/exec'
import {exportVariable} from '@actions/core'
import * as io from '@actions/io'
import {existsSync, writeFileSync} from 'fs'
import * as path from 'path'
import * as utils from './cacheUtils'
import {ArchiveTool} from './contracts'
import {
CompressionMethod,
SystemTarPathOnWindows,
ArchiveToolType,
TarFilename,
ManifestFilename
} from './constants'
import {CompressionMethod} from './constants'
const IS_WINDOWS = process.platform === 'win32'
exportVariable('MSYS', 'winsymlinks:nativestrict')
// Returns tar path and type: BSD or GNU
async function getTarPath(): Promise<ArchiveTool> {
async function getTarPath(
args: string[],
compressionMethod: CompressionMethod
): Promise<string> {
switch (process.platform) {
case 'win32': {
const gnuTar = await utils.getGnuTarPathOnWindows()
const systemTar = SystemTarPathOnWindows
if (gnuTar) {
// Use GNUtar as default on windows
return <ArchiveTool>{path: gnuTar, type: ArchiveToolType.GNU}
const systemTar = `${process.env['windir']}\\System32\\tar.exe`
if (compressionMethod !== CompressionMethod.Gzip) {
// We only use zstandard compression on windows when gnu tar is installed due to
// a bug with compressing large files with bsdtar + zstd
args.push('--force-local')
} else if (existsSync(systemTar)) {
return <ArchiveTool>{path: systemTar, type: ArchiveToolType.BSD}
return systemTar
} else if (await utils.isGnuTarInstalled()) {
args.push('--force-local')
}
break
}
@@ -34,133 +29,27 @@ async function getTarPath(): Promise<ArchiveTool> {
const gnuTar = await io.which('gtar', false)
if (gnuTar) {
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
return <ArchiveTool>{path: gnuTar, type: ArchiveToolType.GNU}
} else {
return <ArchiveTool>{
path: await io.which('tar', true),
type: ArchiveToolType.BSD
}
args.push('--delay-directory-restore')
return gnuTar
}
break
}
default:
break
}
// Default assumption is GNU tar is present in path
return <ArchiveTool>{
path: await io.which('tar', true),
type: ArchiveToolType.GNU
}
return await io.which('tar', true)
}
// Return arguments for tar as per tarPath, compressionMethod, method type and os
async function getTarArgs(
tarPath: ArchiveTool,
async function execTar(
args: string[],
compressionMethod: CompressionMethod,
type: string,
archivePath = ''
): Promise<string[]> {
const args = [`"${tarPath.path}"`]
const cacheFileName = utils.getCacheFileName(compressionMethod)
const tarFile = 'cache.tar'
const workingDirectory = getWorkingDirectory()
// Speficic args for BSD tar on windows for workaround
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
// Method specific args
switch (type) {
case 'create':
args.push(
'--posix',
'-cf',
BSD_TAR_ZSTD
? tarFile
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--exclude',
BSD_TAR_ZSTD
? tarFile
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--files-from',
ManifestFilename
)
break
case 'extract':
args.push(
'-xf',
BSD_TAR_ZSTD
? tarFile
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
)
break
case 'list':
args.push(
'-tf',
BSD_TAR_ZSTD
? tarFile
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
)
break
cwd?: string
): Promise<void> {
try {
await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd})
} catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`)
}
// Platform specific args
if (tarPath.type === ArchiveToolType.GNU) {
switch (process.platform) {
case 'win32':
args.push('--force-local')
break
case 'darwin':
args.push('--delay-directory-restore')
break
}
}
return args
}
// Returns commands to run tar and compression program
async function getCommands(
compressionMethod: CompressionMethod,
type: string,
archivePath = ''
): Promise<string[]> {
let args
const tarPath = await getTarPath()
const tarArgs = await getTarArgs(
tarPath,
compressionMethod,
type,
archivePath
)
const compressionArgs =
type !== 'create'
? await getDecompressionProgram(tarPath, compressionMethod, archivePath)
: await getCompressionProgram(tarPath, compressionMethod)
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
if (BSD_TAR_ZSTD && type !== 'create') {
args = [[...compressionArgs].join(' '), [...tarArgs].join(' ')]
} else {
args = [[...tarArgs].join(' '), [...compressionArgs].join(' ')]
}
if (BSD_TAR_ZSTD) {
return args
}
return [args.join(' ')]
}
function getWorkingDirectory(): string {
@@ -168,107 +57,37 @@ function getWorkingDirectory(): string {
}
// Common function for extractTar and listTar to get the compression method
async function getDecompressionProgram(
tarPath: ArchiveTool,
compressionMethod: CompressionMethod,
archivePath: string
): Promise<string[]> {
function getCompressionProgram(compressionMethod: CompressionMethod): string[] {
// -d: Decompress.
// unzstd is equivalent to 'zstd -d'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
switch (compressionMethod) {
case CompressionMethod.Zstd:
return BSD_TAR_ZSTD
? [
'zstd -d --long=30 --force -o',
TarFilename,
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
]
: [
'--use-compress-program',
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
]
return [
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return BSD_TAR_ZSTD
? [
'zstd -d --force -o',
TarFilename,
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
]
: ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']
return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']
default:
return ['-z']
}
}
// Used for creating the archive
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// zstdmt is equivalent to 'zstd -T0'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
async function getCompressionProgram(
tarPath: ArchiveTool,
compressionMethod: CompressionMethod
): Promise<string[]> {
const cacheFileName = utils.getCacheFileName(compressionMethod)
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
switch (compressionMethod) {
case CompressionMethod.Zstd:
return BSD_TAR_ZSTD
? [
'zstd -T0 --long=30 --force -o',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
TarFilename
]
: [
'--use-compress-program',
IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return BSD_TAR_ZSTD
? [
'zstd -T0 --force -o',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
TarFilename
]
: ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt']
default:
return ['-z']
}
}
// Executes all commands as separate processes
async function execCommands(commands: string[], cwd?: string): Promise<void> {
for (const command of commands) {
try {
await exec(command, undefined, {cwd})
} catch (error) {
throw new Error(
`${command.split(' ')[0]} failed with error: ${error?.message}`
)
}
}
}
// List the contents of a tar
export async function listTar(
archivePath: string,
compressionMethod: CompressionMethod
): Promise<void> {
const commands = await getCommands(compressionMethod, 'list', archivePath)
await execCommands(commands)
const args = [
...getCompressionProgram(compressionMethod),
'-tf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
]
await execTar(args, compressionMethod)
}
// Extract a tar
export async function extractTar(
archivePath: string,
compressionMethod: CompressionMethod
@@ -276,21 +95,61 @@ export async function extractTar(
// Create directory to extract tar into
const workingDirectory = getWorkingDirectory()
await io.mkdirP(workingDirectory)
const commands = await getCommands(compressionMethod, 'extract', archivePath)
await execCommands(commands)
const args = [
...getCompressionProgram(compressionMethod),
'-xf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
]
await execTar(args, compressionMethod)
}
// Create a tar
export async function createTar(
archiveFolder: string,
sourceDirectories: string[],
compressionMethod: CompressionMethod
): Promise<void> {
// Write source directories to manifest.txt to avoid command length limits
const manifestFilename = 'manifest.txt'
const cacheFileName = utils.getCacheFileName(compressionMethod)
writeFileSync(
path.join(archiveFolder, ManifestFilename),
path.join(archiveFolder, manifestFilename),
sourceDirectories.join('\n')
)
const commands = await getCommands(compressionMethod, 'create')
await execCommands(commands, archiveFolder)
const workingDirectory = getWorkingDirectory()
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// zstdmt is equivalent to 'zstd -T0'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
function getCompressionProgram(): string[] {
switch (compressionMethod) {
case CompressionMethod.Zstd:
return [
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']
default:
return ['-z']
}
}
const args = [
'--posix',
...getCompressionProgram(),
'-cf',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--exclude',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--files-from',
manifestFilename
]
await execTar(args, compressionMethod, archiveFolder)
}