Compare commits

..
Author SHA1 Message Date
Sankalp KotewarandGitHub d556ae3fc4 Merge remote-tracking branch 'origin/master' into kotewar/allow-http-client-for-cache-download 2023-02-10 06:19:41 +00:00
74906bea83 Add documentation for notice (#1105)
Code Scanning - Action / CodeQL-Build (push) Has been cancelled
* Add documentation for notice

This is described in documentation elsewhere.

* Update docs/commands.md

Co-authored-by: Konrad Pabjan <[email protected]>

Co-authored-by: Konrad Pabjan <[email protected]>
2023-01-11 14:09:01 -05:00
Sankalp KotewarandGitHub 59d682457a Merge remote-tracking branch 'origin/master' into kotewar/allow-http-client-for-cache-download 2022-12-27 17:57:58 +00:00
Sankalp KotewarandGitHub a64055103d Added env var for using Http client 2022-12-07 15:09:30 +00:00
Sampark SharmaandGitHub 819157bf87 Merge pull request #1238 from actions/revert-1232-phantsure/gnutar_windows
Revert "Add GNUtar as default in windows"
2022-11-17 12:42:10 +05:30
Sampark SharmaandGitHub 86102e88e9 Revert "Add GNUtar as default in windows" 2022-11-17 12:26:27 +05:30
Josh SorefandGitHub 2f164000dc Fix debug logging link (#820) 2021-05-25 14:28:45 -04:00
Federico GrandiandGitHub dc8e290405 docs(README): fix minor formatting issue (#819) 2021-05-24 10:23:40 -04:00
Linus UnnebäckandGitHub e9c6ee99a5 Simplify mkdirP implementation (#444) 2021-04-28 14:24:23 -04:00
madheadandGitHub 4bf916289e master → main (#596) 2021-02-19 10:02:58 -05:00
8 changed files with 130 additions and 500 deletions
+1 -1
View File
@@ -17,7 +17,7 @@ Binding to a major version is the latest of that major version ( e.g. `v1` == "1
Major versions should guarantee compatibility. A major version can add net new capabilities but should not break existing input compatibility or break existing workflows.
Major version binding allows you to take advantage of bug fixes and critical functionality and security fixes. The `main` branch has the latest code and is unstable to bind to since changes get committed to main and released to the market place by creating a tag. In addition, a new major version carrying breaking changes will get implemented in main after branching off the previous major version.
Major version binding allows you to take advantage of bug fixes and critical functionality and security fixes. The `main` branch has the latest code and is unstable to bind to since changes get committed to `main` and released to the market place by creating a tag. In addition, a new major version carrying breaking changes will get implemented in `main` after branching off the previous major version.
> Warning: do not reference `main` since that is the latest code and can be carrying breaking changes of the next major version.
+3
View File
@@ -100,9 +100,12 @@ There are several commands to emit different levels of log output:
| log level | example usage |
|---|---|
| [debug](action-debugging.md) | `echo "::debug::My debug message"` |
| notice | `echo "::notice::My notice message"` |
| warning | `echo "::warning::My warning message"` |
| error | `echo "::error::My error message"` |
Additional syntax options are described at [the workflow command documentation](https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-a-debug-message).
### Command Echoing
By default, the echoing of commands to stdout only occurs if [Step Debugging is enabled](./action-debugging.md#How-to-Access-Step-Debug-Logs)
+2
View File
@@ -47,3 +47,5 @@ const cacheKey = await cache.restoreCache(paths, key, restoreKeys)
A cache gets downloaded in multiple segments of fixed sizes (`1GB` for a `32-bit` runner and `2GB` for a `64-bit` runner). Sometimes, a segment download gets stuck which causes the workflow job to be stuck forever and fail. Version `v3.0.4` of cache package introduces a segment download timeout. The segment download timeout will allow the segment download to get aborted and hence allow the job to proceed with a cache miss.
Default value of this timeout is 60 minutes and can be customized by specifying an [environment variable](https://docs.github.com/en/actions/learn-github-actions/environment-variables) named `SEGMENT_DOWNLOAD_TIMEOUT_MINS` with timeout value in minutes.
We are also introducing a new variable `DOWNLOAD_WITH_HTTP_CLIENT`, which if set to `true` downloads the cache using plain http-client and not Azure SDK. This has been introduced keeping in mind the problem Azure SDK is causing in downloads getting stuck infinitely.
+56 -173
View File
@@ -1,12 +1,7 @@
import * as exec from '@actions/exec'
import * as io from '@actions/io'
import * as path from 'path'
import {
CacheFilename,
CompressionMethod,
GnuTarPathOnWindows,
SystemTarPathOnWindows
} from '../src/internal/constants'
import {CacheFilename, CompressionMethod} from '../src/internal/constants'
import * as tar from '../src/internal/tar'
import * as utils from '../src/internal/cacheUtils'
// eslint-disable-next-line @typescript-eslint/no-require-imports
@@ -18,7 +13,7 @@ jest.mock('@actions/io')
const IS_WINDOWS = process.platform === 'win32'
const IS_MAC = process.platform === 'darwin'
const defaultTarPath = IS_MAC ? 'gtar' : 'tar'
const defaultTarPath = process.platform === 'darwin' ? 'gtar' : 'tar'
function getTempDir(): string {
return path.join(__dirname, '_temp', 'tar')
@@ -33,10 +28,6 @@ beforeAll(async () => {
await jest.requireActual('@actions/io').rmRF(getTempDir())
})
beforeEach(async () => {
jest.restoreAllMocks()
})
afterAll(async () => {
delete process.env['GITHUB_WORKSPACE']
await jest.requireActual('@actions/io').rmRF(getTempDir())
@@ -50,15 +41,16 @@ test('zstd extract tar', async () => {
? `${process.env['windir']}\\fakepath\\cache.tar`
: 'cache.tar'
const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
await tar.extractTar(archivePath, CompressionMethod.Zstd)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
`"${defaultTarPath}"`,
[
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
'-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P',
@@ -66,50 +58,11 @@ test('zstd extract tar', async () => {
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
]),
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
)
})
test('zstd extract tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const mkdirMock = jest.spyOn(io, 'mkdirP')
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = SystemTarPathOnWindows
const tarFilename = 'cache.tar'
await tar.extractTar(archivePath, CompressionMethod.Zstd)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
'zstd -d --long=30 -o',
[
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&',
`${tarPath}`,
'-xf',
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/')
],
{cwd: undefined}
)
}
})
test('gzip extract tar', async () => {
const mkdirMock = jest.spyOn(io, 'mkdirP')
const execMock = jest.spyOn(exec, 'exec')
@@ -121,47 +74,49 @@ test('gzip extract tar', async () => {
await tar.extractTar(archivePath, CompressionMethod.Gzip)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[
'-z',
'-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P',
'-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z']),
].concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
)
})
test('gzip extract GNU tar on windows with GNUtar in path', async () => {
test('gzip extract GNU tar on windows', async () => {
if (IS_WINDOWS) {
// GNU tar present in path but not at default location
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve('tar'))
jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false)
const isGnuMock = jest
.spyOn(utils, 'isGnuTarInstalled')
.mockReturnValue(Promise.resolve(true))
const execMock = jest.spyOn(exec, 'exec')
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
const workspace = process.env['GITHUB_WORKSPACE']
await tar.extractTar(archivePath, CompressionMethod.Gzip)
expect(isGnuMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"tar"`,
[
'-z',
'-xf',
archivePath.replace(/\\/g, '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/'),
'--force-local',
'-z'
'--force-local'
],
{cwd: undefined}
)
@@ -179,13 +134,13 @@ test('zstd create tar', async () => {
await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
`"${defaultTarPath}"`,
[
'--posix',
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30',
'-cf',
IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd,
'--exclude',
@@ -197,65 +152,13 @@ test('zstd create tar', async () => {
'manifest.txt'
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
]),
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
{
cwd: archiveFolder
}
)
})
test('zstd create tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archiveFolder = getTempDir()
const workspace = process.env['GITHUB_WORKSPACE']
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
const tarFilename = 'cache.tar'
await fs.promises.mkdir(archiveFolder, {recursive: true})
await tar.createTar(
archiveFolder,
sourceDirectories,
CompressionMethod.Zstd
)
const tarPath = SystemTarPathOnWindows
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[
'--posix',
'-cf',
tarFilename.replace(/\\/g, '/'),
'--exclude',
tarFilename.replace(/\\/g, '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/'),
'--files-from',
'manifest.txt',
'&&',
'zstd -T0 --long=30 -o',
CacheFilename.Zstd.replace(/\\/g, '/'),
tarFilename.replace(/\\/g, '/')
],
{
cwd: archiveFolder
}
)
}
})
test('gzip create tar', async () => {
const execMock = jest.spyOn(exec, 'exec')
@@ -267,13 +170,16 @@ test('gzip create tar', async () => {
await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Gzip)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[
'--posix',
'-z',
'-cf',
IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip,
'--exclude',
@@ -283,10 +189,7 @@ test('gzip create tar', async () => {
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from',
'manifest.txt'
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z']),
].concat(IS_MAC ? ['--delay-directory-restore'] : []),
{
cwd: archiveFolder
}
@@ -302,50 +205,22 @@ test('zstd list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.Zstd)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P']
`"${defaultTarPath}"`,
[
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
]),
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
)
})
test('zstd list tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
await tar.listTar(archivePath, CompressionMethod.Zstd)
const tarFilename = 'cache.tar'
const tarPath = SystemTarPathOnWindows
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
'zstd -d --long=30 -o',
[
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&',
`${tarPath}`,
'-tf',
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
],
{cwd: undefined}
)
}
})
test('zstdWithoutLong list tar', async () => {
const execMock = jest.spyOn(exec, 'exec')
@@ -355,14 +230,18 @@ test('zstdWithoutLong list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.ZstdWithoutLong)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P']
`"${defaultTarPath}"`,
[
'--use-compress-program',
IS_WINDOWS ? 'zstd -d' : 'unzstd',
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']),
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
)
})
@@ -375,14 +254,18 @@ test('gzip list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.Gzip)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P']
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z']),
[
'-z',
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
].concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
)
})
+1 -1
View File
@@ -132,7 +132,7 @@ export async function downloadCache(
if (
downloadOptions.useAzureSdk &&
archiveUrl.hostname.endsWith('.blob.core.windows.net')
archiveUrl.hostname.endsWith('.blob.core.windows.net') && !process.env['DOWNLOAD_WITH_HTTP_CLIENT']
) {
// Use Azure storage SDK to download caches hosted on Azure to improve speed and reliability.
await downloadCacheStorageSDK(archiveLocation, archivePath, downloadOptions)
+9 -11
View File
@@ -7,11 +7,7 @@ import * as path from 'path'
import * as semver from 'semver'
import * as util from 'util'
import {v4 as uuidV4} from 'uuid'
import {
CacheFilename,
CompressionMethod,
GnuTarPathOnWindows
} from './constants'
import {CacheFilename, CompressionMethod} from './constants'
// From https://github.com/actions/toolkit/blob/main/packages/tool-cache/src/tool-cache.ts#L23
export async function createTempDirectory(): Promise<string> {
@@ -71,7 +67,7 @@ export async function unlinkFile(filePath: fs.PathLike): Promise<void> {
return util.promisify(fs.unlink)(filePath)
}
export async function getVersion(app: string): Promise<string> {
async function getVersion(app: string): Promise<string> {
core.debug(`Checking ${app} --version`)
let versionOutput = ''
try {
@@ -94,6 +90,11 @@ export async function getVersion(app: string): Promise<string> {
// Use zstandard if possible to maximize cache performance
export async function getCompressionMethod(): Promise<CompressionMethod> {
if (process.platform === 'win32' && !(await isGnuTarInstalled())) {
// Disable zstd due to bug https://github.com/actions/cache/issues/301
return CompressionMethod.Gzip
}
const versionOutput = await getVersion('zstd')
const version = semver.clean(versionOutput)
@@ -115,12 +116,9 @@ export function getCacheFileName(compressionMethod: CompressionMethod): string {
: CacheFilename.Zstd
}
export async function getGnuTarPathOnWindows(): Promise<string> {
if (fs.existsSync(GnuTarPathOnWindows)) {
return GnuTarPathOnWindows
}
export async function isGnuTarInstalled(): Promise<boolean> {
const versionOutput = await getVersion('tar')
return versionOutput.toLowerCase().includes('gnu tar') ? io.which('tar') : ''
return versionOutput.toLowerCase().includes('gnu tar')
}
export function assertDefined<T>(name: string, value?: T): T {
-5
View File
@@ -21,8 +21,3 @@ export const DefaultRetryDelay = 5000
// over the socket during this period, the socket is destroyed and the download
// is aborted.
export const SocketTimeout = 5000
// The default path of GNUtar on hosted Windows runners
export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe`
export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe`
+58 -309
View File
@@ -3,21 +3,25 @@ import * as io from '@actions/io'
import {existsSync, writeFileSync} from 'fs'
import * as path from 'path'
import * as utils from './cacheUtils'
import {CompressionMethod, SystemTarPathOnWindows} from './constants'
import {CompressionMethod} from './constants'
const IS_WINDOWS = process.platform === 'win32'
// Function also mutates the args array. For non-mutation call with passing an empty array.
async function getTarPath(): Promise<string> {
async function getTarPath(
args: string[],
compressionMethod: CompressionMethod
): Promise<string> {
switch (process.platform) {
case 'win32': {
const gnuTar = await utils.getGnuTarPathOnWindows()
const systemTar = SystemTarPathOnWindows
if (gnuTar) {
// Use GNUtar as default on windows
return gnuTar
const systemTar = `${process.env['windir']}\\System32\\tar.exe`
if (compressionMethod !== CompressionMethod.Gzip) {
// We only use zstandard compression on windows when gnu tar is installed due to
// a bug with compressing large files with bsdtar + zstd
args.push('--force-local')
} else if (existsSync(systemTar)) {
return systemTar
} else if (await utils.isGnuTarInstalled()) {
args.push('--force-local')
}
break
}
@@ -25,6 +29,7 @@ async function getTarPath(): Promise<string> {
const gnuTar = await io.which('gtar', false)
if (gnuTar) {
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
args.push('--delay-directory-restore')
return gnuTar
}
break
@@ -35,101 +40,13 @@ async function getTarPath(): Promise<string> {
return await io.which('tar', true)
}
async function getTarArgs(
compressionMethod: CompressionMethod,
type: string,
archivePath = ''
): Promise<string[]> {
const args = []
const manifestFilename = 'manifest.txt'
const cacheFileName = utils.getCacheFileName(compressionMethod)
const tarFile = 'cache.tar'
const tarPath = await getTarPath()
const workingDirectory = getWorkingDirectory()
const BSD_TAR_ZSTD =
tarPath === SystemTarPathOnWindows &&
compressionMethod !== CompressionMethod.Gzip
// Method specific args
switch (type) {
case 'create':
args.push(
'--posix',
'-cf',
BSD_TAR_ZSTD
? tarFile
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--exclude',
BSD_TAR_ZSTD
? tarFile
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--files-from',
manifestFilename
)
break
case 'extract':
args.push(
'-xf',
BSD_TAR_ZSTD
? tarFile
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
)
break
case 'list':
args.push(
'-tf',
BSD_TAR_ZSTD
? tarFile
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
)
break
}
// Platform specific args
switch (process.platform) {
case 'win32': {
const gnuTar = await utils.getGnuTarPathOnWindows()
if (gnuTar) {
// Use GNUtar as default on windows
args.push('--force-local')
}
break
}
case 'darwin': {
const gnuTar = await io.which('gtar', false)
if (gnuTar) {
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
args.push('--delay-directory-restore')
}
break
}
}
return args
}
async function execTar(args: string[], cwd?: string): Promise<void> {
try {
await exec(`"${await getTarPath()}"`, args, {cwd})
} catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`)
}
}
async function execCommand(
command: string,
async function execTar(
args: string[],
compressionMethod: CompressionMethod,
cwd?: string
): Promise<void> {
try {
await exec(command, args, {cwd})
await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd})
} catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`)
}
@@ -140,41 +57,19 @@ function getWorkingDirectory(): string {
}
// Common function for extractTar and listTar to get the compression method
async function getCompressionProgram(
compressionMethod: CompressionMethod,
archivePath: string
): Promise<string[]> {
function getCompressionProgram(compressionMethod: CompressionMethod): string[] {
// -d: Decompress.
// unzstd is equivalent to 'zstd -d'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
const tarPath = await getTarPath()
const tarFile = 'cache.tar'
const BSD_TAR_ZSTD =
tarPath === SystemTarPathOnWindows &&
compressionMethod !== CompressionMethod.Gzip
switch (compressionMethod) {
case CompressionMethod.Zstd:
return BSD_TAR_ZSTD
? [
'zstd -d --long=30 -o',
tarFile,
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&'
]
: [
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
]
return [
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return BSD_TAR_ZSTD
? [
'zstd -d -o',
tarFile,
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&'
]
: ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']
return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']
default:
return ['-z']
}
@@ -184,27 +79,13 @@ export async function listTar(
archivePath: string,
compressionMethod: CompressionMethod
): Promise<void> {
const tarPath = await getTarPath()
const BSD_TAR_ZSTD =
tarPath === SystemTarPathOnWindows &&
compressionMethod !== CompressionMethod.Gzip
const compressionArgs = await getCompressionProgram(
compressionMethod,
archivePath
)
const tarArgs = await getTarArgs(compressionMethod, 'list', archivePath)
// TODO: Add a test for BSD tar on windows
if (BSD_TAR_ZSTD) {
const command = compressionArgs[0]
const args = compressionArgs
.slice(1)
.concat([tarPath])
.concat(tarArgs)
await execCommand(command, args)
} else {
const args = tarArgs.concat(compressionArgs)
await execTar(args)
}
const args = [
...getCompressionProgram(compressionMethod),
'-tf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
]
await execTar(args, compressionMethod)
}
export async function extractTar(
@@ -213,27 +94,16 @@ export async function extractTar(
): Promise<void> {
// Create directory to extract tar into
const workingDirectory = getWorkingDirectory()
const tarPath = await getTarPath()
const BSD_TAR_ZSTD =
tarPath === SystemTarPathOnWindows &&
compressionMethod !== CompressionMethod.Gzip
await io.mkdirP(workingDirectory)
const tarArgs = await getTarArgs(compressionMethod, 'extract', archivePath)
const compressionArgs = await getCompressionProgram(
compressionMethod,
archivePath
)
if (BSD_TAR_ZSTD) {
const command = compressionArgs[0]
const args = compressionArgs
.slice(1)
.concat([tarPath])
.concat(tarArgs)
await execCommand(command, args)
} else {
const args = tarArgs.concat(compressionArgs)
await execTar(args)
}
const args = [
...getCompressionProgram(compressionMethod),
'-xf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
]
await execTar(args, compressionMethod)
}
export async function createTar(
@@ -244,15 +114,11 @@ export async function createTar(
// Write source directories to manifest.txt to avoid command length limits
const manifestFilename = 'manifest.txt'
const cacheFileName = utils.getCacheFileName(compressionMethod)
const tarFile = 'cache.tar'
const tarPath = await getTarPath()
const BSD_TAR_ZSTD =
tarPath === SystemTarPathOnWindows &&
compressionMethod !== CompressionMethod.Gzip
writeFileSync(
path.join(archiveFolder, manifestFilename),
sourceDirectories.join('\n')
)
const workingDirectory = getWorkingDirectory()
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// zstdmt is equivalent to 'zstd -T0'
@@ -262,145 +128,28 @@ export async function createTar(
function getCompressionProgram(): string[] {
switch (compressionMethod) {
case CompressionMethod.Zstd:
return BSD_TAR_ZSTD
? [
'&&',
'zstd -T0 --long=30 -o',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
tarFile
]
: [
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
]
return [
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return BSD_TAR_ZSTD
? [
'&&',
'zstd -T0 -o',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
tarFile
]
: ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']
return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']
default:
return ['-z']
}
}
const tarArgs = await getTarArgs(compressionMethod, 'create')
const compressionArgs = getCompressionProgram()
const args = tarArgs.concat(compressionArgs)
await execTar(args, archiveFolder)
const args = [
'--posix',
...getCompressionProgram(),
'-cf',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--exclude',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--files-from',
manifestFilename
]
await execTar(args, compressionMethod, archiveFolder)
}
const ARCHIVE_TOOL_GNU: string = 'gnu'
const ARCHIVE_TOOL_BSD: string = 'bsd'
const manifestFilename = 'manifest.txt'
export async function createTar2(
archiveFolder: string,
sourceDirectories: string[]
): Promise<void> {
// 1. decide the compression algo - zstd otherwise gzip
// 2. decide tar - gnutar otherwise bsdtar or systemtar
// 3. deide archiving/unarchiving args depending on tar
// 4. decide compression/decompression args depending on tar and compression
// 5. decide the exec command
let tar = getTarPath2()
let compression = getCompressionMethod2()
let args = argsMap.get({tar: tar, compression: compression, os: process.platform, "create"})
}
// return archive tool name and path
async function getTarPath2(): Promise<ArchiveTool> {
switch (process.platform) {
case 'win32': {
const gnuTar = await utils.getGnuTarPathOnWindows()
const systemTar = SystemTarPathOnWindows
if (gnuTar) {
// Use GNUtar as default on windows
return <ArchiveTool>{ name: ARCHIVE_TOOL_GNU, path: gnuTar }
} else if (existsSync(systemTar)) {
return <ArchiveTool>{name: ARCHIVE_TOOL_BSD, path: systemTar}
}
break
}
case 'darwin': {
const gnuTar = await io.which('gtar', false)
if (gnuTar) {
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
return <ArchiveTool>{ name: ARCHIVE_TOOL_GNU, path: gnuTar }
} else {
const path = await io.which('tar', true)
return <ArchiveTool>{name: ARCHIVE_TOOL_BSD, path: path}
}
}
default:
break
}
const path = await io.which('tar', true)
return <ArchiveTool>{name: ARCHIVE_TOOL_GNU, path: path}
}
// Use zstandard if possible to maximize cache performance
export async function getCompressionMethod2(): Promise<CompressionMethod> {
const versionOutput = await utils.getVersion('zstd')
const version = semver.clean(versionOutput)
if (!versionOutput.toLowerCase().includes('zstd command line interface')) {
// zstd is not installed
return CompressionMethod.Gzip
} else if (!version || semver.lt(version, 'v1.3.2')) {
// zstd is installed but using a version earlier than v1.3.2
// v1.3.2 is required to use the `--long` options in zstd
return CompressionMethod.ZstdWithoutLong
} else {
return CompressionMethod.Zstd
}
}
interface ArchiveTool {
name: string
path: string
}
// tar, compression, os, operation -> tar args + compression args
const argsMap: Map<ArgsLookupKey, string[][]> = new Map()
argsMap.set({tar: ARCHIVE_TOOL_GNU, compression: CompressionMethod.Zstd, os: "windows", operation: "create"},
[[
'--posix',
'-cf',
'%%cacheFileName%%',
'--exclude',
'%%cacheFileName%%',
'-P',
'-C',
'%%workingDirectory%%',
'--files-from',
manifestFilename],
['--use-compress-program',
'zstd -T0 --long=30']
])
argsMap.set({tar: ARCHIVE_TOOL_GNU, compression: CompressionMethod.Zstd, os: "linux", operation: "create"},
[
'--use-compress-program',
'zstdmt --long=30'
])
interface ArgsLookupKey {
tar: string
compression: string
os: string
operation: string
}