Compare commits

..

1 Commits

Author SHA1 Message Date
Bishal Prasad 44cd5d150b args map 2022-11-24 08:50:57 +00:00
7 changed files with 308 additions and 307 deletions
-75
View File
@@ -161,81 +161,6 @@ test('restore with gzip compressed cache found', async () => {
expect(getCompressionMock).toHaveBeenCalledTimes(1)
})
test('restore with zstd as default but gzip compressed cache found on windows', async () => {
if (process.platform === 'win32') {
const paths = ['node_modules']
const key = 'node-test'
const cacheEntry: ArtifactCacheEntry = {
cacheKey: key,
scope: 'refs/heads/main',
archiveLocation: 'www.actionscache.test/download'
}
const getCacheMock = jest.spyOn(cacheHttpClient, 'getCacheEntry')
getCacheMock
.mockImplementationOnce(async () => {
throw new Error('Cache not found.')
})
.mockImplementationOnce(async () => {
return Promise.resolve(cacheEntry)
})
const tempPath = '/foo/bar'
const createTempDirectoryMock = jest.spyOn(
cacheUtils,
'createTempDirectory'
)
createTempDirectoryMock.mockImplementation(async () => {
return Promise.resolve(tempPath)
})
const archivePath = path.join(tempPath, CacheFilename.Gzip)
const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache')
const fileSize = 142
const getArchiveFileSizeInBytesMock = jest
.spyOn(cacheUtils, 'getArchiveFileSizeInBytes')
.mockReturnValue(fileSize)
const extractTarMock = jest.spyOn(tar, 'extractTar')
const unlinkFileMock = jest.spyOn(cacheUtils, 'unlinkFile')
const compression = CompressionMethod.Zstd
const getCompressionMock = jest
.spyOn(cacheUtils, 'getCompressionMethod')
.mockReturnValue(Promise.resolve(compression))
const cacheKey = await restoreCache(paths, key)
expect(cacheKey).toBe(key)
expect(getCacheMock).toHaveBeenNthCalledWith(1, [key], paths, {
compressionMethod: compression
})
expect(getCacheMock).toHaveBeenNthCalledWith(2, [key], paths, {
compressionMethod: CompressionMethod.Gzip
})
expect(createTempDirectoryMock).toHaveBeenCalledTimes(1)
expect(downloadCacheMock).toHaveBeenCalledWith(
cacheEntry.archiveLocation,
archivePath,
undefined
)
expect(getArchiveFileSizeInBytesMock).toHaveBeenCalledWith(archivePath)
expect(extractTarMock).toHaveBeenCalledTimes(1)
expect(extractTarMock).toHaveBeenCalledWith(
archivePath,
CompressionMethod.Gzip
)
expect(unlinkFileMock).toHaveBeenCalledTimes(1)
expect(unlinkFileMock).toHaveBeenCalledWith(archivePath)
expect(getCompressionMock).toHaveBeenCalledTimes(1)
}
})
test('restore with zstd compressed cache found', async () => {
const paths = ['node_modules']
const key = 'node-test'
+52 -65
View File
@@ -5,9 +5,7 @@ import {
CacheFilename,
CompressionMethod,
GnuTarPathOnWindows,
ManifestFilename,
SystemTarPathOnWindows,
TarFilename
SystemTarPathOnWindows
} from '../src/internal/constants'
import * as tar from '../src/internal/tar'
import * as utils from '../src/internal/cacheUtils'
@@ -59,8 +57,8 @@ test('zstd extract tar', async () => {
expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[
`"${tarPath}"`,
'-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P',
@@ -71,9 +69,9 @@ test('zstd extract tar', async () => {
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
])
.join(' ')
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
]),
{cwd: undefined}
)
})
@@ -88,24 +86,26 @@ test('zstd extract tar with windows BSDtar', async () => {
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = SystemTarPathOnWindows
const tarFilename = 'cache.tar'
await tar.extractTar(archivePath, CompressionMethod.Zstd)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
'zstd -d --long=30 -o',
[
'zstd -d --long=30 -o',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&',
`"${tarPath}"`,
`${tarPath}`,
'-xf',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/')
].join(' ')
],
{cwd: undefined}
)
}
})
@@ -124,8 +124,8 @@ test('gzip extract tar', async () => {
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[
`"${tarPath}"`,
'-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P',
@@ -134,8 +134,8 @@ test('gzip extract tar', async () => {
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z'])
.join(' ')
.concat(['-z']),
{cwd: undefined}
)
})
@@ -153,8 +153,8 @@ test('gzip extract GNU tar on windows with GNUtar in path', async () => {
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"tar"`,
[
`"tar"`,
'-xf',
archivePath.replace(/\\/g, '/'),
'-P',
@@ -162,7 +162,8 @@ test('gzip extract GNU tar on windows with GNUtar in path', async () => {
workspace?.replace(/\\/g, '/'),
'--force-local',
'-z'
].join(' ')
],
{cwd: undefined}
)
}
})
@@ -182,8 +183,8 @@ test('zstd create tar', async () => {
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[
`"${tarPath}"`,
'--posix',
'-cf',
IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd,
@@ -193,16 +194,14 @@ test('zstd create tar', async () => {
'-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from',
ManifestFilename
'manifest.txt'
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30'
])
.join(' '),
undefined, // args
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
]),
{
cwd: archiveFolder
}
@@ -219,6 +218,7 @@ test('zstd create tar with windows BSDtar', async () => {
const archiveFolder = getTempDir()
const workspace = process.env['GITHUB_WORKSPACE']
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
const tarFilename = 'cache.tar'
await fs.promises.mkdir(archiveFolder, {recursive: true})
@@ -232,24 +232,23 @@ test('zstd create tar with windows BSDtar', async () => {
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[
`"${tarPath}"`,
'--posix',
'-cf',
TarFilename.replace(/\\/g, '/'),
tarFilename.replace(/\\/g, '/'),
'--exclude',
TarFilename.replace(/\\/g, '/'),
tarFilename.replace(/\\/g, '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/'),
'--files-from',
ManifestFilename,
'manifest.txt',
'&&',
'zstd -T0 --long=30 -o',
CacheFilename.Zstd.replace(/\\/g, '/'),
TarFilename.replace(/\\/g, '/')
].join(' '),
undefined, // args
tarFilename.replace(/\\/g, '/')
],
{
cwd: archiveFolder
}
@@ -272,8 +271,8 @@ test('gzip create tar', async () => {
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[
`"${tarPath}"`,
'--posix',
'-cf',
IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip,
@@ -283,13 +282,11 @@ test('gzip create tar', async () => {
'-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from',
ManifestFilename
'manifest.txt'
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z'])
.join(' '),
undefined, // args
.concat(['-z']),
{
cwd: archiveFolder
}
@@ -308,19 +305,15 @@ test('zstd list tar', async () => {
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
[
`"${tarPath}"`,
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
]
`"${tarPath}"`,
['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P']
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
])
.join(' ')
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
]),
{cwd: undefined}
)
})
@@ -334,19 +327,21 @@ test('zstd list tar with windows BSDtar', async () => {
await tar.listTar(archivePath, CompressionMethod.Zstd)
const tarFilename = 'cache.tar'
const tarPath = SystemTarPathOnWindows
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
'zstd -d --long=30 -o',
[
'zstd -d --long=30 -o',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&',
`"${tarPath}"`,
`${tarPath}`,
'-tf',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
].join(' ')
],
{cwd: undefined}
)
}
})
@@ -363,16 +358,12 @@ test('zstdWithoutLong list tar', async () => {
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
[
`"${tarPath}"`,
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
]
`"${tarPath}"`,
['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P']
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd'])
.join(' ')
.concat(['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']),
{cwd: undefined}
)
})
@@ -387,15 +378,11 @@ test('gzip list tar', async () => {
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
[
`"${tarPath}"`,
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
]
`"${tarPath}"`,
['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P']
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z'])
.join(' ')
.concat(['-z']),
{cwd: undefined}
)
})
+6 -27
View File
@@ -4,8 +4,6 @@ import * as utils from './internal/cacheUtils'
import * as cacheHttpClient from './internal/cacheHttpClient'
import {createTar, extractTar, listTar} from './internal/tar'
import {DownloadOptions, UploadOptions} from './options'
import {CompressionMethod} from './internal/constants'
import {ArtifactCacheEntry} from './internal/contracts'
export class ValidationError extends Error {
constructor(message: string) {
@@ -87,38 +85,19 @@ export async function restoreCache(
checkKey(key)
}
let cacheEntry: ArtifactCacheEntry | null
let compressionMethod = await utils.getCompressionMethod()
const compressionMethod = await utils.getCompressionMethod()
let archivePath = ''
try {
try {
// path are needed to compute version
cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, {
compressionMethod
})
} catch (error) {
// This is to support the old cache entry created
// by the old version of the cache action on windows.
if (
process.platform === 'win32' &&
compressionMethod !== CompressionMethod.Gzip
) {
compressionMethod = CompressionMethod.Gzip
cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, {
compressionMethod
})
if (!cacheEntry?.archiveLocation) {
throw error
}
} else {
throw error
}
}
// path are needed to compute version
const cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, {
compressionMethod
})
if (!cacheEntry?.archiveLocation) {
// Cache not found
return undefined
}
archivePath = path.join(
await utils.createTempDirectory(),
utils.getCacheFileName(compressionMethod)
+1 -1
View File
@@ -71,7 +71,7 @@ export async function unlinkFile(filePath: fs.PathLike): Promise<void> {
return util.promisify(fs.unlink)(filePath)
}
async function getVersion(app: string): Promise<string> {
export async function getVersion(app: string): Promise<string> {
core.debug(`Checking ${app} --version`)
let versionOutput = ''
try {
-10
View File
@@ -11,11 +11,6 @@ export enum CompressionMethod {
Zstd = 'zstd'
}
export enum ArchiveToolType {
GNU = 'gnu',
BSD = 'bsd'
}
// The default number of retry attempts.
export const DefaultRetryAttempts = 2
@@ -30,9 +25,4 @@ export const SocketTimeout = 5000
// The default path of GNUtar on hosted Windows runners
export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe`
// The default path of BSDtar on hosted Windows runners
export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe`
export const TarFilename = 'cache.tar'
export const ManifestFilename = 'manifest.txt'
-5
View File
@@ -31,8 +31,3 @@ export interface InternalCacheOptions {
compressionMethod?: CompressionMethod
cacheSize?: number
}
export interface ArchiveTool {
path: string
type: string
}
+249 -124
View File
@@ -3,28 +3,21 @@ import * as io from '@actions/io'
import {existsSync, writeFileSync} from 'fs'
import * as path from 'path'
import * as utils from './cacheUtils'
import {ArchiveTool} from './contracts'
import {
CompressionMethod,
SystemTarPathOnWindows,
ArchiveToolType,
TarFilename,
ManifestFilename
} from './constants'
import {CompressionMethod, SystemTarPathOnWindows} from './constants'
const IS_WINDOWS = process.platform === 'win32'
// Function also mutates the args array. For non-mutation call with passing an empty array.
async function getTarPath(): Promise<ArchiveTool> {
async function getTarPath(): Promise<string> {
switch (process.platform) {
case 'win32': {
const gnuTar = await utils.getGnuTarPathOnWindows()
const systemTar = SystemTarPathOnWindows
if (gnuTar) {
// Use GNUtar as default on windows
return <ArchiveTool>{path: gnuTar, type: ArchiveToolType.GNU}
return gnuTar
} else if (existsSync(systemTar)) {
return <ArchiveTool>{path: systemTar, type: ArchiveToolType.BSD}
return systemTar
}
break
}
@@ -32,38 +25,30 @@ async function getTarPath(): Promise<ArchiveTool> {
const gnuTar = await io.which('gtar', false)
if (gnuTar) {
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
return <ArchiveTool>{path: gnuTar, type: ArchiveToolType.GNU}
} else {
return <ArchiveTool>{
path: await io.which('tar', true),
type: ArchiveToolType.BSD
}
return gnuTar
}
break
}
default:
break
}
return <ArchiveTool>{
path: await io.which('tar', true),
type: ArchiveToolType.GNU
}
return await io.which('tar', true)
}
// Return arguments for tar as per tarPath, compressionMethod, method type and os
async function getTarArgs(
tarPath: ArchiveTool,
compressionMethod: CompressionMethod,
type: string,
archivePath = ''
): Promise<string[]> {
const args = [`"${tarPath.path}"`]
const args = []
const manifestFilename = 'manifest.txt'
const cacheFileName = utils.getCacheFileName(compressionMethod)
const tarFile = 'cache.tar'
const tarPath = await getTarPath()
const workingDirectory = getWorkingDirectory()
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
tarPath === SystemTarPathOnWindows &&
compressionMethod !== CompressionMethod.Gzip
// Method specific args
switch (type) {
@@ -82,7 +67,7 @@ async function getTarArgs(
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--files-from',
ManifestFilename
manifestFilename
)
break
case 'extract':
@@ -108,44 +93,45 @@ async function getTarArgs(
}
// Platform specific args
if (tarPath.type === ArchiveToolType.GNU) {
switch (process.platform) {
case 'win32':
switch (process.platform) {
case 'win32': {
const gnuTar = await utils.getGnuTarPathOnWindows()
if (gnuTar) {
// Use GNUtar as default on windows
args.push('--force-local')
break
case 'darwin':
}
break
}
case 'darwin': {
const gnuTar = await io.which('gtar', false)
if (gnuTar) {
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
args.push('--delay-directory-restore')
break
}
break
}
}
return args
}
async function getArgs(
compressionMethod: CompressionMethod,
type: string,
archivePath = ''
): Promise<string> {
const tarPath = await getTarPath()
const tarArgs = await getTarArgs(
tarPath,
compressionMethod,
type,
archivePath
)
const compressionArgs =
type !== 'create'
? await getDecompressionProgram(tarPath, compressionMethod, archivePath)
: await getCompressionProgram(tarPath, compressionMethod)
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
if (BSD_TAR_ZSTD && type !== 'create') {
return [...compressionArgs, ...tarArgs].join(' ')
} else {
return [...tarArgs, ...compressionArgs].join(' ')
async function execTar(args: string[], cwd?: string): Promise<void> {
try {
await exec(`"${await getTarPath()}"`, args, {cwd})
} catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`)
}
}
async function execCommand(
command: string,
args: string[],
cwd?: string
): Promise<void> {
try {
await exec(command, args, {cwd})
} catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`)
}
}
@@ -154,8 +140,7 @@ function getWorkingDirectory(): string {
}
// Common function for extractTar and listTar to get the compression method
async function getDecompressionProgram(
tarPath: ArchiveTool,
async function getCompressionProgram(
compressionMethod: CompressionMethod,
archivePath: string
): Promise<string[]> {
@@ -163,73 +148,33 @@ async function getDecompressionProgram(
// unzstd is equivalent to 'zstd -d'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
const tarPath = await getTarPath()
const tarFile = 'cache.tar'
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
tarPath === SystemTarPathOnWindows &&
compressionMethod !== CompressionMethod.Gzip
switch (compressionMethod) {
case CompressionMethod.Zstd:
return BSD_TAR_ZSTD
? [
'zstd -d --long=30 -o',
TarFilename,
tarFile,
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&'
]
: [
'--use-compress-program',
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return BSD_TAR_ZSTD
? [
'zstd -d -o',
TarFilename,
tarFile,
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&'
]
: ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']
default:
return ['-z']
}
}
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// zstdmt is equivalent to 'zstd -T0'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
async function getCompressionProgram(
tarPath: ArchiveTool,
compressionMethod: CompressionMethod
): Promise<string[]> {
const cacheFileName = utils.getCacheFileName(compressionMethod)
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
switch (compressionMethod) {
case CompressionMethod.Zstd:
return BSD_TAR_ZSTD
? [
'&&',
'zstd -T0 --long=30 -o',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
TarFilename
]
: [
'--use-compress-program',
IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return BSD_TAR_ZSTD
? [
'&&',
'zstd -T0 -o',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
TarFilename
]
: ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt']
: ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']
default:
return ['-z']
}
@@ -239,11 +184,26 @@ export async function listTar(
archivePath: string,
compressionMethod: CompressionMethod
): Promise<void> {
const args = await getArgs(compressionMethod, 'list', archivePath)
try {
await exec(args)
} catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`)
const tarPath = await getTarPath()
const BSD_TAR_ZSTD =
tarPath === SystemTarPathOnWindows &&
compressionMethod !== CompressionMethod.Gzip
const compressionArgs = await getCompressionProgram(
compressionMethod,
archivePath
)
const tarArgs = await getTarArgs(compressionMethod, 'list', archivePath)
// TODO: Add a test for BSD tar on windows
if (BSD_TAR_ZSTD) {
const command = compressionArgs[0]
const args = compressionArgs
.slice(1)
.concat([tarPath])
.concat(tarArgs)
await execCommand(command, args)
} else {
const args = tarArgs.concat(compressionArgs)
await execTar(args)
}
}
@@ -253,12 +213,26 @@ export async function extractTar(
): Promise<void> {
// Create directory to extract tar into
const workingDirectory = getWorkingDirectory()
const tarPath = await getTarPath()
const BSD_TAR_ZSTD =
tarPath === SystemTarPathOnWindows &&
compressionMethod !== CompressionMethod.Gzip
await io.mkdirP(workingDirectory)
const args = await getArgs(compressionMethod, 'extract', archivePath)
try {
await exec(args)
} catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`)
const tarArgs = await getTarArgs(compressionMethod, 'extract', archivePath)
const compressionArgs = await getCompressionProgram(
compressionMethod,
archivePath
)
if (BSD_TAR_ZSTD) {
const command = compressionArgs[0]
const args = compressionArgs
.slice(1)
.concat([tarPath])
.concat(tarArgs)
await execCommand(command, args)
} else {
const args = tarArgs.concat(compressionArgs)
await execTar(args)
}
}
@@ -268,14 +242,165 @@ export async function createTar(
compressionMethod: CompressionMethod
): Promise<void> {
// Write source directories to manifest.txt to avoid command length limits
const manifestFilename = 'manifest.txt'
const cacheFileName = utils.getCacheFileName(compressionMethod)
const tarFile = 'cache.tar'
const tarPath = await getTarPath()
const BSD_TAR_ZSTD =
tarPath === SystemTarPathOnWindows &&
compressionMethod !== CompressionMethod.Gzip
writeFileSync(
path.join(archiveFolder, ManifestFilename),
path.join(archiveFolder, manifestFilename),
sourceDirectories.join('\n')
)
const args = await getArgs(compressionMethod, 'create')
try {
await exec(args, undefined, {cwd: archiveFolder})
} catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`)
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// zstdmt is equivalent to 'zstd -T0'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
function getCompressionProgram(): string[] {
switch (compressionMethod) {
case CompressionMethod.Zstd:
return BSD_TAR_ZSTD
? [
'&&',
'zstd -T0 --long=30 -o',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
tarFile
]
: [
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return BSD_TAR_ZSTD
? [
'&&',
'zstd -T0 -o',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
tarFile
]
: ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']
default:
return ['-z']
}
}
const tarArgs = await getTarArgs(compressionMethod, 'create')
const compressionArgs = getCompressionProgram()
const args = tarArgs.concat(compressionArgs)
await execTar(args, archiveFolder)
}
const ARCHIVE_TOOL_GNU: string = 'gnu'
const ARCHIVE_TOOL_BSD: string = 'bsd'
const manifestFilename = 'manifest.txt'
export async function createTar2(
archiveFolder: string,
sourceDirectories: string[]
): Promise<void> {
// 1. decide the compression algo - zstd otherwise gzip
// 2. decide tar - gnutar otherwise bsdtar or systemtar
// 3. deide archiving/unarchiving args depending on tar
// 4. decide compression/decompression args depending on tar and compression
// 5. decide the exec command
let tar = getTarPath2()
let compression = getCompressionMethod2()
let args = argsMap.get({tar: tar, compression: compression, os: process.platform, "create"})
}
// return archive tool name and path
async function getTarPath2(): Promise<ArchiveTool> {
switch (process.platform) {
case 'win32': {
const gnuTar = await utils.getGnuTarPathOnWindows()
const systemTar = SystemTarPathOnWindows
if (gnuTar) {
// Use GNUtar as default on windows
return <ArchiveTool>{ name: ARCHIVE_TOOL_GNU, path: gnuTar }
} else if (existsSync(systemTar)) {
return <ArchiveTool>{name: ARCHIVE_TOOL_BSD, path: systemTar}
}
break
}
case 'darwin': {
const gnuTar = await io.which('gtar', false)
if (gnuTar) {
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
return <ArchiveTool>{ name: ARCHIVE_TOOL_GNU, path: gnuTar }
} else {
const path = await io.which('tar', true)
return <ArchiveTool>{name: ARCHIVE_TOOL_BSD, path: path}
}
}
default:
break
}
const path = await io.which('tar', true)
return <ArchiveTool>{name: ARCHIVE_TOOL_GNU, path: path}
}
// Use zstandard if possible to maximize cache performance
export async function getCompressionMethod2(): Promise<CompressionMethod> {
const versionOutput = await utils.getVersion('zstd')
const version = semver.clean(versionOutput)
if (!versionOutput.toLowerCase().includes('zstd command line interface')) {
// zstd is not installed
return CompressionMethod.Gzip
} else if (!version || semver.lt(version, 'v1.3.2')) {
// zstd is installed but using a version earlier than v1.3.2
// v1.3.2 is required to use the `--long` options in zstd
return CompressionMethod.ZstdWithoutLong
} else {
return CompressionMethod.Zstd
}
}
interface ArchiveTool {
name: string
path: string
}
// tar, compression, os, operation -> tar args + compression args
const argsMap: Map<ArgsLookupKey, string[][]> = new Map()
argsMap.set({tar: ARCHIVE_TOOL_GNU, compression: CompressionMethod.Zstd, os: "windows", operation: "create"},
[[
'--posix',
'-cf',
'%%cacheFileName%%',
'--exclude',
'%%cacheFileName%%',
'-P',
'-C',
'%%workingDirectory%%',
'--files-from',
manifestFilename],
['--use-compress-program',
'zstd -T0 --long=30']
])
argsMap.set({tar: ARCHIVE_TOOL_GNU, compression: CompressionMethod.Zstd, os: "linux", operation: "create"},
[
'--use-compress-program',
'zstdmt --long=30'
])
interface ArgsLookupKey {
tar: string
compression: string
os: string
operation: string
}