Compare commits
19 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 44cd5d150b | |||
| 0822441ee0 | |||
| 187781e273 | |||
| 1f3371766a | |||
| 39b7a86b23 | |||
| 2f73afa843 | |||
| ffde3e4bd5 | |||
| 32f538163d | |||
| 32b95825ba | |||
| 54eb9b8055 | |||
| f9dfb05bd2 | |||
| b3bd482c0f | |||
| 4fa5b7d133 | |||
| ea9856079f | |||
| 5f89653f1b | |||
| e9e146bbf8 | |||
| 27bf8304ea | |||
| db3517fe3b | |||
| 6349c3ca3a |
Vendored
+144
-39
@@ -4,7 +4,8 @@ import * as path from 'path'
|
|||||||
import {
|
import {
|
||||||
CacheFilename,
|
CacheFilename,
|
||||||
CompressionMethod,
|
CompressionMethod,
|
||||||
GnuTarPathOnWindows
|
GnuTarPathOnWindows,
|
||||||
|
SystemTarPathOnWindows
|
||||||
} from '../src/internal/constants'
|
} from '../src/internal/constants'
|
||||||
import * as tar from '../src/internal/tar'
|
import * as tar from '../src/internal/tar'
|
||||||
import * as utils from '../src/internal/cacheUtils'
|
import * as utils from '../src/internal/cacheUtils'
|
||||||
@@ -17,7 +18,7 @@ jest.mock('@actions/io')
|
|||||||
const IS_WINDOWS = process.platform === 'win32'
|
const IS_WINDOWS = process.platform === 'win32'
|
||||||
const IS_MAC = process.platform === 'darwin'
|
const IS_MAC = process.platform === 'darwin'
|
||||||
|
|
||||||
const defaultTarPath = process.platform === 'darwin' ? 'gtar' : 'tar'
|
const defaultTarPath = IS_MAC ? 'gtar' : 'tar'
|
||||||
|
|
||||||
function getTempDir(): string {
|
function getTempDir(): string {
|
||||||
return path.join(__dirname, '_temp', 'tar')
|
return path.join(__dirname, '_temp', 'tar')
|
||||||
@@ -58,8 +59,6 @@ test('zstd extract tar', async () => {
|
|||||||
expect(execMock).toHaveBeenCalledWith(
|
expect(execMock).toHaveBeenCalledWith(
|
||||||
`"${tarPath}"`,
|
`"${tarPath}"`,
|
||||||
[
|
[
|
||||||
'--use-compress-program',
|
|
||||||
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
|
|
||||||
'-xf',
|
'-xf',
|
||||||
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
|
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
|
||||||
'-P',
|
'-P',
|
||||||
@@ -67,11 +66,50 @@ test('zstd extract tar', async () => {
|
|||||||
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
|
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
|
||||||
]
|
]
|
||||||
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
||||||
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
|
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
|
||||||
|
.concat([
|
||||||
|
'--use-compress-program',
|
||||||
|
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
|
||||||
|
]),
|
||||||
{cwd: undefined}
|
{cwd: undefined}
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test('zstd extract tar with windows BSDtar', async () => {
|
||||||
|
if (IS_WINDOWS) {
|
||||||
|
const mkdirMock = jest.spyOn(io, 'mkdirP')
|
||||||
|
const execMock = jest.spyOn(exec, 'exec')
|
||||||
|
jest
|
||||||
|
.spyOn(utils, 'getGnuTarPathOnWindows')
|
||||||
|
.mockReturnValue(Promise.resolve(''))
|
||||||
|
|
||||||
|
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
|
||||||
|
const workspace = process.env['GITHUB_WORKSPACE']
|
||||||
|
const tarPath = SystemTarPathOnWindows
|
||||||
|
const tarFilename = 'cache.tar'
|
||||||
|
|
||||||
|
await tar.extractTar(archivePath, CompressionMethod.Zstd)
|
||||||
|
|
||||||
|
expect(mkdirMock).toHaveBeenCalledWith(workspace)
|
||||||
|
expect(execMock).toHaveBeenCalledTimes(1)
|
||||||
|
expect(execMock).toHaveBeenCalledWith(
|
||||||
|
'zstd -d --long=30 -o',
|
||||||
|
[
|
||||||
|
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||||
|
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||||
|
'&&',
|
||||||
|
`${tarPath}`,
|
||||||
|
'-xf',
|
||||||
|
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||||
|
'-P',
|
||||||
|
'-C',
|
||||||
|
workspace?.replace(/\\/g, '/')
|
||||||
|
],
|
||||||
|
{cwd: undefined}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
test('gzip extract tar', async () => {
|
test('gzip extract tar', async () => {
|
||||||
const mkdirMock = jest.spyOn(io, 'mkdirP')
|
const mkdirMock = jest.spyOn(io, 'mkdirP')
|
||||||
const execMock = jest.spyOn(exec, 'exec')
|
const execMock = jest.spyOn(exec, 'exec')
|
||||||
@@ -88,7 +126,6 @@ test('gzip extract tar', async () => {
|
|||||||
expect(execMock).toHaveBeenCalledWith(
|
expect(execMock).toHaveBeenCalledWith(
|
||||||
`"${tarPath}"`,
|
`"${tarPath}"`,
|
||||||
[
|
[
|
||||||
'-z',
|
|
||||||
'-xf',
|
'-xf',
|
||||||
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
|
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
|
||||||
'-P',
|
'-P',
|
||||||
@@ -96,7 +133,8 @@ test('gzip extract tar', async () => {
|
|||||||
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
|
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
|
||||||
]
|
]
|
||||||
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
||||||
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
|
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
|
||||||
|
.concat(['-z']),
|
||||||
{cwd: undefined}
|
{cwd: undefined}
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
@@ -104,7 +142,7 @@ test('gzip extract tar', async () => {
|
|||||||
test('gzip extract GNU tar on windows with GNUtar in path', async () => {
|
test('gzip extract GNU tar on windows with GNUtar in path', async () => {
|
||||||
if (IS_WINDOWS) {
|
if (IS_WINDOWS) {
|
||||||
// GNU tar present in path but not at default location
|
// GNU tar present in path but not at default location
|
||||||
const isGnuMock = jest
|
jest
|
||||||
.spyOn(utils, 'getGnuTarPathOnWindows')
|
.spyOn(utils, 'getGnuTarPathOnWindows')
|
||||||
.mockReturnValue(Promise.resolve('tar'))
|
.mockReturnValue(Promise.resolve('tar'))
|
||||||
const execMock = jest.spyOn(exec, 'exec')
|
const execMock = jest.spyOn(exec, 'exec')
|
||||||
@@ -113,18 +151,17 @@ test('gzip extract GNU tar on windows with GNUtar in path', async () => {
|
|||||||
|
|
||||||
await tar.extractTar(archivePath, CompressionMethod.Gzip)
|
await tar.extractTar(archivePath, CompressionMethod.Gzip)
|
||||||
|
|
||||||
expect(isGnuMock).toHaveBeenCalledTimes(1)
|
|
||||||
expect(execMock).toHaveBeenCalledTimes(1)
|
expect(execMock).toHaveBeenCalledTimes(1)
|
||||||
expect(execMock).toHaveBeenCalledWith(
|
expect(execMock).toHaveBeenCalledWith(
|
||||||
`"tar"`,
|
`"tar"`,
|
||||||
[
|
[
|
||||||
'-z',
|
|
||||||
'-xf',
|
'-xf',
|
||||||
archivePath.replace(/\\/g, '/'),
|
archivePath.replace(/\\/g, '/'),
|
||||||
'-P',
|
'-P',
|
||||||
'-C',
|
'-C',
|
||||||
workspace?.replace(/\\/g, '/'),
|
workspace?.replace(/\\/g, '/'),
|
||||||
'--force-local'
|
'--force-local',
|
||||||
|
'-z'
|
||||||
],
|
],
|
||||||
{cwd: undefined}
|
{cwd: undefined}
|
||||||
)
|
)
|
||||||
@@ -149,8 +186,6 @@ test('zstd create tar', async () => {
|
|||||||
`"${tarPath}"`,
|
`"${tarPath}"`,
|
||||||
[
|
[
|
||||||
'--posix',
|
'--posix',
|
||||||
'--use-compress-program',
|
|
||||||
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30',
|
|
||||||
'-cf',
|
'-cf',
|
||||||
IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd,
|
IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd,
|
||||||
'--exclude',
|
'--exclude',
|
||||||
@@ -162,13 +197,65 @@ test('zstd create tar', async () => {
|
|||||||
'manifest.txt'
|
'manifest.txt'
|
||||||
]
|
]
|
||||||
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
||||||
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
|
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
|
||||||
|
.concat([
|
||||||
|
'--use-compress-program',
|
||||||
|
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
|
||||||
|
]),
|
||||||
{
|
{
|
||||||
cwd: archiveFolder
|
cwd: archiveFolder
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test('zstd create tar with windows BSDtar', async () => {
|
||||||
|
if (IS_WINDOWS) {
|
||||||
|
const execMock = jest.spyOn(exec, 'exec')
|
||||||
|
jest
|
||||||
|
.spyOn(utils, 'getGnuTarPathOnWindows')
|
||||||
|
.mockReturnValue(Promise.resolve(''))
|
||||||
|
|
||||||
|
const archiveFolder = getTempDir()
|
||||||
|
const workspace = process.env['GITHUB_WORKSPACE']
|
||||||
|
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
|
||||||
|
const tarFilename = 'cache.tar'
|
||||||
|
|
||||||
|
await fs.promises.mkdir(archiveFolder, {recursive: true})
|
||||||
|
|
||||||
|
await tar.createTar(
|
||||||
|
archiveFolder,
|
||||||
|
sourceDirectories,
|
||||||
|
CompressionMethod.Zstd
|
||||||
|
)
|
||||||
|
|
||||||
|
const tarPath = SystemTarPathOnWindows
|
||||||
|
|
||||||
|
expect(execMock).toHaveBeenCalledTimes(1)
|
||||||
|
expect(execMock).toHaveBeenCalledWith(
|
||||||
|
`"${tarPath}"`,
|
||||||
|
[
|
||||||
|
'--posix',
|
||||||
|
'-cf',
|
||||||
|
tarFilename.replace(/\\/g, '/'),
|
||||||
|
'--exclude',
|
||||||
|
tarFilename.replace(/\\/g, '/'),
|
||||||
|
'-P',
|
||||||
|
'-C',
|
||||||
|
workspace?.replace(/\\/g, '/'),
|
||||||
|
'--files-from',
|
||||||
|
'manifest.txt',
|
||||||
|
'&&',
|
||||||
|
'zstd -T0 --long=30 -o',
|
||||||
|
CacheFilename.Zstd.replace(/\\/g, '/'),
|
||||||
|
tarFilename.replace(/\\/g, '/')
|
||||||
|
],
|
||||||
|
{
|
||||||
|
cwd: archiveFolder
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
test('gzip create tar', async () => {
|
test('gzip create tar', async () => {
|
||||||
const execMock = jest.spyOn(exec, 'exec')
|
const execMock = jest.spyOn(exec, 'exec')
|
||||||
|
|
||||||
@@ -187,7 +274,6 @@ test('gzip create tar', async () => {
|
|||||||
`"${tarPath}"`,
|
`"${tarPath}"`,
|
||||||
[
|
[
|
||||||
'--posix',
|
'--posix',
|
||||||
'-z',
|
|
||||||
'-cf',
|
'-cf',
|
||||||
IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip,
|
IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip,
|
||||||
'--exclude',
|
'--exclude',
|
||||||
@@ -199,7 +285,8 @@ test('gzip create tar', async () => {
|
|||||||
'manifest.txt'
|
'manifest.txt'
|
||||||
]
|
]
|
||||||
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
||||||
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
|
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
|
||||||
|
.concat(['-z']),
|
||||||
{
|
{
|
||||||
cwd: archiveFolder
|
cwd: archiveFolder
|
||||||
}
|
}
|
||||||
@@ -219,19 +306,46 @@ test('zstd list tar', async () => {
|
|||||||
expect(execMock).toHaveBeenCalledTimes(1)
|
expect(execMock).toHaveBeenCalledTimes(1)
|
||||||
expect(execMock).toHaveBeenCalledWith(
|
expect(execMock).toHaveBeenCalledWith(
|
||||||
`"${tarPath}"`,
|
`"${tarPath}"`,
|
||||||
[
|
['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P']
|
||||||
'--use-compress-program',
|
|
||||||
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
|
|
||||||
'-tf',
|
|
||||||
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
|
|
||||||
'-P'
|
|
||||||
]
|
|
||||||
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
||||||
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
|
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
|
||||||
|
.concat([
|
||||||
|
'--use-compress-program',
|
||||||
|
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
|
||||||
|
]),
|
||||||
{cwd: undefined}
|
{cwd: undefined}
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test('zstd list tar with windows BSDtar', async () => {
|
||||||
|
if (IS_WINDOWS) {
|
||||||
|
const execMock = jest.spyOn(exec, 'exec')
|
||||||
|
jest
|
||||||
|
.spyOn(utils, 'getGnuTarPathOnWindows')
|
||||||
|
.mockReturnValue(Promise.resolve(''))
|
||||||
|
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
|
||||||
|
|
||||||
|
await tar.listTar(archivePath, CompressionMethod.Zstd)
|
||||||
|
|
||||||
|
const tarFilename = 'cache.tar'
|
||||||
|
const tarPath = SystemTarPathOnWindows
|
||||||
|
expect(execMock).toHaveBeenCalledTimes(1)
|
||||||
|
expect(execMock).toHaveBeenCalledWith(
|
||||||
|
'zstd -d --long=30 -o',
|
||||||
|
[
|
||||||
|
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||||
|
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||||
|
'&&',
|
||||||
|
`${tarPath}`,
|
||||||
|
'-tf',
|
||||||
|
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||||
|
'-P'
|
||||||
|
],
|
||||||
|
{cwd: undefined}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
test('zstdWithoutLong list tar', async () => {
|
test('zstdWithoutLong list tar', async () => {
|
||||||
const execMock = jest.spyOn(exec, 'exec')
|
const execMock = jest.spyOn(exec, 'exec')
|
||||||
|
|
||||||
@@ -245,15 +359,10 @@ test('zstdWithoutLong list tar', async () => {
|
|||||||
expect(execMock).toHaveBeenCalledTimes(1)
|
expect(execMock).toHaveBeenCalledTimes(1)
|
||||||
expect(execMock).toHaveBeenCalledWith(
|
expect(execMock).toHaveBeenCalledWith(
|
||||||
`"${tarPath}"`,
|
`"${tarPath}"`,
|
||||||
[
|
['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P']
|
||||||
'--use-compress-program',
|
|
||||||
IS_WINDOWS ? 'zstd -d' : 'unzstd',
|
|
||||||
'-tf',
|
|
||||||
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
|
|
||||||
'-P'
|
|
||||||
]
|
|
||||||
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
||||||
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
|
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
|
||||||
|
.concat(['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']),
|
||||||
{cwd: undefined}
|
{cwd: undefined}
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
@@ -270,14 +379,10 @@ test('gzip list tar', async () => {
|
|||||||
expect(execMock).toHaveBeenCalledTimes(1)
|
expect(execMock).toHaveBeenCalledTimes(1)
|
||||||
expect(execMock).toHaveBeenCalledWith(
|
expect(execMock).toHaveBeenCalledWith(
|
||||||
`"${tarPath}"`,
|
`"${tarPath}"`,
|
||||||
[
|
['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P']
|
||||||
'-z',
|
|
||||||
'-tf',
|
|
||||||
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
|
|
||||||
'-P'
|
|
||||||
]
|
|
||||||
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
.concat(IS_WINDOWS ? ['--force-local'] : [])
|
||||||
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
|
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
|
||||||
|
.concat(['-z']),
|
||||||
{cwd: undefined}
|
{cwd: undefined}
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|||||||
+1
-6
@@ -71,7 +71,7 @@ export async function unlinkFile(filePath: fs.PathLike): Promise<void> {
|
|||||||
return util.promisify(fs.unlink)(filePath)
|
return util.promisify(fs.unlink)(filePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
async function getVersion(app: string): Promise<string> {
|
export async function getVersion(app: string): Promise<string> {
|
||||||
core.debug(`Checking ${app} --version`)
|
core.debug(`Checking ${app} --version`)
|
||||||
let versionOutput = ''
|
let versionOutput = ''
|
||||||
try {
|
try {
|
||||||
@@ -94,11 +94,6 @@ async function getVersion(app: string): Promise<string> {
|
|||||||
|
|
||||||
// Use zstandard if possible to maximize cache performance
|
// Use zstandard if possible to maximize cache performance
|
||||||
export async function getCompressionMethod(): Promise<CompressionMethod> {
|
export async function getCompressionMethod(): Promise<CompressionMethod> {
|
||||||
if (process.platform === 'win32' && !(await getGnuTarPathOnWindows())) {
|
|
||||||
// Disable zstd due to bug https://github.com/actions/cache/issues/301
|
|
||||||
return CompressionMethod.Gzip
|
|
||||||
}
|
|
||||||
|
|
||||||
const versionOutput = await getVersion('zstd')
|
const versionOutput = await getVersion('zstd')
|
||||||
const version = semver.clean(versionOutput)
|
const version = semver.clean(versionOutput)
|
||||||
|
|
||||||
|
|||||||
+2
@@ -24,3 +24,5 @@ export const SocketTimeout = 5000
|
|||||||
|
|
||||||
// The default path of GNUtar on hosted Windows runners
|
// The default path of GNUtar on hosted Windows runners
|
||||||
export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe`
|
export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe`
|
||||||
|
|
||||||
|
export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe`
|
||||||
|
|||||||
Vendored
+298
-39
@@ -3,18 +3,18 @@ import * as io from '@actions/io'
|
|||||||
import {existsSync, writeFileSync} from 'fs'
|
import {existsSync, writeFileSync} from 'fs'
|
||||||
import * as path from 'path'
|
import * as path from 'path'
|
||||||
import * as utils from './cacheUtils'
|
import * as utils from './cacheUtils'
|
||||||
import {CompressionMethod} from './constants'
|
import {CompressionMethod, SystemTarPathOnWindows} from './constants'
|
||||||
|
|
||||||
const IS_WINDOWS = process.platform === 'win32'
|
const IS_WINDOWS = process.platform === 'win32'
|
||||||
|
|
||||||
async function getTarPath(args: string[]): Promise<string> {
|
// Function also mutates the args array. For non-mutation call with passing an empty array.
|
||||||
|
async function getTarPath(): Promise<string> {
|
||||||
switch (process.platform) {
|
switch (process.platform) {
|
||||||
case 'win32': {
|
case 'win32': {
|
||||||
const gnuTar = await utils.getGnuTarPathOnWindows()
|
const gnuTar = await utils.getGnuTarPathOnWindows()
|
||||||
const systemTar = `${process.env['windir']}\\System32\\tar.exe`
|
const systemTar = SystemTarPathOnWindows
|
||||||
if (gnuTar) {
|
if (gnuTar) {
|
||||||
// Use GNUtar as default on windows
|
// Use GNUtar as default on windows
|
||||||
args.push('--force-local')
|
|
||||||
return gnuTar
|
return gnuTar
|
||||||
} else if (existsSync(systemTar)) {
|
} else if (existsSync(systemTar)) {
|
||||||
return systemTar
|
return systemTar
|
||||||
@@ -25,7 +25,6 @@ async function getTarPath(args: string[]): Promise<string> {
|
|||||||
const gnuTar = await io.which('gtar', false)
|
const gnuTar = await io.which('gtar', false)
|
||||||
if (gnuTar) {
|
if (gnuTar) {
|
||||||
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
|
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
|
||||||
args.push('--delay-directory-restore')
|
|
||||||
return gnuTar
|
return gnuTar
|
||||||
}
|
}
|
||||||
break
|
break
|
||||||
@@ -36,9 +35,101 @@ async function getTarPath(args: string[]): Promise<string> {
|
|||||||
return await io.which('tar', true)
|
return await io.which('tar', true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function getTarArgs(
|
||||||
|
compressionMethod: CompressionMethod,
|
||||||
|
type: string,
|
||||||
|
archivePath = ''
|
||||||
|
): Promise<string[]> {
|
||||||
|
const args = []
|
||||||
|
const manifestFilename = 'manifest.txt'
|
||||||
|
const cacheFileName = utils.getCacheFileName(compressionMethod)
|
||||||
|
const tarFile = 'cache.tar'
|
||||||
|
const tarPath = await getTarPath()
|
||||||
|
const workingDirectory = getWorkingDirectory()
|
||||||
|
const BSD_TAR_ZSTD =
|
||||||
|
tarPath === SystemTarPathOnWindows &&
|
||||||
|
compressionMethod !== CompressionMethod.Gzip
|
||||||
|
|
||||||
|
// Method specific args
|
||||||
|
switch (type) {
|
||||||
|
case 'create':
|
||||||
|
args.push(
|
||||||
|
'--posix',
|
||||||
|
'-cf',
|
||||||
|
BSD_TAR_ZSTD
|
||||||
|
? tarFile
|
||||||
|
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||||
|
'--exclude',
|
||||||
|
BSD_TAR_ZSTD
|
||||||
|
? tarFile
|
||||||
|
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||||
|
'-P',
|
||||||
|
'-C',
|
||||||
|
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||||
|
'--files-from',
|
||||||
|
manifestFilename
|
||||||
|
)
|
||||||
|
break
|
||||||
|
case 'extract':
|
||||||
|
args.push(
|
||||||
|
'-xf',
|
||||||
|
BSD_TAR_ZSTD
|
||||||
|
? tarFile
|
||||||
|
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||||
|
'-P',
|
||||||
|
'-C',
|
||||||
|
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
|
||||||
|
)
|
||||||
|
break
|
||||||
|
case 'list':
|
||||||
|
args.push(
|
||||||
|
'-tf',
|
||||||
|
BSD_TAR_ZSTD
|
||||||
|
? tarFile
|
||||||
|
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||||
|
'-P'
|
||||||
|
)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// Platform specific args
|
||||||
|
switch (process.platform) {
|
||||||
|
case 'win32': {
|
||||||
|
const gnuTar = await utils.getGnuTarPathOnWindows()
|
||||||
|
if (gnuTar) {
|
||||||
|
// Use GNUtar as default on windows
|
||||||
|
args.push('--force-local')
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
case 'darwin': {
|
||||||
|
const gnuTar = await io.which('gtar', false)
|
||||||
|
if (gnuTar) {
|
||||||
|
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
|
||||||
|
args.push('--delay-directory-restore')
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return args
|
||||||
|
}
|
||||||
|
|
||||||
async function execTar(args: string[], cwd?: string): Promise<void> {
|
async function execTar(args: string[], cwd?: string): Promise<void> {
|
||||||
try {
|
try {
|
||||||
await exec(`"${await getTarPath(args)}"`, args, {cwd})
|
await exec(`"${await getTarPath()}"`, args, {cwd})
|
||||||
|
} catch (error) {
|
||||||
|
throw new Error(`Tar failed with error: ${error?.message}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function execCommand(
|
||||||
|
command: string,
|
||||||
|
args: string[],
|
||||||
|
cwd?: string
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await exec(command, args, {cwd})
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
throw new Error(`Tar failed with error: ${error?.message}`)
|
throw new Error(`Tar failed with error: ${error?.message}`)
|
||||||
}
|
}
|
||||||
@@ -49,19 +140,41 @@ function getWorkingDirectory(): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Common function for extractTar and listTar to get the compression method
|
// Common function for extractTar and listTar to get the compression method
|
||||||
function getCompressionProgram(compressionMethod: CompressionMethod): string[] {
|
async function getCompressionProgram(
|
||||||
|
compressionMethod: CompressionMethod,
|
||||||
|
archivePath: string
|
||||||
|
): Promise<string[]> {
|
||||||
// -d: Decompress.
|
// -d: Decompress.
|
||||||
// unzstd is equivalent to 'zstd -d'
|
// unzstd is equivalent to 'zstd -d'
|
||||||
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
|
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
|
||||||
// Using 30 here because we also support 32-bit self-hosted runners.
|
// Using 30 here because we also support 32-bit self-hosted runners.
|
||||||
|
const tarPath = await getTarPath()
|
||||||
|
const tarFile = 'cache.tar'
|
||||||
|
const BSD_TAR_ZSTD =
|
||||||
|
tarPath === SystemTarPathOnWindows &&
|
||||||
|
compressionMethod !== CompressionMethod.Gzip
|
||||||
switch (compressionMethod) {
|
switch (compressionMethod) {
|
||||||
case CompressionMethod.Zstd:
|
case CompressionMethod.Zstd:
|
||||||
return [
|
return BSD_TAR_ZSTD
|
||||||
|
? [
|
||||||
|
'zstd -d --long=30 -o',
|
||||||
|
tarFile,
|
||||||
|
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||||
|
'&&'
|
||||||
|
]
|
||||||
|
: [
|
||||||
'--use-compress-program',
|
'--use-compress-program',
|
||||||
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
|
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
|
||||||
]
|
]
|
||||||
case CompressionMethod.ZstdWithoutLong:
|
case CompressionMethod.ZstdWithoutLong:
|
||||||
return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']
|
return BSD_TAR_ZSTD
|
||||||
|
? [
|
||||||
|
'zstd -d -o',
|
||||||
|
tarFile,
|
||||||
|
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||||
|
'&&'
|
||||||
|
]
|
||||||
|
: ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']
|
||||||
default:
|
default:
|
||||||
return ['-z']
|
return ['-z']
|
||||||
}
|
}
|
||||||
@@ -71,14 +184,28 @@ export async function listTar(
|
|||||||
archivePath: string,
|
archivePath: string,
|
||||||
compressionMethod: CompressionMethod
|
compressionMethod: CompressionMethod
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const args = [
|
const tarPath = await getTarPath()
|
||||||
...getCompressionProgram(compressionMethod),
|
const BSD_TAR_ZSTD =
|
||||||
'-tf',
|
tarPath === SystemTarPathOnWindows &&
|
||||||
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
compressionMethod !== CompressionMethod.Gzip
|
||||||
'-P'
|
const compressionArgs = await getCompressionProgram(
|
||||||
]
|
compressionMethod,
|
||||||
|
archivePath
|
||||||
|
)
|
||||||
|
const tarArgs = await getTarArgs(compressionMethod, 'list', archivePath)
|
||||||
|
// TODO: Add a test for BSD tar on windows
|
||||||
|
if (BSD_TAR_ZSTD) {
|
||||||
|
const command = compressionArgs[0]
|
||||||
|
const args = compressionArgs
|
||||||
|
.slice(1)
|
||||||
|
.concat([tarPath])
|
||||||
|
.concat(tarArgs)
|
||||||
|
await execCommand(command, args)
|
||||||
|
} else {
|
||||||
|
const args = tarArgs.concat(compressionArgs)
|
||||||
await execTar(args)
|
await execTar(args)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function extractTar(
|
export async function extractTar(
|
||||||
archivePath: string,
|
archivePath: string,
|
||||||
@@ -86,17 +213,28 @@ export async function extractTar(
|
|||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
// Create directory to extract tar into
|
// Create directory to extract tar into
|
||||||
const workingDirectory = getWorkingDirectory()
|
const workingDirectory = getWorkingDirectory()
|
||||||
|
const tarPath = await getTarPath()
|
||||||
|
const BSD_TAR_ZSTD =
|
||||||
|
tarPath === SystemTarPathOnWindows &&
|
||||||
|
compressionMethod !== CompressionMethod.Gzip
|
||||||
await io.mkdirP(workingDirectory)
|
await io.mkdirP(workingDirectory)
|
||||||
const args = [
|
const tarArgs = await getTarArgs(compressionMethod, 'extract', archivePath)
|
||||||
...getCompressionProgram(compressionMethod),
|
const compressionArgs = await getCompressionProgram(
|
||||||
'-xf',
|
compressionMethod,
|
||||||
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
archivePath
|
||||||
'-P',
|
)
|
||||||
'-C',
|
if (BSD_TAR_ZSTD) {
|
||||||
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
|
const command = compressionArgs[0]
|
||||||
]
|
const args = compressionArgs
|
||||||
|
.slice(1)
|
||||||
|
.concat([tarPath])
|
||||||
|
.concat(tarArgs)
|
||||||
|
await execCommand(command, args)
|
||||||
|
} else {
|
||||||
|
const args = tarArgs.concat(compressionArgs)
|
||||||
await execTar(args)
|
await execTar(args)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function createTar(
|
export async function createTar(
|
||||||
archiveFolder: string,
|
archiveFolder: string,
|
||||||
@@ -106,11 +244,15 @@ export async function createTar(
|
|||||||
// Write source directories to manifest.txt to avoid command length limits
|
// Write source directories to manifest.txt to avoid command length limits
|
||||||
const manifestFilename = 'manifest.txt'
|
const manifestFilename = 'manifest.txt'
|
||||||
const cacheFileName = utils.getCacheFileName(compressionMethod)
|
const cacheFileName = utils.getCacheFileName(compressionMethod)
|
||||||
|
const tarFile = 'cache.tar'
|
||||||
|
const tarPath = await getTarPath()
|
||||||
|
const BSD_TAR_ZSTD =
|
||||||
|
tarPath === SystemTarPathOnWindows &&
|
||||||
|
compressionMethod !== CompressionMethod.Gzip
|
||||||
writeFileSync(
|
writeFileSync(
|
||||||
path.join(archiveFolder, manifestFilename),
|
path.join(archiveFolder, manifestFilename),
|
||||||
sourceDirectories.join('\n')
|
sourceDirectories.join('\n')
|
||||||
)
|
)
|
||||||
const workingDirectory = getWorkingDirectory()
|
|
||||||
|
|
||||||
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
|
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
|
||||||
// zstdmt is equivalent to 'zstd -T0'
|
// zstdmt is equivalent to 'zstd -T0'
|
||||||
@@ -120,28 +262,145 @@ export async function createTar(
|
|||||||
function getCompressionProgram(): string[] {
|
function getCompressionProgram(): string[] {
|
||||||
switch (compressionMethod) {
|
switch (compressionMethod) {
|
||||||
case CompressionMethod.Zstd:
|
case CompressionMethod.Zstd:
|
||||||
return [
|
return BSD_TAR_ZSTD
|
||||||
|
? [
|
||||||
|
'&&',
|
||||||
|
'zstd -T0 --long=30 -o',
|
||||||
|
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||||
|
tarFile
|
||||||
|
]
|
||||||
|
: [
|
||||||
'--use-compress-program',
|
'--use-compress-program',
|
||||||
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
|
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
|
||||||
]
|
]
|
||||||
case CompressionMethod.ZstdWithoutLong:
|
case CompressionMethod.ZstdWithoutLong:
|
||||||
return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']
|
return BSD_TAR_ZSTD
|
||||||
|
? [
|
||||||
|
'&&',
|
||||||
|
'zstd -T0 -o',
|
||||||
|
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
||||||
|
tarFile
|
||||||
|
]
|
||||||
|
: ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']
|
||||||
default:
|
default:
|
||||||
return ['-z']
|
return ['-z']
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const args = [
|
const tarArgs = await getTarArgs(compressionMethod, 'create')
|
||||||
'--posix',
|
const compressionArgs = getCompressionProgram()
|
||||||
...getCompressionProgram(),
|
const args = tarArgs.concat(compressionArgs)
|
||||||
'-cf',
|
|
||||||
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
|
||||||
'--exclude',
|
|
||||||
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
|
||||||
'-P',
|
|
||||||
'-C',
|
|
||||||
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
|
|
||||||
'--files-from',
|
|
||||||
manifestFilename
|
|
||||||
]
|
|
||||||
await execTar(args, archiveFolder)
|
await execTar(args, archiveFolder)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const ARCHIVE_TOOL_GNU: string = 'gnu'
|
||||||
|
const ARCHIVE_TOOL_BSD: string = 'bsd'
|
||||||
|
|
||||||
|
const manifestFilename = 'manifest.txt'
|
||||||
|
|
||||||
|
export async function createTar2(
|
||||||
|
archiveFolder: string,
|
||||||
|
sourceDirectories: string[]
|
||||||
|
): Promise<void> {
|
||||||
|
|
||||||
|
// 1. decide the compression algo - zstd otherwise gzip
|
||||||
|
// 2. decide tar - gnutar otherwise bsdtar or systemtar
|
||||||
|
// 3. deide archiving/unarchiving args depending on tar
|
||||||
|
// 4. decide compression/decompression args depending on tar and compression
|
||||||
|
// 5. decide the exec command
|
||||||
|
|
||||||
|
let tar = getTarPath2()
|
||||||
|
let compression = getCompressionMethod2()
|
||||||
|
let args = argsMap.get({tar: tar, compression: compression, os: process.platform, "create"})
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// return archive tool name and path
|
||||||
|
async function getTarPath2(): Promise<ArchiveTool> {
|
||||||
|
switch (process.platform) {
|
||||||
|
case 'win32': {
|
||||||
|
const gnuTar = await utils.getGnuTarPathOnWindows()
|
||||||
|
const systemTar = SystemTarPathOnWindows
|
||||||
|
if (gnuTar) {
|
||||||
|
// Use GNUtar as default on windows
|
||||||
|
return <ArchiveTool>{ name: ARCHIVE_TOOL_GNU, path: gnuTar }
|
||||||
|
} else if (existsSync(systemTar)) {
|
||||||
|
return <ArchiveTool>{name: ARCHIVE_TOOL_BSD, path: systemTar}
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
case 'darwin': {
|
||||||
|
const gnuTar = await io.which('gtar', false)
|
||||||
|
if (gnuTar) {
|
||||||
|
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
|
||||||
|
return <ArchiveTool>{ name: ARCHIVE_TOOL_GNU, path: gnuTar }
|
||||||
|
} else {
|
||||||
|
const path = await io.which('tar', true)
|
||||||
|
return <ArchiveTool>{name: ARCHIVE_TOOL_BSD, path: path}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
break
|
||||||
|
}
|
||||||
|
const path = await io.which('tar', true)
|
||||||
|
return <ArchiveTool>{name: ARCHIVE_TOOL_GNU, path: path}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use zstandard if possible to maximize cache performance
|
||||||
|
export async function getCompressionMethod2(): Promise<CompressionMethod> {
|
||||||
|
const versionOutput = await utils.getVersion('zstd')
|
||||||
|
const version = semver.clean(versionOutput)
|
||||||
|
|
||||||
|
if (!versionOutput.toLowerCase().includes('zstd command line interface')) {
|
||||||
|
// zstd is not installed
|
||||||
|
return CompressionMethod.Gzip
|
||||||
|
} else if (!version || semver.lt(version, 'v1.3.2')) {
|
||||||
|
// zstd is installed but using a version earlier than v1.3.2
|
||||||
|
// v1.3.2 is required to use the `--long` options in zstd
|
||||||
|
return CompressionMethod.ZstdWithoutLong
|
||||||
|
} else {
|
||||||
|
return CompressionMethod.Zstd
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ArchiveTool {
|
||||||
|
name: string
|
||||||
|
path: string
|
||||||
|
}
|
||||||
|
|
||||||
|
// tar, compression, os, operation -> tar args + compression args
|
||||||
|
const argsMap: Map<ArgsLookupKey, string[][]> = new Map()
|
||||||
|
argsMap.set({tar: ARCHIVE_TOOL_GNU, compression: CompressionMethod.Zstd, os: "windows", operation: "create"},
|
||||||
|
[[
|
||||||
|
'--posix',
|
||||||
|
'-cf',
|
||||||
|
'%%cacheFileName%%',
|
||||||
|
'--exclude',
|
||||||
|
'%%cacheFileName%%',
|
||||||
|
'-P',
|
||||||
|
'-C',
|
||||||
|
'%%workingDirectory%%',
|
||||||
|
'--files-from',
|
||||||
|
manifestFilename],
|
||||||
|
['--use-compress-program',
|
||||||
|
'zstd -T0 --long=30']
|
||||||
|
])
|
||||||
|
|
||||||
|
argsMap.set({tar: ARCHIVE_TOOL_GNU, compression: CompressionMethod.Zstd, os: "linux", operation: "create"},
|
||||||
|
[
|
||||||
|
'--use-compress-program',
|
||||||
|
'zstdmt --long=30'
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
interface ArgsLookupKey {
|
||||||
|
tar: string
|
||||||
|
compression: string
|
||||||
|
os: string
|
||||||
|
operation: string
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user