Compare commits

..
Author SHA1 Message Date
Vallie Joseph a99da03c77 . 2022-12-12 05:26:28 +00:00
Vallie Joseph e0023baa89 trying execFileSync 2022-12-12 05:12:53 +00:00
Vallie Joseph 17bd5c18e9 adding windows verbatim args 2022-12-12 05:05:46 +00:00
Vallie Joseph 4b6dd2fbd0 Test 2022-12-08 20:44:39 +00:00
Vallie Joseph d6c6f72a70 Test 2022-12-08 20:38:15 +00:00
Sampark Sharma 819157bf87 Merge pull request #1238 from actions/revert-1232-phantsure/gnutar_windows
Revert "Add GNUtar as default in windows"
2022-11-17 12:42:10 +05:30
Sampark Sharma 86102e88e9 Revert "Add GNUtar as default in windows" 2022-11-17 12:26:27 +05:30
8 changed files with 139 additions and 505 deletions
+1 -1
View File
@@ -221,7 +221,7 @@ console.log(`We can even get context data, like the repo: ${context.repo.repo}`)
## Contributing ## Contributing
We welcome contributions. See [how to contribute](.github/CONTRIBUTING.md). We welcome contributions. See [how to contribute](.github/CONTRIBUTING.md).
hi
## Code of Conduct ## Code of Conduct
See [our code of conduct](CODE_OF_CONDUCT.md). See [our code of conduct](CODE_OF_CONDUCT.md).
+56 -173
View File
@@ -1,12 +1,7 @@
import * as exec from '@actions/exec' import * as exec from '@actions/exec'
import * as io from '@actions/io' import * as io from '@actions/io'
import * as path from 'path' import * as path from 'path'
import { import {CacheFilename, CompressionMethod} from '../src/internal/constants'
CacheFilename,
CompressionMethod,
GnuTarPathOnWindows,
SystemTarPathOnWindows
} from '../src/internal/constants'
import * as tar from '../src/internal/tar' import * as tar from '../src/internal/tar'
import * as utils from '../src/internal/cacheUtils' import * as utils from '../src/internal/cacheUtils'
// eslint-disable-next-line @typescript-eslint/no-require-imports // eslint-disable-next-line @typescript-eslint/no-require-imports
@@ -18,7 +13,7 @@ jest.mock('@actions/io')
const IS_WINDOWS = process.platform === 'win32' const IS_WINDOWS = process.platform === 'win32'
const IS_MAC = process.platform === 'darwin' const IS_MAC = process.platform === 'darwin'
const defaultTarPath = IS_MAC ? 'gtar' : 'tar' const defaultTarPath = process.platform === 'darwin' ? 'gtar' : 'tar'
function getTempDir(): string { function getTempDir(): string {
return path.join(__dirname, '_temp', 'tar') return path.join(__dirname, '_temp', 'tar')
@@ -33,10 +28,6 @@ beforeAll(async () => {
await jest.requireActual('@actions/io').rmRF(getTempDir()) await jest.requireActual('@actions/io').rmRF(getTempDir())
}) })
beforeEach(async () => {
jest.restoreAllMocks()
})
afterAll(async () => { afterAll(async () => {
delete process.env['GITHUB_WORKSPACE'] delete process.env['GITHUB_WORKSPACE']
await jest.requireActual('@actions/io').rmRF(getTempDir()) await jest.requireActual('@actions/io').rmRF(getTempDir())
@@ -50,15 +41,16 @@ test('zstd extract tar', async () => {
? `${process.env['windir']}\\fakepath\\cache.tar` ? `${process.env['windir']}\\fakepath\\cache.tar`
: 'cache.tar' : 'cache.tar'
const workspace = process.env['GITHUB_WORKSPACE'] const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
await tar.extractTar(archivePath, CompressionMethod.Zstd) await tar.extractTar(archivePath, CompressionMethod.Zstd)
expect(mkdirMock).toHaveBeenCalledWith(workspace) expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`, `"${defaultTarPath}"`,
[ [
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
'-xf', '-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P', '-P',
@@ -66,50 +58,11 @@ test('zstd extract tar', async () => {
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
] ]
.concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []),
.concat([
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
]),
{cwd: undefined} {cwd: undefined}
) )
}) })
test('zstd extract tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const mkdirMock = jest.spyOn(io, 'mkdirP')
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = SystemTarPathOnWindows
const tarFilename = 'cache.tar'
await tar.extractTar(archivePath, CompressionMethod.Zstd)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
'zstd -d --long=30 -o',
[
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&',
`${tarPath}`,
'-xf',
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/')
],
{cwd: undefined}
)
}
})
test('gzip extract tar', async () => { test('gzip extract tar', async () => {
const mkdirMock = jest.spyOn(io, 'mkdirP') const mkdirMock = jest.spyOn(io, 'mkdirP')
const execMock = jest.spyOn(exec, 'exec') const execMock = jest.spyOn(exec, 'exec')
@@ -121,47 +74,49 @@ test('gzip extract tar', async () => {
await tar.extractTar(archivePath, CompressionMethod.Gzip) await tar.extractTar(archivePath, CompressionMethod.Gzip)
expect(mkdirMock).toHaveBeenCalledWith(workspace) expect(mkdirMock).toHaveBeenCalledWith(workspace)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`, `"${tarPath}"`,
[ [
'-z',
'-xf', '-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P', '-P',
'-C', '-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
] ].concat(IS_MAC ? ['--delay-directory-restore'] : []),
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z']),
{cwd: undefined} {cwd: undefined}
) )
}) })
test('gzip extract GNU tar on windows with GNUtar in path', async () => { test('gzip extract GNU tar on windows', async () => {
if (IS_WINDOWS) { if (IS_WINDOWS) {
// GNU tar present in path but not at default location jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false)
jest
.spyOn(utils, 'getGnuTarPathOnWindows') const isGnuMock = jest
.mockReturnValue(Promise.resolve('tar')) .spyOn(utils, 'isGnuTarInstalled')
.mockReturnValue(Promise.resolve(true))
const execMock = jest.spyOn(exec, 'exec') const execMock = jest.spyOn(exec, 'exec')
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
const workspace = process.env['GITHUB_WORKSPACE'] const workspace = process.env['GITHUB_WORKSPACE']
await tar.extractTar(archivePath, CompressionMethod.Gzip) await tar.extractTar(archivePath, CompressionMethod.Gzip)
expect(isGnuMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"tar"`, `"tar"`,
[ [
'-z',
'-xf', '-xf',
archivePath.replace(/\\/g, '/'), archivePath.replace(/\\/g, '/'),
'-P', '-P',
'-C', '-C',
workspace?.replace(/\\/g, '/'), workspace?.replace(/\\/g, '/'),
'--force-local', '--force-local'
'-z'
], ],
{cwd: undefined} {cwd: undefined}
) )
@@ -179,13 +134,13 @@ test('zstd create tar', async () => {
await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd) await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`, `"${defaultTarPath}"`,
[ [
'--posix', '--posix',
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30',
'-cf', '-cf',
IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd,
'--exclude', '--exclude',
@@ -197,65 +152,13 @@ test('zstd create tar', async () => {
'manifest.txt' 'manifest.txt'
] ]
.concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []),
.concat([
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
]),
{ {
cwd: archiveFolder cwd: archiveFolder
} }
) )
}) })
test('zstd create tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archiveFolder = getTempDir()
const workspace = process.env['GITHUB_WORKSPACE']
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
const tarFilename = 'cache.tar'
await fs.promises.mkdir(archiveFolder, {recursive: true})
await tar.createTar(
archiveFolder,
sourceDirectories,
CompressionMethod.Zstd
)
const tarPath = SystemTarPathOnWindows
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[
'--posix',
'-cf',
tarFilename.replace(/\\/g, '/'),
'--exclude',
tarFilename.replace(/\\/g, '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/'),
'--files-from',
'manifest.txt',
'&&',
'zstd -T0 --long=30 -o',
CacheFilename.Zstd.replace(/\\/g, '/'),
tarFilename.replace(/\\/g, '/')
],
{
cwd: archiveFolder
}
)
}
})
test('gzip create tar', async () => { test('gzip create tar', async () => {
const execMock = jest.spyOn(exec, 'exec') const execMock = jest.spyOn(exec, 'exec')
@@ -267,13 +170,16 @@ test('gzip create tar', async () => {
await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Gzip) await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Gzip)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`, `"${tarPath}"`,
[ [
'--posix', '--posix',
'-z',
'-cf', '-cf',
IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip, IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip,
'--exclude', '--exclude',
@@ -283,10 +189,7 @@ test('gzip create tar', async () => {
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from', '--files-from',
'manifest.txt' 'manifest.txt'
] ].concat(IS_MAC ? ['--delay-directory-restore'] : []),
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z']),
{ {
cwd: archiveFolder cwd: archiveFolder
} }
@@ -302,50 +205,22 @@ test('zstd list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.Zstd) await tar.listTar(archivePath, CompressionMethod.Zstd)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`, `"${defaultTarPath}"`,
['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P'] [
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
]
.concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []),
.concat([
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
]),
{cwd: undefined} {cwd: undefined}
) )
}) })
test('zstd list tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
await tar.listTar(archivePath, CompressionMethod.Zstd)
const tarFilename = 'cache.tar'
const tarPath = SystemTarPathOnWindows
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
'zstd -d --long=30 -o',
[
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&',
`${tarPath}`,
'-tf',
tarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
],
{cwd: undefined}
)
}
})
test('zstdWithoutLong list tar', async () => { test('zstdWithoutLong list tar', async () => {
const execMock = jest.spyOn(exec, 'exec') const execMock = jest.spyOn(exec, 'exec')
@@ -355,14 +230,18 @@ test('zstdWithoutLong list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.ZstdWithoutLong) await tar.listTar(archivePath, CompressionMethod.ZstdWithoutLong)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`, `"${defaultTarPath}"`,
['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P'] [
'--use-compress-program',
IS_WINDOWS ? 'zstd -d' : 'unzstd',
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
]
.concat(IS_WINDOWS ? ['--force-local'] : []) .concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []) .concat(IS_MAC ? ['--delay-directory-restore'] : []),
.concat(['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']),
{cwd: undefined} {cwd: undefined}
) )
}) })
@@ -375,14 +254,18 @@ test('gzip list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.Gzip) await tar.listTar(archivePath, CompressionMethod.Gzip)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith( expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`, `"${tarPath}"`,
['-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P'] [
.concat(IS_WINDOWS ? ['--force-local'] : []) '-z',
.concat(IS_MAC ? ['--delay-directory-restore'] : []) '-tf',
.concat(['-z']), IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
].concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined} {cwd: undefined}
) )
}) })
+9 -11
View File
@@ -7,11 +7,7 @@ import * as path from 'path'
import * as semver from 'semver' import * as semver from 'semver'
import * as util from 'util' import * as util from 'util'
import {v4 as uuidV4} from 'uuid' import {v4 as uuidV4} from 'uuid'
import { import {CacheFilename, CompressionMethod} from './constants'
CacheFilename,
CompressionMethod,
GnuTarPathOnWindows
} from './constants'
// From https://github.com/actions/toolkit/blob/main/packages/tool-cache/src/tool-cache.ts#L23 // From https://github.com/actions/toolkit/blob/main/packages/tool-cache/src/tool-cache.ts#L23
export async function createTempDirectory(): Promise<string> { export async function createTempDirectory(): Promise<string> {
@@ -71,7 +67,7 @@ export async function unlinkFile(filePath: fs.PathLike): Promise<void> {
return util.promisify(fs.unlink)(filePath) return util.promisify(fs.unlink)(filePath)
} }
export async function getVersion(app: string): Promise<string> { async function getVersion(app: string): Promise<string> {
core.debug(`Checking ${app} --version`) core.debug(`Checking ${app} --version`)
let versionOutput = '' let versionOutput = ''
try { try {
@@ -94,6 +90,11 @@ export async function getVersion(app: string): Promise<string> {
// Use zstandard if possible to maximize cache performance // Use zstandard if possible to maximize cache performance
export async function getCompressionMethod(): Promise<CompressionMethod> { export async function getCompressionMethod(): Promise<CompressionMethod> {
if (process.platform === 'win32' && !(await isGnuTarInstalled())) {
// Disable zstd due to bug https://github.com/actions/cache/issues/301
return CompressionMethod.Gzip
}
const versionOutput = await getVersion('zstd') const versionOutput = await getVersion('zstd')
const version = semver.clean(versionOutput) const version = semver.clean(versionOutput)
@@ -115,12 +116,9 @@ export function getCacheFileName(compressionMethod: CompressionMethod): string {
: CacheFilename.Zstd : CacheFilename.Zstd
} }
export async function getGnuTarPathOnWindows(): Promise<string> { export async function isGnuTarInstalled(): Promise<boolean> {
if (fs.existsSync(GnuTarPathOnWindows)) {
return GnuTarPathOnWindows
}
const versionOutput = await getVersion('tar') const versionOutput = await getVersion('tar')
return versionOutput.toLowerCase().includes('gnu tar') ? io.which('tar') : '' return versionOutput.toLowerCase().includes('gnu tar')
} }
export function assertDefined<T>(name: string, value?: T): T { export function assertDefined<T>(name: string, value?: T): T {
-5
View File
@@ -21,8 +21,3 @@ export const DefaultRetryDelay = 5000
// over the socket during this period, the socket is destroyed and the download // over the socket during this period, the socket is destroyed and the download
// is aborted. // is aborted.
export const SocketTimeout = 5000 export const SocketTimeout = 5000
// The default path of GNUtar on hosted Windows runners
export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe`
export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe`
+58 -309
View File
@@ -3,21 +3,25 @@ import * as io from '@actions/io'
import {existsSync, writeFileSync} from 'fs' import {existsSync, writeFileSync} from 'fs'
import * as path from 'path' import * as path from 'path'
import * as utils from './cacheUtils' import * as utils from './cacheUtils'
import {CompressionMethod, SystemTarPathOnWindows} from './constants' import {CompressionMethod} from './constants'
const IS_WINDOWS = process.platform === 'win32' const IS_WINDOWS = process.platform === 'win32'
// Function also mutates the args array. For non-mutation call with passing an empty array. async function getTarPath(
async function getTarPath(): Promise<string> { args: string[],
compressionMethod: CompressionMethod
): Promise<string> {
switch (process.platform) { switch (process.platform) {
case 'win32': { case 'win32': {
const gnuTar = await utils.getGnuTarPathOnWindows() const systemTar = `${process.env['windir']}\\System32\\tar.exe`
const systemTar = SystemTarPathOnWindows if (compressionMethod !== CompressionMethod.Gzip) {
if (gnuTar) { // We only use zstandard compression on windows when gnu tar is installed due to
// Use GNUtar as default on windows // a bug with compressing large files with bsdtar + zstd
return gnuTar args.push('--force-local')
} else if (existsSync(systemTar)) { } else if (existsSync(systemTar)) {
return systemTar return systemTar
} else if (await utils.isGnuTarInstalled()) {
args.push('--force-local')
} }
break break
} }
@@ -25,6 +29,7 @@ async function getTarPath(): Promise<string> {
const gnuTar = await io.which('gtar', false) const gnuTar = await io.which('gtar', false)
if (gnuTar) { if (gnuTar) {
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
args.push('--delay-directory-restore')
return gnuTar return gnuTar
} }
break break
@@ -35,101 +40,13 @@ async function getTarPath(): Promise<string> {
return await io.which('tar', true) return await io.which('tar', true)
} }
async function getTarArgs( async function execTar(
compressionMethod: CompressionMethod,
type: string,
archivePath = ''
): Promise<string[]> {
const args = []
const manifestFilename = 'manifest.txt'
const cacheFileName = utils.getCacheFileName(compressionMethod)
const tarFile = 'cache.tar'
const tarPath = await getTarPath()
const workingDirectory = getWorkingDirectory()
const BSD_TAR_ZSTD =
tarPath === SystemTarPathOnWindows &&
compressionMethod !== CompressionMethod.Gzip
// Method specific args
switch (type) {
case 'create':
args.push(
'--posix',
'-cf',
BSD_TAR_ZSTD
? tarFile
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--exclude',
BSD_TAR_ZSTD
? tarFile
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--files-from',
manifestFilename
)
break
case 'extract':
args.push(
'-xf',
BSD_TAR_ZSTD
? tarFile
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
)
break
case 'list':
args.push(
'-tf',
BSD_TAR_ZSTD
? tarFile
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
)
break
}
// Platform specific args
switch (process.platform) {
case 'win32': {
const gnuTar = await utils.getGnuTarPathOnWindows()
if (gnuTar) {
// Use GNUtar as default on windows
args.push('--force-local')
}
break
}
case 'darwin': {
const gnuTar = await io.which('gtar', false)
if (gnuTar) {
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
args.push('--delay-directory-restore')
}
break
}
}
return args
}
async function execTar(args: string[], cwd?: string): Promise<void> {
try {
await exec(`"${await getTarPath()}"`, args, {cwd})
} catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`)
}
}
async function execCommand(
command: string,
args: string[], args: string[],
compressionMethod: CompressionMethod,
cwd?: string cwd?: string
): Promise<void> { ): Promise<void> {
try { try {
await exec(command, args, {cwd}) await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd})
} catch (error) { } catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`) throw new Error(`Tar failed with error: ${error?.message}`)
} }
@@ -140,41 +57,19 @@ function getWorkingDirectory(): string {
} }
// Common function for extractTar and listTar to get the compression method // Common function for extractTar and listTar to get the compression method
async function getCompressionProgram( function getCompressionProgram(compressionMethod: CompressionMethod): string[] {
compressionMethod: CompressionMethod,
archivePath: string
): Promise<string[]> {
// -d: Decompress. // -d: Decompress.
// unzstd is equivalent to 'zstd -d' // unzstd is equivalent to 'zstd -d'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners. // Using 30 here because we also support 32-bit self-hosted runners.
const tarPath = await getTarPath()
const tarFile = 'cache.tar'
const BSD_TAR_ZSTD =
tarPath === SystemTarPathOnWindows &&
compressionMethod !== CompressionMethod.Gzip
switch (compressionMethod) { switch (compressionMethod) {
case CompressionMethod.Zstd: case CompressionMethod.Zstd:
return BSD_TAR_ZSTD return [
? [ '--use-compress-program',
'zstd -d --long=30 -o', IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
tarFile, ]
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&'
]
: [
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
]
case CompressionMethod.ZstdWithoutLong: case CompressionMethod.ZstdWithoutLong:
return BSD_TAR_ZSTD return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']
? [
'zstd -d -o',
tarFile,
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&'
]
: ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']
default: default:
return ['-z'] return ['-z']
} }
@@ -184,27 +79,13 @@ export async function listTar(
archivePath: string, archivePath: string,
compressionMethod: CompressionMethod compressionMethod: CompressionMethod
): Promise<void> { ): Promise<void> {
const tarPath = await getTarPath() const args = [
const BSD_TAR_ZSTD = ...getCompressionProgram(compressionMethod),
tarPath === SystemTarPathOnWindows && '-tf',
compressionMethod !== CompressionMethod.Gzip archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
const compressionArgs = await getCompressionProgram( '-P'
compressionMethod, ]
archivePath await execTar(args, compressionMethod)
)
const tarArgs = await getTarArgs(compressionMethod, 'list', archivePath)
// TODO: Add a test for BSD tar on windows
if (BSD_TAR_ZSTD) {
const command = compressionArgs[0]
const args = compressionArgs
.slice(1)
.concat([tarPath])
.concat(tarArgs)
await execCommand(command, args)
} else {
const args = tarArgs.concat(compressionArgs)
await execTar(args)
}
} }
export async function extractTar( export async function extractTar(
@@ -213,27 +94,16 @@ export async function extractTar(
): Promise<void> { ): Promise<void> {
// Create directory to extract tar into // Create directory to extract tar into
const workingDirectory = getWorkingDirectory() const workingDirectory = getWorkingDirectory()
const tarPath = await getTarPath()
const BSD_TAR_ZSTD =
tarPath === SystemTarPathOnWindows &&
compressionMethod !== CompressionMethod.Gzip
await io.mkdirP(workingDirectory) await io.mkdirP(workingDirectory)
const tarArgs = await getTarArgs(compressionMethod, 'extract', archivePath) const args = [
const compressionArgs = await getCompressionProgram( ...getCompressionProgram(compressionMethod),
compressionMethod, '-xf',
archivePath archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
) '-P',
if (BSD_TAR_ZSTD) { '-C',
const command = compressionArgs[0] workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
const args = compressionArgs ]
.slice(1) await execTar(args, compressionMethod)
.concat([tarPath])
.concat(tarArgs)
await execCommand(command, args)
} else {
const args = tarArgs.concat(compressionArgs)
await execTar(args)
}
} }
export async function createTar( export async function createTar(
@@ -244,15 +114,11 @@ export async function createTar(
// Write source directories to manifest.txt to avoid command length limits // Write source directories to manifest.txt to avoid command length limits
const manifestFilename = 'manifest.txt' const manifestFilename = 'manifest.txt'
const cacheFileName = utils.getCacheFileName(compressionMethod) const cacheFileName = utils.getCacheFileName(compressionMethod)
const tarFile = 'cache.tar'
const tarPath = await getTarPath()
const BSD_TAR_ZSTD =
tarPath === SystemTarPathOnWindows &&
compressionMethod !== CompressionMethod.Gzip
writeFileSync( writeFileSync(
path.join(archiveFolder, manifestFilename), path.join(archiveFolder, manifestFilename),
sourceDirectories.join('\n') sourceDirectories.join('\n')
) )
const workingDirectory = getWorkingDirectory()
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// zstdmt is equivalent to 'zstd -T0' // zstdmt is equivalent to 'zstd -T0'
@@ -262,145 +128,28 @@ export async function createTar(
function getCompressionProgram(): string[] { function getCompressionProgram(): string[] {
switch (compressionMethod) { switch (compressionMethod) {
case CompressionMethod.Zstd: case CompressionMethod.Zstd:
return BSD_TAR_ZSTD return [
? [ '--use-compress-program',
'&&', IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
'zstd -T0 --long=30 -o', ]
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
tarFile
]
: [
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
]
case CompressionMethod.ZstdWithoutLong: case CompressionMethod.ZstdWithoutLong:
return BSD_TAR_ZSTD return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']
? [
'&&',
'zstd -T0 -o',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
tarFile
]
: ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']
default: default:
return ['-z'] return ['-z']
} }
} }
const tarArgs = await getTarArgs(compressionMethod, 'create') const args = [
const compressionArgs = getCompressionProgram() '--posix',
const args = tarArgs.concat(compressionArgs) ...getCompressionProgram(),
await execTar(args, archiveFolder) '-cf',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--exclude',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--files-from',
manifestFilename
]
await execTar(args, compressionMethod, archiveFolder)
} }
const ARCHIVE_TOOL_GNU: string = 'gnu'
const ARCHIVE_TOOL_BSD: string = 'bsd'
const manifestFilename = 'manifest.txt'
export async function createTar2(
archiveFolder: string,
sourceDirectories: string[]
): Promise<void> {
// 1. decide the compression algo - zstd otherwise gzip
// 2. decide tar - gnutar otherwise bsdtar or systemtar
// 3. deide archiving/unarchiving args depending on tar
// 4. decide compression/decompression args depending on tar and compression
// 5. decide the exec command
let tar = getTarPath2()
let compression = getCompressionMethod2()
let args = argsMap.get({tar: tar, compression: compression, os: process.platform, "create"})
}
// return archive tool name and path
async function getTarPath2(): Promise<ArchiveTool> {
switch (process.platform) {
case 'win32': {
const gnuTar = await utils.getGnuTarPathOnWindows()
const systemTar = SystemTarPathOnWindows
if (gnuTar) {
// Use GNUtar as default on windows
return <ArchiveTool>{ name: ARCHIVE_TOOL_GNU, path: gnuTar }
} else if (existsSync(systemTar)) {
return <ArchiveTool>{name: ARCHIVE_TOOL_BSD, path: systemTar}
}
break
}
case 'darwin': {
const gnuTar = await io.which('gtar', false)
if (gnuTar) {
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
return <ArchiveTool>{ name: ARCHIVE_TOOL_GNU, path: gnuTar }
} else {
const path = await io.which('tar', true)
return <ArchiveTool>{name: ARCHIVE_TOOL_BSD, path: path}
}
}
default:
break
}
const path = await io.which('tar', true)
return <ArchiveTool>{name: ARCHIVE_TOOL_GNU, path: path}
}
// Use zstandard if possible to maximize cache performance
export async function getCompressionMethod2(): Promise<CompressionMethod> {
const versionOutput = await utils.getVersion('zstd')
const version = semver.clean(versionOutput)
if (!versionOutput.toLowerCase().includes('zstd command line interface')) {
// zstd is not installed
return CompressionMethod.Gzip
} else if (!version || semver.lt(version, 'v1.3.2')) {
// zstd is installed but using a version earlier than v1.3.2
// v1.3.2 is required to use the `--long` options in zstd
return CompressionMethod.ZstdWithoutLong
} else {
return CompressionMethod.Zstd
}
}
interface ArchiveTool {
name: string
path: string
}
// tar, compression, os, operation -> tar args + compression args
const argsMap: Map<ArgsLookupKey, string[][]> = new Map()
argsMap.set({tar: ARCHIVE_TOOL_GNU, compression: CompressionMethod.Zstd, os: "windows", operation: "create"},
[[
'--posix',
'-cf',
'%%cacheFileName%%',
'--exclude',
'%%cacheFileName%%',
'-P',
'-C',
'%%workingDirectory%%',
'--files-from',
manifestFilename],
['--use-compress-program',
'zstd -T0 --long=30']
])
argsMap.set({tar: ARCHIVE_TOOL_GNU, compression: CompressionMethod.Zstd, os: "linux", operation: "create"},
[
'--use-compress-program',
'zstdmt --long=30'
])
interface ArgsLookupKey {
tar: string
compression: string
os: string
operation: string
}
+2 -2
View File
@@ -1,12 +1,12 @@
{ {
"name": "@actions/io", "name": "@actions/io",
"version": "1.1.2", "version": "1.1.3",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@actions/io", "name": "@actions/io",
"version": "1.1.2", "version": "1.1.3",
"license": "MIT" "license": "MIT"
} }
} }
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@actions/io", "name": "@actions/io",
"version": "1.1.2", "version": "1.1.3",
"description": "Actions io lib", "description": "Actions io lib",
"keywords": [ "keywords": [
"github", "github",
+12 -3
View File
@@ -4,7 +4,8 @@ import * as path from 'path'
import {promisify} from 'util' import {promisify} from 'util'
import * as ioUtil from './io-util' import * as ioUtil from './io-util'
const exec = promisify(childProcess.exec) // const exec = promisify(childProcess.exec)
// const fork = promisify(childProcess.fork)
const execFile = promisify(childProcess.execFile) const execFile = promisify(childProcess.execFile)
/** /**
@@ -129,12 +130,20 @@ export async function rmRF(inputPath: string): Promise<void> {
try { try {
const cmdPath = ioUtil.getCmdPath() const cmdPath = ioUtil.getCmdPath()
if (await ioUtil.isDirectory(inputPath, true)) { if (await ioUtil.isDirectory(inputPath, true)) {
await exec(`${cmdPath} /s /c "rd /s /q "%inputPath%""`, { await execFile(`${cmdPath} /s /c "rd /s /q "%inputPath%""`, {
env: {inputPath} env: {inputPath}
}).catch(err => {
// if you try to delete a file that doesn't exist, desired result is achieved
// other errors are valid
if (err.code !== 'ENOENT') throw err
}) })
} else { } else {
await exec(`${cmdPath} /s /c "del /f /a "%inputPath%""`, { await execFile(`${cmdPath} /s /c "del /f /a "%inputPath%""`, {
env: {inputPath} env: {inputPath}
}).catch(err => {
// if you try to delete a file that doesn't exist, desired result is achieved
// other errors are valid
if (err.code !== 'ENOENT') throw err
}) })
} }
} catch (err) { } catch (err) {