Compare commits

...
Author SHA1 Message Date
Sampark SharmaandGitHub fb3b6981af Update release version to patch 2022-12-27 10:19:21 +00:00
Sampark SharmaandGitHub 85bd49f5ed Revert "Cache package release for compression change in windows (#1281)"
This reverts commit b228732644.
2022-12-27 14:06:04 +05:30
b228732644 Cache package release for compression change in windows (#1281)
* bsd + zstd fallback implementation

* bsd + zstd fallback implementation

* Fix tar operations

* Add -v option for testing

* Fix order of args for tar

* Add GNUtar as default on windows

* Fix test

* Fix tar tests

* Fix lint issues

* Fix windows gnutar test case

* Temporarily remove thhe condition that prevents zstd usage on windows unless with GNUtar

* Address some comments and correct compression commands

* Add windows bsdtar test

* Fix windows test

* Fix test

* Separate args

* Fix old tests

* Add new tests

* Fix tests

* Fix lint test

* Refactor code

* Address review comments

* Fix test

* Fix tar test

* Add await to async function calls

* Fix test

* Update for beta release

* Fix audit issues

* Add fallback to gzip compression if cache not found

* Fix test

* Add test

* Address review comments

* Revert Address review comments

* Release 3.1.0-beta.2 cache package

* Fix issues

* Reconfigure catch block

* Add debug logging for gzip fall back

* Fix test

* Add end to end test for cache using bsd on windows
and address review comments

* Fix test

* Fix test

* Fix tests

* Add better comments

* Update packages/cache/src/internal/cacheHttpClient.ts

Co-authored-by: Bishal Prasad <[email protected]>

* Address review comments

* Update for new beta cache package release

* Address bugbash issues

* Fix tests

* Release new actions/cache minor version

Co-authored-by: Lovepreet Singh <[email protected]>
Co-authored-by: Bishal Prasad <[email protected]>
2022-12-22 20:47:35 +05:30
Sampark SharmaandGitHub 2a4f3544ad Merge pull request #1234 from actions/phantsure/version-logging
Add logs for cache version on miss
2022-12-22 11:48:58 +05:30
Sampark SharmaandGitHub c23fe4b81f FIx test 2022-12-21 10:30:22 +00:00
Sampark SharmaandGitHub 034d154f88 Merge branch 'phantsure/version-logging' of https://github.com/actions/toolkit into phantsure/version-logging 2022-12-21 10:27:13 +00:00
Sampark SharmaandGitHub ccfa36f304 Address review comments 2022-12-21 10:24:52 +00:00
e96dc8a69a Update packages/cache/src/internal/cacheHttpClient.ts
Co-authored-by: Bishal Prasad <[email protected]>
2022-12-16 18:17:37 +05:30
Sampark SharmaandGitHub b8c50aa82d Fix response code 2022-12-16 05:55:53 +00:00
Sampark SharmaandGitHub 24685611e2 Add current scope from github ref 2022-12-16 05:54:48 +00:00
Sampark SharmaandGitHub e559a15ca6 Fix test 2022-12-13 11:36:10 +00:00
Sampark SharmaandGitHub 816c1b3760 Fix tests 2022-12-13 11:25:40 +00:00
Sampark SharmaandGitHub 819157bf87 Merge pull request #1238 from actions/revert-1232-phantsure/gnutar_windows
Revert "Add GNUtar as default in windows"
2022-11-17 12:42:10 +05:30
Sampark SharmaandGitHub 86102e88e9 Revert "Add GNUtar as default in windows" 2022-11-17 12:26:27 +05:30
Sampark SharmaandGitHub b9d1dd898e Address review comments 2022-11-15 11:04:24 +00:00
Sampark SharmaandGitHub aaac0e6c98 Address review comments 2022-11-15 10:32:24 +00:00
Sampark SharmaandGitHub a735d9bcd4 Add logs for cache version on miss 2022-11-14 06:31:26 +00:00
8 changed files with 90 additions and 68 deletions
+22 -34
View File
@@ -1,11 +1,7 @@
import * as exec from '@actions/exec'
import * as io from '@actions/io'
import * as path from 'path'
import {
CacheFilename,
CompressionMethod,
GnuTarPathOnWindows
} from '../src/internal/constants'
import {CacheFilename, CompressionMethod} from '../src/internal/constants'
import * as tar from '../src/internal/tar'
import * as utils from '../src/internal/cacheUtils'
// eslint-disable-next-line @typescript-eslint/no-require-imports
@@ -32,10 +28,6 @@ beforeAll(async () => {
await jest.requireActual('@actions/io').rmRF(getTempDir())
})
beforeEach(async () => {
jest.restoreAllMocks()
})
afterAll(async () => {
delete process.env['GITHUB_WORKSPACE']
await jest.requireActual('@actions/io').rmRF(getTempDir())
@@ -49,14 +41,13 @@ test('zstd extract tar', async () => {
? `${process.env['windir']}\\fakepath\\cache.tar`
: 'cache.tar'
const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
await tar.extractTar(archivePath, CompressionMethod.Zstd)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
`"${defaultTarPath}"`,
[
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
@@ -83,7 +74,9 @@ test('gzip extract tar', async () => {
await tar.extractTar(archivePath, CompressionMethod.Gzip)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
@@ -94,19 +87,18 @@ test('gzip extract tar', async () => {
'-P',
'-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
].concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
)
})
test('gzip extract GNU tar on windows with GNUtar in path', async () => {
test('gzip extract GNU tar on windows', async () => {
if (IS_WINDOWS) {
// GNU tar present in path but not at default location
jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false)
const isGnuMock = jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve('tar'))
.spyOn(utils, 'isGnuTarInstalled')
.mockReturnValue(Promise.resolve(true))
const execMock = jest.spyOn(exec, 'exec')
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
const workspace = process.env['GITHUB_WORKSPACE']
@@ -142,11 +134,9 @@ test('zstd create tar', async () => {
await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
`"${defaultTarPath}"`,
[
'--posix',
'--use-compress-program',
@@ -180,7 +170,9 @@ test('gzip create tar', async () => {
await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Gzip)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
@@ -197,9 +189,7 @@ test('gzip create tar', async () => {
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from',
'manifest.txt'
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
].concat(IS_MAC ? ['--delay-directory-restore'] : []),
{
cwd: archiveFolder
}
@@ -215,10 +205,9 @@ test('zstd list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.Zstd)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
`"${defaultTarPath}"`,
[
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
@@ -241,10 +230,9 @@ test('zstdWithoutLong list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.ZstdWithoutLong)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
`"${defaultTarPath}"`,
[
'--use-compress-program',
IS_WINDOWS ? 'zstd -d' : 'unzstd',
@@ -266,7 +254,9 @@ test('gzip list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.Gzip)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
@@ -275,9 +265,7 @@ test('gzip list tar', async () => {
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
].concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
)
})
+8 -8
View File
@@ -1,12 +1,12 @@
{
"name": "@actions/cache",
"version": "3.0.6",
"version": "3.1.1",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@actions/cache",
"version": "3.0.6",
"version": "3.1.1",
"license": "MIT",
"dependencies": {
"@actions/core": "^1.10.0",
@@ -457,9 +457,9 @@
}
},
"node_modules/minimatch": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"dependencies": {
"brace-expansion": "^1.1.7"
},
@@ -998,9 +998,9 @@
}
},
"minimatch": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"requires": {
"brace-expansion": "^1.1.7"
}
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@actions/cache",
"version": "3.0.6",
"version": "3.1.1",
"preview": true,
"description": "Actions cache lib",
"keywords": [
+31 -1
View File
@@ -17,7 +17,8 @@ import {
CommitCacheRequest,
ReserveCacheRequest,
ReserveCacheResponse,
ITypedResponseWithError
ITypedResponseWithError,
ArtifactCacheList
} from './contracts'
import {downloadCacheHttpClient, downloadCacheStorageSDK} from './downloadUtils'
import {
@@ -104,6 +105,10 @@ export async function getCacheEntry(
httpClient.getJson<ArtifactCacheEntry>(getCacheApiUrl(resource))
)
if (response.statusCode === 204) {
// List cache for primary key only if cache miss occurs
if (core.isDebug()) {
await printCachesListForDiagnostics(keys[0], httpClient, version)
}
return null
}
if (!isSuccessStatusCode(response.statusCode)) {
@@ -122,6 +127,31 @@ export async function getCacheEntry(
return cacheResult
}
async function printCachesListForDiagnostics(
key: string,
httpClient: HttpClient,
version: string
): Promise<void> {
const resource = `caches?key=${encodeURIComponent(key)}`
const response = await retryTypedResponse('listCache', async () =>
httpClient.getJson<ArtifactCacheList>(getCacheApiUrl(resource))
)
if (response.statusCode === 200) {
const cacheListResult = response.result
const totalCount = cacheListResult?.totalCount
if (totalCount && totalCount > 0) {
core.debug(
`No matching cache found for cache key '${key}', version '${version} and scope ${process.env['GITHUB_REF']}. There exist one or more cache(s) with similar key but they have different version or scope. See more info on cache matching here: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#matching-a-cache-key \nOther caches with similar key:`
)
for (const cacheEntry of cacheListResult?.artifactCaches || []) {
core.debug(
`Cache Key: ${cacheEntry?.cacheKey}, Cache Version: ${cacheEntry?.cacheVersion}, Cache Scope: ${cacheEntry?.scope}, Cache Created: ${cacheEntry?.creationTime}`
)
}
}
}
}
export async function downloadCache(
archiveLocation: string,
archivePath: string,
+4 -11
View File
@@ -7,11 +7,7 @@ import * as path from 'path'
import * as semver from 'semver'
import * as util from 'util'
import {v4 as uuidV4} from 'uuid'
import {
CacheFilename,
CompressionMethod,
GnuTarPathOnWindows
} from './constants'
import {CacheFilename, CompressionMethod} from './constants'
// From https://github.com/actions/toolkit/blob/main/packages/tool-cache/src/tool-cache.ts#L23
export async function createTempDirectory(): Promise<string> {
@@ -94,7 +90,7 @@ async function getVersion(app: string): Promise<string> {
// Use zstandard if possible to maximize cache performance
export async function getCompressionMethod(): Promise<CompressionMethod> {
if (process.platform === 'win32' && !(await getGnuTarPathOnWindows())) {
if (process.platform === 'win32' && !(await isGnuTarInstalled())) {
// Disable zstd due to bug https://github.com/actions/cache/issues/301
return CompressionMethod.Gzip
}
@@ -120,12 +116,9 @@ export function getCacheFileName(compressionMethod: CompressionMethod): string {
: CacheFilename.Zstd
}
export async function getGnuTarPathOnWindows(): Promise<string> {
if (fs.existsSync(GnuTarPathOnWindows)) {
return GnuTarPathOnWindows
}
export async function isGnuTarInstalled(): Promise<boolean> {
const versionOutput = await getVersion('tar')
return versionOutput.toLowerCase().includes('gnu tar') ? io.which('tar') : ''
return versionOutput.toLowerCase().includes('gnu tar')
}
export function assertDefined<T>(name: string, value?: T): T {
-3
View File
@@ -21,6 +21,3 @@ export const DefaultRetryDelay = 5000
// over the socket during this period, the socket is destroyed and the download
// is aborted.
export const SocketTimeout = 5000
// The default path of GNUtar on hosted Windows runners
export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe`
+6
View File
@@ -9,10 +9,16 @@ export interface ITypedResponseWithError<T> extends TypedResponse<T> {
export interface ArtifactCacheEntry {
cacheKey?: string
scope?: string
cacheVersion?: string
creationTime?: string
archiveLocation?: string
}
export interface ArtifactCacheList {
totalCount: number
artifactCaches?: ArtifactCacheEntry[]
}
export interface CommitCacheRequest {
size: number
}
+18 -10
View File
@@ -7,17 +7,21 @@ import {CompressionMethod} from './constants'
const IS_WINDOWS = process.platform === 'win32'
async function getTarPath(args: string[]): Promise<string> {
async function getTarPath(
args: string[],
compressionMethod: CompressionMethod
): Promise<string> {
switch (process.platform) {
case 'win32': {
const gnuTar = await utils.getGnuTarPathOnWindows()
const systemTar = `${process.env['windir']}\\System32\\tar.exe`
if (gnuTar) {
// Use GNUtar as default on windows
if (compressionMethod !== CompressionMethod.Gzip) {
// We only use zstandard compression on windows when gnu tar is installed due to
// a bug with compressing large files with bsdtar + zstd
args.push('--force-local')
return gnuTar
} else if (existsSync(systemTar)) {
return systemTar
} else if (await utils.isGnuTarInstalled()) {
args.push('--force-local')
}
break
}
@@ -36,9 +40,13 @@ async function getTarPath(args: string[]): Promise<string> {
return await io.which('tar', true)
}
async function execTar(args: string[], cwd?: string): Promise<void> {
async function execTar(
args: string[],
compressionMethod: CompressionMethod,
cwd?: string
): Promise<void> {
try {
await exec(`"${await getTarPath(args)}"`, args, {cwd})
await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd})
} catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`)
}
@@ -77,7 +85,7 @@ export async function listTar(
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
]
await execTar(args)
await execTar(args, compressionMethod)
}
export async function extractTar(
@@ -95,7 +103,7 @@ export async function extractTar(
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
]
await execTar(args)
await execTar(args, compressionMethod)
}
export async function createTar(
@@ -143,5 +151,5 @@ export async function createTar(
'--files-from',
manifestFilename
]
await execTar(args, archiveFolder)
await execTar(args, compressionMethod, archiveFolder)
}