Remove unused cache API

Prepare `@actions/cache` `4.0.0` release

packages/cache/README.md

@@ -6,6 +6,20 @@ See ["Caching dependencies to speed up workflows"](https://docs.github.com/en/ac
 
 Note that GitHub will remove any cache entries that have not been accessed in over 7 days. There is no limit on the number of caches you can store, but the total size of all caches in a repository is limited to 10 GB. If you exceed this limit, GitHub will save your cache but will begin evicting caches until the total size is less than 10 GB.
 
+## ⚠️ Important changes
+
+The cache backend service has been rewritten from the ground up for improved performance and reliability. The [@actions/cache](https://github.com/actions/toolkit/tree/main/packages/cache) package now integrates with the new cache service (v2) APIs.
+
+The new service will gradually roll out as of **February 1st, 2025**. The legacy service will also be sunset on the same date. Changes in this release are **fully backward compatible**.
+
+**All previous versions of this package will be deprecated**. We recommend upgrading to version `4.0.0` as soon as possible before **February 1st, 2025.**
+
+If you do not upgrade, all workflow runs using any of the deprecated [@actions/cache](https://github.com/actions/toolkit/tree/main/packages/cache) packages will fail.
+
+Upgrading to the recommended version should not break or require any changes to your workflows beyond updating your `package.json` to version `4.0.0`.
+
+Read more about the change & access the migration guide: [reference to the announcement](https://github.com/actions/toolkit/discussions/1890).
+
 ## Usage
 
 This package is used by the v2+ versions of our first party cache action. You can find an example implementation in the cache repo [here](https://github.com/actions/cache).
@@ -47,5 +61,3 @@ const cacheKey = await cache.restoreCache(paths, key, restoreKeys)
 A cache gets downloaded in multiple segments of fixed sizes (now `128MB` to fail-fast, previously `1GB` for a `32-bit` runner and `2GB` for a `64-bit` runner were used). Sometimes, a segment download gets stuck which causes the workflow job to be stuck forever and fail. Version `v3.0.4` of cache package introduces a segment download timeout. The segment download timeout will allow the segment download to get aborted and hence allow the job to proceed with a cache miss.
 
 Default value of this timeout is 10 minutes (starting `v3.2.1` and higher, previously 60 minutes in versions between `v.3.0.4` and `v3.2.0`, both included) and can be customized by specifying an [environment variable](https://docs.github.com/en/actions/learn-github-actions/environment-variables) named `SEGMENT_DOWNLOAD_TIMEOUT_MINS` with timeout value in minutes.
-
-

packages/cache/RELEASES.md

@@ -1,6 +1,29 @@
 # @actions/cache Releases
 
+### 4.0.0
+
+#### Important changes
+
+The cache backend service has been rewritten from the ground up for improved performance and reliability. The [@actions/cache](https://github.com/actions/toolkit/tree/main/packages/cache) package now integrates with the new cache service (v2) APIs.
+
+The new service will gradually roll out as of **February 1st, 2025**. The legacy service will also be sunset on the same date. Changes in this release are **fully backward compatible**.
+
+**All previous versions of this package will be deprecated**. We recommend upgrading to version `4.0.0` as soon as possible before **February 1st, 2025.**
+
+If you do not upgrade, all workflow runs using any of the deprecated [@actions/cache](https://github.com/actions/toolkit/tree/main/packages/cache) packages will fail.
+
+Upgrading to the recommended version should not break or require any changes to your workflows beyond updating your `package.json` to version `4.0.0`.
+
+Read more about the change & access the migration guide: [reference to the announcement](https://github.com/actions/toolkit/discussions/1890).
+
+#### Minor changes
+
+- Update `@actions/core` to `1.11.0`
+- Update `semver` `6.3.1`
+- Add `twirp-ts` `2.5.0` to dependencies
+
 ### 3.3.0
+
 - Update `@actions/core` to `1.11.1`
 - Remove dependency on `uuid` package [#1824](https://github.com/actions/toolkit/pull/1824), [#1842](https://github.com/actions/toolkit/pull/1842)
 

packages/cache/__tests__/options.test.ts

@@ -11,8 +11,6 @@ const downloadConcurrency = 8
 const timeoutInMs = 30000
 const segmentTimeoutInMs = 600000
 const lookupOnly = false
-const uploadConcurrency = 4
-const uploadChunkSize = 32 * 1024 * 1024
 
 test('getDownloadOptions sets defaults', async () => {
   const actualOptions = getDownloadOptions()
@@ -43,18 +41,21 @@ test('getDownloadOptions overrides all settings', async () => {
 })
 
 test('getUploadOptions sets defaults', async () => {
+  const expectedOptions: UploadOptions = {
+    uploadConcurrency: 4,
+    uploadChunkSize: 32 * 1024 * 1024,
+    useAzureSdk: false
+  }
   const actualOptions = getUploadOptions()
 
-  expect(actualOptions).toEqual({
-    uploadConcurrency,
-    uploadChunkSize
-  })
+  expect(actualOptions).toEqual(expectedOptions)
 })
 
 test('getUploadOptions overrides all settings', async () => {
   const expectedOptions: UploadOptions = {
     uploadConcurrency: 2,
-    uploadChunkSize: 16 * 1024 * 1024
+    uploadChunkSize: 16 * 1024 * 1024,
+    useAzureSdk: true
   }
 
   const actualOptions = getUploadOptions(expectedOptions)
@@ -62,6 +63,34 @@ test('getUploadOptions overrides all settings', async () => {
   expect(actualOptions).toEqual(expectedOptions)
 })
 
+test('env variables override all getUploadOptions settings', async () => {
+  const expectedOptions: UploadOptions = {
+    uploadConcurrency: 16,
+    uploadChunkSize: 64 * 1024 * 1024,
+    useAzureSdk: true
+  }
+
+  process.env.CACHE_UPLOAD_CONCURRENCY = '16'
+  process.env.CACHE_UPLOAD_CHUNK_SIZE = '64'
+
+  const actualOptions = getUploadOptions(expectedOptions)
+  expect(actualOptions).toEqual(expectedOptions)
+})
+
+test('env variables override all getUploadOptions settings but do not exceed caps', async () => {
+  const expectedOptions: UploadOptions = {
+    uploadConcurrency: 32,
+    uploadChunkSize: 128 * 1024 * 1024,
+    useAzureSdk: true
+  }
+
+  process.env.CACHE_UPLOAD_CONCURRENCY = '64'
+  process.env.CACHE_UPLOAD_CHUNK_SIZE = '256'
+
+  const actualOptions = getUploadOptions(expectedOptions)
+  expect(actualOptions).toEqual(expectedOptions)
+})
+
 test('getDownloadOptions overrides download timeout minutes', async () => {
   const expectedOptions: DownloadOptions = {
     useAzureSdk: false,

packages/cache/__tests__/restoreCacheV2.test.ts

@@ -3,11 +3,11 @@ import * as path from 'path'
 import * as tar from '../src/internal/tar'
 import * as config from '../src/internal/config'
 import * as cacheUtils from '../src/internal/cacheUtils'
-import * as downloadCacheModule from '../src/internal/blob/download-cache'
+import * as cacheHttpClient from '../src/internal/cacheHttpClient'
 import {restoreCache} from '../src/cache'
 import {CacheFilename, CompressionMethod} from '../src/internal/constants'
 import {CacheServiceClientJSON} from '../src/generated/results/api/v1/cache.twirp'
-import {BlobDownloadResponseParsed} from '@azure/storage-blob'
+import {DownloadOptions} from '../src/options'
 
 jest.mock('../src/internal/cacheHttpClient')
 jest.mock('../src/internal/cacheUtils')
@@ -142,6 +142,7 @@ test('restore with gzip compressed cache found', async () => {
   const signedDownloadUrl = 'https://blob-storage.local?signed=true'
   const cacheVersion =
     'd90f107aaeb22920dba0c637a23c37b5bc497b4dfa3b07fe3f79bf88a273c11b'
+  const options = {useAzureSdk: true} as DownloadOptions
 
   const getCacheVersionMock = jest.spyOn(cacheUtils, 'getCacheVersion')
   getCacheVersionMock.mockReturnValue(cacheVersion)
@@ -169,17 +170,7 @@ test('restore with gzip compressed cache found', async () => {
   })
 
   const archivePath = path.join(tempPath, CacheFilename.Gzip)
-  const downloadCacheFileMock = jest.spyOn(
-    downloadCacheModule,
-    'downloadCacheFile'
-  )
-  downloadCacheFileMock.mockReturnValue(
-    Promise.resolve({
-      _response: {
-        status: 200
-      }
-    } as BlobDownloadResponseParsed)
-  )
+  const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache')
 
   const fileSize = 142
   const getArchiveFileSizeInBytesMock = jest
@@ -189,7 +180,7 @@ test('restore with gzip compressed cache found', async () => {
   const extractTarMock = jest.spyOn(tar, 'extractTar')
   const unlinkFileMock = jest.spyOn(cacheUtils, 'unlinkFile')
 
-  const cacheKey = await restoreCache(paths, key)
+  const cacheKey = await restoreCache(paths, key, [], options)
 
   expect(cacheKey).toBe(key)
   expect(getCacheVersionMock).toHaveBeenCalledWith(
@@ -203,9 +194,10 @@ test('restore with gzip compressed cache found', async () => {
     version: cacheVersion
   })
   expect(createTempDirectoryMock).toHaveBeenCalledTimes(1)
-  expect(downloadCacheFileMock).toHaveBeenCalledWith(
+  expect(downloadCacheMock).toHaveBeenCalledWith(
     signedDownloadUrl,
-    archivePath
+    archivePath,
+    options
   )
   expect(getArchiveFileSizeInBytesMock).toHaveBeenCalledWith(archivePath)
   expect(logInfoMock).toHaveBeenCalledWith(`Cache Size: ~0 MB (142 B)`)
@@ -226,6 +218,7 @@ test('restore with zstd compressed cache found', async () => {
   const signedDownloadUrl = 'https://blob-storage.local?signed=true'
   const cacheVersion =
     '8e2e96a184cb0cd6b48285b176c06a418f3d7fce14c29d9886fd1bb4f05c513d'
+  const options = {useAzureSdk: true} as DownloadOptions
 
   const getCacheVersionMock = jest.spyOn(cacheUtils, 'getCacheVersion')
   getCacheVersionMock.mockReturnValue(cacheVersion)
@@ -253,17 +246,7 @@ test('restore with zstd compressed cache found', async () => {
   })
 
   const archivePath = path.join(tempPath, CacheFilename.Zstd)
-  const downloadCacheFileMock = jest.spyOn(
-    downloadCacheModule,
-    'downloadCacheFile'
-  )
-  downloadCacheFileMock.mockReturnValue(
-    Promise.resolve({
-      _response: {
-        status: 200
-      }
-    } as BlobDownloadResponseParsed)
-  )
+  const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache')
 
   const fileSize = 62915000
   const getArchiveFileSizeInBytesMock = jest
@@ -273,7 +256,7 @@ test('restore with zstd compressed cache found', async () => {
   const extractTarMock = jest.spyOn(tar, 'extractTar')
   const unlinkFileMock = jest.spyOn(cacheUtils, 'unlinkFile')
 
-  const cacheKey = await restoreCache(paths, key)
+  const cacheKey = await restoreCache(paths, key, [], options)
 
   expect(cacheKey).toBe(key)
   expect(getCacheVersionMock).toHaveBeenCalledWith(
@@ -287,9 +270,10 @@ test('restore with zstd compressed cache found', async () => {
     version: cacheVersion
   })
   expect(createTempDirectoryMock).toHaveBeenCalledTimes(1)
-  expect(downloadCacheFileMock).toHaveBeenCalledWith(
+  expect(downloadCacheMock).toHaveBeenCalledWith(
     signedDownloadUrl,
-    archivePath
+    archivePath,
+    options
   )
   expect(getArchiveFileSizeInBytesMock).toHaveBeenCalledWith(archivePath)
   expect(logInfoMock).toHaveBeenCalledWith(`Cache Size: ~60 MB (62915000 B)`)
@@ -311,6 +295,7 @@ test('restore with cache found for restore key', async () => {
   const signedDownloadUrl = 'https://blob-storage.local?signed=true'
   const cacheVersion =
     'b8b58e9bd7b1e8f83d9f05c7e06ea865ba44a0330e07a14db74ac74386677bed'
+  const options = {useAzureSdk: true} as DownloadOptions
 
   const getCacheVersionMock = jest.spyOn(cacheUtils, 'getCacheVersion')
   getCacheVersionMock.mockReturnValue(cacheVersion)
@@ -338,17 +323,7 @@ test('restore with cache found for restore key', async () => {
   })
 
   const archivePath = path.join(tempPath, CacheFilename.Gzip)
-  const downloadCacheFileMock = jest.spyOn(
-    downloadCacheModule,
-    'downloadCacheFile'
-  )
-  downloadCacheFileMock.mockReturnValue(
-    Promise.resolve({
-      _response: {
-        status: 200
-      }
-    } as BlobDownloadResponseParsed)
-  )
+  const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache')
 
   const fileSize = 142
   const getArchiveFileSizeInBytesMock = jest
@@ -358,7 +333,7 @@ test('restore with cache found for restore key', async () => {
   const extractTarMock = jest.spyOn(tar, 'extractTar')
   const unlinkFileMock = jest.spyOn(cacheUtils, 'unlinkFile')
 
-  const cacheKey = await restoreCache(paths, key, restoreKeys)
+  const cacheKey = await restoreCache(paths, key, restoreKeys, options)
 
   expect(cacheKey).toBe(restoreKeys[0])
   expect(getCacheVersionMock).toHaveBeenCalledWith(
@@ -372,9 +347,10 @@ test('restore with cache found for restore key', async () => {
     version: cacheVersion
   })
   expect(createTempDirectoryMock).toHaveBeenCalledTimes(1)
-  expect(downloadCacheFileMock).toHaveBeenCalledWith(
+  expect(downloadCacheMock).toHaveBeenCalledWith(
     signedDownloadUrl,
-    archivePath
+    archivePath,
+    options
   )
   expect(getArchiveFileSizeInBytesMock).toHaveBeenCalledWith(archivePath)
   expect(logInfoMock).toHaveBeenCalledWith(`Cache Size: ~0 MB (142 B)`)
@@ -388,14 +364,14 @@ test('restore with cache found for restore key', async () => {
   expect(compressionMethodMock).toHaveBeenCalledTimes(1)
 })
 
-test('restore with dry run', async () => {
+test('restore with lookup only enabled', async () => {
   const paths = ['node_modules']
   const key = 'node-test'
-  const options = {lookupOnly: true}
   const compressionMethod = CompressionMethod.Gzip
   const signedDownloadUrl = 'https://blob-storage.local?signed=true'
   const cacheVersion =
     'd90f107aaeb22920dba0c637a23c37b5bc497b4dfa3b07fe3f79bf88a273c11b'
+  const options = {lookupOnly: true, useAzureSdk: true} as DownloadOptions
 
   const getCacheVersionMock = jest.spyOn(cacheUtils, 'getCacheVersion')
   getCacheVersionMock.mockReturnValue(cacheVersion)
@@ -416,10 +392,7 @@ test('restore with dry run', async () => {
   )
 
   const createTempDirectoryMock = jest.spyOn(cacheUtils, 'createTempDirectory')
-  const downloadCacheFileMock = jest.spyOn(
-    downloadCacheModule,
-    'downloadCacheFile'
-  )
+  const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache')
 
   const cacheKey = await restoreCache(paths, key, undefined, options)
 
@@ -438,5 +411,5 @@ test('restore with dry run', async () => {
 
   // creating a tempDir and downloading the cache are skipped
   expect(createTempDirectoryMock).toHaveBeenCalledTimes(0)
-  expect(downloadCacheFileMock).toHaveBeenCalledTimes(0)
+  expect(downloadCacheMock).toHaveBeenCalledTimes(0)
 })

packages/cache/__tests__/saveCache.test.ts

@@ -270,7 +270,12 @@ test('save with server error should fail', async () => {
     compression
   )
   expect(saveCacheMock).toHaveBeenCalledTimes(1)
-  expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archiveFile, undefined)
+  expect(saveCacheMock).toHaveBeenCalledWith(
+    cacheId,
+    archiveFile,
+    '',
+    undefined
+  )
   expect(getCompressionMock).toHaveBeenCalledTimes(1)
 })
 
@@ -315,7 +320,12 @@ test('save with valid inputs uploads a cache', async () => {
     compression
   )
   expect(saveCacheMock).toHaveBeenCalledTimes(1)
-  expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archiveFile, undefined)
+  expect(saveCacheMock).toHaveBeenCalledWith(
+    cacheId,
+    archiveFile,
+    '',
+    undefined
+  )
   expect(getCompressionMock).toHaveBeenCalledTimes(1)
 })
 

packages/cache/__tests__/saveCacheV2.test.ts

@@ -0,0 +1,339 @@
+import * as core from '@actions/core'
+import * as path from 'path'
+import {saveCache} from '../src/cache'
+import * as cacheUtils from '../src/internal/cacheUtils'
+import {CacheFilename, CompressionMethod} from '../src/internal/constants'
+import * as config from '../src/internal/config'
+import * as tar from '../src/internal/tar'
+import {CacheServiceClientJSON} from '../src/generated/results/api/v1/cache.twirp'
+import * as cacheHttpClient from '../src/internal/cacheHttpClient'
+import {UploadOptions} from '../src/options'
+
+let logDebugMock: jest.SpyInstance
+
+jest.mock('../src/internal/tar')
+
+const uploadFileMock = jest.fn()
+const blockBlobClientMock = jest.fn().mockImplementation(() => ({
+  uploadFile: uploadFileMock
+}))
+jest.mock('@azure/storage-blob', () => ({
+  BlobClient: jest.fn().mockImplementation(() => {
+    return {
+      getBlockBlobClient: blockBlobClientMock
+    }
+  })
+}))
+
+beforeAll(() => {
+  process.env['ACTIONS_RUNTIME_TOKEN'] = 'token'
+  jest.spyOn(console, 'log').mockImplementation(() => {})
+  jest.spyOn(core, 'debug').mockImplementation(() => {})
+  jest.spyOn(core, 'info').mockImplementation(() => {})
+  jest.spyOn(core, 'warning').mockImplementation(() => {})
+  jest.spyOn(core, 'error').mockImplementation(() => {})
+  jest.spyOn(cacheUtils, 'resolvePaths').mockImplementation(async filePaths => {
+    return filePaths.map(x => path.resolve(x))
+  })
+  jest.spyOn(cacheUtils, 'createTempDirectory').mockImplementation(async () => {
+    return Promise.resolve('/foo/bar')
+  })
+
+  // Ensure that we're using v2 for these tests
+  jest.spyOn(config, 'getCacheServiceVersion').mockReturnValue('v2')
+
+  logDebugMock = jest.spyOn(core, 'debug')
+})
+
+afterEach(() => {
+  expect(logDebugMock).toHaveBeenCalledWith('Cache service version: v2')
+  jest.clearAllMocks()
+})
+
+test('save with missing input should fail', async () => {
+  const paths: string[] = []
+  const key = 'Linux-node-bb828da54c148048dd17899ba9fda624811cfb43'
+
+  await expect(saveCache(paths, key)).rejects.toThrowError(
+    `Path Validation Error: At least one directory or file path is required`
+  )
+})
+
+test('save with large cache outputs should fail using', async () => {
+  const paths = 'node_modules'
+  const key = 'Linux-node-bb828da54c148048dd17899ba9fda624811cfb43'
+  const cachePaths = [path.resolve(paths)]
+
+  const createTarMock = jest.spyOn(tar, 'createTar')
+  const logWarningMock = jest.spyOn(core, 'warning')
+
+  const cacheSize = 11 * 1024 * 1024 * 1024 //~11GB, over the 10GB limit
+  jest
+    .spyOn(cacheUtils, 'getArchiveFileSizeInBytes')
+    .mockReturnValueOnce(cacheSize)
+  const compression = CompressionMethod.Gzip
+  const getCompressionMock = jest
+    .spyOn(cacheUtils, 'getCompressionMethod')
+    .mockReturnValueOnce(Promise.resolve(compression))
+
+  const cacheId = await saveCache([paths], key)
+  expect(cacheId).toBe(-1)
+  expect(logWarningMock).toHaveBeenCalledWith(
+    'Failed to save: Cache size of ~11264 MB (11811160064 B) is over the 10GB limit, not saving cache.'
+  )
+
+  const archiveFolder = '/foo/bar'
+
+  expect(createTarMock).toHaveBeenCalledWith(
+    archiveFolder,
+    cachePaths,
+    compression
+  )
+  expect(getCompressionMock).toHaveBeenCalledTimes(1)
+})
+
+test('create cache entry failure', async () => {
+  const paths = ['node_modules']
+  const key = 'Linux-node-bb828da54c148048dd17899ba9fda624811cfb43'
+  const infoLogMock = jest.spyOn(core, 'info')
+
+  const createCacheEntryMock = jest
+    .spyOn(CacheServiceClientJSON.prototype, 'CreateCacheEntry')
+    .mockReturnValue(Promise.resolve({ok: false, signedUploadUrl: ''}))
+
+  const createTarMock = jest.spyOn(tar, 'createTar')
+  const finalizeCacheEntryMock = jest.spyOn(
+    CacheServiceClientJSON.prototype,
+    'FinalizeCacheEntryUpload'
+  )
+  const compression = CompressionMethod.Zstd
+  const getCompressionMock = jest
+    .spyOn(cacheUtils, 'getCompressionMethod')
+    .mockReturnValueOnce(Promise.resolve(compression))
+  const archiveFileSize = 1024
+  jest
+    .spyOn(cacheUtils, 'getArchiveFileSizeInBytes')
+    .mockReturnValueOnce(archiveFileSize)
+  const cacheVersion = cacheUtils.getCacheVersion(paths, compression)
+  const saveCacheMock = jest.spyOn(cacheHttpClient, 'saveCache')
+
+  const cacheId = await saveCache(paths, key)
+  expect(cacheId).toBe(-1)
+  expect(infoLogMock).toHaveBeenCalledWith(
+    `Failed to save: Unable to reserve cache with key ${key}, another job may be creating this cache.`
+  )
+
+  expect(createCacheEntryMock).toHaveBeenCalledWith({
+    key,
+    version: cacheVersion
+  })
+  expect(createTarMock).toHaveBeenCalledTimes(1)
+  expect(getCompressionMock).toHaveBeenCalledTimes(1)
+  expect(finalizeCacheEntryMock).toHaveBeenCalledTimes(0)
+  expect(saveCacheMock).toHaveBeenCalledTimes(0)
+})
+
+test('save cache fails if a signedUploadURL was not passed', async () => {
+  const paths = 'node_modules'
+  const key = 'Linux-node-bb828da54c148048dd17899ba9fda624811cfb43'
+  const cachePaths = [path.resolve(paths)]
+  const signedUploadURL = ''
+  const archiveFileSize = 1024
+  const options: UploadOptions = {
+    archiveSizeBytes: archiveFileSize, // These should always match
+    useAzureSdk: true,
+    uploadChunkSize: 64 * 1024 * 1024,
+    uploadConcurrency: 8
+  }
+
+  const createCacheEntryMock = jest
+    .spyOn(CacheServiceClientJSON.prototype, 'CreateCacheEntry')
+    .mockReturnValue(
+      Promise.resolve({ok: true, signedUploadUrl: signedUploadURL})
+    )
+
+  const createTarMock = jest.spyOn(tar, 'createTar')
+  const saveCacheMock = jest.spyOn(cacheHttpClient, 'saveCache')
+
+  const compression = CompressionMethod.Zstd
+  const getCompressionMock = jest
+    .spyOn(cacheUtils, 'getCompressionMethod')
+    .mockReturnValueOnce(Promise.resolve(compression))
+
+  const cacheVersion = cacheUtils.getCacheVersion([paths], compression)
+  jest
+    .spyOn(cacheUtils, 'getArchiveFileSizeInBytes')
+    .mockReturnValueOnce(archiveFileSize)
+
+  const cacheId = await saveCache([paths], key, options)
+
+  expect(cacheId).toBe(-1)
+  expect(createCacheEntryMock).toHaveBeenCalledWith({
+    key,
+    version: cacheVersion
+  })
+
+  const archiveFolder = '/foo/bar'
+  const archiveFile = path.join(archiveFolder, CacheFilename.Zstd)
+  expect(createTarMock).toHaveBeenCalledWith(
+    archiveFolder,
+    cachePaths,
+    compression
+  )
+
+  expect(saveCacheMock).toHaveBeenCalledWith(
+    -1,
+    archiveFile,
+    signedUploadURL,
+    options
+  )
+  expect(getCompressionMock).toHaveBeenCalledTimes(1)
+})
+
+test('finalize save cache failure', async () => {
+  const paths = 'node_modules'
+  const key = 'Linux-node-bb828da54c148048dd17899ba9fda624811cfb43'
+  const cachePaths = [path.resolve(paths)]
+  const logWarningMock = jest.spyOn(core, 'warning')
+  const signedUploadURL = 'https://blob-storage.local?signed=true'
+  const archiveFileSize = 1024
+  const options: UploadOptions = {
+    archiveSizeBytes: archiveFileSize, // These should always match
+    useAzureSdk: true,
+    uploadChunkSize: 64 * 1024 * 1024,
+    uploadConcurrency: 8
+  }
+
+  const createCacheEntryMock = jest
+    .spyOn(CacheServiceClientJSON.prototype, 'CreateCacheEntry')
+    .mockReturnValue(
+      Promise.resolve({ok: true, signedUploadUrl: signedUploadURL})
+    )
+
+  const createTarMock = jest.spyOn(tar, 'createTar')
+  const saveCacheMock = jest
+    .spyOn(cacheHttpClient, 'saveCache')
+    .mockResolvedValue(Promise.resolve())
+
+  const compression = CompressionMethod.Zstd
+  const getCompressionMock = jest
+    .spyOn(cacheUtils, 'getCompressionMethod')
+    .mockReturnValueOnce(Promise.resolve(compression))
+
+  const cacheVersion = cacheUtils.getCacheVersion([paths], compression)
+  jest
+    .spyOn(cacheUtils, 'getArchiveFileSizeInBytes')
+    .mockReturnValueOnce(archiveFileSize)
+
+  const finalizeCacheEntryMock = jest
+    .spyOn(CacheServiceClientJSON.prototype, 'FinalizeCacheEntryUpload')
+    .mockReturnValue(Promise.resolve({ok: false, entryId: ''}))
+
+  const cacheId = await saveCache([paths], key, options)
+
+  expect(createCacheEntryMock).toHaveBeenCalledWith({
+    key,
+    version: cacheVersion
+  })
+
+  const archiveFolder = '/foo/bar'
+  const archiveFile = path.join(archiveFolder, CacheFilename.Zstd)
+  expect(createTarMock).toHaveBeenCalledWith(
+    archiveFolder,
+    cachePaths,
+    compression
+  )
+
+  expect(saveCacheMock).toHaveBeenCalledWith(
+    -1,
+    archiveFile,
+    signedUploadURL,
+    options
+  )
+  expect(getCompressionMock).toHaveBeenCalledTimes(1)
+
+  expect(finalizeCacheEntryMock).toHaveBeenCalledWith({
+    key,
+    version: cacheVersion,
+    sizeBytes: archiveFileSize.toString()
+  })
+
+  expect(cacheId).toBe(-1)
+  expect(logWarningMock).toHaveBeenCalledWith(
+    `Failed to save: Unable to finalize cache with key ${key}, another job may be finalizing this cache.`
+  )
+})
+
+test('save with valid inputs uploads a cache', async () => {
+  const paths = 'node_modules'
+  const key = 'Linux-node-bb828da54c148048dd17899ba9fda624811cfb43'
+  const cachePaths = [path.resolve(paths)]
+  const signedUploadURL = 'https://blob-storage.local?signed=true'
+  const createTarMock = jest.spyOn(tar, 'createTar')
+  const archiveFileSize = 1024
+  const options: UploadOptions = {
+    archiveSizeBytes: archiveFileSize, // These should always match
+    useAzureSdk: true,
+    uploadChunkSize: 64 * 1024 * 1024,
+    uploadConcurrency: 8
+  }
+
+  jest
+    .spyOn(cacheUtils, 'getArchiveFileSizeInBytes')
+    .mockReturnValueOnce(archiveFileSize)
+
+  const cacheId = 4
+  jest
+    .spyOn(CacheServiceClientJSON.prototype, 'CreateCacheEntry')
+    .mockReturnValue(
+      Promise.resolve({ok: true, signedUploadUrl: signedUploadURL})
+    )
+
+  const saveCacheMock = jest.spyOn(cacheHttpClient, 'saveCache')
+
+  const compression = CompressionMethod.Zstd
+  const getCompressionMock = jest
+    .spyOn(cacheUtils, 'getCompressionMethod')
+    .mockReturnValue(Promise.resolve(compression))
+  const cacheVersion = cacheUtils.getCacheVersion([paths], compression)
+
+  const finalizeCacheEntryMock = jest
+    .spyOn(CacheServiceClientJSON.prototype, 'FinalizeCacheEntryUpload')
+    .mockReturnValue(Promise.resolve({ok: true, entryId: cacheId.toString()}))
+
+  const expectedCacheId = await saveCache([paths], key)
+
+  const archiveFolder = '/foo/bar'
+  const archiveFile = path.join(archiveFolder, CacheFilename.Zstd)
+  expect(saveCacheMock).toHaveBeenCalledWith(
+    -1,
+    archiveFile,
+    signedUploadURL,
+    options
+  )
+  expect(createTarMock).toHaveBeenCalledWith(
+    archiveFolder,
+    cachePaths,
+    compression
+  )
+
+  expect(finalizeCacheEntryMock).toHaveBeenCalledWith({
+    key,
+    version: cacheVersion,
+    sizeBytes: archiveFileSize.toString()
+  })
+
+  expect(getCompressionMock).toHaveBeenCalledTimes(1)
+  expect(expectedCacheId).toBe(cacheId)
+})
+
+test('save with non existing path should not save cache using v2 saveCache', async () => {
+  const path = 'node_modules'
+  const key = 'Linux-node-bb828da54c148048dd17899ba9fda624811cfb43'
+  jest.spyOn(cacheUtils, 'resolvePaths').mockImplementation(async () => {
+    return []
+  })
+  await expect(saveCache([path], key)).rejects.toThrowError(
+    `Path Validation Error: Path(s) specified in the action for caching do(es) not exist, hence no cache is being saved.`
+  )
+})

packages/cache/__tests__/uploadUtils.test.ts

@@ -0,0 +1,58 @@
+import * as uploadUtils from '../src/internal/uploadUtils'
+import {TransferProgressEvent} from '@azure/ms-rest-js'
+
+test('upload progress tracked correctly', () => {
+  const progress = new uploadUtils.UploadProgress(1000)
+
+  expect(progress.contentLength).toBe(1000)
+  expect(progress.sentBytes).toBe(0)
+  expect(progress.displayedComplete).toBe(false)
+  expect(progress.timeoutHandle).toBeUndefined()
+  expect(progress.getTransferredBytes()).toBe(0)
+  expect(progress.isDone()).toBe(false)
+
+  progress.onProgress()({loadedBytes: 0} as TransferProgressEvent)
+
+  expect(progress.contentLength).toBe(1000)
+  expect(progress.sentBytes).toBe(0)
+  expect(progress.displayedComplete).toBe(false)
+  expect(progress.timeoutHandle).toBeUndefined()
+  expect(progress.getTransferredBytes()).toBe(0)
+  expect(progress.isDone()).toBe(false)
+
+  progress.onProgress()({loadedBytes: 250} as TransferProgressEvent)
+
+  expect(progress.contentLength).toBe(1000)
+  expect(progress.sentBytes).toBe(250)
+  expect(progress.displayedComplete).toBe(false)
+  expect(progress.timeoutHandle).toBeUndefined()
+  expect(progress.getTransferredBytes()).toBe(250)
+  expect(progress.isDone()).toBe(false)
+
+  progress.onProgress()({loadedBytes: 500} as TransferProgressEvent)
+
+  expect(progress.contentLength).toBe(1000)
+  expect(progress.sentBytes).toBe(500)
+  expect(progress.displayedComplete).toBe(false)
+  expect(progress.timeoutHandle).toBeUndefined()
+  expect(progress.getTransferredBytes()).toBe(500)
+  expect(progress.isDone()).toBe(false)
+
+  progress.onProgress()({loadedBytes: 750} as TransferProgressEvent)
+
+  expect(progress.contentLength).toBe(1000)
+  expect(progress.sentBytes).toBe(750)
+  expect(progress.displayedComplete).toBe(false)
+  expect(progress.timeoutHandle).toBeUndefined()
+  expect(progress.getTransferredBytes()).toBe(750)
+  expect(progress.isDone()).toBe(false)
+
+  progress.onProgress()({loadedBytes: 1000} as TransferProgressEvent)
+
+  expect(progress.contentLength).toBe(1000)
+  expect(progress.sentBytes).toBe(1000)
+  expect(progress.displayedComplete).toBe(false)
+  expect(progress.timeoutHandle).toBeUndefined()
+  expect(progress.getTransferredBytes()).toBe(1000)
+  expect(progress.isDone()).toBe(true)
+})

packages/cache/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@actions/cache",
-  "version": "3.3.0",
+  "version": "4.0.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@actions/cache",
-      "version": "3.3.0",
+      "version": "4.0.0",
       "license": "MIT",
       "dependencies": {
         "@actions/core": "^1.11.1",

packages/cache/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@actions/cache",
-  "version": "3.3.0",
+  "version": "4.0.0",
   "preview": true,
   "description": "Actions cache lib",
   "keywords": [

packages/cache/src/cache.ts

@@ -13,8 +13,6 @@ import {
   GetCacheEntryDownloadURLRequest
 } from './generated/results/api/v1/cache'
 import {CacheFileSizeLimit} from './internal/constants'
-import {uploadCacheFile} from './internal/blob/upload-cache'
-import {downloadCacheFile} from './internal/blob/download-cache'
 export class ValidationError extends Error {
   constructor(message: string) {
     super(message)
@@ -66,8 +64,8 @@ export function isFeatureAvailable(): boolean {
  * Restores cache from keys
  *
  * @param paths a list of file paths to restore from the cache
- * @param primaryKey an explicit key for restoring the cache
- * @param restoreKeys an optional ordered list of keys to use for restoring the cache if no cache hit occurred for key
+ * @param primaryKey an explicit key for restoring the cache. Lookup is done with prefix matching.
+ * @param restoreKeys an optional ordered list of keys to use for restoring the cache if no cache hit occurred for primaryKey
  * @param downloadOptions cache download options
  * @param enableCrossOsArchive an optional boolean enabled to restore on windows any cache created on any platform
  * @returns string returns the key for the cache hit, otherwise returns undefined
@@ -108,12 +106,12 @@ export async function restoreCache(
 /**
  * Restores cache using the legacy Cache Service
  *
- * @param paths
- * @param primaryKey
- * @param restoreKeys
- * @param options
- * @param enableCrossOsArchive
- * @returns
+ * @param paths a list of file paths to restore from the cache
+ * @param primaryKey an explicit key for restoring the cache. Lookup is done with prefix matching.
+ * @param restoreKeys an optional ordered list of keys to use for restoring the cache if no cache hit occurred for primaryKey
+ * @param options cache download options
+ * @param enableCrossOsArchive an optional boolean enabled to restore on Windows any cache created on any platform
+ * @returns string returns the key for the cache hit, otherwise returns undefined
  */
 async function restoreCacheV1(
   paths: string[],
@@ -204,11 +202,11 @@ async function restoreCacheV1(
 }
 
 /**
- * Restores cache using the new Cache Service
+ * Restores cache using Cache Service v2
  *
  * @param paths a list of file paths to restore from the cache
- * @param primaryKey an explicit key for restoring the cache
- * @param restoreKeys an optional ordered list of keys to use for restoring the cache if no cache hit occurred for key
+ * @param primaryKey an explicit key for restoring the cache. Lookup is done with prefix matching
+ * @param restoreKeys an optional ordered list of keys to use for restoring the cache if no cache hit occurred for primaryKey
  * @param downloadOptions cache download options
  * @param enableCrossOsArchive an optional boolean enabled to restore on windows any cache created on any platform
  * @returns string returns the key for the cache hit, otherwise returns undefined
@@ -220,6 +218,11 @@ async function restoreCacheV2(
   options?: DownloadOptions,
   enableCrossOsArchive = false
 ): Promise<string | undefined> {
+  // Override UploadOptions to force the use of Azure
+  options = {
+    ...options,
+    useAzureSdk: true
+  }
   restoreKeys = restoreKeys || []
   const keys = [primaryKey, ...restoreKeys]
 
@@ -271,11 +274,11 @@ async function restoreCacheV2(
     core.debug(`Archive path: ${archivePath}`)
     core.debug(`Starting download of archive to: ${archivePath}`)
 
-    const downloadResponse = await downloadCacheFile(
+    await cacheHttpClient.downloadCache(
       response.signedDownloadUrl,
-      archivePath
+      archivePath,
+      options
     )
-    core.debug(`Download response status: ${downloadResponse._response.status}`)
 
     const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath)
     core.info(
@@ -328,10 +331,10 @@ export async function saveCache(
   options?: UploadOptions,
   enableCrossOsArchive = false
 ): Promise<number> {
+  const cacheServiceVersion: string = getCacheServiceVersion()
+  core.debug(`Cache service version: ${cacheServiceVersion}`)
   checkPaths(paths)
   checkKey(key)
-
-  const cacheServiceVersion: string = getCacheServiceVersion()
   switch (cacheServiceVersion) {
     case 'v2':
       return await saveCacheV2(paths, key, options, enableCrossOsArchive)
@@ -422,7 +425,7 @@ async function saveCacheV1(
     }
 
     core.debug(`Saving Cache (ID: ${cacheId})`)
-    await cacheHttpClient.saveCache(cacheId, archivePath, options)
+    await cacheHttpClient.saveCache(cacheId, archivePath, '', options)
   } catch (error) {
     const typedError = error as Error
     if (typedError.name === ValidationError.name) {
@@ -445,12 +448,12 @@ async function saveCacheV1(
 }
 
 /**
- * Save cache using the new Cache Service
+ * Save cache using Cache Service v2
  *
- * @param paths
- * @param key
- * @param options
- * @param enableCrossOsArchive
+ * @param paths a list of file paths to restore from the cache
+ * @param key an explicit key for restoring the cache
+ * @param options cache upload options
+ * @param enableCrossOsArchive an optional boolean enabled to save cache on windows which could be restored on any platform
  * @returns
  */
 async function saveCacheV2(
@@ -459,6 +462,15 @@ async function saveCacheV2(
   options?: UploadOptions,
   enableCrossOsArchive = false
 ): Promise<number> {
+  // Override UploadOptions to force the use of Azure
+  // ...options goes first because we want to override the default values
+  // set in UploadOptions with these specific figures
+  options = {
+    ...options,
+    uploadChunkSize: 64 * 1024 * 1024, // 64 MiB
+    uploadConcurrency: 8, // 8 workers for parallel upload
+    useAzureSdk: true
+  }
   const compressionMethod = await utils.getCompressionMethod()
   const twirpClient = cacheTwirpClient.internalCacheTwirpClient()
   let cacheId = -1
@@ -499,6 +511,9 @@ async function saveCacheV2(
       )
     }
 
+    // Set the archive size in the options, will be used to display the upload progress
+    options.archiveSizeBytes = archiveFileSize
+
     core.debug('Reserving Cache')
     const version = utils.getCacheVersion(
       paths,
@@ -518,7 +533,12 @@ async function saveCacheV2(
     }
 
     core.debug(`Attempting to upload cache located at: ${archivePath}`)
-    await uploadCacheFile(response.signedUploadUrl, archivePath)
+    await cacheHttpClient.saveCache(
+      cacheId,
+      archivePath,
+      response.signedUploadUrl,
+      options
+    )
 
     const finalizeRequest: FinalizeCacheEntryUploadRequest = {
       key,

packages/cache/src/generated/results/api/v1/cache.ts

@@ -212,52 +212,6 @@ export interface ListCacheEntriesResponse {
      */
     entries: CacheEntry[];
 }
-/**
- * @generated from protobuf message github.actions.results.api.v1.LookupCacheEntryRequest
- */
-export interface LookupCacheEntryRequest {
-    /**
-     * Scope and other metadata for the cache entry
-     *
-     * @generated from protobuf field: github.actions.results.entities.v1.CacheMetadata metadata = 1;
-     */
-    metadata?: CacheMetadata;
-    /**
-     * An explicit key for a cache entry
-     *
-     * @generated from protobuf field: string key = 2;
-     */
-    key: string;
-    /**
-     * Restore keys used for prefix searching
-     *
-     * @generated from protobuf field: repeated string restore_keys = 3;
-     */
-    restoreKeys: string[];
-    /**
-     * Hash of the compression tool, runner OS and paths cached
-     *
-     * @generated from protobuf field: string version = 4;
-     */
-    version: string;
-}
-/**
- * @generated from protobuf message github.actions.results.api.v1.LookupCacheEntryResponse
- */
-export interface LookupCacheEntryResponse {
-    /**
-     * Indicates whether the cache entry exists or not
-     *
-     * @generated from protobuf field: bool exists = 1;
-     */
-    exists: boolean;
-    /**
-     * Matched cache entry metadata
-     *
-     * @generated from protobuf field: github.actions.results.entities.v1.CacheEntry entry = 2;
-     */
-    entry?: CacheEntry;
-}
 // @generated message type with reflection information, may provide speed optimized methods
 class CreateCacheEntryRequest$Type extends MessageType<CreateCacheEntryRequest> {
     constructor() {
@@ -840,128 +794,6 @@ class ListCacheEntriesResponse$Type extends MessageType<ListCacheEntriesResponse
  * @generated MessageType for protobuf message github.actions.results.api.v1.ListCacheEntriesResponse
  */
 export const ListCacheEntriesResponse = new ListCacheEntriesResponse$Type();
-// @generated message type with reflection information, may provide speed optimized methods
-class LookupCacheEntryRequest$Type extends MessageType<LookupCacheEntryRequest> {
-    constructor() {
-        super("github.actions.results.api.v1.LookupCacheEntryRequest", [
-            { no: 1, name: "metadata", kind: "message", T: () => CacheMetadata },
-            { no: 2, name: "key", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
-            { no: 3, name: "restore_keys", kind: "scalar", repeat: 2 /*RepeatType.UNPACKED*/, T: 9 /*ScalarType.STRING*/ },
-            { no: 4, name: "version", kind: "scalar", T: 9 /*ScalarType.STRING*/ }
-        ]);
-    }
-    create(value?: PartialMessage<LookupCacheEntryRequest>): LookupCacheEntryRequest {
-        const message = { key: "", restoreKeys: [], version: "" };
-        globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this });
-        if (value !== undefined)
-            reflectionMergePartial<LookupCacheEntryRequest>(this, message, value);
-        return message;
-    }
-    internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: LookupCacheEntryRequest): LookupCacheEntryRequest {
-        let message = target ?? this.create(), end = reader.pos + length;
-        while (reader.pos < end) {
-            let [fieldNo, wireType] = reader.tag();
-            switch (fieldNo) {
-                case /* github.actions.results.entities.v1.CacheMetadata metadata */ 1:
-                    message.metadata = CacheMetadata.internalBinaryRead(reader, reader.uint32(), options, message.metadata);
-                    break;
-                case /* string key */ 2:
-                    message.key = reader.string();
-                    break;
-                case /* repeated string restore_keys */ 3:
-                    message.restoreKeys.push(reader.string());
-                    break;
-                case /* string version */ 4:
-                    message.version = reader.string();
-                    break;
-                default:
-                    let u = options.readUnknownField;
-                    if (u === "throw")
-                        throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
-                    let d = reader.skip(wireType);
-                    if (u !== false)
-                        (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
-            }
-        }
-        return message;
-    }
-    internalBinaryWrite(message: LookupCacheEntryRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
-        /* github.actions.results.entities.v1.CacheMetadata metadata = 1; */
-        if (message.metadata)
-            CacheMetadata.internalBinaryWrite(message.metadata, writer.tag(1, WireType.LengthDelimited).fork(), options).join();
-        /* string key = 2; */
-        if (message.key !== "")
-            writer.tag(2, WireType.LengthDelimited).string(message.key);
-        /* repeated string restore_keys = 3; */
-        for (let i = 0; i < message.restoreKeys.length; i++)
-            writer.tag(3, WireType.LengthDelimited).string(message.restoreKeys[i]);
-        /* string version = 4; */
-        if (message.version !== "")
-            writer.tag(4, WireType.LengthDelimited).string(message.version);
-        let u = options.writeUnknownFields;
-        if (u !== false)
-            (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
-        return writer;
-    }
-}
-/**
- * @generated MessageType for protobuf message github.actions.results.api.v1.LookupCacheEntryRequest
- */
-export const LookupCacheEntryRequest = new LookupCacheEntryRequest$Type();
-// @generated message type with reflection information, may provide speed optimized methods
-class LookupCacheEntryResponse$Type extends MessageType<LookupCacheEntryResponse> {
-    constructor() {
-        super("github.actions.results.api.v1.LookupCacheEntryResponse", [
-            { no: 1, name: "exists", kind: "scalar", T: 8 /*ScalarType.BOOL*/ },
-            { no: 2, name: "entry", kind: "message", T: () => CacheEntry }
-        ]);
-    }
-    create(value?: PartialMessage<LookupCacheEntryResponse>): LookupCacheEntryResponse {
-        const message = { exists: false };
-        globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this });
-        if (value !== undefined)
-            reflectionMergePartial<LookupCacheEntryResponse>(this, message, value);
-        return message;
-    }
-    internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: LookupCacheEntryResponse): LookupCacheEntryResponse {
-        let message = target ?? this.create(), end = reader.pos + length;
-        while (reader.pos < end) {
-            let [fieldNo, wireType] = reader.tag();
-            switch (fieldNo) {
-                case /* bool exists */ 1:
-                    message.exists = reader.bool();
-                    break;
-                case /* github.actions.results.entities.v1.CacheEntry entry */ 2:
-                    message.entry = CacheEntry.internalBinaryRead(reader, reader.uint32(), options, message.entry);
-                    break;
-                default:
-                    let u = options.readUnknownField;
-                    if (u === "throw")
-                        throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
-                    let d = reader.skip(wireType);
-                    if (u !== false)
-                        (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
-            }
-        }
-        return message;
-    }
-    internalBinaryWrite(message: LookupCacheEntryResponse, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
-        /* bool exists = 1; */
-        if (message.exists !== false)
-            writer.tag(1, WireType.Varint).bool(message.exists);
-        /* github.actions.results.entities.v1.CacheEntry entry = 2; */
-        if (message.entry)
-            CacheEntry.internalBinaryWrite(message.entry, writer.tag(2, WireType.LengthDelimited).fork(), options).join();
-        let u = options.writeUnknownFields;
-        if (u !== false)
-            (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
-        return writer;
-    }
-}
-/**
- * @generated MessageType for protobuf message github.actions.results.api.v1.LookupCacheEntryResponse
- */
-export const LookupCacheEntryResponse = new LookupCacheEntryResponse$Type();
 /**
  * @generated ServiceType for protobuf service github.actions.results.api.v1.CacheService
  */
@@ -970,6 +802,5 @@ export const CacheService = new ServiceType("github.actions.results.api.v1.Cache
     { name: "FinalizeCacheEntryUpload", options: {}, I: FinalizeCacheEntryUploadRequest, O: FinalizeCacheEntryUploadResponse },
     { name: "GetCacheEntryDownloadURL", options: {}, I: GetCacheEntryDownloadURLRequest, O: GetCacheEntryDownloadURLResponse },
     { name: "DeleteCacheEntry", options: {}, I: DeleteCacheEntryRequest, O: DeleteCacheEntryResponse },
-    { name: "ListCacheEntries", options: {}, I: ListCacheEntriesRequest, O: ListCacheEntriesResponse },
-    { name: "LookupCacheEntry", options: {}, I: LookupCacheEntryRequest, O: LookupCacheEntryResponse }
+    { name: "ListCacheEntries", options: {}, I: ListCacheEntriesRequest, O: ListCacheEntriesResponse }
 ]);

packages/cache/src/generated/results/api/v1/cache.twirp.ts

@@ -19,8 +19,6 @@ import {
   DeleteCacheEntryResponse,
   ListCacheEntriesRequest,
   ListCacheEntriesResponse,
-  LookupCacheEntryRequest,
-  LookupCacheEntryResponse,
 } from "./cache";
 
 //==================================//
@@ -52,9 +50,6 @@ export interface CacheServiceClient {
   ListCacheEntries(
     request: ListCacheEntriesRequest
   ): Promise<ListCacheEntriesResponse>;
-  LookupCacheEntry(
-    request: LookupCacheEntryRequest
-  ): Promise<LookupCacheEntryResponse>;
 }
 
 export class CacheServiceClientJSON implements CacheServiceClient {
@@ -66,7 +61,6 @@ export class CacheServiceClientJSON implements CacheServiceClient {
     this.GetCacheEntryDownloadURL.bind(this);
     this.DeleteCacheEntry.bind(this);
     this.ListCacheEntries.bind(this);
-    this.LookupCacheEntry.bind(this);
   }
   CreateCacheEntry(
     request: CreateCacheEntryRequest
@@ -167,26 +161,6 @@ export class CacheServiceClientJSON implements CacheServiceClient {
       })
     );
   }
-
-  LookupCacheEntry(
-    request: LookupCacheEntryRequest
-  ): Promise<LookupCacheEntryResponse> {
-    const data = LookupCacheEntryRequest.toJson(request, {
-      useProtoFieldName: true,
-      emitDefaultValues: false,
-    });
-    const promise = this.rpc.request(
-      "github.actions.results.api.v1.CacheService",
-      "LookupCacheEntry",
-      "application/json",
-      data as object
-    );
-    return promise.then((data) =>
-      LookupCacheEntryResponse.fromJson(data as any, {
-        ignoreUnknownFields: true,
-      })
-    );
-  }
 }
 
 export class CacheServiceClientProtobuf implements CacheServiceClient {
@@ -198,7 +172,6 @@ export class CacheServiceClientProtobuf implements CacheServiceClient {
     this.GetCacheEntryDownloadURL.bind(this);
     this.DeleteCacheEntry.bind(this);
     this.ListCacheEntries.bind(this);
-    this.LookupCacheEntry.bind(this);
   }
   CreateCacheEntry(
     request: CreateCacheEntryRequest
@@ -274,21 +247,6 @@ export class CacheServiceClientProtobuf implements CacheServiceClient {
       ListCacheEntriesResponse.fromBinary(data as Uint8Array)
     );
   }
-
-  LookupCacheEntry(
-    request: LookupCacheEntryRequest
-  ): Promise<LookupCacheEntryResponse> {
-    const data = LookupCacheEntryRequest.toBinary(request);
-    const promise = this.rpc.request(
-      "github.actions.results.api.v1.CacheService",
-      "LookupCacheEntry",
-      "application/protobuf",
-      data
-    );
-    return promise.then((data) =>
-      LookupCacheEntryResponse.fromBinary(data as Uint8Array)
-    );
-  }
 }
 
 //==================================//
@@ -316,10 +274,6 @@ export interface CacheServiceTwirp<T extends TwirpContext = TwirpContext> {
     ctx: T,
     request: ListCacheEntriesRequest
   ): Promise<ListCacheEntriesResponse>;
-  LookupCacheEntry(
-    ctx: T,
-    request: LookupCacheEntryRequest
-  ): Promise<LookupCacheEntryResponse>;
 }
 
 export enum CacheServiceMethod {
@@ -328,7 +282,6 @@ export enum CacheServiceMethod {
   GetCacheEntryDownloadURL = "GetCacheEntryDownloadURL",
   DeleteCacheEntry = "DeleteCacheEntry",
   ListCacheEntries = "ListCacheEntries",
-  LookupCacheEntry = "LookupCacheEntry",
 }
 
 export const CacheServiceMethodList = [
@@ -337,7 +290,6 @@ export const CacheServiceMethodList = [
   CacheServiceMethod.GetCacheEntryDownloadURL,
   CacheServiceMethod.DeleteCacheEntry,
   CacheServiceMethod.ListCacheEntries,
-  CacheServiceMethod.LookupCacheEntry,
 ];
 
 export function createCacheServiceServer<T extends TwirpContext = TwirpContext>(
@@ -457,26 +409,6 @@ function matchCacheServiceRoute<T extends TwirpContext = TwirpContext>(
           interceptors
         );
       };
-    case "LookupCacheEntry":
-      return async (
-        ctx: T,
-        service: CacheServiceTwirp,
-        data: Buffer,
-        interceptors?: Interceptor<
-          T,
-          LookupCacheEntryRequest,
-          LookupCacheEntryResponse
-        >[]
-      ) => {
-        ctx = { ...ctx, methodName: "LookupCacheEntry" };
-        await events.onMatch(ctx);
-        return handleCacheServiceLookupCacheEntryRequest(
-          ctx,
-          service,
-          data,
-          interceptors
-        );
-      };
     default:
       events.onNotFound();
       const msg = `no handler found`;
@@ -648,39 +580,6 @@ function handleCacheServiceListCacheEntriesRequest<
       throw new TwirpError(TwirpErrorCode.BadRoute, msg);
   }
 }
-
-function handleCacheServiceLookupCacheEntryRequest<
-  T extends TwirpContext = TwirpContext
->(
-  ctx: T,
-  service: CacheServiceTwirp,
-  data: Buffer,
-  interceptors?: Interceptor<
-    T,
-    LookupCacheEntryRequest,
-    LookupCacheEntryResponse
-  >[]
-): Promise<string | Uint8Array> {
-  switch (ctx.contentType) {
-    case TwirpContentType.JSON:
-      return handleCacheServiceLookupCacheEntryJSON<T>(
-        ctx,
-        service,
-        data,
-        interceptors
-      );
-    case TwirpContentType.Protobuf:
-      return handleCacheServiceLookupCacheEntryProtobuf<T>(
-        ctx,
-        service,
-        data,
-        interceptors
-      );
-    default:
-      const msg = "unexpected Content-Type";
-      throw new TwirpError(TwirpErrorCode.BadRoute, msg);
-  }
-}
 async function handleCacheServiceCreateCacheEntryJSON<
   T extends TwirpContext = TwirpContext
 >(
@@ -920,54 +819,6 @@ async function handleCacheServiceListCacheEntriesJSON<
     }) as string
   );
 }
-
-async function handleCacheServiceLookupCacheEntryJSON<
-  T extends TwirpContext = TwirpContext
->(
-  ctx: T,
-  service: CacheServiceTwirp,
-  data: Buffer,
-  interceptors?: Interceptor<
-    T,
-    LookupCacheEntryRequest,
-    LookupCacheEntryResponse
-  >[]
-) {
-  let request: LookupCacheEntryRequest;
-  let response: LookupCacheEntryResponse;
-
-  try {
-    const body = JSON.parse(data.toString() || "{}");
-    request = LookupCacheEntryRequest.fromJson(body, {
-      ignoreUnknownFields: true,
-    });
-  } catch (e) {
-    if (e instanceof Error) {
-      const msg = "the json request could not be decoded";
-      throw new TwirpError(TwirpErrorCode.Malformed, msg).withCause(e, true);
-    }
-  }
-
-  if (interceptors && interceptors.length > 0) {
-    const interceptor = chainInterceptors(...interceptors) as Interceptor<
-      T,
-      LookupCacheEntryRequest,
-      LookupCacheEntryResponse
-    >;
-    response = await interceptor(ctx, request!, (ctx, inputReq) => {
-      return service.LookupCacheEntry(ctx, inputReq);
-    });
-  } else {
-    response = await service.LookupCacheEntry(ctx, request!);
-  }
-
-  return JSON.stringify(
-    LookupCacheEntryResponse.toJson(response, {
-      useProtoFieldName: true,
-      emitDefaultValues: false,
-    }) as string
-  );
-}
 async function handleCacheServiceCreateCacheEntryProtobuf<
   T extends TwirpContext = TwirpContext
 >(
@@ -1167,43 +1018,3 @@ async function handleCacheServiceListCacheEntriesProtobuf<
 
   return Buffer.from(ListCacheEntriesResponse.toBinary(response));
 }
-
-async function handleCacheServiceLookupCacheEntryProtobuf<
-  T extends TwirpContext = TwirpContext
->(
-  ctx: T,
-  service: CacheServiceTwirp,
-  data: Buffer,
-  interceptors?: Interceptor<
-    T,
-    LookupCacheEntryRequest,
-    LookupCacheEntryResponse
-  >[]
-) {
-  let request: LookupCacheEntryRequest;
-  let response: LookupCacheEntryResponse;
-
-  try {
-    request = LookupCacheEntryRequest.fromBinary(data);
-  } catch (e) {
-    if (e instanceof Error) {
-      const msg = "the protobuf request could not be decoded";
-      throw new TwirpError(TwirpErrorCode.Malformed, msg).withCause(e, true);
-    }
-  }
-
-  if (interceptors && interceptors.length > 0) {
-    const interceptor = chainInterceptors(...interceptors) as Interceptor<
-      T,
-      LookupCacheEntryRequest,
-      LookupCacheEntryResponse
-    >;
-    response = await interceptor(ctx, request!, (ctx, inputReq) => {
-      return service.LookupCacheEntry(ctx, inputReq);
-    });
-  } else {
-    response = await service.LookupCacheEntry(ctx, request!);
-  }
-
-  return Buffer.from(LookupCacheEntryResponse.toBinary(response));
-}

packages/cache/src/internal/blob/download-cache.ts

@@ -1,31 +0,0 @@
-import * as core from '@actions/core'
-
-import {
-  BlobClient,
-  BlockBlobClient,
-  BlobDownloadOptions,
-  BlobDownloadResponseParsed
-} from '@azure/storage-blob'
-
-export async function downloadCacheFile(
-  signedUploadURL: string,
-  archivePath: string
-): Promise<BlobDownloadResponseParsed> {
-  const downloadOptions: BlobDownloadOptions = {
-    maxRetryRequests: 5
-  }
-
-  const blobClient: BlobClient = new BlobClient(signedUploadURL)
-  const blockBlobClient: BlockBlobClient = blobClient.getBlockBlobClient()
-
-  core.debug(
-    `BlobClient: ${blobClient.name}:${blobClient.accountName}:${blobClient.containerName}`
-  )
-
-  return blockBlobClient.downloadToFile(
-    archivePath,
-    0,
-    undefined,
-    downloadOptions
-  )
-}

packages/cache/src/internal/blob/upload-cache.ts

@@ -1,27 +0,0 @@
-import * as core from '@actions/core'
-import {
-  BlobClient,
-  BlockBlobClient,
-  BlockBlobParallelUploadOptions
-} from '@azure/storage-blob'
-
-export async function uploadCacheFile(
-  signedUploadURL: string,
-  archivePath: string
-): Promise<{}> {
-  // Specify data transfer options
-  const uploadOptions: BlockBlobParallelUploadOptions = {
-    blockSize: 4 * 1024 * 1024, // 4 MiB max block size
-    concurrency: 4, // maximum number of parallel transfer workers
-    maxSingleShotSize: 8 * 1024 * 1024 // 8 MiB initial transfer size
-  }
-
-  const blobClient: BlobClient = new BlobClient(signedUploadURL)
-  const blockBlobClient: BlockBlobClient = blobClient.getBlockBlobClient()
-
-  core.debug(
-    `BlobClient: ${blobClient.name}:${blobClient.accountName}:${blobClient.containerName}`
-  )
-
-  return blockBlobClient.uploadFile(archivePath, uploadOptions)
-}

packages/cache/src/internal/cacheHttpClient.ts

@@ -8,6 +8,7 @@ import {
 import * as fs from 'fs'
 import {URL} from 'url'
 import * as utils from './cacheUtils'
+import {uploadCacheArchiveSDK} from './uploadUtils'
 import {
   ArtifactCacheEntry,
   InternalCacheOptions,
@@ -34,6 +35,7 @@ import {
   retryTypedResponse
 } from './requestUtils'
 import {getCacheServiceURL} from './config'
+import {getUserAgentString} from './shared/user-agent'
 
 function getCacheApiUrl(resource: string): string {
   const baseUrl: string = getCacheServiceURL()
@@ -65,7 +67,7 @@ function createHttpClient(): HttpClient {
   const bearerCredentialHandler = new BearerCredentialHandler(token)
 
   return new HttpClient(
-    'actions/cache',
+    getUserAgentString(),
     [bearerCredentialHandler],
     getRequestOptions()
   )
@@ -325,26 +327,45 @@ async function commitCache(
 export async function saveCache(
   cacheId: number,
   archivePath: string,
+  signedUploadURL?: string,
   options?: UploadOptions
 ): Promise<void> {
-  const httpClient = createHttpClient()
+  const uploadOptions = getUploadOptions(options)
 
-  core.debug('Upload cache')
-  await uploadFile(httpClient, cacheId, archivePath, options)
+  if (uploadOptions.useAzureSdk) {
+    // Use Azure storage SDK to upload caches directly to Azure
+    if (!signedUploadURL) {
+      throw new Error(
+        'Azure Storage SDK can only be used when a signed URL is provided.'
+      )
+    }
+    await uploadCacheArchiveSDK(signedUploadURL, archivePath, options)
+  } else {
+    const httpClient = createHttpClient()
 
-  // Commit Cache
-  core.debug('Commiting cache')
-  const cacheSize = utils.getArchiveFileSizeInBytes(archivePath)
-  core.info(
-    `Cache Size: ~${Math.round(cacheSize / (1024 * 1024))} MB (${cacheSize} B)`
-  )
+    core.debug('Upload cache')
+    await uploadFile(httpClient, cacheId, archivePath, options)
 
-  const commitCacheResponse = await commitCache(httpClient, cacheId, cacheSize)
-  if (!isSuccessStatusCode(commitCacheResponse.statusCode)) {
-    throw new Error(
-      `Cache service responded with ${commitCacheResponse.statusCode} during commit cache.`
+    // Commit Cache
+    core.debug('Commiting cache')
+    const cacheSize = utils.getArchiveFileSizeInBytes(archivePath)
+    core.info(
+      `Cache Size: ~${Math.round(
+        cacheSize / (1024 * 1024)
+      )} MB (${cacheSize} B)`
     )
-  }
 
-  core.info('Cache saved successfully')
+    const commitCacheResponse = await commitCache(
+      httpClient,
+      cacheId,
+      cacheSize
+    )
+    if (!isSuccessStatusCode(commitCacheResponse.statusCode)) {
+      throw new Error(
+        `Cache service responded with ${commitCacheResponse.statusCode} during commit cache.`
+      )
+    }
+
+    core.info('Cache saved successfully')
+  }
 }

packages/cache/src/internal/uploadUtils.ts

@@ -0,0 +1,177 @@
+import * as core from '@actions/core'
+import {
+  BlobClient,
+  BlobUploadCommonResponse,
+  BlockBlobClient,
+  BlockBlobParallelUploadOptions
+} from '@azure/storage-blob'
+import {TransferProgressEvent} from '@azure/ms-rest-js'
+import {InvalidResponseError} from './shared/errors'
+import {UploadOptions} from '../options'
+
+/**
+ * Class for tracking the upload state and displaying stats.
+ */
+export class UploadProgress {
+  contentLength: number
+  sentBytes: number
+  startTime: number
+  displayedComplete: boolean
+  timeoutHandle?: ReturnType<typeof setTimeout>
+
+  constructor(contentLength: number) {
+    this.contentLength = contentLength
+    this.sentBytes = 0
+    this.displayedComplete = false
+    this.startTime = Date.now()
+  }
+
+  /**
+   * Sets the number of bytes sent
+   *
+   * @param sentBytes the number of bytes sent
+   */
+  setSentBytes(sentBytes: number): void {
+    this.sentBytes = sentBytes
+  }
+
+  /**
+   * Returns the total number of bytes transferred.
+   */
+  getTransferredBytes(): number {
+    return this.sentBytes
+  }
+
+  /**
+   * Returns true if the upload is complete.
+   */
+  isDone(): boolean {
+    return this.getTransferredBytes() === this.contentLength
+  }
+
+  /**
+   * Prints the current upload stats. Once the upload completes, this will print one
+   * last line and then stop.
+   */
+  display(): void {
+    if (this.displayedComplete) {
+      return
+    }
+
+    const transferredBytes = this.sentBytes
+    const percentage = (100 * (transferredBytes / this.contentLength)).toFixed(
+      1
+    )
+    const elapsedTime = Date.now() - this.startTime
+    const uploadSpeed = (
+      transferredBytes /
+      (1024 * 1024) /
+      (elapsedTime / 1000)
+    ).toFixed(1)
+
+    core.info(
+      `Sent ${transferredBytes} of ${this.contentLength} (${percentage}%), ${uploadSpeed} MBs/sec`
+    )
+
+    if (this.isDone()) {
+      this.displayedComplete = true
+    }
+  }
+
+  /**
+   * Returns a function used to handle TransferProgressEvents.
+   */
+  onProgress(): (progress: TransferProgressEvent) => void {
+    return (progress: TransferProgressEvent) => {
+      this.setSentBytes(progress.loadedBytes)
+    }
+  }
+
+  /**
+   * Starts the timer that displays the stats.
+   *
+   * @param delayInMs the delay between each write
+   */
+  startDisplayTimer(delayInMs = 1000): void {
+    const displayCallback = (): void => {
+      this.display()
+
+      if (!this.isDone()) {
+        this.timeoutHandle = setTimeout(displayCallback, delayInMs)
+      }
+    }
+
+    this.timeoutHandle = setTimeout(displayCallback, delayInMs)
+  }
+
+  /**
+   * Stops the timer that displays the stats. As this typically indicates the upload
+   * is complete, this will display one last line, unless the last line has already
+   * been written.
+   */
+  stopDisplayTimer(): void {
+    if (this.timeoutHandle) {
+      clearTimeout(this.timeoutHandle)
+      this.timeoutHandle = undefined
+    }
+
+    this.display()
+  }
+}
+
+/**
+ * Uploads a cache archive directly to Azure Blob Storage using the Azure SDK.
+ * This function will display progress information to the console. Concurrency of the
+ * upload is determined by the calling functions.
+ *
+ * @param signedUploadURL
+ * @param archivePath
+ * @param options
+ * @returns
+ */
+export async function uploadCacheArchiveSDK(
+  signedUploadURL: string,
+  archivePath: string,
+  options?: UploadOptions
+): Promise<BlobUploadCommonResponse> {
+  const blobClient: BlobClient = new BlobClient(signedUploadURL)
+  const blockBlobClient: BlockBlobClient = blobClient.getBlockBlobClient()
+  const uploadProgress = new UploadProgress(options?.archiveSizeBytes ?? 0)
+
+  // Specify data transfer options
+  const uploadOptions: BlockBlobParallelUploadOptions = {
+    blockSize: options?.uploadChunkSize,
+    concurrency: options?.uploadConcurrency, // maximum number of parallel transfer workers
+    maxSingleShotSize: 128 * 1024 * 1024, // 128 MiB initial transfer size
+    onProgress: uploadProgress.onProgress()
+  }
+
+  try {
+    uploadProgress.startDisplayTimer()
+
+    core.debug(
+      `BlobClient: ${blobClient.name}:${blobClient.accountName}:${blobClient.containerName}`
+    )
+
+    const response = await blockBlobClient.uploadFile(
+      archivePath,
+      uploadOptions
+    )
+
+    // TODO: better management of non-retryable errors
+    if (response._response.status >= 400) {
+      throw new InvalidResponseError(
+        `uploadCacheArchiveSDK: upload failed with status code ${response._response.status}`
+      )
+    }
+
+    return response
+  } catch (error) {
+    core.warning(
+      `uploadCacheArchiveSDK: internal error uploading cache archive: ${error.message}`
+    )
+    throw error
+  } finally {
+    uploadProgress.stopDisplayTimer()
+  }
+}

packages/cache/src/options.ts

@@ -4,6 +4,14 @@ import * as core from '@actions/core'
  * Options to control cache upload
  */
 export interface UploadOptions {
+  /**
+   * Indicates whether to use the Azure Blob SDK to download caches
+   * that are stored on Azure Blob Storage to improve reliability and
+   * performance
+   *
+   * @default false
+   */
+  useAzureSdk?: boolean
   /**
    * Number of parallel cache upload
    *
@@ -16,6 +24,10 @@ export interface UploadOptions {
    * @default 32MB
    */
   uploadChunkSize?: number
+  /**
+   * Archive size in bytes
+   */
+  archiveSizeBytes?: number
 }
 
 /**
@@ -76,12 +88,18 @@ export interface DownloadOptions {
  * @param copy the original upload options
  */
 export function getUploadOptions(copy?: UploadOptions): UploadOptions {
+  // Defaults if not overriden
   const result: UploadOptions = {
+    useAzureSdk: false,
     uploadConcurrency: 4,
     uploadChunkSize: 32 * 1024 * 1024
   }
 
   if (copy) {
+    if (typeof copy.useAzureSdk === 'boolean') {
+      result.useAzureSdk = copy.useAzureSdk
+    }
+
     if (typeof copy.uploadConcurrency === 'number') {
       result.uploadConcurrency = copy.uploadConcurrency
     }
@@ -91,6 +109,26 @@ export function getUploadOptions(copy?: UploadOptions): UploadOptions {
     }
   }
 
+  /**
+   * Add env var overrides
+   */
+  // Cap the uploadConcurrency at 32
+  result.uploadConcurrency = !isNaN(
+    Number(process.env['CACHE_UPLOAD_CONCURRENCY'])
+  )
+    ? Math.min(32, Number(process.env['CACHE_UPLOAD_CONCURRENCY']))
+    : result.uploadConcurrency
+  // Cap the uploadChunkSize at 128MiB
+  result.uploadChunkSize = !isNaN(
+    Number(process.env['CACHE_UPLOAD_CHUNK_SIZE'])
+  )
+    ? Math.min(
+        128 * 1024 * 1024,
+        Number(process.env['CACHE_UPLOAD_CHUNK_SIZE']) * 1024 * 1024
+      )
+    : result.uploadChunkSize
+
+  core.debug(`Use Azure SDK: ${result.useAzureSdk}`)
   core.debug(`Upload concurrency: ${result.uploadConcurrency}`)
   core.debug(`Upload chunk size: ${result.uploadChunkSize}`)