diff --git a/packages/cache/RELEASES.md b/packages/cache/RELEASES.md index 920a20e8ed..66c248ca10 100644 --- a/packages/cache/RELEASES.md +++ b/packages/cache/RELEASES.md @@ -2,4 +2,7 @@ ### 0.1.0 -- Initial release \ No newline at end of file +- Initial release + +### 0.2.0 +- Fixes two issues around using zstd compression algorithm \ No newline at end of file diff --git a/packages/cache/__tests__/tar.test.ts b/packages/cache/__tests__/tar.test.ts index 0aa6c78435..ecb6799bc5 100644 --- a/packages/cache/__tests__/tar.test.ts +++ b/packages/cache/__tests__/tar.test.ts @@ -38,13 +38,11 @@ test('zstd extract tar', async () => { ? `${process.env['windir']}\\fakepath\\cache.tar` : 'cache.tar' const workspace = process.env['GITHUB_WORKSPACE'] + const tarPath = 'tar' await tar.extractTar(archivePath, CompressionMethod.Zstd) expect(mkdirMock).toHaveBeenCalledWith(workspace) - const tarPath = IS_WINDOWS - ? `${process.env['windir']}\\System32\\tar.exe` - : 'tar' expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( `"${tarPath}"`, @@ -56,7 +54,7 @@ test('zstd extract tar', async () => { '-P', '-C', IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace - ], + ].concat(IS_WINDOWS ? ['--force-local'] : []), {cwd: undefined} ) }) @@ -95,7 +93,7 @@ test('gzip extract GNU tar on windows', async () => { jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false) const isGnuMock = jest - .spyOn(utils, 'useGnuTar') + .spyOn(utils, 'isGnuTarInstalled') .mockReturnValue(Promise.resolve(true)) const execMock = jest.spyOn(exec, 'exec') const archivePath = `${process.env['windir']}\\fakepath\\cache.tar` @@ -127,15 +125,12 @@ test('zstd create tar', async () => { const archiveFolder = getTempDir() const workspace = process.env['GITHUB_WORKSPACE'] const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`] + const tarPath = 'tar' await fs.promises.mkdir(archiveFolder, {recursive: true}) await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd) - const tarPath = IS_WINDOWS - ? `${process.env['windir']}\\System32\\tar.exe` - : 'tar' - expect(execMock).toHaveBeenCalledTimes(1) expect(execMock).toHaveBeenCalledWith( `"${tarPath}"`, @@ -149,7 +144,7 @@ test('zstd create tar', async () => { IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace, '--files-from', 'manifest.txt' - ], + ].concat(IS_WINDOWS ? ['--force-local'] : []), { cwd: archiveFolder } diff --git a/packages/cache/package-lock.json b/packages/cache/package-lock.json index 08f18a7c8a..80ef7664b0 100644 --- a/packages/cache/package-lock.json +++ b/packages/cache/package-lock.json @@ -1,92 +1,103 @@ { - "name": "@actions/cache", - "version": "0.1.0", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "@actions/core": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.4.tgz", - "integrity": "sha512-YJCEq8BE3CdN8+7HPZ/4DxJjk/OkZV2FFIf+DlZTC/4iBlzYCD5yjRR6eiOS5llO11zbRltIRuKAjMKaWTE6cg==" - }, - "@actions/exec": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.0.4.tgz", - "integrity": "sha512-4DPChWow9yc9W3WqEbUj8Nr86xkpyE29ZzWjXucHItclLbEW6jr80Zx4nqv18QL6KK65+cifiQZXvnqgTV6oHw==", - "requires": { - "@actions/io": "^1.0.1" - } - }, - "@actions/glob": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/@actions/glob/-/glob-0.1.0.tgz", - "integrity": "sha512-lx8SzyQ2FE9+UUvjqY1f28QbTJv+w8qP7kHHbfQRhphrlcx0Mdmm1tZdGJzfxv1jxREa/sLW4Oy8CbGQKCJySA==", - "requires": { - "@actions/core": "^1.2.0", - "minimatch": "^3.0.4" - } - }, - "@actions/http-client": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.8.tgz", - "integrity": "sha512-G4JjJ6f9Hb3Zvejj+ewLLKLf99ZC+9v+yCxoYf9vSyH+WkzPLB2LuUtRMGNkooMqdugGBFStIKXOuvH1W+EctA==", - "requires": { - "tunnel": "0.0.6" - } - }, - "@actions/io": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.0.2.tgz", - "integrity": "sha512-J8KuFqVPr3p6U8W93DOXlXW6zFvrQAJANdS+vw0YhusLIq+bszW8zmK2Fh1C2kDPX8FMvwIl1OUcFgvJoXLbAg==" - }, - "@types/uuid": { - "version": "3.4.9", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.9.tgz", - "integrity": "sha512-XDwyIlt/47l2kWLTzw/mtrpLdB+GPSskR2n/PIcPn+VYhVO77rGhRncIR5GPU0KRzXuqkDO+J5qqrG0Y8P6jzQ==", - "dev": true - }, - "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" - }, - "brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" - }, - "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "tunnel": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", - "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" - }, - "typescript": { - "version": "3.8.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz", - "integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==", - "dev": true - }, - "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" - } - } + "name": "@actions/cache", + "version": "0.2.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@actions/core": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.4.tgz", + "integrity": "sha512-YJCEq8BE3CdN8+7HPZ/4DxJjk/OkZV2FFIf+DlZTC/4iBlzYCD5yjRR6eiOS5llO11zbRltIRuKAjMKaWTE6cg==" + }, + "@actions/exec": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.0.4.tgz", + "integrity": "sha512-4DPChWow9yc9W3WqEbUj8Nr86xkpyE29ZzWjXucHItclLbEW6jr80Zx4nqv18QL6KK65+cifiQZXvnqgTV6oHw==", + "requires": { + "@actions/io": "^1.0.1" + } + }, + "@actions/glob": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@actions/glob/-/glob-0.1.0.tgz", + "integrity": "sha512-lx8SzyQ2FE9+UUvjqY1f28QbTJv+w8qP7kHHbfQRhphrlcx0Mdmm1tZdGJzfxv1jxREa/sLW4Oy8CbGQKCJySA==", + "requires": { + "@actions/core": "^1.2.0", + "minimatch": "^3.0.4" + } + }, + "@actions/http-client": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.8.tgz", + "integrity": "sha512-G4JjJ6f9Hb3Zvejj+ewLLKLf99ZC+9v+yCxoYf9vSyH+WkzPLB2LuUtRMGNkooMqdugGBFStIKXOuvH1W+EctA==", + "requires": { + "tunnel": "0.0.6" + } + }, + "@actions/io": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.0.2.tgz", + "integrity": "sha512-J8KuFqVPr3p6U8W93DOXlXW6zFvrQAJANdS+vw0YhusLIq+bszW8zmK2Fh1C2kDPX8FMvwIl1OUcFgvJoXLbAg==" + }, + "@types/semver": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-6.2.1.tgz", + "integrity": "sha512-+beqKQOh9PYxuHvijhVl+tIHvT6tuwOrE9m14zd+MT2A38KoKZhh7pYJ0SNleLtwDsiIxHDsIk9bv01oOxvSvA==", + "dev": true + }, + "@types/uuid": { + "version": "3.4.9", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.9.tgz", + "integrity": "sha512-XDwyIlt/47l2kWLTzw/mtrpLdB+GPSskR2n/PIcPn+VYhVO77rGhRncIR5GPU0KRzXuqkDO+J5qqrG0Y8P6jzQ==", + "dev": true + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + }, + "tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" + }, + "typescript": { + "version": "3.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz", + "integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==", + "dev": true + }, + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + } + } } diff --git a/packages/cache/package.json b/packages/cache/package.json index 69e9318121..928a02831a 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -1,6 +1,6 @@ { "name": "@actions/cache", - "version": "0.1.0", + "version": "0.2.0", "preview": true, "description": "Actions cache lib", "keywords": [ @@ -41,10 +41,12 @@ "@actions/glob": "^0.1.0", "@actions/http-client": "^1.0.8", "@actions/io": "^1.0.1", + "semver": "^6.1.0", "uuid": "^3.3.3" }, "devDependencies": { "typescript": "^3.8.3", + "@types/semver": "^6.0.0", "@types/uuid": "^3.4.5" } } diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index b7d34448d3..21af8031d0 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -96,7 +96,9 @@ export function getCacheVersion( compressionMethod?: CompressionMethod ): string { const components = paths.concat( - compressionMethod === CompressionMethod.Zstd ? [compressionMethod] : [] + !compressionMethod || compressionMethod === CompressionMethod.Gzip + ? [] + : [compressionMethod] ) // Add salt to cache version to support breaking changes in cache entry diff --git a/packages/cache/src/internal/cacheUtils.ts b/packages/cache/src/internal/cacheUtils.ts index f3f8500641..53b35afae7 100644 --- a/packages/cache/src/internal/cacheUtils.ts +++ b/packages/cache/src/internal/cacheUtils.ts @@ -4,6 +4,7 @@ import * as glob from '@actions/glob' import * as io from '@actions/io' import * as fs from 'fs' import * as path from 'path' +import * as semver from 'semver' import * as util from 'util' import {v4 as uuidV4} from 'uuid' import {CacheFilename, CompressionMethod} from './constants' @@ -82,19 +83,33 @@ async function getVersion(app: string): Promise { // Use zstandard if possible to maximize cache performance export async function getCompressionMethod(): Promise { + if (process.platform === 'win32' && !isGnuTarInstalled()) { + // Disable zstd due to bug https://github.com/actions/cache/issues/301 + return CompressionMethod.Gzip + } + const versionOutput = await getVersion('zstd') - return versionOutput.toLowerCase().includes('zstd command line interface') - ? CompressionMethod.Zstd - : CompressionMethod.Gzip + const version = semver.clean(versionOutput) + + if (!versionOutput.toLowerCase().includes('zstd command line interface')) { + // zstd is not installed + return CompressionMethod.Gzip + } else if (!version || semver.lt(version, 'v1.3.2')) { + // zstd is installed but using a version earlier than v1.3.2 + // v1.3.2 is required to use the `--long` options in zstd + return CompressionMethod.ZstdWithoutLong + } else { + return CompressionMethod.Zstd + } } export function getCacheFileName(compressionMethod: CompressionMethod): string { - return compressionMethod === CompressionMethod.Zstd - ? CacheFilename.Zstd - : CacheFilename.Gzip + return compressionMethod === CompressionMethod.Gzip + ? CacheFilename.Gzip + : CacheFilename.Zstd } -export async function useGnuTar(): Promise { +export async function isGnuTarInstalled(): Promise { const versionOutput = await getVersion('tar') return versionOutput.toLowerCase().includes('gnu tar') } diff --git a/packages/cache/src/internal/constants.ts b/packages/cache/src/internal/constants.ts index b3d2a57746..06c8c8f0bb 100644 --- a/packages/cache/src/internal/constants.ts +++ b/packages/cache/src/internal/constants.ts @@ -5,6 +5,9 @@ export enum CacheFilename { export enum CompressionMethod { Gzip = 'gzip', + // Long range mode was added to zstd in v1.3.2. + // This enum is for earlier version of zstd that does not have --long support + ZstdWithoutLong = 'zstd-without-long', Zstd = 'zstd' } diff --git a/packages/cache/src/internal/tar.ts b/packages/cache/src/internal/tar.ts index 221c7c704d..7f836b83d3 100644 --- a/packages/cache/src/internal/tar.ts +++ b/packages/cache/src/internal/tar.ts @@ -5,23 +5,33 @@ import * as path from 'path' import * as utils from './cacheUtils' import {CompressionMethod} from './constants' -async function getTarPath(args: string[]): Promise { - // Explicitly use BSD Tar on Windows +async function getTarPath( + args: string[], + compressionMethod: CompressionMethod +): Promise { const IS_WINDOWS = process.platform === 'win32' if (IS_WINDOWS) { const systemTar = `${process.env['windir']}\\System32\\tar.exe` - if (existsSync(systemTar)) { + if (compressionMethod !== CompressionMethod.Gzip) { + // We only use zstandard compression on windows when gnu tar is installed due to + // a bug with compressing large files with bsdtar + zstd + args.push('--force-local') + } else if (existsSync(systemTar)) { return systemTar - } else if (await utils.useGnuTar()) { + } else if (await utils.isGnuTarInstalled()) { args.push('--force-local') } } return await io.which('tar', true) } -async function execTar(args: string[], cwd?: string): Promise { +async function execTar( + args: string[], + compressionMethod: CompressionMethod, + cwd?: string +): Promise { try { - await exec(`"${await getTarPath(args)}"`, args, {cwd}) + await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd}) } catch (error) { throw new Error(`Tar failed with error: ${error?.message}`) } @@ -41,17 +51,25 @@ export async function extractTar( // --d: Decompress. // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. + function getCompressionProgram(): string[] { + switch (compressionMethod) { + case CompressionMethod.Zstd: + return ['--use-compress-program', 'zstd -d --long=30'] + case CompressionMethod.ZstdWithoutLong: + return ['--use-compress-program', 'zstd -d'] + default: + return ['-z'] + } + } const args = [ - ...(compressionMethod === CompressionMethod.Zstd - ? ['--use-compress-program', 'zstd -d --long=30'] - : ['-z']), + ...getCompressionProgram(), '-xf', archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', '-C', workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') ] - await execTar(args) + await execTar(args, compressionMethod) } export async function createTar( @@ -66,14 +84,24 @@ export async function createTar( path.join(archiveFolder, manifestFilename), sourceDirectories.join('\n') ) + const workingDirectory = getWorkingDirectory() + // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. - const workingDirectory = getWorkingDirectory() + // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. + function getCompressionProgram(): string[] { + switch (compressionMethod) { + case CompressionMethod.Zstd: + return ['--use-compress-program', 'zstd -T0 --long=30'] + case CompressionMethod.ZstdWithoutLong: + return ['--use-compress-program', 'zstd -T0'] + default: + return ['-z'] + } + } const args = [ - ...(compressionMethod === CompressionMethod.Zstd - ? ['--use-compress-program', 'zstd -T0 --long=30'] - : ['-z']), + ...getCompressionProgram(), '-cf', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', @@ -82,5 +110,5 @@ export async function createTar( '--files-from', manifestFilename ] - await execTar(args, archiveFolder) + await execTar(args, compressionMethod, archiveFolder) }