From 1a8c0dd6c5f7daec79d80f376c72871ba1072c23 Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Mon, 17 Oct 2022 12:45:43 +0100 Subject: [PATCH] Support custom extension matching This commit adds support for customising extension matching. This can be used to install releases that are in alternative file formats, or lack a file extension entirely e.g. pure binaries. When a download release asset is not an archive it is copied into the tool cache rather than being extracted to there. The copied asset may optionally be renamed and chmod'd if desired. Also adds support for extracting release archives that are .tar.bz2 format. --- .github/workflows/test.yml | 56 +++++++++++++++++++++++++ README.md | 51 +++++++++++++++++++++++ action.yml | 13 ++++++ lib/main.js | 83 +++++++++++++++++++++++++++++++++---- src/main.ts | 85 ++++++++++++++++++++++++++++++++++---- 5 files changed, 273 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 57fa906f..f1c30806 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,7 @@ name: "Test typescript-action" on: pull_request: + workflow_dispatch: push: branches: - master @@ -74,3 +75,58 @@ jobs: platform: ${{ matrix.platform }} arch: ${{ matrix.arch }} - run: tfsec --version + opentelemetry-ocb: + strategy: + matrix: + version: [ "v0.62.1", "v0.62.0", "latest" ] + runs-on: [ "ubuntu-latest", "macos-latest"] + arch: [ "amd64" ] + include: + - runs-on: "ubuntu-latest" + platform: linux + - runs-on: "macos-latest" + platform: darwin + runs-on: ${{ matrix.runs-on }} + steps: + - uses: actions/checkout@v1 + - run: npm ci + - run: npm run build + - uses: ./ + with: + repo: open-telemetry/opentelemetry-collector + tag: ${{ matrix.version }} + platform: ${{ matrix.platform }} + arch: ${{ matrix.arch }} + extension-matching: disable + rename-to: ocb + chmod: 0755 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: ocb version + mozilla-grcov: + strategy: + matrix: + version: [ "v0.8.12", "v0.8.7", "latest" ] + runs-on: [ "ubuntu-latest", "macos-latest" ] + arch: [ "x86_64" ] + include: + - runs-on: "ubuntu-latest" + platform: linux + - runs-on: "macos-latest" + platform: darwin + runs-on: ${{ matrix.runs-on }} + steps: + - uses: actions/checkout@v1 + - run: npm ci + - run: npm run build + - uses: ./ + with: + repo: mozilla/grcov + tag: ${{ matrix.version }} + platform: ${{ matrix.platform }} + arch: ${{ matrix.arch }} + extension: "\\.bz2" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: grcov --version + diff --git a/README.md b/README.md index 88c9abde..99f51b1a 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,57 @@ steps: Caching helps avoid [Rate limiting](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#requests-from-github-actions), since this action does not need to scan tags and releases on a cache hit. Caching currently is not expected to speed up installation. +### Changing Release File Extensions + +As described below this action defaults to assuming that a release is either a `.tar.gz` or a `.zip` archive but this +may not always be true for all releases. For example a project might release a pure binary, a different archive format, +custom file extension etc. + +This action can change its extension matching behaviour via the `extension-matching` and `extension` parameters. For +example to match on a `.bz2` extension: + +```yaml +# ... +jobs: + my_job: + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Github token scoped to job + steps: + - name: Install Mozilla grcov + uses: jaxxstorm/action-install-gh-release@v1.5.0 + with: # Grab a specific file extension + repo: mozilla/grcov + tag: v0.8.12 + extension: "\\.bz2" +``` + +Here the `extension` parameter is used to provide a regular expression for the file extension(s) you want to match. If +this is not specified then the action defaults to `\.(tag.gz|zip)`. Since this a regular expression being embedded into +YAML be aware that you may need to provide an extra level of character escaping, in the above example we have a `\\` +used in order to escape the backslash and get an actual `\.` (literal match of the period character) in the regular +expression passed into the action. + +Alternatively if a project produces pure binary releases with no file extension then you can install as follows: + +```yaml +# ... +jobs: + my_job: + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Github token scoped to job + steps: + - name: Install Open Telemetry Collector Builder (ocb) + uses: jaxxstorm/action-install-gh-release@v1.5.0 + with: # Grab a pure binary + repo: open-telemetry/opentelemetry-collector + tag: v0.62.1 + extension-matching: disable + rename-to: ocb + chmod: 0755 +``` + +Note the use of the `rename-to` and `chmod` parameters to rename the downloaded binary and make it executable. + ## Finding a release By default, this action will lookup the Platform and Architecture of the runner and use those values to interpolate and match a release package. **The release package name is first converted to lowercase**. The match pattern is: diff --git a/action.yml b/action.yml index 3522b68e..6895155c 100644 --- a/action.yml +++ b/action.yml @@ -18,6 +18,19 @@ inputs: arch: description: "OS Architecture to match in release package. Specify this parameter if the repository releases do not follow a normal convention otherwise it will be auto-detected." required: false + extension: + description: "Custom file extension to match in release package. Specify this parameter if the repository releases do not provide a .tar.gz or .zip format release." + required: false + extension-matching: + description: "Enable/disable file extension matching in release package. Specify this parameter if the repository releases do not have a file extension e.g. they are pure binaries." + required: false + default: enable + rename-to: + description: "When installing a release that is not an archive, e.g. a pure binary, this controls how the downloaded release asset is renamed. Specify this parameter if installing a non-archive release." + required: false + chmod: + description: "When installing a release that is not an archive, e.g. a pure binary, this controls how the downloaded release asset is chmod'd. Specify this parameter if installing a non-archive release and you need to change its permissions e.g. make it executable." + required: false cache: description: "When set to 'enable', caches the downloads of known tags with actions/cache" required: false diff --git a/lib/main.js b/lib/main.js index f20059a1..471f28da 100644 --- a/lib/main.js +++ b/lib/main.js @@ -30,6 +30,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge Object.defineProperty(exports, "__esModule", { value: true }); const os = __importStar(require("os")); const path = __importStar(require("path")); +const fs = __importStar(require("fs")); const cache = __importStar(require("@actions/cache")); const core = __importStar(require("@actions/core")); const tc = __importStar(require("@actions/tool-cache")); @@ -105,6 +106,32 @@ function run() { } core.info(`==> System reported arch: ${os.arch()}`); core.info(`==> Using arch: ${osArch}`); + // Determine File Extensions (if any) + const extMatching = core.getInput("extension-matching") === "enable"; + let extension = core.getInput("extension"); + let extMatchRegexForm = ""; + if (extMatching) { + if (extension === "") { + extMatchRegexForm = "\.(tar.gz|zip)"; + core.info(`==> Using default file extension matching: ${extMatchRegexForm}`); + } + else { + extMatchRegexForm = extension; + core.info(`==> Using custom file extension matching: ${extMatchRegexForm}`); + } + } + else { + core.info("==> File extension matching disabled"); + } + // Determine whether renaming is in use + let renameTo = core.getInput("rename-to"); + if (renameTo !== "") { + core.info(`==> Will rename downloaded release to ${renameTo}`); + } + let chmodTo = core.getInput("chmod"); + if (chmodTo !== "") { + core.info(`==> Will chmod downloaded release asset to ${chmodTo}`); + } let toolInfo = { owner: owner, project: project, @@ -138,7 +165,7 @@ function run() { }); } let osMatchRegexForm = `(${osMatch.join('|')})`; - let re = new RegExp(`${osMatchRegexForm}.*${osMatchRegexForm}.*\.(tar.gz|zip)`); + let re = new RegExp(`${osMatchRegexForm}.*${osMatchRegexForm}.*${extMatchRegexForm}`); let asset = getReleaseUrl.data.assets.find(obj => { core.info(`searching for ${obj.name} with ${re.source}`); let normalized_obj_name = obj.name.toLowerCase(); @@ -148,13 +175,55 @@ function run() { const found = getReleaseUrl.data.assets.map(f => f.name); throw new Error(`Could not find a release for ${tag}. Found: ${found}`); } - const extractFn = getExtractFn(asset.name); const url = asset.url; core.info(`Downloading ${project} from ${url}`); const binPath = yield tc.downloadTool(url, undefined, `token ${token}`, { accept: 'application/octet-stream' }); - yield extractFn(binPath, dest); + const extractFn = getExtractFn(asset.name); + if (extractFn !== undefined) { + // Release is an archive file so extract it to the destination + yield extractFn(binPath, dest); + core.info(`Automatically extracted release asset ${asset.name} to ${dest}`); + if (renameTo !== "") { + core.warning("rename-to parameter ignored when installing a release from an archive"); + } + if (chmodTo !== "") { + core.warning("chmod parameter ignored when installing a release from an archive"); + } + } + else { + // As it wasn't an archive we've just downloaded it as a blob, this uses an auto-assigned name which will + // be a UUID which is likely meaningless to the caller. If they have specified a rename-to and a chmod + // parameter then this is where we apply those. + // Regardless of any rename-to parameter we still need to move the download to the actual destination + // otherwise it won't end up on the path as expected + core.warning(`Release asset ${asset.name} did not have a recognised file extension, unable to automatically extract it`); + try { + fs.mkdirSync(dest, { 'recursive': true }); + const outputPath = path.join(dest, renameTo !== "" ? renameTo : path.basename(binPath)); + core.info(`Created output directory ${dest}`); + try { + fs.renameSync(binPath, outputPath); + core.info(`Moved release asset ${asset.name} to ${outputPath}`); + if (chmodTo !== "") { + try { + fs.chmodSync(outputPath, chmodTo); + core.info(`chmod'd ${outputPath} to ${chmodTo}`); + } + catch (chmodErr) { + core.setFailed(`Failed to chmod ${outputPath} to ${chmodTo}: ${chmodErr}`); + } + } + } + catch (renameErr) { + core.setFailed(`Failed to move downloaded release asset ${asset.name} from ${binPath} to ${outputPath}: ${renameErr}`); + } + } + catch (err) { + core.setFailed(`Failed to create required output directory ${dest}`); + } + } if (cacheEnabled && cacheKey !== undefined) { try { yield cache.saveCache([dest], cacheKey); @@ -173,7 +242,7 @@ function run() { } } core.addPath(dest); - core.info(`Successfully extracted ${project} to ${dest}`); + core.info(`Successfully installed ${project} to ${dest}`); } catch (error) { if (error instanceof Error) { @@ -186,7 +255,7 @@ function run() { }); } function cachePrimaryKey(info) { - // Currently not caching "latest" verisons of the tool. + // Currently not caching "latest" versions of the tool. if (info.tag === "latest") { return undefined; } @@ -204,14 +273,14 @@ function getCacheDirectory() { return cacheDirectory; } function getExtractFn(assetName) { - if (assetName.endsWith('.tar.gz')) { + if (assetName.endsWith('.tar.gz') || assetName.endsWith('.tar.bz2')) { return tc.extractTar; } else if (assetName.endsWith('.zip')) { return tc.extractZip; } else { - throw new Error(`Unreachable error? File is neither .tar.gz nor .zip, got: ${assetName}`); + return undefined; } } run(); diff --git a/src/main.ts b/src/main.ts index 6f1633d8..677a0ce5 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,5 +1,6 @@ import * as os from "os"; import * as path from "path"; +import * as fs from "fs"; import * as cache from "@actions/cache"; import * as core from "@actions/core"; @@ -104,6 +105,32 @@ async function run() { core.info(`==> System reported arch: ${os.arch()}`) core.info(`==> Using arch: ${osArch}`) + // Determine File Extensions (if any) + const extMatching = core.getInput("extension-matching") === "enable"; + let extension = core.getInput("extension"); + let extMatchRegexForm = ""; + if (extMatching) { + if (extension === "") { + extMatchRegexForm = "\.(tar.gz|zip)"; + core.info(`==> Using default file extension matching: ${extMatchRegexForm}`); + } else { + extMatchRegexForm = extension; + core.info(`==> Using custom file extension matching: ${extMatchRegexForm}`); + } + } else { + core.info("==> File extension matching disabled"); + } + + // Determine whether renaming is in use + let renameTo = core.getInput("rename-to"); + if (renameTo !== "") { + core.info(`==> Will rename downloaded release to ${renameTo}`); + } + let chmodTo = core.getInput("chmod"); + if (chmodTo !== "") { + core.info(`==> Will chmod downloaded release asset to ${chmodTo}`); + } + let toolInfo: ToolInfo = { owner: owner, project: project, @@ -139,7 +166,7 @@ async function run() { } let osMatchRegexForm = `(${osMatch.join('|')})` - let re = new RegExp(`${osMatchRegexForm}.*${osMatchRegexForm}.*\.(tar.gz|zip)`) + let re = new RegExp(`${osMatchRegexForm}.*${osMatchRegexForm}.*${extMatchRegexForm}`) let asset = getReleaseUrl.data.assets.find(obj => { core.info(`searching for ${obj.name} with ${re.source}`) let normalized_obj_name = obj.name.toLowerCase() @@ -153,8 +180,6 @@ async function run() { ) } - const extractFn = getExtractFn(asset.name); - const url = asset.url core.info(`Downloading ${project} from ${url}`) @@ -165,7 +190,51 @@ async function run() { accept: 'application/octet-stream' } ); - await extractFn(binPath, dest); + + const extractFn = getExtractFn(asset.name) + if (extractFn !== undefined) { + // Release is an archive file so extract it to the destination + await extractFn(binPath, dest); + core.info(`Automatically extracted release asset ${asset.name} to ${dest}`); + + if (renameTo !== "") { + core.warning("rename-to parameter ignored when installing a release from an archive"); + } + if (chmodTo !== "") { + core.warning("chmod parameter ignored when installing a release from an archive"); + } + } else { + // As it wasn't an archive we've just downloaded it as a blob, this uses an auto-assigned name which will + // be a UUID which is likely meaningless to the caller. If they have specified a rename-to and a chmod + // parameter then this is where we apply those. + // Regardless of any rename-to parameter we still need to move the download to the actual destination + // otherwise it won't end up on the path as expected + core.warning( + `Release asset ${asset.name} did not have a recognised file extension, unable to automatically extract it`) + try { + fs.mkdirSync(dest, {'recursive': true}); + + const outputPath = path.join(dest, renameTo !== "" ? renameTo : path.basename(binPath)); + core.info(`Created output directory ${dest}`); + try { + fs.renameSync(binPath, outputPath); + core.info(`Moved release asset ${asset.name} to ${outputPath}`); + + if (chmodTo !== "") { + try { + fs.chmodSync(outputPath, chmodTo); + core.info(`chmod'd ${outputPath} to ${chmodTo}`) + } catch (chmodErr) { + core.setFailed(`Failed to chmod ${outputPath} to ${chmodTo}: ${chmodErr}`); + } + } + } catch (renameErr) { + core.setFailed(`Failed to move downloaded release asset ${asset.name} from ${binPath} to ${outputPath}: ${renameErr}`); + } + } catch (err) { + core.setFailed(`Failed to create required output directory ${dest}`); + } + } if (cacheEnabled && cacheKey !== undefined) { try { @@ -183,7 +252,7 @@ async function run() { } core.addPath(dest); - core.info(`Successfully extracted ${project} to ${dest}`) + core.info(`Successfully installed ${project} to ${dest}`) } catch (error) { if (error instanceof Error) { core.setFailed(error.message); @@ -194,7 +263,7 @@ async function run() { } function cachePrimaryKey(info: ToolInfo): string|undefined { - // Currently not caching "latest" verisons of the tool. + // Currently not caching "latest" versions of the tool. if (info.tag === "latest") { return undefined; } @@ -217,12 +286,12 @@ function getCacheDirectory() { } function getExtractFn(assetName: any) { - if (assetName.endsWith('.tar.gz')) { + if (assetName.endsWith('.tar.gz') || assetName.endsWith('.tar.bz2')) { return tc.extractTar; } else if (assetName.endsWith('.zip')) { return tc.extractZip; } else { - throw new Error(`Unreachable error? File is neither .tar.gz nor .zip, got: ${assetName}`); + return undefined; } }