feat: remove qq and parallelize build/pack/promote

.gitignore

@@ -6,4 +6,6 @@
 /lib
 /node_modules
 /tmp
+/test/tmp
 package-lock.json
+/dist

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "request": "attach",
+      "name": "Attach",
+      "port": 9229,
+      "skipFiles": ["<node_internals>/**"]
+    }
+  ]
+}

package.json

@@ -1,145 +1 @@
-{
-  "name": "oclif",
-  "description": "oclif: create your own CLI",
-  "version": "3.2.29",
-  "author": "Salesforce",
-  "bin": {
-    "oclif": "bin/run"
-  },
-  "bugs": "https://github.com/oclif/oclif/issues",
-  "dependencies": {
-    "@oclif/core": "^1.20.4",
-    "@oclif/plugin-help": "^5.1.19",
-    "@oclif/plugin-not-found": "^2.3.7",
-    "@oclif/plugin-warn-if-update-available": "^2.0.14",
-    "aws-sdk": "^2.1231.0",
-    "concurrently": "^7.5.0",
-    "debug": "^4.3.3",
-    "find-yarn-workspace-root": "^2.0.0",
-    "fs-extra": "^8.1",
-    "github-slugger": "^1.4.0",
-    "lodash": "^4.17.21",
-    "normalize-package-data": "^3.0.3",
-    "qqjs": "^0.3.11",
-    "semver": "^7.3.8",
-    "tslib": "^2.3.1",
-    "yeoman-environment": "^3.11.1",
-    "yeoman-generator": "^5.6.1",
-    "yosay": "^2.0.2"
-  },
-  "devDependencies": {
-    "@oclif/plugin-legacy": "^1.2.7",
-    "@oclif/test": "^2.2.10",
-    "@types/chai": "^4.3.4",
-    "@types/execa": "^0.9.0",
-    "@types/fs-extra": "^9.0",
-    "@types/lodash": "^4.14.186",
-    "@types/lodash.template": "^4.5.0",
-    "@types/mocha": "^8.2.3",
-    "@types/node": "^14.18.32",
-    "@types/read-pkg": "^5.1.0",
-    "@types/semver": "^7.3.13",
-    "@types/shelljs": "^0.8.11",
-    "@types/supports-color": "^7.2.1",
-    "@types/write-json-file": "^3.2.1",
-    "@types/yeoman-generator": "^5.2.10",
-    "@types/yosay": "^2.0.1",
-    "chai": "^4.3.7",
-    "conventional-changelog-cli": "^2.2.2",
-    "eslint": "^7.32.0",
-    "eslint-config-oclif": "^4.0.0",
-    "eslint-config-oclif-typescript": "^1.0.2",
-    "execa": "^0.11.0",
-    "fancy-test": "^1.4.10",
-    "globby": "^11.1.0",
-    "mocha": "^9.2.2",
-    "npm-run-path": "^4.0.1",
-    "nps": "^5.10.0",
-    "shelljs": "^0.8.5",
-    "shx": "^0.3.4",
-    "tmp": "^0.2.1",
-    "ts-node": "^10.7.0",
-    "typescript": "4.5.5"
-  },
-  "resolutions": {
-    "colors": "1.4.0",
-    "@oclif/core": "^1.16.1"
-  },
-  "overrides": {
-    "colors": "1.4.0",
-    "@oclif/core": "^1.16.1"
-  },
-  "engines": {
-    "node": ">=12.0.0"
-  },
-  "files": [
-    ".oclif.manifest.json",
-    "/bin",
-    "/lib",
-    "/templates"
-  ],
-  "homepage": "https://github.com/oclif/oclif",
-  "keywords": [
-    "oclif"
-  ],
-  "license": "MIT",
-  "main": "lib/index.js",
-  "oclif": {
-    "commands": "./lib/commands",
-    "plugins": [
-      "@oclif/plugin-help",
-      "@oclif/plugin-warn-if-update-available",
-      "@oclif/plugin-not-found"
-    ],
-    "bin": "oclif",
-    "dirname": "oclif",
-    "topicSeparator": " ",
-    "macos": {
-      "identifier": "com.oclif.cli"
-    },
-    "update": {
-      "autoupdate": {
-        "rollout": 50,
-        "debounce": 60
-      },
-      "node": {
-        "version": "12.12.0"
-      },
-      "s3": {
-        "bucket": "dfc-data-production",
-        "indexVersionLimit": 20,
-        "folder": "media/salesforce-cli/oclif-testing",
-        "acl": " ",
-        "host": "https://developer.salesforce.com",
-        "xz": true
-      }
-    },
-    "topics": {
-      "pack": {
-        "description": "package an oclif CLI into installable artifacts"
-      },
-      "upload": {
-        "description": "upload installable CLI artifacts to AWS S3"
-      }
-    }
-  },
-  "repository": "oclif/oclif",
-  "scripts": {
-    "build": "shx rm -rf lib && tsc",
-    "devcli:lint": "eslint . --ext .ts --config .eslintrc",
-    "devcli:test": "mocha --forbid-only \"test/unit/*.test.ts\"",
-    "devcli": "yarn build --noEmit && yarn run devcli:test && yarn run devcli:lint",
-    "lint": "nps lint",
-    "postpack": "shx rm .oclif.manifest.json",
-    "posttest": "yarn run lint",
-    "prepack": "shx rm -rf lib && tsc && bin/dev manifest .",
-    "test": "nps test && yarn test:unit && yarn test:integration",
-    "test:integration": "mocha --forbid-only \"test/integration/*.test.ts\"",
-    "test:unit": "mocha --forbid-only \"test/unit/*.test.ts\"",
-    "version": "bin/dev readme && git add README.md"
-  },
-  "publishConfig": {
-    "registry": "https://registry.npmjs.org"
-  },
-  "types": "lib/index.d.ts"
-}
+{"name":"oclif","description":"oclif: create your own CLI","version":"3.2.29","author":"Salesforce","bin":{"oclif":"bin/run"},"bugs":"https://github.com/oclif/oclif/issues","dependencies":{"@oclif/core":"^1.20.4","@oclif/plugin-help":"^5.1.19","@oclif/plugin-not-found":"^2.3.7","@oclif/plugin-warn-if-update-available":"^2.0.14","aws-sdk":"^2.1231.0","concurrently":"^7.5.0","debug":"^4.3.3","find-yarn-workspace-root":"^2.0.0","fs-extra":"^8.1","github-slugger":"^1.4.0","got":"^11","lodash":"^4.17.21","normalize-package-data":"^3.0.3","semver":"^7.3.8","tslib":"^2.3.1","yeoman-environment":"^3.11.1","yeoman-generator":"^5.6.1","yosay":"^2.0.2"},"devDependencies":{"@oclif/plugin-legacy":"^1.2.7","@oclif/test":"^2.2.10","@types/chai":"^4.3.4","@types/execa":"^0.9.0","@types/fs-extra":"^9.0","@types/lodash":"^4.14.186","@types/lodash.template":"^4.5.0","@types/mocha":"^8.2.3","@types/node":"^14.18.32","@types/read-pkg":"^5.1.0","@types/semver":"^7.3.13","@types/shelljs":"^0.8.11","@types/supports-color":"^7.2.1","@types/write-json-file":"^3.2.1","@types/yeoman-generator":"^5.2.10","@types/yosay":"^2.0.1","chai":"^4.3.7","conventional-changelog-cli":"^2.2.2","eslint":"^7.32.0","eslint-config-oclif":"^4.0.0","eslint-config-oclif-typescript":"^1.0.2","execa":"^0.11.0","fancy-test":"^1.4.10","globby":"^11.1.0","mocha":"^9.2.2","npm-run-path":"^4.0.1","nps":"^5.10.0","shelljs":"^0.8.5","shx":"^0.3.4","tmp":"^0.2.1","ts-node":"^10.7.0","typescript":"4.5.5"},"resolutions":{"colors":"1.4.0","@oclif/core":"^1.16.1"},"overrides":{"colors":"1.4.0","@oclif/core":"^1.16.1"},"engines":{"node":">=12.0.0"},"files":[".oclif.manifest.json","/bin","/lib","/templates"],"homepage":"https://github.com/oclif/oclif","keywords":["oclif"],"license":"MIT","main":"lib/index.js","oclif":{"commands":"./lib/commands","plugins":["@oclif/plugin-help","@oclif/plugin-warn-if-update-available","@oclif/plugin-not-found"],"bin":"oclif","dirname":"oclif","topicSeparator":" ","macos":{"identifier":"com.oclif.cli"},"update":{"autoupdate":{"rollout":50,"debounce":60},"node":{"version":"16.13.2"},"s3":{"bucket":"dfc-data-production","indexVersionLimit":20,"folder":"media/salesforce-cli/oclif-testing","acl":" ","host":"https://developer.salesforce.com","xz":true}},"topics":{"pack":{"description":"package an oclif CLI into installable artifacts"},"upload":{"description":"upload installable CLI artifacts to AWS S3"}}},"repository":"oclif/oclif","scripts":{"build":"shx rm -rf lib && tsc","devcli:lint":"eslint . --ext .ts --config .eslintrc","devcli:test":"mocha --forbid-only \"test/unit/*.test.ts\"","devcli":"yarn build --noEmit && yarn run devcli:test && yarn run devcli:lint","lint":"nps lint","postpack":"shx rm .oclif.manifest.json","posttest":"yarn run lint","prepack":"shx rm -rf lib && tsc && bin/dev manifest .","test":"nps test && yarn test:unit && yarn test:integration","test:integration":"mocha --forbid-only \"test/integration/*.test.ts\"","test:unit":"mocha --forbid-only \"test/unit/*.test.ts\"","version":"bin/dev readme && git add README.md"},"publishConfig":{"registry":"https://registry.npmjs.org"},"types":"lib/index.d.ts"}

src/aws.ts

@@ -1,9 +1,9 @@
 import * as CloudFront from 'aws-sdk/clients/cloudfront'
 import * as S3 from 'aws-sdk/clients/s3'
 import * as fs from 'fs-extra'
-import * as qq from 'qqjs'
 
 import {debug as Debug, log} from './log'
+import {prettifyPaths} from './util'
 
 const debug = Debug.new('aws')
 
@@ -65,7 +65,7 @@ export default {
   get s3() {
     return {
       uploadFile: (local: string, options: S3.Types.PutObjectRequest) => new Promise((resolve, reject) => {
-        log('s3:uploadFile', qq.prettifyPaths(local), `s3://${options.Bucket}/${options.Key}`)
+        log('s3:uploadFile', prettifyPaths(local), `s3://${options.Bucket}/${options.Key}`)
         options.Body = fs.createReadStream(local)
         aws.s3.upload(options, err => {
           if (err) reject(err)

src/commands/pack/deb.ts

@@ -1,11 +1,14 @@
 import {Command, Flags} from '@oclif/core'
 import {Interfaces} from '@oclif/core'
-
+import * as fs from 'fs-extra'
 import * as _ from 'lodash'
-import * as qq from 'qqjs'
-
+import * as path from 'path'
 import * as Tarballs from '../../tarballs'
 import {templateShortKey, debVersion, debArch} from '../../upload-util'
+import {exec as execSync} from 'child_process'
+import {promisify} from 'node:util'
+
+const exec = promisify(execSync)
 
 const scripts = {
   /* eslint-disable no-useless-escape */
@@ -61,23 +64,22 @@ export default class PackDeb extends Command {
     const {config} = buildConfig
     await Tarballs.build(buildConfig, {platform: 'linux', pack: false, tarball: flags.tarball})
     const dist = buildConfig.dist('deb')
-    await qq.emptyDir(dist)
+    await fs.emptyDir(dist)
     const build = async (arch: Interfaces.ArchTypes) => {
       const target: { platform: 'linux'; arch: Interfaces.ArchTypes} = {platform: 'linux', arch}
       const versionedDebBase = templateShortKey('deb', {bin: config.bin, versionShaRevision: debVersion(buildConfig), arch: debArch(arch) as any})
-      const workspace = qq.join(buildConfig.tmp, 'apt', versionedDebBase.replace('.deb', '.apt'))
-      await qq.rm(workspace)
-      await qq.mkdirp([workspace, 'DEBIAN'])
-      await qq.mkdirp([workspace, 'usr/bin'])
-      await qq.mkdirp([workspace, 'usr/lib'])
-      await qq.mv(buildConfig.workspace(target), [workspace, 'usr/lib', config.dirname])
-      await qq.write([workspace, 'usr/lib', config.dirname, 'bin', config.bin], scripts.bin(config))
-      await qq.write([workspace, 'DEBIAN/control'], scripts.control(buildConfig, debArch(arch)))
-      await qq.chmod([workspace, 'usr/lib', config.dirname, 'bin', config.bin], 0o755)
-      await qq.x(`ln -s "../lib/${config.dirname}/bin/${config.bin}" "${workspace}/usr/bin/${config.bin}"`)
-      await qq.x(`sudo chown -R root "${workspace}"`)
-      await qq.x(`sudo chgrp -R root "${workspace}"`)
-      await qq.x(`dpkg --build "${workspace}" "${qq.join(dist, versionedDebBase)}"`)
+      const workspace = path.join(buildConfig.tmp, 'apt', versionedDebBase.replace('.deb', '.apt'))
+      await fs.rm(workspace)
+      await fs.mkdirp(path.join(workspace, 'DEBIAN'))
+      await fs.mkdirp(path.join(workspace, 'usr', 'bin'))
+      await fs.mkdirp(path.join(workspace, 'usr', 'lib'))
+      await fs.move(buildConfig.workspace(target), path.join(workspace, 'usr', 'lib', config.dirname))
+      await fs.writeFile(path.join(workspace, 'usr', 'lib', config.dirname, 'bin', config.bin), scripts.bin(config), {mode: 0o755})
+      await fs.writeFile(path.join(workspace, 'DEBIAN', 'control'), scripts.control(buildConfig, debArch(arch)))
+      await exec(`ln -s "../lib/${config.dirname}/bin/${config.bin}" "${workspace}/usr/bin/${config.bin}"`)
+      await exec(`sudo chown -R root "${workspace}"`)
+      await exec(`sudo chgrp -R root "${workspace}"`)
+      await exec(`dpkg --build "${workspace}" "${path.join(dist, versionedDebBase)}"`)
     }
 
     const arches = _.uniq(buildConfig.targets
@@ -86,17 +88,17 @@ export default class PackDeb extends Command {
     // eslint-disable-next-line no-await-in-loop
     for (const a of arches) await build(a)
 
-    await qq.x('apt-ftparchive packages . > Packages', {cwd: dist})
-    await qq.x('gzip -c Packages > Packages.gz', {cwd: dist})
-    await qq.x('bzip2 -k Packages', {cwd: dist})
-    await qq.x('xz -k Packages', {cwd: dist})
-    const ftparchive = qq.join(buildConfig.tmp, 'apt', 'apt-ftparchive.conf')
-    await qq.write(ftparchive, scripts.ftparchive(config))
-    await qq.x(`apt-ftparchive -c "${ftparchive}" release . > Release`, {cwd: dist})
+    await exec('apt-ftparchive packages . > Packages', {cwd: dist})
+    await exec('gzip -c Packages > Packages.gz', {cwd: dist})
+    await exec('bzip2 -k Packages', {cwd: dist})
+    await exec('xz -k Packages', {cwd: dist})
+    const ftparchive = path.join(buildConfig.tmp, 'apt', 'apt-ftparchive.conf')
+    await fs.writeFile(ftparchive, scripts.ftparchive(config))
+    await exec(`apt-ftparchive -c "${ftparchive}" release . > Release`, {cwd: dist})
     const gpgKey = config.scopedEnvVar('DEB_KEY')
     if (gpgKey) {
-      await qq.x(`gpg --digest-algo SHA512 --clearsign -u ${gpgKey} -o InRelease Release`, {cwd: dist})
-      await qq.x(`gpg --digest-algo SHA512 -abs -u ${gpgKey} -o Release.gpg Release`, {cwd: dist})
+      await exec(`gpg --digest-algo SHA512 --clearsign -u ${gpgKey} -o InRelease Release`, {cwd: dist})
+      await exec(`gpg --digest-algo SHA512 -abs -u ${gpgKey} -o Release.gpg Release`, {cwd: dist})
     }
   }
 }

src/commands/pack/macos.ts

@@ -1,14 +1,16 @@
 import * as path from 'path'
 
 import * as _ from 'lodash'
-import * as qq from 'qqjs'
-
+import * as fs from 'fs-extra'
 import {Command, Flags} from '@oclif/core'
 import {Interfaces} from '@oclif/core'
 
 import * as Tarballs from '../../tarballs'
 import {templateShortKey} from '../../upload-util'
+import {exec as execSync} from 'child_process'
+import {promisify} from 'node:util'
 
+const exec = promisify(execSync)
 type OclifConfig = {
   macos?: {
     identifier?: string;
@@ -77,7 +79,7 @@ while [ "$1" != "-y" ]; do
 done
 
 echo "Application uninstalling process started"
-# remove link to shorcut file
+# remove link to shortcut file
 find "/usr/local/bin/" -name "${config.bin}" | xargs rm
 ${additionalCLI ? `find "/usr/local/bin/" -name "${additionalCLI}" | xargs rm` : ''}
 if [ $? -eq 0 ]
@@ -153,30 +155,33 @@ the CLI should already exist in a directory named after the CLI that is the root
     if (!c.macos.identifier) this.error('package.json must have oclif.macos.identifier set')
     const macos = c.macos
     const packageIdentifier = macos.identifier
-    await Tarballs.build(buildConfig, {platform: 'darwin', pack: false, tarball: flags.tarball})
-    const scriptsDir = qq.join(buildConfig.tmp, 'macos/scripts')
-    await qq.emptyDir(buildConfig.dist('macos'))
-    const noBundleConfigurationPath = qq.join(buildConfig.tmp, 'macos/no-bundle.plist')
+    await Tarballs.build(buildConfig, {platform: 'darwin', pack: false, tarball: flags.tarball, parallel: true})
+    const scriptsDir = path.join(buildConfig.tmp, 'macos/scripts')
+    await fs.emptyDir(buildConfig.dist('macos'))
+    const noBundleConfigurationPath = path.join(buildConfig.tmp, 'macos', 'no-bundle.plist')
 
     const build = async (arch: Interfaces.ArchTypes) => {
       const templateKey = templateShortKey('macos', {bin: config.bin, version: config.version, sha: buildConfig.gitSha, arch})
       const dist = buildConfig.dist(`macos/${templateKey}`)
       const rootDir = buildConfig.workspace({platform: 'darwin', arch})
       const writeNoBundleConfiguration = async () => {
-        await qq.write(noBundleConfigurationPath, noBundleConfiguration)
-        await qq.chmod(noBundleConfigurationPath, 0o755)
+        await fs.mkdir(path.dirname(noBundleConfigurationPath), {recursive: true})
+        await fs.writeFile(noBundleConfigurationPath, noBundleConfiguration, {mode: 0o755})
       }
+
       const writeScript = async (script: 'preinstall' | 'postinstall' | 'uninstall') => {
-        const path = script === 'uninstall' ? [rootDir, 'bin'] : [scriptsDir]
-        path.push(script)
-        await qq.write(path, scripts[script](config, flags['additional-cli']))
-        await qq.chmod(path, 0o755)
+        const scriptLocation = script === 'uninstall' ? [rootDir, 'bin'] : [scriptsDir]
+        scriptLocation.push(script)
+        await fs.mkdir(path.dirname(path.join(...scriptLocation)), {recursive: true})
+        await fs.writeFile(path.join(...scriptLocation), scripts[script](config, flags['additional-cli']), {mode: 0o755})
       }
 
-      await writeNoBundleConfiguration();
-      await writeScript('preinstall')
-      await writeScript('postinstall')
-      await writeScript('uninstall')
+      await Promise.all([
+        writeNoBundleConfiguration(),
+        writeScript('preinstall'),
+        writeScript('postinstall'),
+        writeScript('uninstall'),
+      ])
       /* eslint-disable array-element-newline */
       const args = [
         '--root', rootDir,
@@ -192,13 +197,12 @@ the CLI should already exist in a directory named after the CLI that is the root
       } else this.debug('Skipping macOS pkg signing')
       if (process.env.OSX_KEYCHAIN) args.push('--keychain', process.env.OSX_KEYCHAIN)
       args.push(dist)
-      await qq.x('pkgbuild', args as string[])
+      await exec(`pkgbuild  ${args.join(' ')}`)
     }
 
     const arches = _.uniq(buildConfig.targets
     .filter(t => t.platform === 'darwin')
     .map(t => t.arch))
-    // eslint-disable-next-line no-await-in-loop
-    for (const a of arches) await build(a)
+    await Promise.all(arches.map(a => build(a)))
   }
 }

src/commands/pack/tarballs.ts

@@ -1,5 +1,4 @@
 import {Command, Flags} from '@oclif/core'
-import * as qq from 'qqjs'
 
 import * as Tarballs from '../../tarballs'
 
@@ -18,7 +17,6 @@ This can be used to create oclif CLIs that use the system node or that come prel
   }
 
   async run(): Promise<void> {
-    const prevCwd = qq.cwd()
     if (process.platform === 'win32') throw new Error('pack does not function on windows')
     const {flags} = await this.parse(PackTarballs)
     const buildConfig = await Tarballs.buildConfig(flags.root, {xz: flags.xz, targets: flags?.targets?.split(',')})
@@ -30,6 +28,5 @@ This can be used to create oclif CLIs that use the system node or that come prel
       tarball: flags.tarball,
       parallel: flags.parallel,
     })
-    qq.cd(prevCwd)
   }
 }