Merge pull request #1397 from skirsdeda/darwin-universal

Darwin universal architecture

cli/src/__test__/parse-triple.spec.ts

@@ -113,6 +113,16 @@ const triples = [
       raw: 'armv7-linux-androideabi',
     },
   } as const,
+  {
+    name: 'universal-apple-darwin',
+    expected: {
+      abi: null,
+      arch: 'universal',
+      platform: 'darwin',
+      platformArchABI: 'darwin-universal',
+      raw: 'universal-apple-darwin',
+    },
+  } as const,
 ]
 
 for (const triple of triples) {

cli/src/artifacts.ts

@@ -6,6 +6,7 @@ import { fdir } from 'fdir'
 
 import { getNapiConfig } from './consts'
 import { debugFactory } from './debug'
+import { UniArchsByPlatform } from './parse-triple'
 import { readFileAsync, writeFileAsync } from './utils'
 
 const debug = debugFactory('artifacts')
@@ -38,6 +39,14 @@ export class ArtifactsCommand extends Command {
       join(process.cwd(), this.distDir, platform.platformArchABI),
     )
 
+    const universalSourceBins = new Set(
+      platforms
+        .filter((platform) => platform.arch === 'universal')
+        .flatMap((p) =>
+          UniArchsByPlatform[p.platform].map((a) => `${p.platform}-${a}`),
+        ),
+    )
+
     await sourceApi.withPromise().then((output) =>
       Promise.all(
         (output as string[]).map(async (filePath) => {
@@ -51,8 +60,17 @@ export class ArtifactsCommand extends Command {
                 _binaryName,
               )}] is not matched with [${chalk.greenBright(binaryName)}], skip`,
             )
+            return
           }
           const dir = distDirs.find((dir) => dir.includes(platformArchABI))
+          if (!dir && universalSourceBins.has(platformArchABI)) {
+            debug(
+              `[${chalk.yellowBright(
+                platformArchABI,
+              )}] has no dist dir but it is source bin for universal arch, skip`,
+            )
+            return
+          }
           if (!dir) {
             throw new TypeError(`No dist dir found for ${filePath}`)
           }

cli/src/create-npm-dir.ts

@@ -47,7 +47,10 @@ export class CreateNpmDirCommand extends Command {
         name: `${packageName}-${platformDetail.platformArchABI}`,
         version,
         os: [platformDetail.platform],
-        cpu: [platformDetail.arch],
+        cpu:
+          platformDetail.arch !== 'universal'
+            ? [platformDetail.arch]
+            : undefined,
         main: binaryFileName,
         files: [binaryFileName],
         ...pick(

cli/src/index.ts

@@ -11,6 +11,7 @@ import { HelpCommand } from './help'
 import { NewProjectCommand } from './new'
 import { PrePublishCommand } from './pre-publish'
 import { RenameCommand } from './rename'
+import { UniversalCommand } from './universal'
 import { VersionCommand } from './version'
 
 const cli = new Cli({
@@ -23,6 +24,7 @@ cli.register(BuildCommand)
 cli.register(CreateNpmDirCommand)
 cli.register(PrePublishCommand)
 cli.register(VersionCommand)
+cli.register(UniversalCommand)
 cli.register(NewProjectCommand)
 cli.register(RenameCommand)
 cli.register(HelpCommand)

cli/src/js-binding-template.ts

@@ -105,6 +105,15 @@ switch (platform) {
     }
     break
   case 'darwin':
+    localFileExisted = existsSync(join(__dirname, '${localName}.darwin-universal.node'))
+    try {
+      if (localFileExisted) {
+        nativeBinding = require('./${localName}.darwin-universal.node')
+      } else {
+        nativeBinding = require('${pkgName}-darwin-universal')
+      }
+      break
+    } catch {}
     switch (arch) {
       case 'x64':
         localFileExisted = existsSync(join(__dirname, '${localName}.darwin-x64.node'))

cli/src/new/ci-template.ts

@@ -57,11 +57,6 @@ jobs:
           - host: macos-latest
             target: 'aarch64-apple-darwin'
             build: |
-              sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/*;
-              export CC=$(xcrun -f clang);
-              export CXX=$(xcrun -f clang++);
-              SYSROOT=$(xcrun --sdk macosx --show-sdk-path);
-              export CFLAGS="-isysroot $SYSROOT -isystem $SYSROOT";
               yarn build --target aarch64-apple-darwin
               strip -x *.node
           - host: ubuntu-latest
@@ -146,9 +141,9 @@ jobs:
           key: \${{ matrix.settings.target }}-cargo-\${{ matrix.settings.host }}
 
       - uses: goto-bus-stop/setup-zig@v2
-          if: \${{ matrix.settings.target == 'armv7-unknown-linux-gnueabihf' }}
-          with:
-            version: 0.10.0
+        if: \${{ matrix.settings.target == 'armv7-unknown-linux-gnueabihf' }}
+        with:
+          version: 0.10.0
 
       - name: Setup toolchain
         run: \${{ matrix.settings.setup }}
@@ -512,6 +507,52 @@ jobs:
             yarn test
             ls -la
 
+  universal-macOS:
+    name: Build universal macOS binary
+    needs:
+      - build
+    runs-on: macos-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+          check-latest: true
+          cache: yarn
+
+      - name: Cache NPM dependencies
+        uses: actions/cache@v3
+        with:
+          path: .yarn/cache
+          key: npm-cache-test-x86_64-apple-darwin-16-\${{ hashFiles('yarn.lock') }}
+
+      - name: 'Install dependencies'
+        run: yarn install
+
+      - name: Download macOS x64 artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: bindings-x86_64-apple-darwin
+          path: artifacts
+      - name: Download macOS arm64 artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: bindings-aarch64-apple-darwin
+          path: artifacts
+
+      - name: Combine binaries
+        run: yarn universal
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: bindings-universal-apple-darwin
+          path: \${{ env.APP_NAME }}.*.node
+          if-no-files-found: error
+
   publish:
     name: Publish
     runs-on: ubuntu-latest

cli/src/new/ci-yml.ts

@@ -1,5 +1,7 @@
 import { load, dump } from 'js-yaml'
 
+import { NodeArchToCpu, UniArchsByPlatform, parseTriple } from '../parse-triple'
+
 import { YAML } from './ci-template'
 
 const BUILD_FREEBSD = 'build-freebsd'
@@ -9,25 +11,39 @@ const TEST_LINUX_X64_MUSL = 'test-linux-x64-musl-binding'
 const TEST_LINUX_AARCH64_GNU = 'test-linux-aarch64-gnu-binding'
 const TEST_LINUX_AARCH64_MUSL = 'test-linux-aarch64-musl-binding'
 const TEST_LINUX_ARM_GNUEABIHF = 'test-linux-arm-gnueabihf-binding'
+const UNIVERSAL_MACOS = 'universal-macOS'
 
 export const createGithubActionsCIYml = (
   binaryName: string,
   targets: string[],
 ) => {
+  const allTargets = new Set(
+    targets.flatMap((t) => {
+      const platform = parseTriple(t)
+      if (platform.arch === 'universal') {
+        const srcTriples = UniArchsByPlatform[platform.platform]?.map((arch) =>
+          t.replace('universal', NodeArchToCpu[arch]),
+        )
+        return [t, ...(srcTriples ?? [])]
+      }
+      return [t]
+    }),
+  )
   const fullTemplate = load(YAML(binaryName)) as any
   const requiredSteps = []
-  const enableWindowsX86 = targets.includes('x86_64-pc-windows-msvc')
-  const enableMacOSX86 = targets.includes('x86_64-apple-darwin')
-  const enableLinuxX86Gnu = targets.includes('x86_64-unknown-linux-gnu')
-  const enableLinuxX86Musl = targets.includes('x86_64-unknown-linux-musl')
-  const enableLinuxArm8Gnu = targets.includes('aarch64-unknown-linux-gnu')
-  const enableLinuxArm8Musl = targets.includes('aarch64-unknown-linux-musl')
-  const enableLinuxArm7 = targets.includes('armv7-unknown-linux-gnueabihf')
-  const enableFreeBSD = targets.includes('x86_64-unknown-freebsd')
+  const enableWindowsX86 = allTargets.has('x86_64-pc-windows-msvc')
+  const enableMacOSX86 = allTargets.has('x86_64-apple-darwin')
+  const enableLinuxX86Gnu = allTargets.has('x86_64-unknown-linux-gnu')
+  const enableLinuxX86Musl = allTargets.has('x86_64-unknown-linux-musl')
+  const enableLinuxArm8Gnu = allTargets.has('aarch64-unknown-linux-gnu')
+  const enableLinuxArm8Musl = allTargets.has('aarch64-unknown-linux-musl')
+  const enableLinuxArm7 = allTargets.has('armv7-unknown-linux-gnueabihf')
+  const enableFreeBSD = allTargets.has('x86_64-unknown-freebsd')
+  const enableMacOSUni = allTargets.has('universal-apple-darwin')
   fullTemplate.env.APP_NAME = binaryName
   fullTemplate.jobs.build.strategy.matrix.settings =
     fullTemplate.jobs.build.strategy.matrix.settings.filter(
-      ({ target }: { target: string }) => targets.includes(target),
+      ({ target }: { target: string }) => allTargets.has(target),
     )
   if (!fullTemplate.jobs.build.strategy.matrix.settings.length) {
     delete fullTemplate.jobs.build.strategy.matrix
@@ -81,6 +97,12 @@ export const createGithubActionsCIYml = (
     requiredSteps.push(TEST_LINUX_ARM_GNUEABIHF)
   }
 
+  if (!enableMacOSUni) {
+    delete fullTemplate.jobs[UNIVERSAL_MACOS]
+  } else {
+    requiredSteps.push(UNIVERSAL_MACOS)
+  }
+
   fullTemplate.jobs.publish.needs = requiredSteps
 
   return dump(fullTemplate, {

cli/src/new/index.ts

@@ -45,6 +45,7 @@ const SupportedPlatforms: string[] = [
   'x86_64-unknown-freebsd',
   'i686-pc-windows-msvc',
   'armv7-linux-androideabi',
+  'universal-apple-darwin',
 ]
 
 export class NewProjectCommand extends Command {

cli/src/new/package.ts

@@ -31,6 +31,7 @@ export const createPackageJson = (
       'build:debug': 'napi build --platform',
       prepublishOnly: 'napi prepublish -t npm',
       test: 'ava',
+      universal: 'napi universal',
       version: 'napi version',
     },
   }

cli/src/parse-triple.ts

@@ -13,6 +13,7 @@ type NodeJSArch =
   | 's390x'
   | 'x32'
   | 'x64'
+  | 'universal'
 
 const CpuToNodeArch: { [index: string]: NodeJSArch } = {
   x86_64: 'x64',
@@ -21,13 +22,24 @@ const CpuToNodeArch: { [index: string]: NodeJSArch } = {
   armv7: 'arm',
 }
 
+export const NodeArchToCpu: { [index: string]: string } = {
+  x64: 'x86_64',
+  arm64: 'aarch64',
+  ia32: 'i686',
+  arm: 'armv7',
+}
+
 const SysToNodePlatform: { [index: string]: NodeJS.Platform } = {
   linux: 'linux',
   freebsd: 'freebsd',
   darwin: 'darwin',
   windows: 'win32',
 }
 
+export const UniArchsByPlatform: Record<string, NodeJSArch[]> = {
+  darwin: ['x64', 'arm64'],
+}
+
 export interface PlatformDetail {
   platform: NodeJS.Platform
   platformArchABI: string

cli/src/universal.ts

@@ -0,0 +1,79 @@
+import { spawnSync } from 'child_process'
+import { join } from 'path'
+
+import chalk from 'chalk'
+import { Command, Option } from 'clipanion'
+
+import { getNapiConfig } from './consts'
+import { debugFactory } from './debug'
+import { UniArchsByPlatform } from './parse-triple'
+import { fileExists } from './utils'
+
+const debug = debugFactory('universal')
+
+export class UniversalCommand extends Command {
+  static usage = Command.Usage({
+    description: 'Combine built binaries to universal binaries',
+  })
+
+  static paths = [['universal']]
+
+  sourceDir = Option.String('-d,--dir', 'artifacts')
+
+  distDir = Option.String('--dist', '.')
+
+  configFileName?: string = Option.String('-c,--config')
+
+  buildUniversal: Record<
+    keyof typeof UniArchsByPlatform,
+    (binName: string, srcFiles: string[]) => string
+  > = {
+    darwin: (binName, srcFiles) => {
+      const outPath = join(
+        this.distDir,
+        `${binName}.${process.platform}-universal.node`,
+      )
+      const srcPaths = srcFiles.map((f) => join(this.sourceDir, f))
+      spawnSync('lipo', ['-create', '-output', outPath, ...srcPaths])
+      return outPath
+    },
+  }
+
+  async execute() {
+    const { platforms, binaryName } = getNapiConfig(this.configFileName)
+
+    const targetPlatform = platforms.find(
+      (p) => p.platform === process.platform && p.arch === 'universal',
+    )
+    if (!targetPlatform) {
+      throw new TypeError(
+        `'universal' arch for platform '${process.platform}' not found in config!`,
+      )
+    }
+
+    const srcFiles = UniArchsByPlatform[process.platform]?.map(
+      (a) => `${binaryName}.${process.platform}-${a}.node`,
+    )
+    if (!srcFiles) {
+      throw new TypeError(
+        `'universal' arch for platform '${process.platform}' not supported.`,
+      )
+    }
+
+    debug(
+      `Looking up source binaries to combine: ${chalk.yellowBright(srcFiles)}`,
+    )
+    const srcFileLookup = await Promise.all(
+      srcFiles.map((f) => fileExists(join(this.sourceDir, f))),
+    )
+    const notFoundFiles = srcFiles.filter((_f, i) => !srcFileLookup[i])
+    if (notFoundFiles.length > 0) {
+      throw new TypeError(
+        `Some binary files were not found: ${JSON.stringify(notFoundFiles)}`,
+      )
+    }
+
+    const outPath = this.buildUniversal[process.platform](binaryName, srcFiles)
+    debug(`Produced universal binary: ${outPath}`)
+  }
+}