Merge pull request #165 from crazy-max/append

append nodes to builder support
docker · Oct 14, 2022 · 59b5ed6 · 59b5ed6
2 parents 95cb08c + bd61d52
commit 59b5ed6
Show file tree

Hide file tree

Showing 15 changed files with 365 additions and 36 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -305,9 +305,13 @@ jobs:
           platforms: ${{ matrix.qemu-platforms }}
       -
         name: Set up Docker Buildx
+        id: buildx
         uses: ./
         with:
           version: ${{ matrix.buildx-version }}
+      -
+        name: List builder platforms
+        run: echo ${{ steps.buildx.outputs.platforms }}
 
   build-ref:
     runs-on: ubuntu-latest
@@ -416,3 +420,47 @@ jobs:
             echo "::error::Should have failed"
             exit 1
           fi
+
+  append:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Create dummy contexts
+        run: |
+          docker context create ctxbuilder2
+          docker context create ctxbuilder3
+      -
+        name: Set up Docker Buildx
+        id: buildx
+        uses: ./
+        with:
+          append: |
+            - name: builder2
+              endpoint: ctxbuilder2
+              platforms: linux/amd64
+              driver-opts:
+                - image=moby/buildkit:master
+                - network=host
+            - endpoint: ctxbuilder3
+              platforms: linux/arm64
+      -
+        name: List builder platforms
+        run: echo ${{ steps.buildx.outputs.platforms }}
+
+  platforms:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      -
+        name: Set up Docker Buildx
+        uses: ./
+        with:
+          platforms: linux/amd64
diff --git a/README.md b/README.md
@@ -21,6 +21,7 @@ ___
 * [Usage](#usage)
 * [Advanced usage](#advanced-usage)
   * [Authentication support](docs/advanced/auth.md)
+  * [Append additional nodes to the builder](docs/advanced/append-nodes.md)
   * [Install by default](docs/advanced/install-default.md)
   * [BuildKit daemon configuration](docs/advanced/buildkit-config.md)
   * [Standalone mode](docs/advanced/standalone.md)
@@ -61,6 +62,7 @@ jobs:
 ## Advanced usage
 
 * [Authentication support](docs/advanced/auth.md)
+* [Append additional nodes to the builder](docs/advanced/append-nodes.md)
 * [Install by default](docs/advanced/install-default.md)
 * [BuildKit daemon configuration](docs/advanced/buildkit-config.md)
 * [Standalone mode](docs/advanced/standalone.md)
@@ -69,32 +71,36 @@ jobs:
 
 ### inputs
 
-Following inputs can be used as `step.with` keys
+Following inputs can be used as `step.with` keys:
 
-| Name              | Type   | Description                                                                                                                                                                                     |
-|-------------------|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `version`         | String | [Buildx](https://github.com/docker/buildx) version. (eg. `v0.3.0`, `latest`, `https://github.com/docker/buildx.git#master`)                                                                     |
-| `driver`          | String | Sets the [builder driver](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver) to be used (default `docker-container`)                                                   |
-| `driver-opts`     | CSV    | List of additional [driver-specific options](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver-opt) (eg. `image=moby/buildkit:master`)                                 |
-| `buildkitd-flags` | String | [Flags for buildkitd](https://docs.docker.com/engine/reference/commandline/buildx_create/#buildkitd-flags) daemon (since [buildx v0.3.0](https://github.com/docker/buildx/releases/tag/v0.3.0)) |
-| `install`         | Bool   | Sets up `docker build` command as an alias to `docker buildx` (default `false`)                                                                                                                 |
-| `use`             | Bool   | Switch to this builder instance (default `true`)                                                                                                                                                |
-| `endpoint`        | String | [Optional address for docker socket](https://docs.docker.com/engine/reference/commandline/buildx_create/#description) or context from `docker context ls`                                       |
-| `config`¹         | String | [BuildKit config file](https://docs.docker.com/engine/reference/commandline/buildx_create/#config)                                                                                              |
-| `config-inline`¹  | String | Same as `config` but inline                                                                                                                                                                     |
-
-> * ¹ `config` and `config-inline` are mutually exclusive
-
-> `CSV` type must be a newline-delimited string
-> ```yaml
-> driver-opts: image=moby/buildkit:master
-> ```
+> `List` type is a newline-delimited string
 > ```yaml
 > driver-opts: |
 >   image=moby/buildkit:master
 >   network=host
 > ```
 
+> `CSV` type must be a newline-delimited string
+> ```yaml
+> platforms: linux/amd64,linux/arm64
+> ```
+
+| Name              | Type     | Description                                                                                                                                                                                     |
+|-------------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `version`         | String   | [Buildx](https://github.com/docker/buildx) version. (eg. `v0.3.0`, `latest`, `https://github.com/docker/buildx.git#master`)                                                                     |
+| `driver`          | String   | Sets the [builder driver](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver) to be used (default `docker-container`)                                                   |
+| `driver-opts`     | List     | List of additional [driver-specific options](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver-opt) (eg. `image=moby/buildkit:master`)                                 |
+| `buildkitd-flags` | String   | [Flags for buildkitd](https://docs.docker.com/engine/reference/commandline/buildx_create/#buildkitd-flags) daemon (since [buildx v0.3.0](https://github.com/docker/buildx/releases/tag/v0.3.0)) |
+| `install`         | Bool     | Sets up `docker build` command as an alias to `docker buildx` (default `false`)                                                                                                                 |
+| `use`             | Bool     | Switch to this builder instance (default `true`)                                                                                                                                                |
+| `endpoint`        | String   | [Optional address for docker socket](https://docs.docker.com/engine/reference/commandline/buildx_create/#description) or context from `docker context ls`                                       |
+| `platforms`       | List/CSV | Fixed [platforms](https://docs.docker.com/engine/reference/commandline/buildx_create/#platform) for current node. If not empty, values take priority over the detected ones.                    |
+| `config`¹         | String   | [BuildKit config file](https://docs.docker.com/engine/reference/commandline/buildx_create/#config)                                                                                              |
+| `config-inline`¹  | String   | Same as `config` but inline                                                                                                                                                                     |
+| `append`          | YAML     | [Append additional nodes](docs/advanced/append-nodes.md) to the builder                                                                                                                         |
+
+> * ¹ `config` and `config-inline` are mutually exclusive
+
 ### outputs
 
 Following outputs are available

diff --git a/__tests__/context.test.ts b/__tests__/context.test.ts
@@ -4,6 +4,7 @@ import * as os from 'os';
 import * as path from 'path';
 import * as uuid from 'uuid';
 import * as context from '../src/context';
+import * as nodes from '../src/nodes';
 
 const tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), 'docker-setup-buildx-')).split(path.sep).join(path.posix.sep);
 jest.spyOn(context, 'tmpDir').mockImplementation((): string => {
@@ -90,6 +91,24 @@ describe('getCreateArgs', () => {
         'tls://foo:1234'
       ]
     ],
+    [
+      4,
+      new Map<string, string>([
+        ['driver', 'remote'],
+        ['platforms', 'linux/arm64,linux/arm/v7'],
+        ['endpoint', 'tls://foo:1234'],
+        ['install', 'false'],
+        ['use', 'true'],
+      ]),
+      [
+        'create',
+        '--name', 'builder-9b1deb4d-3b7d-4bad-9bdd-2b0d7b3dcb6d',
+        '--driver', 'remote',
+        '--platform', 'linux/arm64,linux/arm/v7',
+        '--use',
+        'tls://foo:1234'
+      ]
+    ],
   ])(
     '[%d] given %p as inputs, returns %p',
     async (num: number, inputs: Map<string, string>, expected: Array<string>) => {
@@ -103,6 +122,57 @@ describe('getCreateArgs', () => {
   );
 });
 
+describe('getAppendArgs', () => {
+  beforeEach(() => {
+    process.env = Object.keys(process.env).reduce((object, key) => {
+      if (!key.startsWith('INPUT_')) {
+        object[key] = process.env[key];
+      }
+      return object;
+    }, {});
+  });
+
+  // prettier-ignore
+  test.each([
+    [
+      0,
+      new Map<string, string>([
+        ['install', 'false'],
+        ['use', 'true'],
+      ]),
+      {
+        "name": "aws_graviton2",
+        "endpoint": "ssh://me@graviton2",
+        "driver-opts": [
+          "image=moby/buildkit:latest"
+        ],
+        "buildkitd-flags": "--allow-insecure-entitlement security.insecure --allow-insecure-entitlement network.host",
+        "platforms": "linux/arm64"
+      },
+      [
+        'create',
+        '--name', 'builder-9b1deb4d-3b7d-4bad-9bdd-2b0d7b3dcb6d',
+        '--append',
+        '--node', 'aws_graviton2',
+        '--driver-opt', 'image=moby/buildkit:latest',
+        '--buildkitd-flags', '--allow-insecure-entitlement security.insecure --allow-insecure-entitlement network.host',
+        '--platform', 'linux/arm64',
+        'ssh://me@graviton2'
+      ]
+    ]
+  ])(
+    '[%d] given %p as inputs, returns %p',
+    async (num: number, inputs: Map<string, string>, node: nodes.Node, expected: Array<string>) => {
+      inputs.forEach((value: string, name: string) => {
+        setInput(name, value);
+      });
+      const inp = await context.getInputs();
+      const res = await context.getAppendArgs(inp, node, '0.9.0');
+      expect(res).toEqual(expected);
+    }
+  );
+});
+
 describe('getInputList', () => {
   it('handles single line correctly', async () => {
     await setInput('foo', 'bar');

diff --git a/action.yml b/action.yml
@@ -32,12 +32,18 @@ inputs:
   endpoint:
     description: 'Optional address for docker socket or context from `docker context ls`'
     required: false
+  platforms:
+    description: 'Fixed platforms for current node. If not empty, values take priority over the detected ones'
+    required: false
   config:
     description: 'BuildKit config file'
     required: false
   config-inline:
     description: 'Inline BuildKit config'
     required: false
+  append:
+    description: 'Append additional nodes to the builder'
+    required: false
 
 outputs:
   name:

diff --git a/dist/index.js b/dist/index.js
diff --git a/dist/index.js.map b/dist/index.js.map
diff --git a/dist/licenses.txt b/dist/licenses.txt
diff --git a/docs/advanced/append-nodes.md b/docs/advanced/append-nodes.md
@@ -0,0 +1,56 @@
+# Append additional nodes to the builder
+
+Buildx also supports running builds on multiple machines. This is useful for
+building [multi-platform images](https://docs.docker.com/build/building/multi-platform/)
+on native nodes for more complicated cases that are not handled by QEMU and
+generally have better performance or for distributing the build across multiple
+machines.
+
+You can append nodes to the builder that is going to be created with the
+`append` input in the form of a YAML string document to remove limitations
+intrinsically linked to GitHub Actions (only string format is handled in the
+input fields):
+
+| Name              | Type   | Description                                                                                                                                                                                                                                                                           |
+|-------------------|--------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `name`            | String | [Name of the node](https://docs.docker.com/engine/reference/commandline/buildx_create/#node). If empty, it is the name of the builder it belongs to, with an index number suffix. This is useful to set it if you want to modify/remove a node in an underlying step of you workflow. |
+| `endpoint`        | String | [Docker context or endpoint](https://docs.docker.com/engine/reference/commandline/buildx_create/#description) of the node to add to the builder                                                                                                                                       |
+| `driver-opts`     | List   | List of additional [driver-specific options](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver-opt)                                                                                                                                                          |
+| `buildkitd-flags` | String | [Flags for buildkitd](https://docs.docker.com/engine/reference/commandline/buildx_create/#buildkitd-flags) daemon                                                                                                                                                                     |
+| `platforms`       | String | Fixed [platforms](https://docs.docker.com/engine/reference/commandline/buildx_create/#platform) for the node. If not empty, values take priority over the detected ones.                                                                                                              |
+
+Here is an example using remote nodes with the [`remote` driver](https://docs.docker.com/build/building/drivers/remote/)
+and [TLS authentication](auth.md#tls-authentication):
+
+```yaml
+name: ci
+
+on:
+  push:
+
+jobs:
+  buildx:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver: remote
+          endpoint: tcp://oneprovider:1234
+          append: |
+            - endpoint: tcp://graviton2:1234
+              platforms: linux/arm64
+            - endpoint: tcp://linuxone:1234
+              platforms: linux/s390x
+        env:
+          BUILDER_NODE_0_AUTH_TLS_CACERT: ${{ secrets.ONEPROVIDER_CA }}
+          BUILDER_NODE_0_AUTH_TLS_CERT: ${{ secrets.ONEPROVIDER_CERT }}
+          BUILDER_NODE_0_AUTH_TLS_KEY: ${{ secrets.ONEPROVIDER_KEY }}
+          BUILDER_NODE_1_AUTH_TLS_CACERT: ${{ secrets.GRAVITON2_CA }}
+          BUILDER_NODE_1_AUTH_TLS_CERT: ${{ secrets.GRAVITON2_CERT }}
+          BUILDER_NODE_1_AUTH_TLS_KEY: ${{ secrets.GRAVITON2_KEY }}
+          BUILDER_NODE_2_AUTH_TLS_CACERT: ${{ secrets.LINUXONE_CA }}
+          BUILDER_NODE_2_AUTH_TLS_CERT: ${{ secrets.LINUXONE_CERT }}
+          BUILDER_NODE_2_AUTH_TLS_KEY: ${{ secrets.LINUXONE_KEY }}
+```
diff --git a/docs/advanced/auth.md b/docs/advanced/auth.md
@@ -41,11 +41,6 @@ the node in the list of nodes:
 * `BUILDER_NODE_<idx>_AUTH_TLS_CERT`
 * `BUILDER_NODE_<idx>_AUTH_TLS_KEY`
 
-> **Note**
-> 
-> The index is always `0` at the moment as we don't support (yet) appending new
-> nodes with this action.
-
 ```yaml
 name: ci
 

diff --git a/jest.config.ts b/jest.config.ts
@@ -1,10 +1,13 @@
 module.exports = {
   clearMocks: true,
   moduleFileExtensions: ['js', 'ts'],
-  setupFiles: ["dotenv/config"],
+  setupFiles: ['dotenv/config'],
   testMatch: ['**/*.test.ts'],
   transform: {
     '^.+\\.ts$': 'ts-jest'
   },
+  moduleNameMapper: {
+    '^csv-parse/sync': '<rootDir>/node_modules/csv-parse/dist/cjs/sync.cjs'
+  },
   verbose: true
-}
+};