feat(core): allow asset bundling on docker remote host / docker in do…

…cker (#23576)

Fixes #8799 

This implements an alternative variant on how to put get files into bundling containers. This is more flexible in its use cases for complex Docker setup scenarios but more complex and slower. Therefore it is not enabled as a default, but as an additional option.

For details to the approach please refer to the linked issue.

----

### All Submissions:

* [X] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md)

### Adding new Construct Runtime Dependencies:

* [ ] This PR adds new construct runtime dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-construct-runtime-dependencies)

### New Features

* [X] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)?
	* [x] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)?

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

packages/@aws-cdk/aws-lambda-go/README.md

@@ -284,3 +284,17 @@ and Go only includes dependencies that are used in the executable. So in this ca
 if `cmd/api` used the `auth` & `middleware` packages, but `cmd/anotherApi` did not, then
 an update to `auth` or `middleware` would only trigger an update to the `cmd/api` Lambda
 Function.
+
+## Docker based bundling in complex Docker configurations
+
+By default the input and output of Docker based bundling is handled via bind mounts.
+In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations.
+
+ ```ts
+new go.GoFunction(this, 'GoFunction', {
+  entry: 'app/cmd/api',
+  bundling: {
+       bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
+   },
+});
+```

packages/@aws-cdk/aws-lambda-go/lib/bundling.ts

@@ -60,6 +60,12 @@ export interface BundlingProps extends BundlingOptions {
    * The system architecture of the lambda function
    */
   readonly architecture: Architecture;
+
+  /**
+   * Which option to use to copy the source files to the docker container and output files back
+   * @default - BundlingFileAccess.BIND_MOUNT
+   */
+  readonly bundlingFileAccess?: cdk.BundlingFileAccess;
 }
 
 /**
@@ -85,6 +91,7 @@ export class Bundling implements cdk.BundlingOptions {
         user: bundling.user,
         securityOpt: bundling.securityOpt,
         network: bundling.network,
+        bundlingFileAccess: bundling.bundlingFileAccess,
       },
     });
   }
@@ -107,6 +114,7 @@ export class Bundling implements cdk.BundlingOptions {
   public readonly user?: string;
   public readonly securityOpt?: string;
   public readonly network?: string;
+  public readonly bundlingFileAccess?: cdk.BundlingFileAccess;
 
   private readonly relativeEntryPath: string;
 
@@ -154,6 +162,7 @@ export class Bundling implements cdk.BundlingOptions {
     this.user = props.user;
     this.securityOpt = props.securityOpt;
     this.network = props.network;
+    this.bundlingFileAccess = props.bundlingFileAccess;
 
     // Local bundling
     if (!props.forcedDockerBundling) { // only if Docker is not forced

packages/@aws-cdk/aws-lambda-go/lib/types.ts

@@ -1,4 +1,4 @@
-import { AssetHashType, DockerImage, DockerRunOptions } from '@aws-cdk/core';
+import { AssetHashType, BundlingFileAccess, DockerImage, DockerRunOptions } from '@aws-cdk/core';
 
 /**
  * Bundling options
@@ -111,6 +111,12 @@ export interface BundlingOptions extends DockerRunOptions {
    * @default - Direct access
    */
   readonly goProxies?: string[];
+
+  /**
+   * Which option to use to copy the source files to the docker container and output files back
+   * @default - BundlingFileAccess.BIND_MOUNT
+   */
+  readonly bundlingFileAccess?: BundlingFileAccess;
 }
 
 /**

packages/@aws-cdk/aws-lambda-go/rosetta/default.ts-fixture

@@ -1,6 +1,6 @@
 // Fixture with packages imported, but nothing else
 import { Construct } from 'constructs';
-import { DockerImage, Stack } from '@aws-cdk/core';
+import { DockerImage, Stack, BundlingFileAccess } from '@aws-cdk/core';
 import * as go from '@aws-cdk/aws-lambda-go';
 
 class Fixture extends Stack {

packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts

@@ -2,7 +2,7 @@ import * as child_process from 'child_process';
 import * as os from 'os';
 import * as path from 'path';
 import { Architecture, Code, Runtime } from '@aws-cdk/aws-lambda';
-import { AssetHashType, DockerImage } from '@aws-cdk/core';
+import { AssetHashType, BundlingFileAccess, DockerImage } from '@aws-cdk/core';
 import { Bundling } from '../lib/bundling';
 import * as util from '../lib/util';
 
@@ -461,3 +461,21 @@ test('Custom bundling network', () => {
     }),
   });
 });
+
+test('Custom bundling file copy variant', () => {
+  Bundling.bundle({
+    entry,
+    moduleDir,
+    runtime: Runtime.GO_1_X,
+    architecture: Architecture.X86_64,
+    forcedDockerBundling: true,
+    bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
+  });
+
+  expect(Code.fromAsset).toHaveBeenCalledWith('/project', {
+    assetHashType: AssetHashType.OUTPUT,
+    bundling: expect.objectContaining({
+      bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
+    }),
+  });
+});

packages/@aws-cdk/aws-lambda-nodejs/README.md

@@ -337,3 +337,16 @@ new nodejs.NodejsFunction(this, 'my-handler', {
 
 If you chose to customize the hash, you will need to make sure it is updated every time the asset
 changes, or otherwise it is possible that some deployments will not be invalidated.
+
+## Docker based bundling in complex Docker configurations
+
+By default the input and output of Docker based bundling is handled via bind mounts.
+In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations.
+
+ ```ts
+ new nodejs.NodejsFunction(this, 'my-handler', {
+  bundling: {
+    bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
+  },
+});
+```

packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts

@@ -43,6 +43,11 @@ export interface BundlingProps extends BundlingOptions {
    */
   readonly preCompilation?: boolean
 
+  /**
+   * Which option to use to copy the source files to the docker container and output files back
+   * @default - BundlingFileAccess.BIND_MOUNT
+   */
+  readonly bundlingFileAccess?: cdk.BundlingFileAccess;
 }
 
 /**
@@ -83,6 +88,7 @@ export class Bundling implements cdk.BundlingOptions {
   public readonly securityOpt?: string;
   public readonly network?: string;
   public readonly local?: cdk.ILocalBundling;
+  public readonly bundlingFileAccess?: cdk.BundlingFileAccess;
 
   private readonly projectRoot: string;
   private readonly relativeEntryPath: string;
@@ -154,6 +160,7 @@ export class Bundling implements cdk.BundlingOptions {
     this.user = props.user;
     this.securityOpt = props.securityOpt;
     this.network = props.network;
+    this.bundlingFileAccess = props.bundlingFileAccess;
 
     // Local bundling
     if (!props.forceDockerBundling) { // only if Docker is not forced

packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts

@@ -1,4 +1,4 @@
-import { DockerImage, DockerRunOptions } from '@aws-cdk/core';
+import { BundlingFileAccess, DockerImage, DockerRunOptions } from '@aws-cdk/core';
 
 /**
  * Bundling options
@@ -301,6 +301,12 @@ export interface BundlingOptions extends DockerRunOptions {
    * @default - no code is injected
    */
   readonly inject?: string[]
+
+  /**
+   * Which option to use to copy the source files to the docker container and output files back
+   * @default - BundlingFileAccess.BIND_MOUNT
+   */
+  readonly bundlingFileAccess?: BundlingFileAccess;
 }
 
 /**

packages/@aws-cdk/aws-lambda-nodejs/rosetta/default.ts-fixture

@@ -1,6 +1,6 @@
 // Fixture with packages imported, but nothing else
 import { Construct } from 'constructs';
-import { DockerImage, Stack } from '@aws-cdk/core';
+import { DockerImage, Stack, BundlingFileAccess } from '@aws-cdk/core';
 import * as nodejs from '@aws-cdk/aws-lambda-nodejs';
 
 class Fixture extends Stack {

packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts

@@ -2,7 +2,7 @@ import * as child_process from 'child_process';
 import * as os from 'os';
 import * as path from 'path';
 import { Architecture, Code, Runtime, RuntimeFamily } from '@aws-cdk/aws-lambda';
-import { AssetHashType, DockerImage } from '@aws-cdk/core';
+import { AssetHashType, BundlingFileAccess, DockerImage } from '@aws-cdk/core';
 import { version as delayVersion } from 'delay/package.json';
 import { Bundling } from '../lib/bundling';
 import { PackageInstallation } from '../lib/package-installation';
@@ -826,3 +826,22 @@ test('Custom bundling network', () => {
     }),
   });
 });
+
+test('Custom bundling file copy variant', () => {
+  Bundling.bundle({
+    entry,
+    projectRoot,
+    depsLockFilePath,
+    runtime: Runtime.NODEJS_14_X,
+    architecture: Architecture.X86_64,
+    forceDockerBundling: true,
+    bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
+  });
+
+  expect(Code.fromAsset).toHaveBeenCalledWith('/project', {
+    assetHashType: AssetHashType.OUTPUT,
+    bundling: expect.objectContaining({
+      bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
+    }),
+  });
+});

packages/@aws-cdk/aws-lambda-python/README.md

@@ -252,3 +252,20 @@ an array of commands to run. Commands are chained with `&&`.
 
 The commands will run in the environment in which bundling occurs: inside the
 container for Docker bundling or on the host OS for local bundling.
+
+## Docker based bundling in complex Docker configurations
+
+By default the input and output of Docker based bundling is handled via bind mounts.
+In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations.
+
+```ts
+const entry = '/path/to/function';
+
+new python.PythonFunction(this, 'function', {
+  entry,
+  runtime: Runtime.PYTHON_3_8,
+  bundling: {
+    bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
+  },
+});
+```

packages/@aws-cdk/aws-lambda-python/lib/bundling.ts

@@ -1,6 +1,6 @@
 import * as path from 'path';
 import { Architecture, AssetCode, Code, Runtime } from '@aws-cdk/aws-lambda';
-import { AssetStaging, BundlingOptions as CdkBundlingOptions, DockerImage, DockerVolume } from '@aws-cdk/core';
+import { AssetStaging, BundlingFileAccess, BundlingOptions as CdkBundlingOptions, DockerImage, DockerVolume } from '@aws-cdk/core';
 import { Packaging, DependenciesFile } from './packaging';
 import { BundlingOptions, ICommandHooks } from './types';
 
@@ -41,6 +41,12 @@ export interface BundlingProps extends BundlingOptions {
    * @default - Does not skip bundling
    */
   readonly skip?: boolean;
+
+  /**
+   * Which option to use to copy the source files to the docker container and output files back
+   * @default - BundlingFileAccess.BIND_MOUNT
+   */
+  bundlingFileAccess?: BundlingFileAccess
 }
 
 /**
@@ -66,6 +72,7 @@ export class Bundling implements CdkBundlingOptions {
   public readonly user?: string;
   public readonly securityOpt?: string;
   public readonly network?: string;
+  public readonly bundlingFileAccess?: BundlingFileAccess;
 
   constructor(props: BundlingProps) {
     const {
@@ -104,6 +111,7 @@ export class Bundling implements CdkBundlingOptions {
     this.user = props.user;
     this.securityOpt = props.securityOpt;
     this.network = props.network;
+    this.bundlingFileAccess = props.bundlingFileAccess;
   }
 
   private createBundlingCommand(options: BundlingCommandOptions): string[] {

packages/@aws-cdk/aws-lambda-python/lib/types.ts

@@ -1,4 +1,4 @@
-import { AssetHashType, DockerImage, DockerRunOptions } from '@aws-cdk/core';
+import { AssetHashType, BundlingFileAccess, DockerImage, DockerRunOptions } from '@aws-cdk/core';
 
 
 /**
@@ -86,6 +86,12 @@ export interface BundlingOptions extends DockerRunOptions {
    * @default - do not run additional commands
    */
   readonly commandHooks?: ICommandHooks;
+
+  /**
+   * Which option to use to copy the source files to the docker container and output files back
+   * @default - BundlingFileAccess.BIND_MOUNT
+   */
+  readonly bundlingFileAccess?: BundlingFileAccess;
 }
 
 /**

packages/@aws-cdk/aws-lambda-python/rosetta/default.ts-fixture

@@ -1,6 +1,6 @@
 // Fixture with packages imported, but nothing else
 import { Construct } from 'constructs';
-import { DockerImage, Stack } from '@aws-cdk/core';
+import { DockerImage, Stack, BundlingFileAccess } from '@aws-cdk/core';
 import { Runtime } from '@aws-cdk/aws-lambda';
 import * as python from '@aws-cdk/aws-lambda-python';
 

packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import { Architecture, Code, Runtime } from '@aws-cdk/aws-lambda';
-import { DockerImage } from '@aws-cdk/core';
+import { BundlingFileAccess, DockerImage } from '@aws-cdk/core';
 import { Bundling } from '../lib/bundling';
 
 jest.spyOn(Code, 'fromAsset');
@@ -374,6 +374,22 @@ test('Bundling with custom network', () => {
   }));
 });
 
+test('Bundling with docker copy variant', () => {
+  const entry = path.join(__dirname, 'lambda-handler');
+  Bundling.bundle({
+    entry: entry,
+    runtime: Runtime.PYTHON_3_7,
+    bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
+
+  });
+
+  expect(Code.fromAsset).toHaveBeenCalledWith(entry, expect.objectContaining({
+    bundling: expect.objectContaining({
+      bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
+    }),
+  }));
+});
+
 test('Do not build docker image when skipping bundling', () => {
   const entry = path.join(__dirname, 'lambda-handler');
   Bundling.bundle({