Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(core): allow asset bundling on docker remote host / docker in docker #23576

Merged
merged 41 commits into from Jan 27, 2023
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
fabddf0
WIP docker bundle with cp
webratz Dec 23, 2022
025e8d9
wip mv
webratz Dec 23, 2022
c2e10b8
adjust cp command
webratz Jan 2, 2023
815edab
Merge branch 'aws:main' into main
webratz Jan 2, 2023
2f71a5b
Merge branch 'main' of https://github.com/webratz/aws-cdk
webratz Jan 3, 2023
8cb6259
fix missing path to copy out
webratz Jan 3, 2023
c006750
slight cleanup and first failed tests
webratz Jan 4, 2023
ac327d0
add basic integration test and pass through config from python lambda
webratz Jan 4, 2023
6fb1dd2
Merge branch 'aws:main' into main
webratz Jan 4, 2023
fd367ef
move to own class and cleanup
webratz Jan 5, 2023
26c1d10
add tests
webratz Jan 5, 2023
87e7a2a
add to all lambda implementations
webratz Jan 5, 2023
dca7457
fix import order
webratz Jan 5, 2023
e804023
add readme
webratz Jan 5, 2023
e667436
add import to readme
webratz Jan 9, 2023
13cba32
Update packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts
webratz Jan 20, 2023
1ca8496
Update packages/@aws-cdk/core/lib/bundling.ts
webratz Jan 20, 2023
813c8b7
Update packages/@aws-cdk/core/lib/bundling.ts
webratz Jan 20, 2023
871da21
Update packages/@aws-cdk/core/lib/bundling.ts
webratz Jan 20, 2023
ed23041
Update packages/@aws-cdk/core/lib/bundling.ts
webratz Jan 20, 2023
6bc93e1
address first MR comments
webratz Jan 20, 2023
7b3798e
fix readme
webratz Jan 20, 2023
d94e232
Merge branch 'main' into webratz/main
mrgrain Jan 20, 2023
54b3a80
WIP: restructure
webratz Jan 20, 2023
5c98592
WIP separate bundling
webratz Jan 23, 2023
056f48b
Merge branch 'main' into main
webratz Jan 23, 2023
ace87a4
move classes to private
webratz Jan 23, 2023
7be1cc5
Update packages/@aws-cdk/core/test/docker-stub-cp.sh
webratz Jan 25, 2023
fa11118
Update packages/@aws-cdk/core/test/bundling.test.ts
webratz Jan 25, 2023
381e5bf
Update packages/@aws-cdk/core/lib/private/asset-staging.ts
webratz Jan 25, 2023
5a9b75e
Update packages/@aws-cdk/core/lib/bundling.ts
webratz Jan 25, 2023
8640305
allow use of different stub files
webratz Jan 25, 2023
4750a5f
Update packages/@aws-cdk/core/lib/bundling.ts
webratz Jan 25, 2023
27026e5
change naming
webratz Jan 25, 2023
974762a
Update packages/@aws-cdk/core/lib/bundling.ts
webratz Jan 25, 2023
2890d54
Update packages/@aws-cdk/core/lib/bundling.ts
webratz Jan 25, 2023
325fdbb
Update packages/@aws-cdk/core/lib/bundling.ts
webratz Jan 25, 2023
6590681
remaining name fixes
webratz Jan 26, 2023
4ecaf47
cleanup
webratz Jan 26, 2023
cb54c28
Merge branch 'main' into webratz/main
mrgrain Jan 26, 2023
c4b09ee
Merge branch 'main' into main
mergify[bot] Jan 27, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 10 additions & 0 deletions packages/@aws-cdk/aws-lambda-go/lib/bundling.ts
Expand Up @@ -2,6 +2,7 @@ import * as os from 'os';
import * as path from 'path';
import { Architecture, AssetCode, Code, Runtime } from '@aws-cdk/aws-lambda';
import * as cdk from '@aws-cdk/core';
import { BundlingFileCopyVariant } from '@aws-cdk/core';
import { BundlingOptions } from './types';
import { exec, findUp, getGoBuildVersion } from './util';

Expand Down Expand Up @@ -60,6 +61,12 @@ export interface BundlingProps extends BundlingOptions {
* The system architecture of the lambda function
*/
readonly architecture: Architecture;

/**
* Which option to use to copy the source files to the docker container and output files back
* @default - BIND_MOUNT
*/
readonly fileCopyVariant?: BundlingFileCopyVariant;
}

/**
Expand All @@ -85,6 +92,7 @@ export class Bundling implements cdk.BundlingOptions {
user: bundling.user,
securityOpt: bundling.securityOpt,
network: bundling.network,
fileCopyVariant: bundling.fileCopyVariant,
},
});
}
Expand All @@ -107,6 +115,7 @@ export class Bundling implements cdk.BundlingOptions {
public readonly user?: string;
public readonly securityOpt?: string;
public readonly network?: string;
public readonly fileCopyVariant?: cdk.BundlingFileCopyVariant;

private readonly relativeEntryPath: string;

Expand Down Expand Up @@ -154,6 +163,7 @@ export class Bundling implements cdk.BundlingOptions {
this.user = props.user;
this.securityOpt = props.securityOpt;
this.network = props.network;
this.fileCopyVariant = props.fileCopyVariant;

// Local bundling
if (!props.forcedDockerBundling) { // only if Docker is not forced
Expand Down
8 changes: 7 additions & 1 deletion packages/@aws-cdk/aws-lambda-go/lib/types.ts
@@ -1,4 +1,4 @@
import { AssetHashType, DockerImage, DockerRunOptions } from '@aws-cdk/core';
import { AssetHashType, BundlingFileCopyVariant, DockerImage, DockerRunOptions } from '@aws-cdk/core';

/**
* Bundling options
Expand Down Expand Up @@ -111,6 +111,12 @@ export interface BundlingOptions extends DockerRunOptions {
* @default - Direct access
*/
readonly goProxies?: string[];

/**
* Which option to use to copy the source files to the docker container and output files back
* @default - BIND_MOUNT
*/
readonly fileCopyVariant?: BundlingFileCopyVariant;
}

/**
Expand Down
20 changes: 19 additions & 1 deletion packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts
Expand Up @@ -2,7 +2,7 @@ import * as child_process from 'child_process';
import * as os from 'os';
import * as path from 'path';
import { Architecture, Code, Runtime } from '@aws-cdk/aws-lambda';
import { AssetHashType, DockerImage } from '@aws-cdk/core';
import { AssetHashType, BundlingFileCopyVariant, DockerImage } from '@aws-cdk/core';
import { Bundling } from '../lib/bundling';
import * as util from '../lib/util';

Expand Down Expand Up @@ -461,3 +461,21 @@ test('Custom bundling network', () => {
}),
});
});

test('Custom bundling file copy variant', () => {
Bundling.bundle({
entry,
moduleDir,
runtime: Runtime.GO_1_X,
architecture: Architecture.X86_64,
forcedDockerBundling: true,
fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY,
});

expect(Code.fromAsset).toHaveBeenCalledWith('/project', {
assetHashType: AssetHashType.OUTPUT,
bundling: expect.objectContaining({
fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY,
}),
});
});
webratz marked this conversation as resolved.
Show resolved Hide resolved
9 changes: 9 additions & 0 deletions packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts
Expand Up @@ -6,6 +6,7 @@ import { PackageInstallation } from './package-installation';
import { LockFile, PackageManager } from './package-manager';
import { BundlingOptions, OutputFormat, SourceMapMode } from './types';
import { exec, extractDependencies, findUp, getTsconfigCompilerOptions } from './util';
import { BundlingFileCopyVariant } from '@aws-cdk/core';

const ESBUILD_MAJOR_VERSION = '0';

Expand Down Expand Up @@ -43,6 +44,12 @@ export interface BundlingProps extends BundlingOptions {
*/
readonly preCompilation?: boolean

/**
* Which option to use to copy the source files to the docker container and output files back
* @default - BIND_MOUNT
*/
readonly fileCopyVariant?: BundlingFileCopyVariant;

}

/**
Expand Down Expand Up @@ -83,6 +90,7 @@ export class Bundling implements cdk.BundlingOptions {
public readonly securityOpt?: string;
public readonly network?: string;
public readonly local?: cdk.ILocalBundling;
public readonly fileCopyVariant?: cdk.BundlingFileCopyVariant;

private readonly projectRoot: string;
private readonly relativeEntryPath: string;
Expand Down Expand Up @@ -154,6 +162,7 @@ export class Bundling implements cdk.BundlingOptions {
this.user = props.user;
this.securityOpt = props.securityOpt;
this.network = props.network;
this.fileCopyVariant = props.fileCopyVariant;

// Local bundling
if (!props.forceDockerBundling) { // only if Docker is not forced
Expand Down
8 changes: 7 additions & 1 deletion packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts
@@ -1,4 +1,4 @@
import { DockerImage, DockerRunOptions } from '@aws-cdk/core';
import { BundlingFileCopyVariant, DockerImage, DockerRunOptions } from '@aws-cdk/core';

/**
* Bundling options
Expand Down Expand Up @@ -301,6 +301,12 @@ export interface BundlingOptions extends DockerRunOptions {
* @default - no code is injected
*/
readonly inject?: string[]

/**
* Which option to use to copy the source files to the docker container and output files back
* @default - BIND_MOUNT
*/
readonly fileCopyVariant?: BundlingFileCopyVariant;
}

/**
Expand Down
21 changes: 20 additions & 1 deletion packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts
Expand Up @@ -2,7 +2,7 @@ import * as child_process from 'child_process';
import * as os from 'os';
import * as path from 'path';
import { Architecture, Code, Runtime } from '@aws-cdk/aws-lambda';
import { AssetHashType, DockerImage } from '@aws-cdk/core';
import { AssetHashType, BundlingFileCopyVariant, DockerImage } from '@aws-cdk/core';
import { version as delayVersion } from 'delay/package.json';
import { Bundling } from '../lib/bundling';
import { PackageInstallation } from '../lib/package-installation';
Expand Down Expand Up @@ -802,3 +802,22 @@ test('Custom bundling network', () => {
}),
});
});

test('Custom bundling file copy variant', () => {
Bundling.bundle({
entry,
projectRoot,
depsLockFilePath,
runtime: Runtime.NODEJS_14_X,
architecture: Architecture.X86_64,
forceDockerBundling: true,
fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY,
});

expect(Code.fromAsset).toHaveBeenCalledWith('/project', {
assetHashType: AssetHashType.OUTPUT,
bundling: expect.objectContaining({
fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY,
}),
});
});
10 changes: 9 additions & 1 deletion packages/@aws-cdk/aws-lambda-python/lib/bundling.ts
@@ -1,6 +1,6 @@
import * as path from 'path';
import { Architecture, AssetCode, Code, Runtime } from '@aws-cdk/aws-lambda';
import { AssetStaging, BundlingOptions as CdkBundlingOptions, DockerImage, DockerVolume } from '@aws-cdk/core';
import { AssetStaging, BundlingFileCopyVariant, BundlingOptions as CdkBundlingOptions, DockerImage, DockerVolume } from '@aws-cdk/core';
import { Packaging, DependenciesFile } from './packaging';
import { BundlingOptions, ICommandHooks } from './types';

Expand Down Expand Up @@ -41,6 +41,12 @@ export interface BundlingProps extends BundlingOptions {
* @default - Does not skip bundling
*/
readonly skip?: boolean;

/**
* Which option to use to copy the source files to the docker container and output files back
* @default - BIND_MOUNT
*/
fileCopyVariant?: BundlingFileCopyVariant
}

/**
Expand All @@ -66,6 +72,7 @@ export class Bundling implements CdkBundlingOptions {
public readonly user?: string;
public readonly securityOpt?: string;
public readonly network?: string;
public readonly fileCopyVariant?: BundlingFileCopyVariant;

constructor(props: BundlingProps) {
const {
Expand Down Expand Up @@ -104,6 +111,7 @@ export class Bundling implements CdkBundlingOptions {
this.user = props.user;
this.securityOpt = props.securityOpt;
this.network = props.network;
this.fileCopyVariant = props.fileCopyVariant;
}

private createBundlingCommand(options: BundlingCommandOptions): string[] {
Expand Down
8 changes: 7 additions & 1 deletion packages/@aws-cdk/aws-lambda-python/lib/types.ts
@@ -1,4 +1,4 @@
import { AssetHashType, DockerImage, DockerRunOptions } from '@aws-cdk/core';
import { AssetHashType, BundlingFileCopyVariant, DockerImage, DockerRunOptions } from '@aws-cdk/core';


/**
Expand Down Expand Up @@ -86,6 +86,12 @@ export interface BundlingOptions extends DockerRunOptions {
* @default - do not run additional commands
*/
readonly commandHooks?: ICommandHooks;

/**
* Which option to use to copy the source files to the docker container and output files back
* @default - BIND_MOUNT
*/
readonly fileCopyVariant?: BundlingFileCopyVariant;
}

/**
Expand Down
18 changes: 17 additions & 1 deletion packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts
@@ -1,7 +1,7 @@
import * as fs from 'fs';
import * as path from 'path';
import { Architecture, Code, Runtime } from '@aws-cdk/aws-lambda';
import { DockerImage } from '@aws-cdk/core';
import { BundlingFileCopyVariant, DockerImage } from '@aws-cdk/core';
import { Bundling } from '../lib/bundling';

jest.spyOn(Code, 'fromAsset');
Expand Down Expand Up @@ -374,6 +374,22 @@ test('Bundling with custom network', () => {
}));
});

test('Bundling with docker copy variant', () => {
const entry = path.join(__dirname, 'lambda-handler');
Bundling.bundle({
entry: entry,
runtime: Runtime.PYTHON_3_7,
fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY,

});

expect(Code.fromAsset).toHaveBeenCalledWith(entry, expect.objectContaining({
bundling: expect.objectContaining({
fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY,
}),
}));
});

test('Do not build docker image when skipping bundling', () => {
const entry = path.join(__dirname, 'lambda-handler');
Bundling.bundle({
Expand Down