From 2972d103f4ce6368f0729c460f39db4620abddeb Mon Sep 17 00:00:00 2001 From: Robert Djurasaj Date: Thu, 31 Mar 2022 22:16:39 -0600 Subject: [PATCH] feat(lambda): add support for ephemeral storage (#19552) Depends on cfnspec bump PR https://github.com/aws/aws-cdk/pull/19553 CFN Docs: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-lambda-function-ephemeralstorage.html Blog: https://aws.amazon.com/blogs/aws/aws-lambda-now-supports-up-to-10-gb-ephemeral-storage/ image Closes #19605. ---- ### All Submissions: * [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/master/CONTRIBUTING.md) ### Adding new Unconventional Dependencies: * [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/master/CONTRIBUTING.md/#adding-new-unconventional-dependencies) ### New Features * [ ] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/master/INTEGRATION_TESTS.md)? * [ ] Did you use `cdk-integ` to deploy the infrastructure and generate the snapshot (i.e. `cdk-integ` without `--dry-run`)? *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/aws-lambda/README.md | 18 ++++++++ packages/@aws-cdk/aws-lambda/lib/function.ts | 17 ++++++- .../@aws-cdk/aws-lambda/test/function.test.ts | 46 +++++++++++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda/README.md b/packages/@aws-cdk/aws-lambda/README.md index 9c9182b174550..e5cc514c9d365 100644 --- a/packages/@aws-cdk/aws-lambda/README.md +++ b/packages/@aws-cdk/aws-lambda/README.md @@ -710,6 +710,24 @@ const fn = new lambda.Function(this, 'MyLambda', { }); ``` +## Ephemeral Storage + +You can configure ephemeral storage on a function to control the amount of storage it gets for reading +or writing data, allowing you to use AWS Lambda for ETL jobs, ML inference, or other data-intensive workloads. +The ephemeral storage will be accessible in the functions' `/tmp` directory. + +```ts +import { Size } from '@aws-cdk/core'; + +const fn = new lambda.Function(this, 'MyFunction', { + runtime: lambda.Runtime.NODEJS_14_X, + handler: 'index.handler', + code: lambda.Code.fromAsset(path.join(__dirname, 'lambda-handler')), + ephemeralStorageSize: Size.mebibytes(1024), +}); +``` + +Read more about using this feature in [this AWS blog post](https://aws.amazon.com/blogs/aws/aws-lambda-now-supports-up-to-10-gb-ephemeral-storage/). ## Singleton Function diff --git a/packages/@aws-cdk/aws-lambda/lib/function.ts b/packages/@aws-cdk/aws-lambda/lib/function.ts index 6bc89845edf3b..904fe515e5a7c 100644 --- a/packages/@aws-cdk/aws-lambda/lib/function.ts +++ b/packages/@aws-cdk/aws-lambda/lib/function.ts @@ -6,7 +6,7 @@ import * as kms from '@aws-cdk/aws-kms'; import * as logs from '@aws-cdk/aws-logs'; import * as sns from '@aws-cdk/aws-sns'; import * as sqs from '@aws-cdk/aws-sqs'; -import { Annotations, ArnFormat, CfnResource, Duration, Fn, Lazy, Names, Stack, Token } from '@aws-cdk/core'; +import { Annotations, ArnFormat, CfnResource, Duration, Fn, Lazy, Names, Size, Stack, Token } from '@aws-cdk/core'; import { Construct } from 'constructs'; import { Architecture } from './architecture'; import { Code, CodeConfig } from './code'; @@ -95,6 +95,13 @@ export interface FunctionOptions extends EventInvokeConfigOptions { */ readonly memorySize?: number; + /** + * The size of the function’s /tmp directory in MB. + * + * @default 512 MiB + */ + readonly ephemeralStorageSize?: Size; + /** * Initial policy statements to add to the created Lambda Role. * @@ -747,6 +754,11 @@ export class Function extends FunctionBase { } this._architecture = props.architecture ?? (props.architectures && props.architectures[0]); + if (props.ephemeralStorageSize && !props.ephemeralStorageSize.isUnresolved() + && (props.ephemeralStorageSize.toMebibytes() < 512 || props.ephemeralStorageSize.toMebibytes() > 10240)) { + throw new Error(`Ephemeral storage size must be between 512 and 10240 MB, received ${props.ephemeralStorageSize}.`); + } + const resource: CfnFunction = new CfnFunction(this, 'Resource', { functionName: this.physicalName, description: props.description, @@ -767,6 +779,9 @@ export class Function extends FunctionBase { // Token, actually *modifies* the 'environment' map. environment: Lazy.uncachedAny({ produce: () => this.renderEnvironment() }), memorySize: props.memorySize, + ephemeralStorage: props.ephemeralStorageSize ? { + size: props.ephemeralStorageSize.toMebibytes(), + } : undefined, vpcConfig: this.configureVpc(props), deadLetterConfig: this.buildDeadLetterConfig(dlqTopicOrQueue), tracingConfig: this.buildTracingConfig(props), diff --git a/packages/@aws-cdk/aws-lambda/test/function.test.ts b/packages/@aws-cdk/aws-lambda/test/function.test.ts index fd9a175d93fb2..f22216222baa2 100644 --- a/packages/@aws-cdk/aws-lambda/test/function.test.ts +++ b/packages/@aws-cdk/aws-lambda/test/function.test.ts @@ -15,6 +15,7 @@ import * as cdk from '@aws-cdk/core'; import * as constructs from 'constructs'; import * as _ from 'lodash'; import * as lambda from '../lib'; +import { Lazy, Size } from '@aws-cdk/core'; describe('function', () => { test('default function', () => { @@ -2584,6 +2585,7 @@ describe('function', () => { architectures: [lambda.Architecture.X86_64, lambda.Architecture.ARM_64], })).toThrow(/one architecture must be specified/); }); + test('Architecture is properly readable from the function', () => { const stack = new cdk.Stack(); const fn = new lambda.Function(stack, 'MyFunction', { @@ -2635,6 +2637,50 @@ describe('function', () => { }); }); +test('throws if ephemeral storage size is out of bound', () => { + const stack = new cdk.Stack(); + expect(() => new lambda.Function(stack, 'MyLambda', { + code: new lambda.InlineCode('foo'), + handler: 'bar', + runtime: lambda.Runtime.NODEJS_14_X, + ephemeralStorageSize: Size.mebibytes(511), + })).toThrow(/Ephemeral storage size must be between 512 and 10240 MB/); +}); + +test('set ephemeral storage to desired size', () => { + const stack = new cdk.Stack(); + new lambda.Function(stack, 'MyLambda', { + code: new lambda.InlineCode('foo'), + handler: 'bar', + runtime: lambda.Runtime.NODEJS_14_X, + ephemeralStorageSize: Size.mebibytes(1024), + }); + + Template.fromStack(stack).hasResource('AWS::Lambda::Function', { + Properties: + { + Code: { ZipFile: 'foo' }, + Handler: 'bar', + Runtime: 'nodejs14.x', + EphemeralStorage: { + Size: 1024, + }, + }, + }); +}); + +test('ephemeral storage allows unresolved tokens', () => { + const stack = new cdk.Stack(); + expect(() => { + new lambda.Function(stack, 'MyLambda', { + code: new lambda.InlineCode('foo'), + handler: 'bar', + runtime: lambda.Runtime.NODEJS_14_X, + ephemeralStorageSize: Size.mebibytes(Lazy.number({ produce: () => 1024 })), + }); + }).not.toThrow(); +}); + function newTestLambda(scope: constructs.Construct) { return new lambda.Function(scope, 'MyLambda', { code: new lambda.InlineCode('foo'),