diff --git a/packages/@aws-cdk/aws-eks/README.md b/packages/@aws-cdk/aws-eks/README.md index cd6e324ecf788..e1d78b774450d 100644 --- a/packages/@aws-cdk/aws-eks/README.md +++ b/packages/@aws-cdk/aws-eks/README.md @@ -102,8 +102,8 @@ The following is a qualitative diagram of the various possible components involv ```text +-----------------------------------------------+ +-----------------+ - | EKS Cluster | kubectl | | - |-----------------------------------------------|<-------------+| Kubectl Handler | + | EKS Cluster | kubectl | | + | ----------- |<-------------+| Kubectl Handler | | | | | | | +-----------------+ | +--------------------+ +-----------------+ | @@ -539,16 +539,21 @@ If the endpoint does not expose private access (via `EndpointAccess.PUBLIC`) **o #### Cluster Handler -The `ClusterHandler` is a Lambda function responsible to interact with the EKS API in order to control the cluster lifecycle. To provision this function inside the VPC, set the `placeClusterHandlerInVpc` property to `true`. This will place the function inside the private subnets of the VPC based on the selection strategy specified in the [`vpcSubnets`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-eks.Cluster.html#vpcsubnetsspan-classapi-icon-api-icon-experimental-titlethis-api-element-is-experimental-it-may-change-without-noticespan) property. +The `ClusterHandler` is a set of Lambda functions (`onEventHandler`, `isCompleteHandler`) responsible for interacting with the EKS API in order to control the cluster lifecycle. To provision these functions inside the VPC, set the `placeClusterHandlerInVpc` property to `true`. This will place the functions inside the private subnets of the VPC based on the selection strategy specified in the [`vpcSubnets`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-eks.Cluster.html#vpcsubnetsspan-classapi-icon-api-icon-experimental-titlethis-api-element-is-experimental-it-may-change-without-noticespan) property. -You can configure the environment of this function by specifying it at cluster instantiation. For example, this can be useful in order to configure an http proxy: +You can configure the environment of the Cluster Handler functions by specifying it at cluster instantiation. For example, this can be useful in order to configure an http proxy: ```ts const cluster = new eks.Cluster(this, 'hello-eks', { version: eks.KubernetesVersion.V1_21, clusterHandlerEnvironment: { - 'http_proxy': 'http://proxy.myproxy.com' - } + https_proxy: 'http://proxy.myproxy.com' + }, + /** + * If the proxy is not open publicly, you can pass a security group to the + * Cluster Handler Lambdas so that it can reach the proxy. + */ + clusterHandlerSecurityGroup: proxyInstanceSecurityGroup }); ``` diff --git a/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/common.ts b/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/common.ts index 8f563de833bf6..21cf958df5a68 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/common.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/common.ts @@ -41,12 +41,6 @@ export abstract class ResourceHandler { } public onEvent() { - // eslint-disable-next-line @typescript-eslint/no-require-imports, import/no-extraneous-dependencies - const ProxyAgent: any = require('proxy-agent'); - aws.config.update({ - httpOptions: { agent: new ProxyAgent() }, - }); - switch (this.requestType) { case 'Create': return this.onCreate(); case 'Update': return this.onUpdate(); diff --git a/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/index.ts b/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/index.ts index e7fc357846259..258f5d8b04545 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/index.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/index.ts @@ -8,7 +8,13 @@ import { EksClient } from './common'; import * as consts from './consts'; import { FargateProfileResourceHandler } from './fargate'; +// eslint-disable-next-line @typescript-eslint/no-require-imports, import/no-extraneous-dependencies +const ProxyAgent = require('proxy-agent'); + aws.config.logger = console; +aws.config.update({ + httpOptions: { agent: new ProxyAgent() }, +}); let eks: aws.EKS | undefined; diff --git a/packages/@aws-cdk/aws-eks/lib/cluster-resource-provider.ts b/packages/@aws-cdk/aws-eks/lib/cluster-resource-provider.ts index f425b0a1eaba6..9bb65be4f56b2 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster-resource-provider.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster-resource-provider.ts @@ -36,11 +36,18 @@ export interface ClusterResourceProviderProps { readonly environment?: { [key: string]: string }; /** - * An AWS Lambda layer that includes the NPM dependency `proxy-agent`. - * - * If not defined, a default layer will be used. - */ + * An AWS Lambda layer that includes the NPM dependency `proxy-agent`. + * + * If not defined, a default layer will be used. + */ readonly onEventLayer?: lambda.ILayerVersion; + + /** + * The security group to associate with the functions. + * + * @default - No security group. + */ + readonly securityGroup?: ec2.ISecurityGroup; } /** @@ -66,6 +73,9 @@ export class ClusterResourceProvider extends NestedStack { private constructor(scope: Construct, id: string, props: ClusterResourceProviderProps) { super(scope as CoreConstruct, id); + // The NPM dependency proxy-agent is required in order to support proxy routing with the AWS JS SDK. + const nodeProxyAgentLayer = new NodeProxyAgentLayer(this, 'NodeProxyAgentLayer'); + const onEvent = new lambda.Function(this, 'OnEventHandler', { code: lambda.Code.fromAsset(HANDLER_DIR), description: 'onEvent handler for EKS cluster resource provider', @@ -75,24 +85,22 @@ export class ClusterResourceProvider extends NestedStack { timeout: Duration.minutes(1), vpc: props.subnets ? props.vpc : undefined, vpcSubnets: props.subnets ? { subnets: props.subnets } : undefined, + securityGroups: props.securityGroup ? [props.securityGroup] : undefined, + // Allow user to override the layer. + layers: props.onEventLayer ? [props.onEventLayer] : [nodeProxyAgentLayer], }); - // Allow user to customize the layer - if (!props.onEventLayer) { - // `NodeProxyAgentLayer` provides `proxy-agent` which is needed to configure `aws-sdk-js` with a user provided proxy. - onEvent.addLayers(new NodeProxyAgentLayer(this, 'NodeProxyAgentLayer')); - } else { - onEvent.addLayers(props.onEventLayer); - } - const isComplete = new lambda.Function(this, 'IsCompleteHandler', { code: lambda.Code.fromAsset(HANDLER_DIR), description: 'isComplete handler for EKS cluster resource provider', runtime: HANDLER_RUNTIME, + environment: props.environment, handler: 'index.isComplete', timeout: Duration.minutes(1), vpc: props.subnets ? props.vpc : undefined, vpcSubnets: props.subnets ? { subnets: props.subnets } : undefined, + securityGroups: props.securityGroup ? [props.securityGroup] : undefined, + layers: [nodeProxyAgentLayer], }); this.provider = new cr.Provider(this, 'Provider', { @@ -102,6 +110,7 @@ export class ClusterResourceProvider extends NestedStack { queryInterval: Duration.minutes(1), vpc: props.subnets ? props.vpc : undefined, vpcSubnets: props.subnets ? { subnets: props.subnets } : undefined, + securityGroups: props.securityGroup ? [props.securityGroup] : undefined, }); props.adminRole.grant(onEvent.role!, 'sts:AssumeRole'); diff --git a/packages/@aws-cdk/aws-eks/lib/cluster-resource.ts b/packages/@aws-cdk/aws-eks/lib/cluster-resource.ts index 88f3cd0138344..ed0852338a527 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster-resource.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster-resource.ts @@ -27,6 +27,7 @@ export interface ClusterResourceProps { readonly subnets?: ec2.ISubnet[]; readonly secretsEncryptionKey?: kms.IKey; readonly onEventLayer?: lambda.ILayerVersion; + readonly clusterHandlerSecurityGroup?: ec2.ISecurityGroup; } /** @@ -66,6 +67,7 @@ export class ClusterResource extends CoreConstruct { vpc: props.vpc, environment: props.environment, onEventLayer: props.onEventLayer, + securityGroup: props.clusterHandlerSecurityGroup, }); const resource = new CustomResource(this, 'Resource', { diff --git a/packages/@aws-cdk/aws-eks/lib/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster.ts index c1d00a9dcd767..2c762ab666327 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster.ts @@ -130,10 +130,21 @@ export interface ICluster extends IResource, ec2.IConnectable { readonly kubectlMemory?: Size; /** - * An AWS Lambda layer that includes the NPM dependency `proxy-agent`. - * - * If not defined, a default layer will be used. - */ + * A security group to associate with the Cluster Handler's Lambdas. + * The Cluster Handler's Lambdas are responsible for calling AWS's EKS API. + * + * Requires `placeClusterHandlerInVpc` to be set to true. + * + * @default - No security group. + * @attribute + */ + readonly clusterHandlerSecurityGroup?: ec2.ISecurityGroup; + + /** + * An AWS Lambda layer that includes the NPM dependency `proxy-agent`. + * + * If not defined, a default layer will be used. + */ readonly onEventLayer?: lambda.ILayerVersion; /** @@ -309,6 +320,14 @@ export interface ClusterAttributes { */ readonly kubectlMemory?: Size; + /** + * A security group id to associate with the Cluster Handler's Lambdas. + * The Cluster Handler's Lambdas are responsible for calling AWS's EKS API. + * + * @default - No security group. + */ + readonly clusterHandlerSecurityGroupId?: string; + /** * An AWS Lambda Layer which includes the NPM dependency `proxy-agent`. This layer * is used by the onEvent handler to route AWS SDK requests through a proxy. @@ -452,13 +471,6 @@ export interface ClusterOptions extends CommonClusterOptions { */ readonly kubectlEnvironment?: { [key: string]: string }; - /** - * Custom environment variables when interacting with the EKS endpoint to manage the cluster lifecycle. - * - * @default - No environment variables. - */ - readonly clusterHandlerEnvironment?: { [key: string]: string }; - /** * An AWS Lambda Layer which includes `kubectl`, Helm and the AWS CLI. * @@ -491,26 +503,40 @@ export interface ClusterOptions extends CommonClusterOptions { readonly kubectlMemory?: Size; /** - * An AWS Lambda Layer which includes the NPM dependency `proxy-agent`. + * Custom environment variables when interacting with the EKS endpoint to manage the cluster lifecycle. + * + * @default - No environment variables. + */ + readonly clusterHandlerEnvironment?: { [key: string]: string }; + + /** + * A security group to associate with the Cluster Handler's Lambdas. + * The Cluster Handler's Lambdas are responsible for calling AWS's EKS API. + * + * Requires `placeClusterHandlerInVpc` to be set to true. + * + * @default - No security group. + */ + readonly clusterHandlerSecurityGroup?: ec2.ISecurityGroup; + + /** + * An AWS Lambda Layer which includes the NPM dependency `proxy-agent`. This layer + * is used by the onEvent handler to route AWS SDK requests through a proxy. * * By default, the provider will use the layer included in the * "aws-lambda-layer-node-proxy-agent" SAR application which is available in all * commercial regions. * - * To deploy the layer locally, visit - * https://github.com/aws-samples/aws-lambda-layer-node-proxy-agent/blob/master/cdk/README.md - * for instructions on how to prepare the .zip file and then define it in your - * app as follows: + * To deploy the layer locally define it in your app as follows: * * ```ts - * const layer = new lambda.LayerVersion(this, 'node-proxy-agent-layer', { + * const layer = new lambda.LayerVersion(this, 'proxy-agent-layer', { * code: lambda.Code.fromAsset(`${__dirname}/layer.zip`)), - * compatibleRuntimes: [lambda.Runtime.NODEJS_14_X] + * compatibleRuntimes: [lambda.Runtime.NODEJS_12_X] * }) * ``` * - * @default - the layer provided by the `aws-lambda-layer-node-proxy-agent` SAR app. - * @see https://github.com/aws-samples/aws-lambda-layer-node-proxy-agent + * @default - a layer bundled with this module. */ readonly onEventLayer?: lambda.ILayerVersion; @@ -749,6 +775,7 @@ abstract class ClusterBase extends Resource implements ICluster { public abstract readonly kubectlSecurityGroup?: ec2.ISecurityGroup; public abstract readonly kubectlPrivateSubnets?: ec2.ISubnet[]; public abstract readonly kubectlMemory?: Size; + public abstract readonly clusterHandlerSecurityGroup?: ec2.ISecurityGroup; public abstract readonly prune: boolean; public abstract readonly openIdConnectProvider: iam.IOpenIdConnectProvider; public abstract readonly awsAuth: AwsAuth; @@ -902,7 +929,7 @@ abstract class ClusterBase extends Resource implements ICluster { // cluster or if `mapRole` is set to false. By default this should happen. let mapRole = options.mapRole ?? true; if (mapRole && !(this instanceof Cluster)) { - // do the mapping... + // do the mapping... Annotations.of(autoScalingGroup).addWarning('Auto-mapping aws-auth role for imported cluster is not supported, please map role manually'); mapRole = false; } @@ -1099,11 +1126,21 @@ export class Cluster extends ClusterBase { */ public readonly kubectlMemory?: Size; + /** + * A security group to associate with the Cluster Handler's Lambdas. + * The Cluster Handler's Lambdas are responsible for calling AWS's EKS API. + * + * Requires `placeClusterHandlerInVpc` to be set to true. + * + * @default - No security group. + */ + public readonly clusterHandlerSecurityGroup?: ec2.ISecurityGroup; + /** * The AWS Lambda layer that contains the NPM dependency `proxy-agent`. If * undefined, a SAR app that contains this layer will be used. */ - public readonly onEventLayer?: lambda.ILayerVersion; + readonly onEventLayer?: lambda.ILayerVersion; /** * Determines if Kubernetes resources can be pruned automatically. @@ -1188,9 +1225,11 @@ export class Cluster extends ClusterBase { this.endpointAccess = props.endpointAccess ?? EndpointAccess.PUBLIC_AND_PRIVATE; this.kubectlEnvironment = props.kubectlEnvironment; this.kubectlLayer = props.kubectlLayer; - this.onEventLayer = props.onEventLayer; this.kubectlMemory = props.kubectlMemory; + this.onEventLayer = props.onEventLayer; + this.clusterHandlerSecurityGroup = props.clusterHandlerSecurityGroup; + const privateSubnets = this.selectPrivateSubnets().slice(0, 16); const publicAccessDisabled = !this.endpointAccess._config.publicAccess; const publicAccessRestricted = !publicAccessDisabled @@ -1215,6 +1254,10 @@ export class Cluster extends ClusterBase { throw new Error('Cannot place cluster handler in the VPC since no private subnets could be selected'); } + if (props.clusterHandlerSecurityGroup && !placeClusterHandlerInVpc) { + throw new Error('Cannot specify clusterHandlerSecurityGroup without placeClusterHandlerInVpc set to true'); + } + const resource = this._clusterResource = new ClusterResource(this, 'Resource', { name: this.physicalName, environment: props.clusterHandlerEnvironment, @@ -1241,6 +1284,7 @@ export class Cluster extends ClusterBase { secretsEncryptionKey: props.secretsEncryptionKey, vpc: this.vpc, subnets: placeClusterHandlerInVpc ? privateSubnets : undefined, + clusterHandlerSecurityGroup: this.clusterHandlerSecurityGroup, onEventLayer: this.onEventLayer, }); @@ -1827,8 +1871,9 @@ class ImportedCluster extends ClusterBase { public readonly kubectlSecurityGroup?: ec2.ISecurityGroup | undefined; public readonly kubectlPrivateSubnets?: ec2.ISubnet[] | undefined; public readonly kubectlLayer?: lambda.ILayerVersion; - public readonly onEventLayer?: lambda.ILayerVersion; public readonly kubectlMemory?: Size; + public readonly clusterHandlerSecurityGroup?: ec2.ISecurityGroup | undefined; + public readonly onEventLayer?: lambda.ILayerVersion; public readonly prune: boolean; // so that `clusterSecurityGroup` on `ICluster` can be configured without optionality, avoiding users from having @@ -1845,8 +1890,9 @@ class ImportedCluster extends ClusterBase { this.kubectlEnvironment = props.kubectlEnvironment; this.kubectlPrivateSubnets = props.kubectlPrivateSubnetIds ? props.kubectlPrivateSubnetIds.map((subnetid, index) => ec2.Subnet.fromSubnetId(this, `KubectlSubnet${index}`, subnetid)) : undefined; this.kubectlLayer = props.kubectlLayer; - this.onEventLayer = props.onEventLayer; this.kubectlMemory = props.kubectlMemory; + this.clusterHandlerSecurityGroup = props.clusterHandlerSecurityGroupId ? ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterHandlerSecurityGroup', props.clusterHandlerSecurityGroupId) : undefined; + this.onEventLayer = props.onEventLayer; this.prune = props.prune ?? true; let i = 1; diff --git a/packages/@aws-cdk/aws-eks/package.json b/packages/@aws-cdk/aws-eks/package.json index e5c7869f9bc13..35db07123cae8 100644 --- a/packages/@aws-cdk/aws-eks/package.json +++ b/packages/@aws-cdk/aws-eks/package.json @@ -93,7 +93,6 @@ "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lambda": "0.0.0", - "@aws-cdk/aws-lambda-nodejs": "0.0.0", "@aws-cdk/aws-ssm": "0.0.0", "@aws-cdk/core": "0.0.0", "@aws-cdk/custom-resources": "0.0.0", @@ -113,7 +112,6 @@ "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lambda": "0.0.0", - "@aws-cdk/aws-lambda-nodejs": "0.0.0", "@aws-cdk/aws-ssm": "0.0.0", "@aws-cdk/core": "0.0.0", "@aws-cdk/custom-resources": "0.0.0", diff --git a/packages/@aws-cdk/aws-eks/test/cluster.test.ts b/packages/@aws-cdk/aws-eks/test/cluster.test.ts index 295092509ffd2..63a17d9c5d64d 100644 --- a/packages/@aws-cdk/aws-eks/test/cluster.test.ts +++ b/packages/@aws-cdk/aws-eks/test/cluster.test.ts @@ -38,6 +38,25 @@ describe('cluster', () => { expect(template.Resources.OnEventHandler42BEBAE0.Properties.Environment).toEqual({ Variables: { foo: 'bar' } }); }); + test('can specify security group to cluster resource handler', () => { + const { stack, vpc } = testFixture(); + const securityGroup = new ec2.SecurityGroup(stack, 'ProxyInstanceSG', { + vpc, + allowAllOutbound: false, + }); + + new eks.Cluster(stack, 'Cluster', { + version: CLUSTER_VERSION, + placeClusterHandlerInVpc: true, + clusterHandlerSecurityGroup: securityGroup, + }); + + const nested = stack.node.tryFindChild('@aws-cdk/aws-eks.ClusterResourceProvider') as cdk.NestedStack; + + const template = SynthUtils.toCloudFormation(nested); + expect(template.Resources.OnEventHandler42BEBAE0.Properties.VpcConfig.SecurityGroupIds).toEqual([{ Ref: 'referencetoStackProxyInstanceSG80B79D87GroupId' }]); + }); + test('throws when trying to place cluster handlers in a vpc with no private subnets', () => { const { stack } = testFixture(); @@ -55,6 +74,21 @@ describe('cluster', () => { }); + test('throws when provided `clusterHandlerSecurityGroup` without `placeClusterHandlerInVpc: true`', () => { + const { stack, vpc } = testFixture(); + const securityGroup = new ec2.SecurityGroup(stack, 'ProxyInstanceSG', { + vpc, + allowAllOutbound: false, + }); + + expect(() => { + new eks.Cluster(stack, 'Cluster', { + version: CLUSTER_VERSION, + clusterHandlerSecurityGroup: securityGroup, + }); + }).toThrow(/Cannot specify clusterHandlerSecurityGroup without placeClusterHandlerInVpc set to true/); + }); + describe('imported Vpc from unparseable list tokens', () => { let stack: cdk.Stack; let vpc: ec2.IVpc;