diff --git a/packages/@aws-cdk/pipelines/README.md b/packages/@aws-cdk/pipelines/README.md index 09679d761cb09..ec6d73b2c292c 100644 --- a/packages/@aws-cdk/pipelines/README.md +++ b/packages/@aws-cdk/pipelines/README.md @@ -350,6 +350,40 @@ const pipeline = new pipelines.CodePipeline(this, 'Pipeline', { You can adapt these examples to your own situation. +#### Migrating from buildspec.yml files + +You may currently have the build instructions for your CodeBuild Projects in a +`buildspec.yml` file in your source repository. In addition to your build +commands, the CodeBuild Project's buildspec als controls some information that +CDK Pipelines manages for you, like artifact identifiers, input artifact +locations, Docker authorization, and exported variables. + +Since there is no way in general for CDK Pipelines to modify the file in your +resource repository, CDK Pipelines configures the BuildSpec directly on the +CodeBuild Project, instead of loading it from the `buildspec.yml` file. +This requires a pipeline self-mutation to update. + +To avoid this, put your build instructions in a separate script, for example +`build.sh`, and call that script from the build `commands` array: + +```ts +declare const source: pipelines.IFileSetProducer; + +const pipeline = new pipelines.CodePipeline(this, 'Pipeline', { + synth: new pipelines.ShellStep('Synth', { + input: source, + commands: [ + // Abstract over doing the build + './build.sh', + ], + }) +}); +``` + +Doing so keeps your exact build instructions in sync with your source code in +the source repository where it belongs, and provides a convenient build script +for developers at the same time. + #### CodePipeline Sources In CodePipeline, *Sources* define where the source of your application lives. @@ -768,6 +802,13 @@ class MyJenkinsStep extends pipelines.Step implements pipelines.ICodePipelineAct private readonly input: pipelines.FileSet, ) { super('MyJenkinsStep'); + + // This is necessary if your step accepts things like environment variables + // that may contain outputs from other steps. It doesn't matter what the + // structure is, as long as it contains the values that may contain outputs. + this.discoverReferencedOutputs({ + env: { /* ... */ } + }); } public produceAction(stage: codepipeline.IStage, options: pipelines.ProduceActionOptions): pipelines.CodePipelineActionFactoryResult { diff --git a/packages/@aws-cdk/pipelines/lib/blueprint/manual-approval.ts b/packages/@aws-cdk/pipelines/lib/blueprint/manual-approval.ts index 859c279533fa3..31b93f5b9cf87 100644 --- a/packages/@aws-cdk/pipelines/lib/blueprint/manual-approval.ts +++ b/packages/@aws-cdk/pipelines/lib/blueprint/manual-approval.ts @@ -33,5 +33,7 @@ export class ManualApprovalStep extends Step { super(id); this.comment = props.comment; + + this.discoverReferencedOutputs(props.comment); } } \ No newline at end of file diff --git a/packages/@aws-cdk/pipelines/lib/blueprint/shell-step.ts b/packages/@aws-cdk/pipelines/lib/blueprint/shell-step.ts index d36f0999fca17..fa01624e9635a 100644 --- a/packages/@aws-cdk/pipelines/lib/blueprint/shell-step.ts +++ b/packages/@aws-cdk/pipelines/lib/blueprint/shell-step.ts @@ -87,7 +87,6 @@ export interface ShellStepProps { * @default - No primary output */ readonly primaryOutputDirectory?: string; - } /** @@ -152,6 +151,11 @@ export class ShellStep extends Step { this.env = props.env ?? {}; this.envFromCfnOutputs = mapValues(props.envFromCfnOutputs ?? {}, StackOutputReference.fromCfnOutput); + // 'env' is the only thing that can contain outputs + this.discoverReferencedOutputs({ + env: this.env, + }); + // Inputs if (props.input) { const fileSet = props.input.primaryOutput; diff --git a/packages/@aws-cdk/pipelines/lib/blueprint/step-output.ts b/packages/@aws-cdk/pipelines/lib/blueprint/step-output.ts new file mode 100644 index 0000000000000..27ba14345229a --- /dev/null +++ b/packages/@aws-cdk/pipelines/lib/blueprint/step-output.ts @@ -0,0 +1,97 @@ +import { IResolvable, IResolveContext, Token, Tokenization } from '@aws-cdk/core'; +import { Step } from './step'; + +const STEP_OUTPUT_SYM = Symbol.for('@aws-cdk/pipelines.StepOutput'); + +/** + * A symbolic reference to a value produced by another step + * + * The engine is responsible for defining how these should be rendered. + */ +export class StepOutput implements IResolvable { + /** + * Return true if the given IResolvable is a StepOutput + */ + public static isStepOutput(resolvable: IResolvable): resolvable is StepOutput { + return !!(resolvable as any)[STEP_OUTPUT_SYM]; + } + + /** + * Find all StepOutputs referenced in the given structure + */ + public static findAll(structure: any): StepOutput[] { + return findAllStepOutputs(structure); + } + + public readonly creationStack: string[] = []; + private resolution: any = undefined; + + constructor(public readonly step: Step, public readonly engineSpecificInformation: any) { + Object.defineProperty(this, STEP_OUTPUT_SYM, { value: true }); + } + + /** + * Define the resolved value for this StepOutput. + * + * Should be called by the engine. + */ + public defineResolution(value: any) { + this.resolution = value; + } + + public resolve(_context: IResolveContext) { + if (this.resolution === undefined) { + throw new Error(`Output for step ${this.step} not configured. Either the step is not in the pipeline, or this engine does not support Outputs for this step.`); + } + return this.resolution; + } + + public toString(): string { + return Token.asString(this); + } +} + +function findAllStepOutputs(structure: any): StepOutput[] { + const ret = new Set(); + recurse(structure); + return Array.from(ret); + + function checkToken(x?: IResolvable) { + if (x && StepOutput.isStepOutput(x)) { + ret.add(x); + return true; + } + + // Return false if it wasn't a Token in the first place (in which case we recurse) + return x !== undefined; + } + + function recurse(x: any): void { + if (!x) { return; } + + if (Tokenization.isResolvable(x)) { + checkToken(x); + return; + } + if (Array.isArray(x)) { + if (!checkToken(Tokenization.reverseList(x))) { + x.forEach(recurse); + } + return; + } + if (typeof x === 'number') { + checkToken(Tokenization.reverseNumber(x)); + return; + } + if (typeof x === 'string') { + Tokenization.reverseString(x).tokens.forEach(checkToken); + return; + } + if (typeof x === 'object') { + for (const [k, v] of Object.entries(x)) { + recurse(k); + recurse(v); + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/pipelines/lib/blueprint/step.ts b/packages/@aws-cdk/pipelines/lib/blueprint/step.ts index 65f4636b2ecbc..797921a620fe7 100644 --- a/packages/@aws-cdk/pipelines/lib/blueprint/step.ts +++ b/packages/@aws-cdk/pipelines/lib/blueprint/step.ts @@ -1,4 +1,5 @@ import { Stack, Token } from '@aws-cdk/core'; +import { StepOutput } from '../blueprint/step-output'; import { FileSet, IFileSetProducer } from './file-set'; /** @@ -30,6 +31,20 @@ export abstract class Step implements IFileSetProducer { */ public readonly dependencyFileSets: FileSet[] = []; + /** + * The list of StepOutputs consumed by this Step + * + * Should be inspected by the pipeline engine. + */ + public readonly consumedStepOutputs: StepOutput[] = []; + + /** + * The list of StepOutputs produced by this Step + * + * Should be inspected by the pipeline engine. + */ + public readonly producedStepOutputs: StepOutput[] = []; + /** * Whether or not this is a Source step * @@ -39,7 +54,7 @@ export abstract class Step implements IFileSetProducer { private _primaryOutput?: FileSet; - private _dependencies: Step[] = []; + private _dependencies = new Set(); constructor( /** Identifier for this step */ @@ -54,7 +69,10 @@ export abstract class Step implements IFileSetProducer { * Return the steps this step depends on, based on the FileSets it requires */ public get dependencies(): Step[] { - return this.dependencyFileSets.map(f => f.producer).concat(this._dependencies); + return Array.from(new Set([ + ...this.dependencyFileSets.map(f => f.producer), + ...this._dependencies, + ])); } /** @@ -79,7 +97,7 @@ export abstract class Step implements IFileSetProducer { * Add a dependency on another step. */ public addStepDependency(step: Step) { - this._dependencies.push(step); + this._dependencies.add(step); } /** @@ -97,6 +115,22 @@ export abstract class Step implements IFileSetProducer { protected configurePrimaryOutput(fs: FileSet) { this._primaryOutput = fs; } + + /** + * Crawl the given structure for references to StepOutputs and add dependencies on all steps found + * + * Should be called by subclasses based on what the user passes in as + * construction properties. The format of the structure passed in here does + * not have to correspond exactly to what gets rendered into the engine, it + * just needs to contain the same amount of data. + */ + protected discoverReferencedOutputs(structure: any) { + for (const output of StepOutput.findAll(structure)) { + this._dependencies.add(output.step); + this.consumedStepOutputs.push(output); + output.step.producedStepOutputs.push(output); + } + } } /** @@ -128,5 +162,4 @@ export interface StackSteps { * @default - no additional steps */ readonly post?: Step[]; - } \ No newline at end of file diff --git a/packages/@aws-cdk/pipelines/lib/codepipeline/codebuild-step.ts b/packages/@aws-cdk/pipelines/lib/codepipeline/codebuild-step.ts index f835e261aba3d..a89305d8d390b 100644 --- a/packages/@aws-cdk/pipelines/lib/codepipeline/codebuild-step.ts +++ b/packages/@aws-cdk/pipelines/lib/codepipeline/codebuild-step.ts @@ -1,8 +1,10 @@ -import { Duration } from '@aws-cdk/core'; import * as codebuild from '@aws-cdk/aws-codebuild'; import * as ec2 from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; +import { Duration } from '@aws-cdk/core'; import { ShellStep, ShellStepProps } from '../blueprint'; +import { StepOutput } from '../blueprint/step-output'; +import { mergeBuildSpecs } from './private/buildspecs'; /** * Construction props for a CodeBuildStep @@ -96,6 +98,17 @@ export interface CodeBuildStepProps extends ShellStepProps { /** * Run a script as a CodeBuild Project + * + * The BuildSpec must be available inline--it cannot reference a file + * on disk. If your current build instructions are in a file like + * `buildspec.yml` in your repository, extract them to a script + * (say, `build.sh`) and invoke that script as part of the build: + * + * ```ts + * new pipelines.CodeBuildStep({ + * commands: ['./build.sh'], + * }); + * ``` */ export class CodeBuildStep extends ShellStep { /** @@ -105,13 +118,6 @@ export class CodeBuildStep extends ShellStep { */ public readonly projectName?: string; - /** - * Additional configuration that can only be configured via BuildSpec - * - * @default - No value specified at construction time, use defaults - */ - public readonly partialBuildSpec?: codebuild.BuildSpec; - /** * The VPC where to execute the SimpleSynth. * @@ -164,13 +170,16 @@ export class CodeBuildStep extends ShellStep { readonly timeout?: Duration; private _project?: codebuild.IProject; + private _partialBuildSpec?: codebuild.BuildSpec; + private readonly exportedVariables = new Set(); + private exportedVarsRendered = false; constructor(id: string, props: CodeBuildStepProps) { super(id, props); this.projectName = props.projectName; this.buildEnvironment = props.buildEnvironment; - this.partialBuildSpec = props.partialBuildSpec; + this._partialBuildSpec = props.partialBuildSpec; this.vpc = props.vpc; this.subnetSelection = props.subnetSelection; this.role = props.role; @@ -198,6 +207,45 @@ export class CodeBuildStep extends ShellStep { return this.project.grantPrincipal; } + /** + * Additional configuration that can only be configured via BuildSpec + * + * Contains exported variables + * + * @default - Contains the exported variables + */ + public get partialBuildSpec(): codebuild.BuildSpec | undefined { + this.exportedVarsRendered = true; + + const varsBuildSpec = this.exportedVariables.size > 0 ? codebuild.BuildSpec.fromObject({ + version: '0.2', + env: { + 'exported-variables': Array.from(this.exportedVariables), + }, + }) : undefined; + + return mergeBuildSpecs(varsBuildSpec, this._partialBuildSpec); + } + + /** + * Reference a CodePipeline variable defined by the CodeBuildStep. + * + * The variable must be set in the shell of the CodeBuild step when + * it finishes its `post_build` phase. + * + * @param variableName the name of the variable for reference. + */ + public exportedVariable(variableName: string): string { + if (this.exportedVarsRendered && !this.exportedVariables.has(variableName)) { + throw new Error('exportVariable(): Pipeline has already been produced, cannot call this function anymore'); + } + + this.exportedVariables.add(variableName); + + // return `#{${this.variablesNamespace}.${variableName}}`; + return new StepOutput(this, variableName).toString(); + } + /** * Set the internal project value * diff --git a/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline-action-factory.ts b/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline-action-factory.ts index 62c9fa86d025b..734c2fefa0128 100644 --- a/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline-action-factory.ts +++ b/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline-action-factory.ts @@ -23,6 +23,15 @@ export interface ProduceActionOptions { */ readonly runOrder: number; + /** + * If this step is producing outputs, the variables namespace assigned to it + * + * Pass this on to the Action you are creating. + * + * @default - Step doesn't produce any outputs + */ + readonly variablesNamespace?: string; + /** * Helper object to translate FileSets to CodePipeline Artifacts */ @@ -87,6 +96,8 @@ export interface ICodePipelineActionFactory { export interface CodePipelineActionFactoryResult { /** * How many RunOrders were consumed + * + * If you add 1 action, return the value 1 here. */ readonly runOrdersConsumed: number; diff --git a/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline-source.ts b/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline-source.ts index dd40d0d6cf0e7..9f9c78de37ad7 100644 --- a/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline-source.ts +++ b/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline-source.ts @@ -8,6 +8,7 @@ import { IBucket } from '@aws-cdk/aws-s3'; import { SecretValue, Token } from '@aws-cdk/core'; import { Node } from 'constructs'; import { FileSet, Step } from '../blueprint'; +import { StepOutput } from '../blueprint/step-output'; import { CodePipelineActionFactoryResult, ProduceActionOptions, ICodePipelineActionFactory } from './codepipeline-action-factory'; /** @@ -104,12 +105,44 @@ export abstract class CodePipelineSource extends Step implements ICodePipelineAc public produceAction(stage: cp.IStage, options: ProduceActionOptions): CodePipelineActionFactoryResult { const output = options.artifacts.toCodePipeline(this.primaryOutput!); - const action = this.getAction(output, options.actionName, options.runOrder); + + const action = this.getAction(output, options.actionName, options.runOrder, options.variablesNamespace); stage.addAction(action); return { runOrdersConsumed: 1 }; } - protected abstract getAction(output: Artifact, actionName: string, runOrder: number): Action; + protected abstract getAction(output: Artifact, actionName: string, runOrder: number, variablesNamespace?: string): Action; + + /** + * Return an attribute of the current source revision + * + * These values can be passed into the environment variables of pipeline steps, + * so your steps can access information about the source revision. + * + * What attributes are available depends on the type of source. These attributes + * are supported: + * + * - GitHub, CodeCommit, and CodeStar connection + * - `AuthorDate` + * - `BranchName` + * - `CommitId` + * - `CommitMessage` + * - GitHub and CodeCommit + * - `CommitterDate` + * - `RepositoryName` + * - GitHub + * - `CommitUrl` + * - CodeStar Connection + * - `FullRepositoryName` + * - S3 + * - `ETag` + * - `VersionId` + * + * @see https://docs.aws.amazon.com/codepipeline/latest/userguide/reference-variables.html#reference-variables-list + */ + public sourceAttribute(name: string): string { + return new StepOutput(this, name).toString(); + } } /** @@ -173,7 +206,7 @@ class GitHubSource extends CodePipelineSource { this.configurePrimaryOutput(new FileSet('Source', this)); } - protected getAction(output: Artifact, actionName: string, runOrder: number) { + protected getAction(output: Artifact, actionName: string, runOrder: number, variablesNamespace?: string) { return new cp_actions.GitHubSourceAction({ output, actionName, @@ -183,6 +216,7 @@ class GitHubSource extends CodePipelineSource { repo: this.repo, branch: this.branch, trigger: this.props.trigger, + variablesNamespace, }); } } @@ -216,7 +250,7 @@ class S3Source extends CodePipelineSource { this.configurePrimaryOutput(new FileSet('Source', this)); } - protected getAction(output: Artifact, _actionName: string, runOrder: number) { + protected getAction(output: Artifact, _actionName: string, runOrder: number, variablesNamespace?: string) { return new cp_actions.S3SourceAction({ output, // Bucket names are guaranteed to conform to ActionName restrictions @@ -225,6 +259,7 @@ class S3Source extends CodePipelineSource { bucketKey: this.objectKey, trigger: this.props.trigger, bucket: this.bucket, + variablesNamespace, }); } } @@ -284,7 +319,7 @@ class CodeStarConnectionSource extends CodePipelineSource { this.configurePrimaryOutput(new FileSet('Source', this)); } - protected getAction(output: Artifact, actionName: string, runOrder: number) { + protected getAction(output: Artifact, actionName: string, runOrder: number, variablesNamespace?: string) { return new cp_actions.CodeStarConnectionsSourceAction({ output, actionName, @@ -295,6 +330,7 @@ class CodeStarConnectionSource extends CodePipelineSource { branch: this.branch, codeBuildCloneOutput: this.props.codeBuildCloneOutput, triggerOnPush: this.props.triggerOnPush, + variablesNamespace, }); } } @@ -341,7 +377,7 @@ class CodeCommitSource extends CodePipelineSource { this.configurePrimaryOutput(new FileSet('Source', this)); } - protected getAction(output: Artifact, _actionName: string, runOrder: number) { + protected getAction(output: Artifact, _actionName: string, runOrder: number, variablesNamespace?: string) { return new cp_actions.CodeCommitSourceAction({ output, // Guaranteed to be okay as action name @@ -352,6 +388,7 @@ class CodeCommitSource extends CodePipelineSource { repository: this.repository, eventRole: this.props.eventRole, codeBuildCloneOutput: this.props.codeBuildCloneOutput, + variablesNamespace, }); } } diff --git a/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline.ts b/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline.ts index 0beae3ea56fa1..507f4ba81db94 100644 --- a/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline.ts +++ b/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline.ts @@ -18,10 +18,10 @@ import { toPosixPath } from '../private/fs'; import { actionName, stackVariableNamespace } from '../private/identifiers'; import { enumerate, flatten, maybeSuffix, noUndefined } from '../private/javascript'; import { writeTemplateConfiguration } from '../private/template-configuration'; -import { CodeBuildFactory, mergeCodeBuildOptions } from './_codebuild-factory'; import { ArtifactMap } from './artifact-map'; import { CodeBuildStep } from './codebuild-step'; import { CodePipelineActionFactoryResult, ICodePipelineActionFactory } from './codepipeline-action-factory'; +import { CodeBuildFactory, mergeCodeBuildOptions } from './private/codebuild-factory'; /** @@ -418,9 +418,14 @@ export class CodePipeline extends PipelineBase { const factory = this.actionFromNode(node); const nodeType = this.nodeTypeFromNode(node); + const name = actionName(node, sharedParent); + + const variablesNamespace = node.data?.type === 'step' + ? handleStepOutputs(node.data.step, pipelineStage, name) + : undefined; const result = factory.produceAction(pipelineStage, { - actionName: actionName(node, sharedParent), + actionName: name, runOrder, artifacts: this.artifacts, scope: obtainScope(this.pipeline, stageName), @@ -429,6 +434,7 @@ export class CodePipeline extends PipelineBase { // If this step happens to produce a CodeBuild job, set the default options codeBuildDefaults: nodeType ? this.codeBuildDefaultsFor(nodeType) : undefined, beforeSelfMutation, + variablesNamespace, }); if (node.data?.type === 'self-update') { @@ -895,3 +901,28 @@ function chunkTranches(n: number, xss: A[][]): A[][][] { function isCodePipelineActionFactory(x: any): x is ICodePipelineActionFactory { return !!(x as ICodePipelineActionFactory).produceAction; } + +/** + * If the step is producing outputs, determine a variableNamespace for it, and configure that on the outputs + */ +function handleStepOutputs(step: Step, stage: cp.IStage, name: string): string | undefined { + let ret: string | undefined; + for (const output of step.producedStepOutputs ?? []) { + ret = namespaceName(stage, name); + if (typeof output.engineSpecificInformation !== 'string') { + throw new Error(`CodePipeline requires that 'engineSpecificInformation' is a string, got: ${JSON.stringify(output.engineSpecificInformation)}`); + } + output.defineResolution(`#{${ret}.${output.engineSpecificInformation}}`); + } + return ret; +} + +/** + * Generate a variable namespace from stage and action names + * + * Variable namespaces cannot have '.', but they can have '@'. Other than that, + * action names are more limited so they translate easily. + */ +export function namespaceName(stage: cp.IStage, name: string) { + return `${stage.stageName}/${name}`.replace(/[^a-zA-Z0-9@_-]/g, '@'); +} \ No newline at end of file diff --git a/packages/@aws-cdk/pipelines/lib/codepipeline/private/buildspecs.ts b/packages/@aws-cdk/pipelines/lib/codepipeline/private/buildspecs.ts new file mode 100644 index 0000000000000..f904d7c174629 --- /dev/null +++ b/packages/@aws-cdk/pipelines/lib/codepipeline/private/buildspecs.ts @@ -0,0 +1,10 @@ +import * as codebuild from '@aws-cdk/aws-codebuild'; + +export function mergeBuildSpecs(a: codebuild.BuildSpec, b?: codebuild.BuildSpec): codebuild.BuildSpec; +export function mergeBuildSpecs(a: codebuild.BuildSpec | undefined, b: codebuild.BuildSpec): codebuild.BuildSpec; +export function mergeBuildSpecs(a?: codebuild.BuildSpec, b?: codebuild.BuildSpec): codebuild.BuildSpec | undefined; +export function mergeBuildSpecs(a?: codebuild.BuildSpec, b?: codebuild.BuildSpec) { + if (!a || !b) { return a ?? b; } + return codebuild.mergeBuildSpecs(a, b); +} + diff --git a/packages/@aws-cdk/pipelines/lib/codepipeline/_codebuild-factory.ts b/packages/@aws-cdk/pipelines/lib/codepipeline/private/codebuild-factory.ts similarity index 94% rename from packages/@aws-cdk/pipelines/lib/codepipeline/_codebuild-factory.ts rename to packages/@aws-cdk/pipelines/lib/codepipeline/private/codebuild-factory.ts index 3414554cd5197..246f48f4b84d5 100644 --- a/packages/@aws-cdk/pipelines/lib/codepipeline/_codebuild-factory.ts +++ b/packages/@aws-cdk/pipelines/lib/codepipeline/private/codebuild-factory.ts @@ -7,15 +7,17 @@ import * as ec2 from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; import { IDependable, Stack, Token } from '@aws-cdk/core'; import { Construct, Node } from 'constructs'; -import { FileSetLocation, ShellStep, StackOutputReference } from '../blueprint'; -import { PipelineQueries } from '../helpers-internal/pipeline-queries'; -import { cloudAssemblyBuildSpecDir, obtainScope } from '../private/construct-internals'; -import { hash, stackVariableNamespace } from '../private/identifiers'; -import { mapValues, mkdict, noEmptyObject, noUndefined, partition } from '../private/javascript'; -import { ArtifactMap } from './artifact-map'; -import { CodeBuildStep } from './codebuild-step'; -import { CodeBuildOptions } from './codepipeline'; -import { ICodePipelineActionFactory, ProduceActionOptions, CodePipelineActionFactoryResult } from './codepipeline-action-factory'; +import { FileSetLocation, ShellStep, StackOutputReference } from '../../blueprint'; +import { StepOutput } from '../../blueprint/step-output'; +import { PipelineQueries } from '../../helpers-internal/pipeline-queries'; +import { cloudAssemblyBuildSpecDir, obtainScope } from '../../private/construct-internals'; +import { hash, stackVariableNamespace } from '../../private/identifiers'; +import { mapValues, mkdict, noEmptyObject, noUndefined, partition } from '../../private/javascript'; +import { ArtifactMap } from '../artifact-map'; +import { CodeBuildStep } from '../codebuild-step'; +import { CodeBuildOptions } from '../codepipeline'; +import { ICodePipelineActionFactory, ProduceActionOptions, CodePipelineActionFactoryResult } from '../codepipeline-action-factory'; +import { mergeBuildSpecs } from './buildspecs'; export interface CodeBuildFactoryProps { /** @@ -110,6 +112,11 @@ export interface CodeBuildFactoryProps { * @default false */ readonly isSynth?: boolean; + + /** + * StepOutputs produced by this CodeBuild step + */ + readonly producedStepOutputs?: StepOutput[]; } /** @@ -129,6 +136,7 @@ export class CodeBuildFactory implements ICodePipelineActionFactory { outputs: shellStep.outputs, stepId: shellStep.id, installCommands: shellStep.installCommands, + producedStepOutputs: shellStep.producedStepOutputs, ...additional, }); } @@ -307,6 +315,7 @@ export class CodeBuildFactory implements ICodePipelineActionFactory { ? { _PROJECT_CONFIG_HASH: projectConfigHash } : {}; + stage.addAction(new codepipeline_actions.CodeBuildAction({ actionName: actionName, input: inputArtifact, @@ -314,6 +323,7 @@ export class CodeBuildFactory implements ICodePipelineActionFactory { outputs: outputArtifacts, project, runOrder: options.runOrder, + variablesNamespace: options.variablesNamespace, // Inclusion of the hash here will lead to the pipeline structure for any changes // made the config of the underlying CodeBuild Project. @@ -421,14 +431,6 @@ function mergeBuildEnvironments(a?: codebuild.BuildEnvironment, b?: codebuild.Bu }; } -export function mergeBuildSpecs(a: codebuild.BuildSpec, b?: codebuild.BuildSpec): codebuild.BuildSpec; -export function mergeBuildSpecs(a: codebuild.BuildSpec | undefined, b: codebuild.BuildSpec): codebuild.BuildSpec; -export function mergeBuildSpecs(a?: codebuild.BuildSpec, b?: codebuild.BuildSpec): codebuild.BuildSpec | undefined; -export function mergeBuildSpecs(a?: codebuild.BuildSpec, b?: codebuild.BuildSpec) { - if (!a || !b) { return a ?? b; } - return codebuild.mergeBuildSpecs(a, b); -} - function isDefined(x: A | undefined): x is NonNullable { return x !== undefined; } @@ -452,7 +454,7 @@ function serializeBuildEnvironment(env: codebuild.BuildEnvironment) { * Whether the given string contains a reference to a CodePipeline variable */ function containsPipelineVariable(s: string) { - return !!s.match(/#\{[^}]+\}/); + return !!s.match(/#\{[^}]+\}/) || StepOutput.findAll(s).length > 0; } /** @@ -507,4 +509,4 @@ function filterBuildSpecCommands(buildSpec: codebuild.BuildSpec, osType: ec2.Ope } return [undefined, x]; } -} \ No newline at end of file +} diff --git a/packages/@aws-cdk/pipelines/test/codepipeline/codebuild-step.test.ts b/packages/@aws-cdk/pipelines/test/codepipeline/codebuild-step.test.ts index 393f1ffb965ba..4980d6c427db7 100644 --- a/packages/@aws-cdk/pipelines/test/codepipeline/codebuild-step.test.ts +++ b/packages/@aws-cdk/pipelines/test/codepipeline/codebuild-step.test.ts @@ -141,4 +141,68 @@ test('envFromOutputs works even with very long stage and stack names', () => { }); // THEN - did not throw an error about identifier lengths +}); + +test('exportedVariables', () => { + const pipeline = new ModernTestGitHubNpmPipeline(pipelineStack, 'Cdk'); + + // GIVEN + const producer = new cdkp.CodeBuildStep('Produce', { + commands: ['export MY_VAR=hello'], + }); + + const consumer = new cdkp.CodeBuildStep('Consume', { + env: { + THE_VAR: producer.exportedVariable('MY_VAR'), + }, + commands: [ + 'echo "The variable was: $THE_VAR"', + ], + }); + + // WHEN + pipeline.addWave('MyWave', { + post: [consumer, producer], + }); + + // THEN + const template = Template.fromStack(pipelineStack); + template.hasResourceProperties('AWS::CodePipeline::Pipeline', { + Stages: [ + { Name: 'Source' }, + { Name: 'Build' }, + { Name: 'UpdatePipeline' }, + { + Name: 'MyWave', + Actions: [ + Match.objectLike({ + Name: 'Produce', + Namespace: 'MyWave@Produce', + RunOrder: 1, + }), + Match.objectLike({ + Name: 'Consume', + RunOrder: 2, + Configuration: Match.objectLike({ + EnvironmentVariables: Match.serializedJson(Match.arrayWith([ + { + name: 'THE_VAR', + type: 'PLAINTEXT', + value: '#{MyWave@Produce.MY_VAR}', + }, + ])), + }), + }), + ], + }, + ], + }); + + template.hasResourceProperties('AWS::CodeBuild::Project', { + BuildSpec: Match.serializedJson(Match.objectLike({ + env: { + 'exported-variables': ['MY_VAR'], + }, + })), + }); }); \ No newline at end of file diff --git a/packages/@aws-cdk/pipelines/test/codepipeline/codepipeline-sources.test.ts b/packages/@aws-cdk/pipelines/test/codepipeline/codepipeline-sources.test.ts index 46fb468c37623..721c65d6d8b21 100644 --- a/packages/@aws-cdk/pipelines/test/codepipeline/codepipeline-sources.test.ts +++ b/packages/@aws-cdk/pipelines/test/codepipeline/codepipeline-sources.test.ts @@ -178,4 +178,45 @@ test('artifact names are never longer than 128 characters', () => { }); expect(artifactId.asString().length).toBeLessThanOrEqual(128); +}); + +test('can use source attributes in pipeline', () => { + const gitHub = cdkp.CodePipelineSource.gitHub('owner/my-repo', 'main'); + + // WHEN + new ModernTestGitHubNpmPipeline(pipelineStack, 'Pipeline', { + input: gitHub, + synth: new cdkp.ShellStep('Synth', { + env: { + GITHUB_URL: gitHub.sourceAttribute('CommitUrl'), + }, + commands: [ + 'echo "Click here: $GITHUB_URL"', + ], + }), + selfMutation: false, + }); + + Template.fromStack(pipelineStack).hasResourceProperties('AWS::CodePipeline::Pipeline', { + Stages: [ + { Name: 'Source' }, + { + Name: 'Build', + Actions: [ + { + Name: 'Synth', + Configuration: Match.objectLike({ + EnvironmentVariables: Match.serializedJson([ + { + name: 'GITHUB_URL', + type: 'PLAINTEXT', + value: '#{Source@owner_my-repo.CommitUrl}', + }, + ]), + }), + }, + ], + }, + ], + }); }); \ No newline at end of file diff --git a/packages/@aws-cdk/pipelines/test/integ.newpipeline-with-vpc.expected.json b/packages/@aws-cdk/pipelines/test/integ.newpipeline-with-vpc.expected.json index 4bd2e638afb4c..6ee8a28d2c76e 100644 --- a/packages/@aws-cdk/pipelines/test/integ.newpipeline-with-vpc.expected.json +++ b/packages/@aws-cdk/pipelines/test/integ.newpipeline-with-vpc.expected.json @@ -870,6 +870,7 @@ } ], "Name": "SelfMutate", + "Namespace": "SelfMutate", "RoleArn": { "Fn::GetAtt": [ "PipelineUpdatePipelineSelfMutateCodePipelineActionRoleD6D4E5CF", @@ -901,6 +902,7 @@ } ], "Name": "FileAsset1", + "Namespace": "FileAsset1", "RoleArn": { "Fn::GetAtt": [ "PipelineAssetsFileAsset1CodePipelineActionRoleC0EC649A", @@ -927,6 +929,7 @@ } ], "Name": "FileAsset2", + "Namespace": "FileAsset2", "RoleArn": { "Fn::GetAtt": [ "PipelineAssetsFileAsset2CodePipelineActionRole06965A59", @@ -2462,4 +2465,4 @@ ] } } -} \ No newline at end of file +} diff --git a/packages/@aws-cdk/pipelines/test/integ.newpipeline.expected.json b/packages/@aws-cdk/pipelines/test/integ.newpipeline.expected.json index 37cd5d99fd7f8..6206239c97d7b 100644 --- a/packages/@aws-cdk/pipelines/test/integ.newpipeline.expected.json +++ b/packages/@aws-cdk/pipelines/test/integ.newpipeline.expected.json @@ -336,6 +336,7 @@ } ], "Name": "SelfMutate", + "Namespace": "SelfMutate", "RoleArn": { "Fn::GetAtt": [ "PipelineUpdatePipelineSelfMutateCodePipelineActionRoleD6D4E5CF", @@ -2383,4 +2384,4 @@ ] } } -} \ No newline at end of file +} diff --git a/packages/@aws-cdk/pipelines/test/integ.pipeline-with-variables.ts b/packages/@aws-cdk/pipelines/test/integ.pipeline-with-variables.ts new file mode 100644 index 0000000000000..a7b62576eb519 --- /dev/null +++ b/packages/@aws-cdk/pipelines/test/integ.pipeline-with-variables.ts @@ -0,0 +1,50 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +/// !cdk-integ PipelineStack pragma:set-context:@aws-cdk/core:newStyleStackSynthesis=true +import { GitHubTrigger } from '@aws-cdk/aws-codepipeline-actions'; +import { App, Stack, StackProps } from '@aws-cdk/core'; +import { Construct } from 'constructs'; +import * as pipelines from '../lib'; + +class PipelineStack extends Stack { + constructor(scope: Construct, id: string, props?: StackProps) { + super(scope, id, props); + + const pipeline = new pipelines.CodePipeline(this, 'Pipeline', { + synth: new pipelines.ShellStep('Synth', { + input: pipelines.CodePipelineSource.gitHub('cdklabs/construct-hub-probe', 'main', { + trigger: GitHubTrigger.POLL, + }), + commands: ['mkdir cdk.out', 'touch cdk.out/dummy'], + }), + selfMutation: false, + }); + + const producer = new pipelines.CodeBuildStep('Produce', { + commands: ['export MY_VAR=hello'], + }); + + const consumer = new pipelines.CodeBuildStep('Consume', { + env: { + THE_VAR: producer.exportedVariable('MY_VAR'), + }, + commands: [ + 'echo "The variable was: $THE_VAR"', + ], + }); + + // WHEN + pipeline.addWave('MyWave', { + post: [consumer, producer], + }); + } +} + +const app = new App({ + context: { + '@aws-cdk/core:newStyleStackSynthesis': '1', + }, +}); +new PipelineStack(app, 'VariablePipelineStack', { + env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION }, +}); +app.synth(); \ No newline at end of file