Skip to content

Commit

Permalink
Squashed commit of the following:
Browse files Browse the repository at this point in the history
feat(pipelines): step outputs

Make it possible to export environment variables from a CodeBuildStep,
and pipeline sources, and use them in the environment variables of
a CodeBuildStep or ShellStep.

Closes #17189, closes #18893, closes #15943, closes #16407.
  • Loading branch information
rix0rrr committed Feb 17, 2022
1 parent 3533ea9 commit 9b0efeb
Show file tree
Hide file tree
Showing 16 changed files with 518 additions and 43 deletions.
41 changes: 41 additions & 0 deletions packages/@aws-cdk/pipelines/README.md
Expand Up @@ -350,6 +350,40 @@ const pipeline = new pipelines.CodePipeline(this, 'Pipeline', {

You can adapt these examples to your own situation.

#### Migrating from buildspec.yml files

You may currently have the build instructions for your CodeBuild Projects in a
`buildspec.yml` file in your source repository. In addition to your build
commands, the CodeBuild Project's buildspec als controls some information that
CDK Pipelines manages for you, like artifact identifiers, input artifact
locations, Docker authorization, and exported variables.

Since there is no way in general for CDK Pipelines to modify the file in your
resource repository, CDK Pipelines configures the BuildSpec directly on the
CodeBuild Project, instead of loading it from the `buildspec.yml` file.
This requires a pipeline self-mutation to update.

To avoid this, put your build instructions in a separate script, for example
`build.sh`, and call that script from the build `commands` array:

```ts
declare const source: pipelines.IFileSetProducer;

const pipeline = new pipelines.CodePipeline(this, 'Pipeline', {
synth: new pipelines.ShellStep('Synth', {
input: source,
commands: [
// Abstract over doing the build
'./build.sh',
],
})
});
```

Doing so keeps your exact build instructions in sync with your source code in
the source repository where it belongs, and provides a convenient build script
for developers at the same time.

#### CodePipeline Sources

In CodePipeline, *Sources* define where the source of your application lives.
Expand Down Expand Up @@ -768,6 +802,13 @@ class MyJenkinsStep extends pipelines.Step implements pipelines.ICodePipelineAct
private readonly input: pipelines.FileSet,
) {
super('MyJenkinsStep');

// This is necessary if your step accepts things like environment variables
// that may contain outputs from other steps. It doesn't matter what the
// structure is, as long as it contains the values that may contain outputs.
this.discoverReferencedOutputs({
env: { /* ... */ }
});
}

public produceAction(stage: codepipeline.IStage, options: pipelines.ProduceActionOptions): pipelines.CodePipelineActionFactoryResult {
Expand Down
2 changes: 2 additions & 0 deletions packages/@aws-cdk/pipelines/lib/blueprint/manual-approval.ts
Expand Up @@ -33,5 +33,7 @@ export class ManualApprovalStep extends Step {
super(id);

this.comment = props.comment;

this.discoverReferencedOutputs(props.comment);
}
}
6 changes: 5 additions & 1 deletion packages/@aws-cdk/pipelines/lib/blueprint/shell-step.ts
Expand Up @@ -87,7 +87,6 @@ export interface ShellStepProps {
* @default - No primary output
*/
readonly primaryOutputDirectory?: string;

}

/**
Expand Down Expand Up @@ -152,6 +151,11 @@ export class ShellStep extends Step {
this.env = props.env ?? {};
this.envFromCfnOutputs = mapValues(props.envFromCfnOutputs ?? {}, StackOutputReference.fromCfnOutput);

// 'env' is the only thing that can contain outputs
this.discoverReferencedOutputs({
env: this.env,
});

// Inputs
if (props.input) {
const fileSet = props.input.primaryOutput;
Expand Down
97 changes: 97 additions & 0 deletions packages/@aws-cdk/pipelines/lib/blueprint/step-output.ts
@@ -0,0 +1,97 @@
import { IResolvable, IResolveContext, Token, Tokenization } from '@aws-cdk/core';
import { Step } from './step';

const STEP_OUTPUT_SYM = Symbol.for('@aws-cdk/pipelines.StepOutput');

/**
* A symbolic reference to a value produced by another step
*
* The engine is responsible for defining how these should be rendered.
*/
export class StepOutput implements IResolvable {
/**
* Return true if the given IResolvable is a StepOutput
*/
public static isStepOutput(resolvable: IResolvable): resolvable is StepOutput {
return !!(resolvable as any)[STEP_OUTPUT_SYM];
}

/**
* Find all StepOutputs referenced in the given structure
*/
public static findAll(structure: any): StepOutput[] {
return findAllStepOutputs(structure);
}

public readonly creationStack: string[] = [];
private resolution: any = undefined;

constructor(public readonly step: Step, public readonly engineSpecificInformation: any) {
Object.defineProperty(this, STEP_OUTPUT_SYM, { value: true });
}

/**
* Define the resolved value for this StepOutput.
*
* Should be called by the engine.
*/
public defineResolution(value: any) {
this.resolution = value;
}

public resolve(_context: IResolveContext) {
if (this.resolution === undefined) {
throw new Error(`Output for step ${this.step} not configured. Either the step is not in the pipeline, or this engine does not support Outputs for this step.`);
}
return this.resolution;
}

public toString(): string {
return Token.asString(this);
}
}

function findAllStepOutputs(structure: any): StepOutput[] {
const ret = new Set<StepOutput>();
recurse(structure);
return Array.from(ret);

function checkToken(x?: IResolvable) {
if (x && StepOutput.isStepOutput(x)) {
ret.add(x);
return true;
}

// Return false if it wasn't a Token in the first place (in which case we recurse)
return x !== undefined;
}

function recurse(x: any): void {
if (!x) { return; }

if (Tokenization.isResolvable(x)) {
checkToken(x);
return;
}
if (Array.isArray(x)) {
if (!checkToken(Tokenization.reverseList(x))) {
x.forEach(recurse);
}
return;
}
if (typeof x === 'number') {
checkToken(Tokenization.reverseNumber(x));
return;
}
if (typeof x === 'string') {
Tokenization.reverseString(x).tokens.forEach(checkToken);
return;
}
if (typeof x === 'object') {
for (const [k, v] of Object.entries(x)) {
recurse(k);
recurse(v);
}
}
}
}
41 changes: 37 additions & 4 deletions packages/@aws-cdk/pipelines/lib/blueprint/step.ts
@@ -1,4 +1,5 @@
import { Stack, Token } from '@aws-cdk/core';
import { StepOutput } from '../blueprint/step-output';
import { FileSet, IFileSetProducer } from './file-set';

/**
Expand Down Expand Up @@ -30,6 +31,20 @@ export abstract class Step implements IFileSetProducer {
*/
public readonly dependencyFileSets: FileSet[] = [];

/**
* The list of StepOutputs consumed by this Step
*
* Should be inspected by the pipeline engine.
*/
public readonly consumedStepOutputs: StepOutput[] = [];

/**
* The list of StepOutputs produced by this Step
*
* Should be inspected by the pipeline engine.
*/
public readonly producedStepOutputs: StepOutput[] = [];

/**
* Whether or not this is a Source step
*
Expand All @@ -39,7 +54,7 @@ export abstract class Step implements IFileSetProducer {

private _primaryOutput?: FileSet;

private _dependencies: Step[] = [];
private _dependencies = new Set<Step>();

constructor(
/** Identifier for this step */
Expand All @@ -54,7 +69,10 @@ export abstract class Step implements IFileSetProducer {
* Return the steps this step depends on, based on the FileSets it requires
*/
public get dependencies(): Step[] {
return this.dependencyFileSets.map(f => f.producer).concat(this._dependencies);
return Array.from(new Set([
...this.dependencyFileSets.map(f => f.producer),
...this._dependencies,
]));
}

/**
Expand All @@ -79,7 +97,7 @@ export abstract class Step implements IFileSetProducer {
* Add a dependency on another step.
*/
public addStepDependency(step: Step) {
this._dependencies.push(step);
this._dependencies.add(step);
}

/**
Expand All @@ -97,6 +115,22 @@ export abstract class Step implements IFileSetProducer {
protected configurePrimaryOutput(fs: FileSet) {
this._primaryOutput = fs;
}

/**
* Crawl the given structure for references to StepOutputs and add dependencies on all steps found
*
* Should be called by subclasses based on what the user passes in as
* construction properties. The format of the structure passed in here does
* not have to correspond exactly to what gets rendered into the engine, it
* just needs to contain the same amount of data.
*/
protected discoverReferencedOutputs(structure: any) {
for (const output of StepOutput.findAll(structure)) {
this._dependencies.add(output.step);
this.consumedStepOutputs.push(output);
output.step.producedStepOutputs.push(output);
}
}
}

/**
Expand Down Expand Up @@ -128,5 +162,4 @@ export interface StackSteps {
* @default - no additional steps
*/
readonly post?: Step[];

}
66 changes: 57 additions & 9 deletions packages/@aws-cdk/pipelines/lib/codepipeline/codebuild-step.ts
@@ -1,8 +1,10 @@
import { Duration } from '@aws-cdk/core';
import * as codebuild from '@aws-cdk/aws-codebuild';
import * as ec2 from '@aws-cdk/aws-ec2';
import * as iam from '@aws-cdk/aws-iam';
import { Duration } from '@aws-cdk/core';
import { ShellStep, ShellStepProps } from '../blueprint';
import { StepOutput } from '../blueprint/step-output';
import { mergeBuildSpecs } from './private/buildspecs';

/**
* Construction props for a CodeBuildStep
Expand Down Expand Up @@ -96,6 +98,17 @@ export interface CodeBuildStepProps extends ShellStepProps {

/**
* Run a script as a CodeBuild Project
*
* The BuildSpec must be available inline--it cannot reference a file
* on disk. If your current build instructions are in a file like
* `buildspec.yml` in your repository, extract them to a script
* (say, `build.sh`) and invoke that script as part of the build:
*
* ```ts
* new pipelines.CodeBuildStep({
* commands: ['./build.sh'],
* });
* ```
*/
export class CodeBuildStep extends ShellStep {
/**
Expand All @@ -105,13 +118,6 @@ export class CodeBuildStep extends ShellStep {
*/
public readonly projectName?: string;

/**
* Additional configuration that can only be configured via BuildSpec
*
* @default - No value specified at construction time, use defaults
*/
public readonly partialBuildSpec?: codebuild.BuildSpec;

/**
* The VPC where to execute the SimpleSynth.
*
Expand Down Expand Up @@ -164,13 +170,16 @@ export class CodeBuildStep extends ShellStep {
readonly timeout?: Duration;

private _project?: codebuild.IProject;
private _partialBuildSpec?: codebuild.BuildSpec;
private readonly exportedVariables = new Set<string>();
private exportedVarsRendered = false;

constructor(id: string, props: CodeBuildStepProps) {
super(id, props);

this.projectName = props.projectName;
this.buildEnvironment = props.buildEnvironment;
this.partialBuildSpec = props.partialBuildSpec;
this._partialBuildSpec = props.partialBuildSpec;
this.vpc = props.vpc;
this.subnetSelection = props.subnetSelection;
this.role = props.role;
Expand Down Expand Up @@ -198,6 +207,45 @@ export class CodeBuildStep extends ShellStep {
return this.project.grantPrincipal;
}

/**
* Additional configuration that can only be configured via BuildSpec
*
* Contains exported variables
*
* @default - Contains the exported variables
*/
public get partialBuildSpec(): codebuild.BuildSpec | undefined {
this.exportedVarsRendered = true;

const varsBuildSpec = this.exportedVariables.size > 0 ? codebuild.BuildSpec.fromObject({
version: '0.2',
env: {
'exported-variables': Array.from(this.exportedVariables),
},
}) : undefined;

return mergeBuildSpecs(varsBuildSpec, this._partialBuildSpec);
}

/**
* Reference a CodePipeline variable defined by the CodeBuildStep.
*
* The variable must be set in the shell of the CodeBuild step when
* it finishes its `post_build` phase.
*
* @param variableName the name of the variable for reference.
*/
public exportedVariable(variableName: string): string {
if (this.exportedVarsRendered && !this.exportedVariables.has(variableName)) {
throw new Error('exportVariable(): Pipeline has already been produced, cannot call this function anymore');
}

this.exportedVariables.add(variableName);

// return `#{${this.variablesNamespace}.${variableName}}`;
return new StepOutput(this, variableName).toString();
}

/**
* Set the internal project value
*
Expand Down

0 comments on commit 9b0efeb

Please sign in to comment.