feat(pipelines): step outputs

packages/@aws-cdk/pipelines/README.md

@@ -350,6 +350,40 @@ const pipeline = new pipelines.CodePipeline(this, 'Pipeline', {
 
 You can adapt these examples to your own situation.
 
+#### Migrating from buildspec.yml files
+
+You may currently have the build instructions for your CodeBuild Projects in a
+`buildspec.yml` file in your source repository. In addition to your build
+commands, the CodeBuild Project's buildspec als controls some information that
+CDK Pipelines manages for you, like artifact identifiers, input artifact
+locations, Docker authorization, and exported variables.
+
+Since there is no way in general for CDK Pipelines to modify the file in your
+resource repository, CDK Pipelines configures the BuildSpec directly on the
+CodeBuild Project, instead of loading it from the `buildspec.yml` file.
+This requires a pipeline self-mutation to update.
+
+To avoid this, put your build instructions in a separate script, for example
+`build.sh`, and call that script from the build `commands` array:
+
+```ts
+declare const source: pipelines.IFileSetProducer;
+
+const pipeline = new pipelines.CodePipeline(this, 'Pipeline', {
+  synth: new pipelines.ShellStep('Synth', {
+    input: source,
+    commands: [
+      // Abstract over doing the build
+      './build.sh',
+    ],
+  })
+});
+```
+
+Doing so keeps your exact build instructions in sync with your source code in
+the source repository where it belongs, and provides a convenient build script
+for developers at the same time.
+
 #### CodePipeline Sources
 
 In CodePipeline, *Sources* define where the source of your application lives.
@@ -768,6 +802,13 @@ class MyJenkinsStep extends pipelines.Step implements pipelines.ICodePipelineAct
     private readonly input: pipelines.FileSet,
   ) {
     super('MyJenkinsStep');
+
+    // This is necessary if your step accepts things like environment variables
+    // that may contain outputs from other steps. It doesn't matter what the
+    // structure is, as long as it contains the values that may contain outputs.
+    this.discoverReferencedOutputs({
+      env: { /* ... */ }
+    });
   }
 
   public produceAction(stage: codepipeline.IStage, options: pipelines.ProduceActionOptions): pipelines.CodePipelineActionFactoryResult {

packages/@aws-cdk/pipelines/lib/blueprint/index.ts

@@ -3,6 +3,7 @@ export * from './file-set';
 export * from './shell-step';
 export * from './stack-deployment';
 export * from './stage-deployment';
+export * from './step-output';
 export * from './step';
 export * from './wave';
 export * from './manual-approval';

packages/@aws-cdk/pipelines/lib/blueprint/manual-approval.ts

@@ -33,5 +33,7 @@ export class ManualApprovalStep extends Step {
     super(id);
 
     this.comment = props.comment;
+
+    this.discoverReferencedOutputs(props.comment);
   }
 }

packages/@aws-cdk/pipelines/lib/blueprint/shell-step.ts

@@ -87,7 +87,6 @@ export interface ShellStepProps {
    * @default - No primary output
    */
   readonly primaryOutputDirectory?: string;
-
 }
 
 /**
@@ -152,6 +151,11 @@ export class ShellStep extends Step {
     this.env = props.env ?? {};
     this.envFromCfnOutputs = mapValues(props.envFromCfnOutputs ?? {}, StackOutputReference.fromCfnOutput);
 
+    // 'env' is the only thing that can contain outputs
+    this.discoverReferencedOutputs({
+      env: this.env,
+    });
+
     // Inputs
     if (props.input) {
       const fileSet = props.input.primaryOutput;

packages/@aws-cdk/pipelines/lib/blueprint/step-output.ts

@@ -0,0 +1,109 @@
+import { IResolvable, IResolveContext, Token, Tokenization } from '@aws-cdk/core';
+import { Step } from './step';
+
+const STEP_OUTPUT_SYM = Symbol.for('@aws-cdk/pipelines.StepOutput');
+
+/**
+ * A symbolic reference to a value produced by another step
+ *
+ * The engine is responsible for defining how these should be rendered.
+ */
+export class StepOutput implements IResolvable {
+  /**
+   * Return true if the given IResolvable is a StepOutput
+   */
+  public static isStepOutput(resolvable: IResolvable): resolvable is StepOutput {
+    return !!(resolvable as any)[STEP_OUTPUT_SYM];
+  }
+
+  /**
+   * Find all StepOutputs referenced in the given structure
+   */
+  public static findAll(structure: any): StepOutput[] {
+    return findAllStepOutputs(structure);
+  }
+
+  /**
+   * The step that produces this output
+   */
+  public readonly step: Step;
+
+  /**
+   * Additional data on the output, to be interpreted by the engine
+   */
+  public readonly engineSpecificInformation: any;
+
+  public readonly creationStack: string[] = [];
+  private resolution: any = undefined;
+
+  constructor(step: Step, engineSpecificInformation: any) {
+    this.step = step;
+    this.engineSpecificInformation = engineSpecificInformation;
+    Object.defineProperty(this, STEP_OUTPUT_SYM, { value: true });
+  }
+
+  /**
+   * Define the resolved value for this StepOutput.
+   *
+   * Should be called by the engine.
+   */
+  public defineResolution(value: any) {
+    this.resolution = value;
+  }
+
+  public resolve(_context: IResolveContext) {
+    if (this.resolution === undefined) {
+      throw new Error(`Output for step ${this.step} not configured. Either the step is not in the pipeline, or this engine does not support Outputs for this step.`);
+    }
+    return this.resolution;
+  }
+
+  public toString(): string {
+    return Token.asString(this);
+  }
+}
+
+function findAllStepOutputs(structure: any): StepOutput[] {
+  const ret = new Set<StepOutput>();
+  recurse(structure);
+  return Array.from(ret);
+
+  function checkToken(x?: IResolvable) {
+    if (x && StepOutput.isStepOutput(x)) {
+      ret.add(x);
+      return true;
+    }
+
+    // Return false if it wasn't a Token in the first place (in which case we recurse)
+    return x !== undefined;
+  }
+
+  function recurse(x: any): void {
+    if (!x) { return; }
+
+    if (Tokenization.isResolvable(x)) {
+      checkToken(x);
+      return;
+    }
+    if (Array.isArray(x)) {
+      if (!checkToken(Tokenization.reverseList(x))) {
+        x.forEach(recurse);
+      }
+      return;
+    }
+    if (typeof x === 'number') {
+      checkToken(Tokenization.reverseNumber(x));
+      return;
+    }
+    if (typeof x === 'string') {
+      Tokenization.reverseString(x).tokens.forEach(checkToken);
+      return;
+    }
+    if (typeof x === 'object') {
+      for (const [k, v] of Object.entries(x)) {
+        recurse(k);
+        recurse(v);
+      }
+    }
+  }
+}

packages/@aws-cdk/pipelines/lib/blueprint/step.ts

@@ -1,4 +1,5 @@
 import { Stack, Token } from '@aws-cdk/core';
+import { StepOutput } from '../blueprint/step-output';
 import { FileSet, IFileSetProducer } from './file-set';
 
 /**
@@ -30,6 +31,20 @@ export abstract class Step implements IFileSetProducer {
    */
   public readonly dependencyFileSets: FileSet[] = [];
 
+  /**
+   * The list of StepOutputs consumed by this Step
+   *
+   * Should be inspected by the pipeline engine.
+   */
+  public readonly consumedStepOutputs: StepOutput[] = [];
+
+  /**
+   * The list of StepOutputs produced by this Step
+   *
+   * Should be inspected by the pipeline engine.
+   */
+  public readonly producedStepOutputs: StepOutput[] = [];
+
   /**
    * Whether or not this is a Source step
    *
@@ -39,7 +54,7 @@ export abstract class Step implements IFileSetProducer {
 
   private _primaryOutput?: FileSet;
 
-  private _dependencies: Step[] = [];
+  private _dependencies = new Set<Step>();
 
   constructor(
     /** Identifier for this step */
@@ -54,7 +69,10 @@ export abstract class Step implements IFileSetProducer {
    * Return the steps this step depends on, based on the FileSets it requires
    */
   public get dependencies(): Step[] {
-    return this.dependencyFileSets.map(f => f.producer).concat(this._dependencies);
+    return Array.from(new Set([
+      ...this.dependencyFileSets.map(f => f.producer),
+      ...this._dependencies,
+    ]));
   }
 
   /**
@@ -79,7 +97,7 @@ export abstract class Step implements IFileSetProducer {
    * Add a dependency on another step.
    */
   public addStepDependency(step: Step) {
-    this._dependencies.push(step);
+    this._dependencies.add(step);
   }
 
   /**
@@ -97,6 +115,22 @@ export abstract class Step implements IFileSetProducer {
   protected configurePrimaryOutput(fs: FileSet) {
     this._primaryOutput = fs;
   }
+
+  /**
+   * Crawl the given structure for references to StepOutputs and add dependencies on all steps found
+   *
+   * Should be called by subclasses based on what the user passes in as
+   * construction properties. The format of the structure passed in here does
+   * not have to correspond exactly to what gets rendered into the engine, it
+   * just needs to contain the same amount of data.
+   */
+  protected discoverReferencedOutputs(structure: any) {
+    for (const output of StepOutput.findAll(structure)) {
+      this._dependencies.add(output.step);
+      this.consumedStepOutputs.push(output);
+      output.step.producedStepOutputs.push(output);
+    }
+  }
 }
 
 /**
@@ -128,5 +162,4 @@ export interface StackSteps {
    * @default - no additional steps
    */
   readonly post?: Step[];
-
 }

packages/@aws-cdk/pipelines/lib/codepipeline/codebuild-step.ts

@@ -1,8 +1,10 @@
-import { Duration } from '@aws-cdk/core';
 import * as codebuild from '@aws-cdk/aws-codebuild';
 import * as ec2 from '@aws-cdk/aws-ec2';
 import * as iam from '@aws-cdk/aws-iam';
+import { Duration } from '@aws-cdk/core';
 import { ShellStep, ShellStepProps } from '../blueprint';
+import { StepOutput } from '../blueprint/step-output';
+import { mergeBuildSpecs } from './private/buildspecs';
 
 /**
  * Construction props for a CodeBuildStep
@@ -96,6 +98,17 @@ export interface CodeBuildStepProps extends ShellStepProps {
 
 /**
  * Run a script as a CodeBuild Project
+ *
+ * The BuildSpec must be available inline--it cannot reference a file
+ * on disk. If your current build instructions are in a file like
+ * `buildspec.yml` in your repository, extract them to a script
+ * (say, `build.sh`) and invoke that script as part of the build:
+ *
+ * ```ts
+ * new pipelines.CodeBuildStep({
+ *   commands: ['./build.sh'],
+ * });
+ * ```
  */
 export class CodeBuildStep extends ShellStep {
   /**
@@ -105,13 +118,6 @@ export class CodeBuildStep extends ShellStep {
    */
   public readonly projectName?: string;
 
-  /**
-   * Additional configuration that can only be configured via BuildSpec
-   *
-   * @default - No value specified at construction time, use defaults
-   */
-  public readonly partialBuildSpec?: codebuild.BuildSpec;
-
   /**
    * The VPC where to execute the SimpleSynth.
    *
@@ -164,13 +170,16 @@ export class CodeBuildStep extends ShellStep {
   readonly timeout?: Duration;
 
   private _project?: codebuild.IProject;
+  private _partialBuildSpec?: codebuild.BuildSpec;
+  private readonly exportedVariables = new Set<string>();
+  private exportedVarsRendered = false;
 
   constructor(id: string, props: CodeBuildStepProps) {
     super(id, props);
 
     this.projectName = props.projectName;
     this.buildEnvironment = props.buildEnvironment;
-    this.partialBuildSpec = props.partialBuildSpec;
+    this._partialBuildSpec = props.partialBuildSpec;
     this.vpc = props.vpc;
     this.subnetSelection = props.subnetSelection;
     this.role = props.role;
@@ -198,6 +207,45 @@ export class CodeBuildStep extends ShellStep {
     return this.project.grantPrincipal;
   }
 
+  /**
+   * Additional configuration that can only be configured via BuildSpec
+   *
+   * Contains exported variables
+   *
+   * @default - Contains the exported variables
+   */
+  public get partialBuildSpec(): codebuild.BuildSpec | undefined {
+    this.exportedVarsRendered = true;
+
+    const varsBuildSpec = this.exportedVariables.size > 0 ? codebuild.BuildSpec.fromObject({
+      version: '0.2',
+      env: {
+        'exported-variables': Array.from(this.exportedVariables),
+      },
+    }) : undefined;
+
+    return mergeBuildSpecs(varsBuildSpec, this._partialBuildSpec);
+  }
+
+  /**
+   * Reference a CodePipeline variable defined by the CodeBuildStep.
+   *
+   * The variable must be set in the shell of the CodeBuild step when
+   * it finishes its `post_build` phase.
+   *
+   * @param variableName the name of the variable for reference.
+   */
+  public exportedVariable(variableName: string): string {
+    if (this.exportedVarsRendered && !this.exportedVariables.has(variableName)) {
+      throw new Error('exportVariable(): Pipeline has already been produced, cannot call this function anymore');
+    }
+
+    this.exportedVariables.add(variableName);
+
+    // return `#{${this.variablesNamespace}.${variableName}}`;
+    return new StepOutput(this, variableName).toString();
+  }
+
   /**
    * Set the internal project value
    *