packages-oss.yml

# packages.yml
#
# packages.yml defines all the packages we are able to build for a single commit
# in this repo. A package means a single zip file containing the executable binary,
# and optionally other files if needed.
#
# packages.yml is a convenience file for the human management of large numbers of
# alternate packages, allowing default and templated values. We generate another
# artifact from this one, called packages.lock which contains the fully expanded set
# of package and layer specs. This fully expanded file is in turn used to drive the
# build process, and as data for templating CI config.

# config contains packagespec config for this repo.
config:
  # product-repo is important to CI providers.
  product-repo:          https://github.com/hashicorp/vault.git
  release-repo:          https://github.com/hashicorp/vault-release.git
  # product-id is used by external systems to identify this product.
  # It can be any unique name, but a golang import path is ideal.
  product-id:            github.com/hashicorp/vault
  circleci-project-slug: gh/hashicorp/vault
  circleci-host:         circleci.com
  on-publish:            create-github-release

# inputs are a set of environment variables that may affect the set of bytes produced
# for a given package. Note that a package is a zip file containing the binary, so
# the name of the binary does affect the package's bytes.
inputs:

  # defaults contains default input values for each package.
  # These values may be overridden on a per-package basis in the packages section.
  defaults:

    # PRODUCT_VERSION is the version of this product. Usually, this should be left
    # as 0.0.0-snapshot. When we build a release candidate, this is overridden in
    # a one-off fashion to produce that build.
    # This should be used in the PACKAGE_NAME template.
    PRODUCT_VERSION: 0.0.0-snapshot

    # GO_VERSION is the version of the Go toolchain to use to compile this package.
    GO_VERSION: 1.15.7

    # YARN_VERSION is the version of Yarn to install for the UI layer.
    YARN_VERSION: 1.19.1-1

    # Standard golang environment variables, passed to the 'go build' command.
    # You can use any standard environment variables here, any that you omit
    # will be ommitted from the go build command too, meaning to use the system
    # default in the build container.
    CGO_ENABLED: 0
    GO111MODULE: "off"

  # templates contain golang template strings. Each of these is rendered per package
  # using that packages values (including any default values), and then added to that
  # package.
  # Note that templates MAY NOT refer to each other, but may refer to any default or
  # package-specific inputs.
  templates:

    # BINARY_NAME is the name of the executable binary we compile and package.
    # It is the name users will use on the CLI to invoke the product.
    BINARY_NAME: 'vault{{if eq .GOOS "windows"}}.exe{{end}}'

    # PRODUCT_VERSION_MMP is just the major.minor.prerelease fields of the PRODUCT_VERSION.
    # Think semantic versioning (semver), although we do not version our binaries
    # using semver.
    PRODUCT_VERSION_MMP: >-
      {{with .PRODUCT_VERSION | strings.SplitN "-" 2}}{{index . 0}}{{end}}
    # PRODUCT_VERSION_PRE is just the prerelease field of the product version (i.e. the bit
    # after any -, if there is one.
    PRODUCT_VERSION_PRE: >-
      {{with .PRODUCT_VERSION | strings.SplitN "-" 2}}{{if gt (len .) 1}}{{index . 1}}{{else}}"''"{{end}}{{end}}

# build-command is a templated bash script to be run in the final builder container
# to produce the package. It may refer to any of the inputs, including rendered templates,
# but not meta data.
#
# The build command is passed 3 environment variables, in addition to all those specified as inputs.
#
#   - PACKAGE_SOURCE_ID       The source ID (usually the git commit SHA, unless build is dirty)
#   - OUTPUT_DIR              Directory to write the executable and zip file to (will exist already)
#   - PACKAGE_ZIP_NAME        The name of the package zip file to create (relative to OUTPUT_DIR)
#
# NOTE: You MUST NOT use single quotes in the build command, because at present we do no escaping.
build-command: VERSION_PKG_PATH=github.com/hashicorp/vault/vendor/github.com/hashicorp/vault/sdk/version;
  go build -v
    -tags ui
    -ldflags "
      -X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID
      -X $VERSION_PKG_PATH.Version={{.PRODUCT_VERSION_MMP}}
      -X $VERSION_PKG_PATH.VersionPrerelease={{.PRODUCT_VERSION_PRE}}"
    -o $OUTPUT_DIR/{{.BINARY_NAME}}
  && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME {{.BINARY_NAME}}

# packages is the full set of packages we are able to build based on a single commit
# in this repo. Each package is a map where the keys are the names of environment
# variables provided to each build (think 'go build' invocation). Each package is
# expanded by first filling in any unspecified variables with those from defaults,
# and then rendering each template and adding the result to the map.
# Each package must result in a unique PACKAGE_NAME.
#
# The fully expanded set of packages are written to packages.lock. That file
# is a useful data source for building CI/CD pipelines.
packages:
  - inputs: { GOOS: darwin, GOARCH: amd64 }
  - inputs: { GOOS: freebsd, GOARCH: 386 }
  - inputs: { GOOS: freebsd, GOARCH: amd64 }
  - inputs: { GOOS: freebsd, GOARCH: arm }
  - inputs: { GOOS: linux, GOARCH: 386 }
  - inputs: { GOOS: linux, GOARCH: amd64 }
  - inputs: { GOOS: linux, GOARCH: arm }
  - inputs: { GOOS: linux, GOARCH: arm64 }
  - inputs: { GOOS: netbsd, GOARCH: 386 }
  - inputs: { GOOS: netbsd, GOARCH: amd64 }
  - inputs: { GOOS: openbsd, GOARCH: 386 }
  - inputs: { GOOS: openbsd, GOARCH: amd64 }
  - inputs: { GOOS: solaris, GOARCH: amd64 }
  - inputs: { GOOS: windows, GOARCH: 386 }
  - inputs: { GOOS: windows, GOARCH: amd64 }

# meta defines additional custom metadata about packages. This metadata does not
# participate in the PACKAGE_SPEC_ID and so changing it does not directly change cache
# keys for layers or packages. In addition, metadata may not be overridden per-package
# and is not available to input or layer dockerfile templates.
meta:

  defaults:
    # No default metadata.

  templates:
    # BUILD_JOB_NAME is the name of a job to build this package in CI. Care must be
    # taken that it is both unique within this set of packages, as well as compatible
    # with the CI system's naming conventions.
    BUILD_JOB_NAME: >-
      {{.GOOS}}_{{.GOARCH}}_package

    # BUNDLE_NAME is used in archive filenames, as well as by downstream processes.
    BUNDLE_NAME: "vault_{{.PRODUCT_VERSION}}"

# package-aliases are a set of paths by which each package may be known, they are
# templates which may refer to any input or meta field defined in this file.
# Package aliases must be unique across all packages defined in this file.
# If any package-alias renders to empty, it is ignored. You can use this
# to produce aliases selectively depending on the package.
#
# Package aliases count as meta data because they do not affect the bytes produced
# per package.
#
# We use package aliases to give human-readable names to packages, and to arrange
# them in a directory hierarchy ready for further processing and distribution.
# Each alias is written as a relative symbolic link in .buildcache/packages/by-alias.
#
# At least one alias must render to a nonempty string.
package-aliases:
  - type: local
    template: >-
      {{.BUNDLE_NAME}}_{{.GOOS}}_{{.GOARCH}}.zip
  # public-hc-releases is the path to use for upload to releases.hashicorp.com
  # it is empty if this package is not public (empty aliases are ignored).
  - type: public-hc-releases
    template: >-
      vault/{{.BUNDLE_NAME}}/{{.BUNDLE_NAME}}_{{.GOOS}}_{{.GOARCH}}.zip

# Layers determines the build layers, which are individually cacheable layers
# in a linear build. Each layer contains a Dockerfile. All the layers
# together produce the final builder image used to compile binaries.
#
# The partial Dockerfiles may contain references to any of the inputs
# including rendered input templates, but may not reference meta data.
# These Dockerfiles, once rendered, count as inputs and affect the
# package spec ID of each package.
#
# The order of layers is significant. The first layer must have a FROM line, and
# forms the base image. Each subsequent layer begins from the previous one.
#
# You can control cacheability by careful use of variables and ordering.
# Try to group things which change infrequently towards the top, and
# things which change more frequently towards the bottom.
#
# If there are things you want to cache that vary between packages defined in
# this file, put them last so that the greater bulk of work can be shared.
#
# NOTE: At present, changing the names and/or adding/removing layers may
# require updating the CI template file at .circleci/config/@build-release.yml.tpl
# which references some of these layers by name.
base-image: "debian@sha256:68f4e2259032a4e6f5035804e64438b52af8dd5889528b305b9059183ea4cd2a"
layers:
  - name: base
    dockerfile: |-
      RUN apt-get update -y && apt-get install --no-install-recommends -y -q \
                               curl \
                               zip \
                               build-essential \
                               gcc-multilib \
                               g++-multilib \
                               ca-certificates \
                               git mercurial bzr \
                               gnupg \
                               libltdl-dev \
                               libltdl7 \
      						             bash \
                            && rm -rf /var/lib/apt/lists/*
  - name: install-go
    dockerfile: |-
      ENV GOPATH /gopath
      ENV GOROOT /goroot
      RUN mkdir $GOROOT && mkdir $GOPATH
      RUN curl https://storage.googleapis.com/golang/go{{.GO_VERSION}}.linux-amd64.tar.gz \
                 | tar xzf - -C $GOROOT --strip-components=1
      ENV PATH $GOROOT/bin:$GOPATH/bin:$PATH
  - name: install-go-tools
    dockerfile: |
      ENV GO111MODULE=off
      RUN go get golang.org/x/tools/cmd/goimports
      RUN go get github.com/hashicorp/go-bindata
      RUN go get github.com/hashicorp/go-bindata/go-bindata
      RUN go get github.com/elazarl/go-bindata-assetfs
      RUN go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs
  - name: set-workdir
    dockerfile: |
      ENV REPO=github.com/hashicorp/vault
      ENV DIR=$GOPATH/src/$REPO
      RUN mkdir -p $DIR
      WORKDIR $DIR
  - name: install-yarn
    dockerfile: |-
      RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
      RUN curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
      RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
      RUN apt-get update -y && apt-get install -y -q nodejs yarn={{.YARN_VERSION}} \
                            && rm -rf /var/lib/apt/lists/*
  - name: ui-dependencies
    source-include: ui/package.json ui/yarn.lock
    dockerfile: |-
      RUN cd ui && yarn install
      RUN cd ui && npm rebuild node-sass
  - name: build-ui
    source-include: ui/
    dockerfile: |-
      RUN cd ui && yarn run build
  - name: build-static-assets
    source-include: Makefile
    dockerfile: |-
      RUN go-bindata-assetfs -o bindata_assetfs.go -pkg http -prefix pkg -modtime 1480000000 -tags ui ./pkg/web_ui/...
      RUN mkdir -p http && mv bindata_assetfs.go http/
      RUN goimports -w http/bindata_assetfs.go
  # Up to now, the layers do not vary between packages. This means the layers above
  # can be built prior to package builds in order to warm the cache and speed up
  # builds.
  #
  # warm-go-build-vendor-cache does vary based on GOOS, GOARCH and CGO_ENABLED though.
  # This means we should cache this in CI per package build.
  - name: warm-go-build-vendor-cache
    source-include: vendor/
    dockerfile: |-
      ENV GOOS={{.GOOS}}
      ENV GOARCH={{.GOARCH}}
      ENV CGO_ENABLED={{.CGO_ENABLED}}
      # Try to build vendored packages. We first filter out packages which report
      # errors in 'go list', because trying to run go build ./vendor/... fails early
      # if we include them. We also don't care about the exit code here, because
      # some of the vendored packages may fail to build, but this won't necessarily
      # mean that the final package will fail to build, and we will still get a
      # usefully warmed cache.
      RUN go list -f '{{"{{.ImportPath}}{{if or .Error .DepsErrors}} ERROR{{end}}"}}' ./vendor/... | grep -v ERROR | xargs go build -v || true
  # The final layer must contain all the source code we've not yet included.
  - name: copy-source
    source-include: .
    source-exclude: vendor/ ui/