config.yml

---
# CI Overview
# -----------
#
# Each night:
#
#   A build image is created (ci_image) from `docker/Dockerfile.ci` and is
#   pushed to `quay.io/influxdb/rust:ci`. This build image is then used to run
#   the CI tasks for the day.
#
# Every commit:
#
# The CI for every PR and merge to main runs tests, fmt, lints and compiles debug binaries
#
# On main if all these checks pass it will then additionally compile in "release" mode and
# publish a docker image to quay.io/influxdb/iox:$COMMIT_SHA
#
# Manual CI Image:
#
# It is possible to manually trigger a rebuild of the image used in CI. To do this, navigate to
# https://app.circleci.com/pipelines/github/influxdata/influxdb_iox?branch=main (overriding the
# branch name if desired). Then:
# - Click "Run Pipeline" in the top-right
# - Expand "Add Parameters"
# - Add a "boolean" parameter called "ci_image" with the value true
# - Click "Run Pipeline"
#
# If you refresh the page you should see a newly running ci_image workflow
#

version: 2.1

orbs:
  win: circleci/windows@4.1

commands:
  rust_components:
    description: Verify installed components
    steps:
      - run:
          name: Verify installed components
          command: |
            rustup --version
            rustup show
            cargo fmt --version
            cargo clippy --version

  cache_restore:
    description: Restore Cargo Cache
    steps:
      - restore_cache:
          name: Restoring Cargo Cache
          keys:
            - cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
            - cargo-cache-{{ arch }}-{{ .Branch }}
            - cargo-cache
  cache_save:
    description: Save Cargo Cache
    steps:
      - save_cache:
          name: Save Cargo Cache
          paths:
            - /usr/local/cargo/registry
          key: cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}

jobs:
  fmt:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Rust fmt
          command: cargo fmt --all -- --check
      - cache_save
  lint:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Clippy
          command: cargo clippy --all-targets --all-features --workspace -- -D warnings
      - cache_save
  cargo_audit:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Install cargo-deny
          command: cargo install --force cargo-deny
      - run:
          name: cargo-deny Checks
          command: cargo deny check -s
      - cache_save
  check:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Install cargo-hack
          command: cargo install cargo-hack
      - run:
          name: Check all features
          command: cargo hack check --feature-powerset --no-dev-deps --workspace
      - cache_save
  doc:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo doc
          # excluding datafusion because it's effectively a dependency masqueraded as workspace crate.
          command: cargo doc --document-private-items --no-deps --workspace --exclude datafusion
      - cache_save
      - run:
          name: Compress Docs
          command: tar -cvzf rustdoc.tar.gz target/doc/
      - store_artifacts:
          path: rustdoc.tar.gz
  test:
    # setup multiple docker images (see https://circleci.com/docs/2.0/configuration-reference/#docker)
    docker:
      - image: quay.io/influxdb/rust:ci
      - image: localstack/localstack:0.14.4
      - image: mcr.microsoft.com/azure-storage/azurite
      - image: fsouza/fake-gcs-server
        command:
          - "-scheme"
          - "http"
    resource_class: 2xlarge # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
      RUST_BACKTRACE: "1"
      # Run integration tests
      TEST_INTEGRATION: 1
      AWS_DEFAULT_REGION: "us-east-1"
      AWS_ACCESS_KEY_ID: test
      AWS_SECRET_ACCESS_KEY: test
      AWS_ENDPOINT: http://127.0.0.1:4566
      AZURE_USE_EMULATOR: "1"
      GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json"
      OBJECT_STORE_BUCKET: test-bucket
    steps:
      - run:
          name: Setup localstack (AWS emulation)
          command: |
            cd /tmp
            curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
            unzip awscliv2.zip
            sudo ./aws/install
            aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket
      - run:
          name: Setup Azurite (Azure emulation)
          # the magical connection string is from https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings
          command: |
            curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
            az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;'
      - run:
          name: Setup fake GCS server
          command: |
            curl -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "http://localhost:4443/storage/v1/b"
            echo '{"gcs_base_url": "http://localhost:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT"
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo test
          command: cargo test --workspace --features=aws,azure,azure_test,gcp
      - cache_save

  test_windows:
    executor:
      name: win/default
      size: medium
    environment:
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - run:
          name: Download rustup
          command: wget https://win.rustup.rs/x86_64 -O rustup-init.exe
      - run:
          name: Install rustup
          command: .\rustup-init.exe -y --default-host=x86_64-pc-windows-msvc
      - run:
          name: Cargo test
          command: cargo test --workspace

workflows:
  version: 2

  # CI for all pull requests.
  ci:
    jobs:
      - check
      - fmt
      - lint
      - cargo_audit
      - test
      - test_windows
      - doc