continue_config.yml

version: 2.1


# the default pipeline parameters, which will be updated according to
# the results of the path-filtering orb
parameters:
  code_change:
    type: boolean
    default: false
  docs_change:
    type: boolean
    default: false
  release_kedro:
    type: boolean
    default: false
  # The parameters below are set in CircleCI UI.
  # https://app.circleci.com/settings/project/github/kedro-org/kedro/triggers?return-to=https%3A%2F%2Fapp.circleci.com%2Fpipelines%2Fgithub%2Fkedro-org%2Fkedro&triggerSource=&scheduledTriggerId=61f7226f-f092-4449-98d9-40420f8c46b2&success=true
  run_hourly:
    type: boolean
    default: false
  run_nightly:
    type: boolean
    default: false

orbs:
  win: circleci/windows@2.4.1

# No windows executor is listed here since windows builds use win/default and modify
# the Python version through the conda environment.
executors:
  docker:
    parameters:
      python_version:
        type: string
    docker:
      - image: public.ecr.aws/g0x0s3o2/kedro-builder:<<parameters.python_version>>
    resource_class: medium+

commands:
  setup_conda:
    steps:
      - run:
          name: Run conda.sh
          command: echo ". /home/circleci/miniconda/etc/profile.d/conda.sh" >> $BASH_ENV
      - run:
          name: Activate conda environment
          command: echo "conda deactivate; conda activate kedro_builder" >> $BASH_ENV

  setup_requirements:
    steps:
      - run:
          name: Install pip setuptools
          command: make install-pip-setuptools
      - run:
          name: Install venv for some pre-commit hooks
          command: conda install -y virtualenv
      - run:
          # pytables does not work properly with python 3.9 to handle our HDFDataSet
          # if pip-installed, so we install this dependency via conda
          name: Install pytables
          command: conda install -c conda-forge pytables -y
      - run:
          name: Install requirements and test requirements
          command: pip install --upgrade -r test_requirements.txt
      - run:
          # this is needed to fix java cacerts so
          # spark can automatically download packages from mvn
          # https://stackoverflow.com/a/50103533/1684058
          name: Fix cacerts
          command: |
            sudo rm /etc/ssl/certs/java/cacerts
            sudo update-ca-certificates -f
      - run:
          # Since recently Spark installation for some reason does not have enough permissions to execute
          # /home/circleci/miniconda/envs/kedro_builder/lib/python3.X/site-packages/pyspark/bin/spark-class.
          # So fixing it manually here.
          name: Fix Spark permissions
          command: sudo chmod -R u+x /home/circleci/miniconda/envs/kedro_builder/lib/
      - run:
          name: Print Python environment
          command: make print-python-env
      - run:
          name: Pip freeze
          command: pip freeze

  setup:
    steps:
      - checkout
      - setup_conda
      - setup_requirements

  # Windows specific commands
  win_setup_conda:
    parameters:
      python_version:
        type: string
    steps:
      - run:
          name: Initialize conda
          command: conda init powershell
      - run:
          name: Create 'kedro_builder' conda environment
          command: conda create -n kedro_builder python=<<parameters.python_version>> -y

  win_setup_env:
    steps:
      - run:
          # Required for Tensorflow tests
          name: Install Microsoft Visual C++ Redistributable
          command: |
            $ProgressPreference = "SilentlyContinue"
            Invoke-WebRequest https://aka.ms/vs/16/release/vc_redist.x64.exe -OutFile vc_redist.x64.exe
            .\vc_redist.x64.exe /S /v/qn
      - run:
          name: Install Java 8
          command: |
            $ProgressPreference = "SilentlyContinue"
            Invoke-WebRequest https://github.com/AdoptOpenJDK/openjdk8-upstream-binaries/releases/download/jdk8u252-b09/OpenJDK8U-jdk_x64_windows_8u252b09.zip -OutFile OpenJDK8U.zip
            Expand-Archive .\OpenJDK8U.zip -DestinationPath C:\OpenJDK8U
      - run:
          name: Create Inbound rules for Java
          command: |
            New-NetFirewallRule -DisplayName "Allow JDK UDP" -Profile "Public" -Protocol "UDP" -Direction Inbound -Program "C:\OpenJDK8U\openjdk-8u252-b09\bin\java.exe" -Action Allow
            New-NetFirewallRule -DisplayName "Allow JDK TCP" -Profile "Public" -Protocol "TCP" -Direction Inbound -Program "C:\OpenJDK8U\openjdk-8u252-b09\bin\java.exe" -Action Allow
      - run:
          name: Set Java environment variables
          command: |
            [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable('Path', 'Machine') + ";C:\OpenJDK8U\openjdk-8u252-b09\bin", "Machine")
            setx /m JAVA_HOME "C:\OpenJDK8U\openjdk-8u252-b09"
      - run:
          name: Setup Hadoop binary
          command: |
            $ProgressPreference = "SilentlyContinue"
            Invoke-WebRequest https://github.com/steveloughran/winutils/raw/master/hadoop-2.6.3/bin/winutils.exe -OutFile winutils.exe
            New-Item -ItemType directory -Path C:\hadoop\bin
            mv .\winutils.exe C:\hadoop\bin
            setx /m HADOOP_HOME "C:\hadoop\"
      - run:
          name: Install 'make' command
          command: choco install make

  win_setup_requirements:
    parameters:
      python_version:
        type: string
    steps:
      - restore_cache:
          name: Restore package cache
          key: kedro-deps-v1-win-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
        # We don't restore the conda environment cache for python 3.10 as it conflicts with the
        # 'Install GDAL, Fiona and pytables' step breaking the conda environment (missing zlib.dll).
      - unless:
          condition:
            equal: [ "3.10", <<parameters.python_version>> ]
          steps:
            - restore_cache:
                name: Restore conda environment cache
                key: kedro-deps-v1-win-<<parameters.python_version>>-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
        # pytables and Fiona have a series of binary dependencies under Windows that
        # are best handled by conda-installing instead of pip-installing them.
        # Dependency resolution works best when installing these altogether in one
        # `conda install` command rather than one at a time in several sequential `conda install`s.
      - run:
          name: Install GDAL, Fiona and pytables
          command: conda activate kedro_builder; conda install gdal fiona pytables -c conda-forge -y
      - run:
          name: Show pip information
          command: conda activate kedro_builder; pip debug --verbose
      - run:
          name: Install all requirements
          command: conda activate kedro_builder; pip install -v -r test_requirements.txt -U
      - run:
          name: Print Python environment
          command: conda activate kedro_builder; make print-python-env
      - run:
          name: Pip freeze
          command: conda activate kedro_builder; pip freeze

  win_setup:
    parameters:
      python_version:
        type: string
    steps:
      - checkout
      - win_setup_conda:
          python_version: <<parameters.python_version>>
      - win_setup_env
      - win_setup_requirements:
          python_version: <<parameters.python_version>>

jobs:
  e2e_tests:
    parameters:
      python_version:
        type: string
    executor:
      name: docker
      python_version: <<parameters.python_version>>
    environment:
      COLUMNS: 120
      LINES: 25
    steps:
      - setup
      - run:
          name: Run e2e tests
          command: make e2e-tests

  win_e2e_tests:
    parameters:
      python_version:
        type: string
    executor: win/default
    environment:
      PIP_DISABLE_PIP_VERSION_CHECK: 1
      COLUMNS: 120
      LINES: 25
      PYTHONIOENCODING: utf-8
    steps:
      - checkout
      - win_setup_conda:
          python_version: <<parameters.python_version>>
      - run:
          name: Install 'make' command
          command: choco install make
      # We don't use the `win_setup` command here, which would install the full set
      # of requirements used by unit tests. Even when those packages are cached
      # it is faster to just install the minimal set of dependencies needed for e2e
      # tests in a new empty environment rather than restore the cache.
      - run:
          name: Install dependencies
          command: conda activate kedro_builder; pip install -r features/windows_reqs.txt
      - run:
          name: Run e2e tests
          command: conda activate kedro_builder; make e2e-tests

  unit_tests:
    parameters:
      python_version:
        type: string
    executor:
      name: docker
      python_version: <<parameters.python_version>>
    steps:
      - setup
      - unless:
          condition:
            equal: ["3.10", <<parameters.python_version>>]
          steps:
            - run:
                name: Run unit tests in parallel
                command: PYTEST_ADDOPTS="-v" make test
      - when:
          condition:
            equal: [ "3.10", <<parameters.python_version>> ]
          steps:
            - run:
                name: Run unit tests sequentially
                command: pytest -v tests --cov-config pyproject.toml


  win_unit_tests:
    parameters:
      python_version:
        type: string
    executor: win/default
    steps:
      - win_setup:
          python_version: <<parameters.python_version>>
      - run:
          # geopandas and tensorflow conflicts when imported simultaneously.
          # The HDF5 header files used to compile this application do not match
          # the version used by the HDF5 library to which this application is linked.
          # Data corruption or segmentation faults may occur if the application continues.
          # This can happen when an application was compiled by one version of HDF5 but
          # linked with a different version of static or shared HDF5 library.
          # You should recompile the application or check your shared library related
          # settings such as 'LD_LIBRARY_PATH'.
          # You can, at your own risk, disable this warning by setting the environment
          # variable 'HDF5_DISABLE_VERSION_CHECK' to a value of '1'.
          # Setting it to 2 or higher will suppress the warning messages totally.
          name: Set HDF5_DISABLE_VERSION_CHECK environment variable
          command: setx /m HDF5_DISABLE_VERSION_CHECK 1
      - unless:
          condition:
            equal: [ "3.10", <<parameters.python_version>> ]
          steps:
            - run:
                name: Run unit tests without spark in parallel
                command: conda activate kedro_builder; make test-no-spark
      - when:
          condition:
            equal: [ "3.10", <<parameters.python_version>> ]
          steps:
            - run:
                name: Run unit tests without spark sequentially
                command: conda activate kedro_builder; pytest tests --no-cov --ignore tests/extras/datasets/spark

  lint:
    parameters:
      python_version:
        type: string
    executor:
      name: docker
      python_version: <<parameters.python_version>>
    steps:
      - setup
      - run:
          name: Run linters
          command: make lint

  pip_compile:
    parameters:
      python_version:
        type: string
    executor:
      name: docker
      python_version: <<parameters.python_version>>
    steps:
      - setup
      - run:
          name: Pip-compile requirements file
          command: make pip-compile

  win_pip_compile:
    parameters:
      python_version:
        type: string
    executor: win/default
    steps:
      - win_setup:
          python_version: <<parameters.python_version>>
      - when:
          # Save Python package cache only for Python 3.7. The conda environment itself
          # is specific to a Python version and is cached separately for each.
          condition:
            equal: ["3.7", <<parameters.python_version>>]
          steps:
          - save_cache:
              name: Save Python package cache
              key: kedro-deps-v1-win-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
              paths:
                # Cache pip cache and conda packages directories
                - c:\tools\miniconda3\pkgs
                - c:\users\circleci\appdata\local\pip\cache
        # We don't save the conda environment cache for python 3.10 due to conflicts with the
        # 'Install GDAL, Fiona and pytables' and 'Restore conda environment cache' steps.
      - unless:
          condition:
            equal: [ "3.10", <<parameters.python_version>> ]
          steps:
            - save_cache:
                name: Save conda environment cache
                key: kedro-deps-v1-win-<<parameters.python_version>>-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
                paths:
                  - c:\tools\miniconda3\envs\kedro_builder
      - run:
          name: Pip-compile requirements file
          command: conda activate kedro_builder; make pip-compile

  sync:
    docker:
      # https://circleci.com/docs/2.0/circleci-images/#circleci-base-image
      - image: cimg/base:2020.01
    steps:
      - checkout
      - add_ssh_keys
      - run:
          name: Set git email and name
          command: |
            git config --global user.email "kedro@kedro.com"
            git config --global user.name "Kedro"
      - run:
          name: Trigger Read The Docs build
          command: ./tools/circleci/rtd-build.sh ${RTD_TOKEN} latest
      - run:
          name: Maybe merge main into develop or raise a PR
          command: ./tools/circleci/github_scripts/merge.sh . "main" "develop" "${GITHUB_TAGGING_TOKEN}"
      - run:
          name: Maybe trigger the release workflow
          command: |
              KEDRO_VERSION=$(./tools/circleci/github_scripts/kedro_version.py ./kedro)
              if ./tools/circleci/check-no-version-pypi.sh "${KEDRO_VERSION}"
              then
                  echo "Starting the release of Kedro ${KEDRO_VERSION}!"
                  ./tools/circleci/circle-release.sh github/kedro-org/kedro
              else
                  echo "Kedro version ${KEDRO_VERSION} already exists on PyPI, skipping..."
              fi

  merge_pr_to_develop:
    docker:
      # https://circleci.com/docs/2.0/circleci-images/#circleci-base-image
      - image: cimg/base:2020.01
    steps:
      - checkout
      - add_ssh_keys
      - run:
          name: Maybe merge an automatic PR into develop
          command: ./tools/circleci/github_scripts/attempt_merge_pr.sh "merge-main-to-develop" "develop" "${GITHUB_TAGGING_TOKEN}"

  build_docker_image:
    parameters:
      python_version:
        type: string
    docker:
      - image: cimg/python:3.8
    steps:
      - setup_remote_docker:
          docker_layer_caching: true
      - checkout
      - run:
          name: Setup AWS CLI
          command: pip install -U awscli
      - run:
          name: Login to AWS ECR
          command: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
      - run:
          name: Build docker images
          command: ./tools/circleci/docker_build_img/build.sh "." "public.ecr.aws/g0x0s3o2/kedro-builder" "<<parameters.python_version>>"
          no_output_timeout: 20m
      - run:
          name: Logout from AWS ECR
          command: docker logout public.ecr.aws
          when: always  # logout even if the previous step has failed

  # This is effectively just a combination of the lint, unit_tests and e2e_tests jobs.
  # It's used to check that the nightly docker image is working ok and before publishing a release.
  build_kedro:
    parameters:
      python_version:
        type: string
    executor:
      name: docker
      python_version: <<parameters.python_version>>
    environment:
      COLUMNS: 120
      LINES: 25
    steps:
      - setup
      - run:
          name: Run linters
          command: make lint
      - unless:
          condition:
            equal: ["3.10", <<parameters.python_version>>]
          steps:
            - run:
                name: Run unit tests in parallel
                command: make test
      - when:
          condition:
            equal: [ "3.10", <<parameters.python_version>> ]
          steps:
            - run:
                name: Run unit tests sequentially
                command: pytest tests --cov-config pyproject.toml
      - run:
          name: Run e2e tests
          command: make e2e-tests

  publish_kedro:
    executor:
      name: docker
      python_version: "3.7"
    steps:
      - setup
      - add_ssh_keys
      - run:
          name: Check Kedro version
          command: |
            KEDRO_VERSION=$(./tools/circleci/github_scripts/kedro_version.py ./kedro)
            if ./tools/circleci/check-no-version-pypi.sh "${KEDRO_VERSION}"
            then
                echo "export KEDRO_VERSION=\"${KEDRO_VERSION}\"" >> $BASH_ENV
            else
                echo "Error: Kedro version ${KEDRO_VERSION} already exists on PyPI"
                exit 1
            fi
      - run:
          name: Tag and publish release on Github
          command: ./tools/circleci/github_scripts/release.sh kedro-org kedro ${GITHUB_TAGGING_TOKEN} ${KEDRO_VERSION}
      - run:
          name: Publish to PyPI
          command: |
            make package
            python -m pip install twine -U
            python -m twine upload --repository-url ${TWINE_REPOSITORY_URL} dist/*
      - run:
          name: Trigger Read The Docs build
          command: |
            ./tools/circleci/rtd-build.sh ${RTD_TOKEN} stable
            # give some time for GitHub release to propagate
            # otherwise RTD fails to build a new tag
            sleep 120
            ./tools/circleci/rtd-build.sh ${RTD_TOKEN} ${KEDRO_VERSION}

  # Trigger kedro-viz build to ensure tests in that project pass
  viz_build:
    docker:
      - image: spotify/alpine # for bash and curl
    steps:
      - run:
          name: Trigger kedro-viz build
          command: |
            curl --location --request POST \
              --url https://circleci.com/api/v2/project/github/kedro-org/kedro-viz/pipeline \
              --header "Circle-Token: $CIRCLE_VIZ_BUILD_TOKEN" \
              --header 'content-type: application/json' \
              --data '{"branch":"main"}'

  all_circleci_checks_succeeded:
    docker:
      - image: circleci/python # any light-weight image
    steps:
      - run:
          name: Success!
          command: echo "All checks passed"

workflows:
  version: 2.1

  lint_only:
    when:
      and:
        - <<pipeline.parameters.docs_change>>
        - not: <<pipeline.parameters.code_change>>
        - not: <<pipeline.parameters.release_kedro>>
        - not: <<pipeline.parameters.run_hourly>>
        - not: <<pipeline.parameters.run_nightly>>
    jobs:
      - lint:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9", "3.10"]
      - all_circleci_checks_succeeded:
          requires:
            - lint

  build_code:
    when:
      and:
        - <<pipeline.parameters.code_change>>
        - not: <<pipeline.parameters.release_kedro>>
        - not: <<pipeline.parameters.run_hourly>>
        - not: <<pipeline.parameters.run_nightly>>
    jobs:
      - e2e_tests:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9", "3.10"]
      - win_e2e_tests:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9", "3.10"]
      - unit_tests:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9", "3.10"]
      - win_unit_tests:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9", "3.10"]
      - lint:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9", "3.10"]
      - pip_compile:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9", "3.10"]
      - win_pip_compile:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9", "3.10"]
      - all_circleci_checks_succeeded:
          requires:
            - e2e_tests
            - win_e2e_tests
            - unit_tests
            - win_unit_tests
            - lint
            - pip_compile
            - win_pip_compile

  main_updated:
    when:
      and:
        - not: <<pipeline.parameters.release_kedro>>
        - not: <<pipeline.parameters.run_hourly>>
        - not: <<pipeline.parameters.run_nightly>>
    jobs:
      - sync:
          filters:
            branches:
              only: main
      - viz_build:
          filters:
            branches:
              only: main

  hourly_pr_merge:
    when:
      and:
        - <<pipeline.parameters.run_hourly>>
        - not: <<pipeline.parameters.release_kedro>>
        - not: <<pipeline.parameters.run_nightly>>
    jobs:
      - merge_pr_to_develop:
        filters:
          branches:
            only: main

  # Python versions that are supported on `main`.
  nightly_build_main:
    when:
      and:
        - <<pipeline.parameters.run_nightly>>
        - not: <<pipeline.parameters.release_kedro>>
        - not: <<pipeline.parameters.run_hourly>>
    jobs:
      - build_docker_image:
          context:
            - kedro-ecr-publish
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9", "3.10"]
          filters:
            branches:
              only: main
      - build_kedro:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9", "3.10"]
          requires:
            - build_docker_image-<<matrix.python_version>>
          filters:
            branches:
              only: main

  # Python versions that are *only* supported on `develop`.
  # If `develop` supports the same versions as `main`, comment this out.
  # nightly_build_develop:
  #   when:
  #     and:
  #       - <<pipeline.parameters.run_nightly>>
  #       - not: <<pipeline.parameters.release_kedro>>
  #       - not: <<pipeline.parameters.run_hourly>>
  #   jobs:
  #     - build_docker_image:
  #         context:
  #           - kedro-ecr-publish
  #         matrix:
  #           parameters:
  #             python_version: []
  #         filters:
  #           branches:
  #             only: develop
  #     - build_kedro:
  #         matrix:
  #           parameters:
  #             python_version: []
  #         requires:
  #           - build_docker_image-<<matrix.python_version>>
  #         filters:
  #           branches:
  #             only: develop

  kedro_release:
    when:
      and:
        - <<pipeline.parameters.release_kedro>>
        - not: <<pipeline.parameters.run_hourly>>
        - not: <<pipeline.parameters.run_nightly>>
    jobs:
      - build_kedro:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9", "3.10"]
      - publish_kedro:
          requires:
            - build_kedro