From 3d2a5cf7f6ed98cef2c398e46628242bb0abb270 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 31 Jul 2021 11:48:15 +0200 Subject: [PATCH] Switches to "/" convention in ghcr.io images with optimisations We are using ghcr.io as image cache for our CI builds and Breeze and it seems ghcr.io is being "rebuilt" while running. We had been using "airflow-.." image convention before, bacause multiple nesting levels of images were not supported, however we experienced errors recently with pushing 2.1 images (https://issues.apache.org/jira/browse/INFRA-22124) and during investigation it turned out, that it is possible now to use "/" in the name of the image, and while it still does not introduce multiple nesting levels and folder structure, the UI of GitHub treats it like that and if you have image which starts wiht "airflow/", the airflow prefix is stripped out and you can also have even more "/" in then name to introduce further hierarchy. Since we have to change image naming convention due to (still unresolved) bug with no permission to push the v2-1-test image we've decided to change naming convention for all our cache images to follow this - now available - "/" connvention to make it better structured and easier to manage/understand. Some more optimisations are implemented - Python, prod-build and ci-manifest images are only pushed when "latest" image is prepared. They are not needed for the COMMIT builds because we only need final images for those builds. This simplified the code quite a bit. CONTINUE_ON_PIP_CHECK_ERROR variable has been removed in favour of ignoring pip error when installing dependencies from branch tip. This might potentially happen for a short while when new changes have been merged, but constraints were not yet regenerated and we have conflicting dependencies. The .dockerignore was reviewed and builds were optimized for people who locally built provider packages and documentation, by excluding unnecessary files. Some instructions which run after the COPY . which did not need sourcer were moved before the COPY command. Those optimisatiions save 30-40 seconds of overhead when building the image (especially when you build images incrementally rather than rebuilding from scratch). PIP and HELM versions have been updated to latest available. Backwards-compatibility was implemented to allow PRs that have not been upgraded to continue building after this one is merged, also a workaround has been implemented to make this change to work even if it is not merged yet to main. This "legacy" mode will be removed in ~week when everybody rebase on top of main. Documentation is updated reflecting those changes. (cherry picked from commit e04c2e3872aa30ed042d3f9bf66d8020cf9c2acb) --- .dockerignore | 4 +- .github/workflows/build-images.yml | 19 ++- .github/workflows/ci.yml | 161 ++++++------------ BREEZE.rst | 4 +- CI.rst | 51 +++--- Dockerfile | 27 ++- Dockerfile.ci | 54 +++--- IMAGES.rst | 30 ++-- README.md | 4 +- breeze | 23 +-- breeze-complete | 2 +- dev/retag_docker_images.py | 9 +- docs/docker-stack/build-arg-ref.rst | 6 - .../ci/images/ci_prepare_ci_image_on_ci.sh | 19 +-- .../ci/images/ci_prepare_prod_image_on_ci.sh | 29 +--- scripts/ci/images/ci_push_legacy_ci_images.sh | 27 +++ .../ci/images/ci_push_legacy_prod_images.sh | 27 +++ .../ci_wait_for_and_verify_all_ci_images.sh | 2 + .../ci_wait_for_and_verify_all_prod_images.sh | 2 + .../images/ci_wait_for_and_verify_ci_image.sh | 27 ++- .../ci_wait_for_and_verify_prod_image.sh | 32 ++-- scripts/ci/libraries/_build_images.sh | 112 +++++++----- scripts/ci/libraries/_initialization.sh | 26 +-- scripts/ci/libraries/_kind.sh | 16 +- scripts/ci/libraries/_parallel.sh | 7 +- .../ci/libraries/_push_pull_remove_images.sh | 117 ++++++++----- scripts/ci/libraries/_script_init.sh | 2 +- scripts/ci/selective_ci_checks.sh | 10 +- scripts/ci/tools/fix_ownership.sh | 14 +- .../docker/install_additional_dependencies.sh | 5 +- scripts/docker/install_airflow.sh | 4 +- ...l_airflow_dependencies_from_branch_tip.sh} | 11 +- .../install_from_docker_context_files.sh | 2 +- 33 files changed, 463 insertions(+), 422 deletions(-) create mode 100755 scripts/ci/images/ci_push_legacy_ci_images.sh create mode 100755 scripts/ci/images/ci_push_legacy_prod_images.sh rename scripts/docker/{install_airflow_from_branch_tip.sh => install_airflow_dependencies_from_branch_tip.sh} (85%) diff --git a/.dockerignore b/.dockerignore index d10cfbcaae575..f6113e2bd6d33 100644 --- a/.dockerignore +++ b/.dockerignore @@ -40,9 +40,6 @@ !scripts/in_container !scripts/docker -# Add provider packages to the context -!provider_packages - # Add tests and kubernetes_tests to context. !tests !kubernetes_tests @@ -129,3 +126,4 @@ airflow/www/static/docs # Exclude docs generated files docs/_build/ docs/_api/ +docs/_doctrees/ diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index f29e199e4fb84..d2a7b97751a61 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -148,7 +148,6 @@ jobs: BACKEND: postgres PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - CONTINUE_ON_PIP_CHECK_FAILURE: "true" DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} CHECK_IF_BASE_PYTHON_IMAGE_UPDATED: > ${{ github.event_name == 'pull_request_target' && 'false' || 'true' }} @@ -204,6 +203,10 @@ jobs: run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Push CI images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}" run: ./scripts/ci/images/ci_push_ci_images.sh + # Remove me ~ 7 August 2021 + - name: "Push Legacy CI images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}" + run: ./scripts/ci/images/ci_push_legacy_ci_images.sh + if: github.event_name == 'pull_request_target' build-prod-images: permissions: @@ -230,8 +233,11 @@ jobs: VERSION_SUFFIX_FOR_PYPI: ".dev0" steps: - name: Set envs + # Set pull image tag for CI image build, in order to pull the image pushed + # Just a moment ago by build-ci-images job run: | echo "GITHUB_REGISTRY_PUSH_IMAGE_TAG=${TARGET_COMMIT_SHA}" >> "$GITHUB_ENV" + echo "GITHUB_REGISTRY_PULL_IMAGE_TAG=${TARGET_COMMIT_SHA}" >> "$GITHUB_ENV" - uses: actions/checkout@v2 with: ref: ${{ env.TARGET_COMMIT_SHA }} @@ -279,10 +285,21 @@ jobs: # Pull images built in the previous step env: GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true" + # Here we are using PULL_IMAGE_TAG set in the environment variables above - name: "Build PROD images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}" run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh + env: + # GITHUB_REGISTRY_PULL_IMAGE_TAG is overriden to latest in order to build PROD image using "latest" + GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest" - name: "Push PROD images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}" run: ./scripts/ci/images/ci_push_production_images.sh + env: + # GITHUB_REGISTRY_PULL_IMAGE_TAG is overriden to latest in order to build PROD image using "latest" + GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest" + # Remove me ~ 7 August 2021 + - name: "Push Legacy PROD images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}" + run: ./scripts/ci/images/ci_push_legacy_prod_images.sh + if: github.event_name == 'pull_request_target' cancel-on-ci-build: permissions: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c317842cfe68..7228f37905a66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -561,7 +561,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" PACKAGE_FORMAT: "sdist" tests-helm: - timeout-minutes: 20 + timeout-minutes: 40 name: "Python unit tests for helm chart" runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} needs: [build-info, ci-images] @@ -1045,108 +1045,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" path: /tmp/kind_logs_* retention-days: 7 - push-prod-images-to-github-registry: - permissions: - packages: write - timeout-minutes: 10 - name: "Push PROD images as cache to GitHub Registry" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: - - build-info - - static-checks - - tests-sqlite - - tests-postgres - - tests-mysql - - tests-kubernetes - - prod-images - - docs - if: > - (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/v1-10-test' || - github.ref == 'refs/heads/v2-0-test' || github.ref == 'refs/heads/v2-1-test') && - github.event_name != 'schedule' - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - - name: "Free space" - run: ./scripts/ci/tools/free_space.sh - - name: Set push-python-image - id: push-python-image - run: | - if [[ "${REF}" == 'refs/head/main' || "${REF}" == 'refs/head/main' ]]; then - echo "::set-output name=wanted::true" - else - echo "::set-output name=wanted::false" - fi - env: - REF: ${{ github.ref }} - - name: - "Prepare PROD image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh - env: - # Since we are going to push both final image and build image segment, we need to pull the - # build image, in case we are pulling from registry rather than building. - WAIT_FOR_PROD_BUILD_IMAGE: "true" - WAIT_FOR_PYTHON_BASE_IMAGE: ${{ steps.push-python-image.outputs.wanted}} - - name: "Push PROD images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_push_production_images.sh - env: - PUSH_PYTHON_BASE_IMAGE: ${{ steps.push-python-image.outputs.wanted}} - - push-ci-images-to-github-registry: - permissions: - packages: write - timeout-minutes: 10 - name: "Push CI images as cache to GitHub Registry" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: - - build-info - - static-checks - - tests-sqlite - - tests-postgres - - tests-mysql - - tests-kubernetes - - ci-images - - docs - if: > - (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/v1-10-test' || - github.ref == 'refs/heads/v2-0-test' || github.ref == 'refs/heads/v2-1-test') && - github.event_name != 'schedule' - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - - name: "Free space" - run: ./scripts/ci/tools/free_space.sh - - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Push CI image ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_push_ci_images.sh - constraints: permissions: contents: write @@ -1166,10 +1064,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: ${{needs.build-info.outputs.pythonVersionsListAsString}} - # Only run it for direct pushes - if: > - github.ref == 'refs/heads/main' || github.ref == 'refs/heads/v1-10-test' || - github.ref == 'refs/heads/v2-0-test' || github.ref == 'refs/heads/v2-1-test' + # Only run it for direct pushes and scheduled builds + if: github.event_name == 'push' || github.event_name == 'schedule' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v2 @@ -1203,17 +1099,68 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: "Set constraints branch name" id: constraints-branch run: ./scripts/ci/constraints/ci_branch_constraints.sh + # only actually push it when we are in apache/airflow repository - name: Checkout ${{ steps.constraints-branch.outputs.branch }} uses: actions/checkout@v2 + if: github.repository == 'apache/airflow' with: path: "repo" ref: ${{ steps.constraints-branch.outputs.branch }} persist-credentials: false - name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}" run: ./scripts/ci/constraints/ci_commit_constraints.sh + if: github.repository == 'apache/airflow' - name: "Push changes" uses: ./.github/actions/github-push-action + if: github.repository == 'apache/airflow' with: github_token: ${{ secrets.GITHUB_TOKEN }} branch: ${{ steps.constraints-branch.outputs.branch }} directory: "repo" + + # Push images to GitHub Registry in Apache repository, if all tests are successful and build + # is executed as result of direct push to "main" or one of the "test" branches + # It actually rebuilds all images using just-pushed constraints if they changed + # It will also check if a new python image was released and will pull the latest one if needed + # Same as build-images.yaml + push-images-to-github-registry: + permissions: + packages: write + timeout-minutes: 10 + name: "Push images as cache to GitHub Registry" + runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + needs: + - build-info + - constraints + - docs + # Only run it for direct pushes and scheduled builds + if: github.event_name == 'push' || github.event_name == 'schedule' + strategy: + matrix: + python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} + env: + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} + PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} + GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest" + GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" + PUSH_PYTHON_BASE_IMAGE: "true" + CHECK_IF_BASE_PYTHON_IMAGE_UPDATED: "true" + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + - name: "Setup python" + uses: actions/setup-python@v2 + with: + python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} + - name: "Free space" + run: ./scripts/ci/tools/free_space.sh + - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:latest" + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh + - name: "Prepare PROD image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:latest" + run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh + - name: "Push CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:latest" + run: ./scripts/ci/images/ci_push_ci_images.sh + - name: "Push PROD images ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:latest" + run: ./scripts/ci/images/ci_push_production_images.sh diff --git a/BREEZE.rst b/BREEZE.rst index 90d3f0b3d90ac..79f27b4b0549f 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -2382,9 +2382,9 @@ This is the current syntax for `./breeze <./breeze>`_: Helm version - only used in case one of kind-cluster commands is used. One of: - v3.2.4 + v3.6.3 - Default: v3.2.4 + Default: v3.6.3 --executor EXECUTOR Executor to use in a kubernetes cluster. diff --git a/CI.rst b/CI.rst index fedd3007b7cbf..6a704d0400aaa 100644 --- a/CI.rst +++ b/CI.rst @@ -568,12 +568,11 @@ This workflow is a regular workflow that performs all checks of Airflow code. +---------------------------+----------------------------------------------+-------+-------+------+ | Tests Kubernetes | Run Kubernetes test | Yes(2)| Yes | Yes | +---------------------------+----------------------------------------------+-------+-------+------+ -| Push PROD images | Pushes PROD images to GitHub Registry (4) | - | Yes | - | -+---------------------------+----------------------------------------------+-------+-------+------+ -| Push CI images | Pushes CI images to GitHub Registry (4) | - | Yes | - | -+---------------------------+----------------------------------------------+-------+-------+------+ | Constraints | Upgrade constraints to latest ones (4) | - | Yes | Yes | +---------------------------+----------------------------------------------+-------+-------+------+ +| Push images | Pushes latest images to GitHub Registry (4) | - | Yes | Yes | ++---------------------------+----------------------------------------------+-------+-------+------+ + Comments: @@ -584,8 +583,8 @@ Comments: You can set it to "false" to disable using shared images - this is slower though as the images are rebuilt in every job that needs them. (4) PROD and CI images are pushed as "latest" to GitHub Container registry and constraints are upgraded - only if all tests are successful. Note that images are not pushed in CRON jobs because they are rebuilt - from scratch and we want to push incremental changes to the Github Container registry. + only if all tests are successful. The images are rebuilt in this step using constraints pushed + in the previous step. CodeQL scan ----------- @@ -620,7 +619,9 @@ with the COMMIT_SHA id for images that were used in particular build. The image names follow the patterns (except the Python image, all the images are stored in https://ghcr.io/ in ``apache`` organization. -The packages are available under: +The packages are available under (CONTAINER_NAME is url-encoded name of the image). Note that "/" are +supported now in the ``ghcr.io`` as apart of the image name within ``apache`` organization, but they +have to be percent-encoded when you access them via UI (/ = %2F) ``https://github.com/apache/airflow/pkgs/container/`` @@ -631,26 +632,30 @@ The packages are available under: | (DockerHub) | | Python maintainer release new versions of those image | | | | with security fixes every few weeks in DockerHub. | +--------------+----------------------------------------------------------+----------------------------------------------------------+ -| Airflow | airflow-python-v2:-slim-buster | Version of python base image used in Airflow Builds | -| python base | or | We keep the "latest" version there and also each build | -| image | airflow-python-v2:-slim-buster- | has an associated specific python version that was used. | +| Airflow | airflow//python:-slim-buster | Version of python base image used in Airflow Builds | +| python base | | We keep the "latest" version only to mark last "good" | +| image | | python base that went through testing and was pushed. | +--------------+----------------------------------------------------------+----------------------------------------------------------+ -| CI image | airflow--python-ci-v2:latest | CI image - this is the image used for most of the tests. | -| | or | Contains all provider dependencies and tools useful | -| | airflow--python-ci-v2: | For testing. This image is used in Breeze. | +| PROD Build | airflow//prod-build/python:latest | Production Build image - this is the "build" stage of | +| image | | production image. It contains build-essentials and all | +| | | necessary apt packages to build/install PIP packages. | +| | | We keep the "latest" version only to speed up builds. | +--------------+----------------------------------------------------------+----------------------------------------------------------+ -| Manifest | airflow--python-ci-v2-manifest:latest | CI manifest image - this is the image used to optimize | -| CI image | or | pulls and builds for Breeze development environment | -| | airflow--python-ci-v2-manifest: | They store hash indicating whether the image will be | +| Manifest | airflow//ci-manifest/python:latest | CI manifest image - this is the image used to optimize | +| CI image | | pulls and builds for Breeze development environment | +| | | They store hash indicating whether the image will be | | | | faster to build or pull. | +| | | We keep the "latest" version only to help breeze to | +| | | check if new image should be pulled. | +--------------+----------------------------------------------------------+----------------------------------------------------------+ -| PROD Build | airflow--python-build-v2:latest | Production Build image - this is the "build" segment of | -| image | or | production image. It contains build-essentials and all | -| | airflow--python-build-v2: | necessary packages to install PIP packages. | +| CI image | airflow//ci/python:latest | CI image - this is the image used for most of the tests. | +| | or | Contains all provider dependencies and tools useful | +| | airflow//ci/python: | For testing. This image is used in Breeze. | +--------------+----------------------------------------------------------+----------------------------------------------------------+ -| PROD image | airflow--python-v2:latest | Production image. This is the actual production image | +| | | faster to build or pull. | +| PROD image | airflow//prod/python:latest | Production image. This is the actual production image | | | or | optimized for size. | -| | airflow--python-v2: | It contains only compiled libraries and minimal set of | +| | airflow//prod/python: | It contains only compiled libraries and minimal set of | | | | dependencies to run Airflow. | +--------------+----------------------------------------------------------+----------------------------------------------------------+ @@ -668,9 +673,9 @@ For example knowing that the CI build was for commit ``cd27124534b46c9688a1d89e7 .. code-block:: bash - docker pull ghcr.io/apache/airflow-main-python3.6-ci:cd27124534b46c9688a1d89e75fcd137ab5137e3 + docker pull ghcr.io/apache/airflow/main/ci/python3.6:cd27124534b46c9688a1d89e75fcd137ab5137e3 - docker run -it ghcr.io/apache/airflow-main-python3.6-ci:cd27124534b46c9688a1d89e75fcd137ab5137e3 + docker run -it ghcr.io/apache/airflow/main/ci/python3.6:cd27124534b46c9688a1d89e75fcd137ab5137e3 But you usually need to pass more variables and complex setup if you want to connect to a database or diff --git a/Dockerfile b/Dockerfile index 66c9649f8be49..210918c684b28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,8 @@ ARG AIRFLOW_GID="50000" ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster" -ARG AIRFLOW_PIP_VERSION=21.1.2 +ARG AIRFLOW_PIP_VERSION=21.2.2 +ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" # By default PIP has progress bar but you can disable it. ARG PIP_PROGRESS_BAR="on" @@ -108,12 +109,13 @@ ARG DEV_APT_COMMAND="\ && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \ && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list" ARG ADDITIONAL_DEV_APT_COMMAND="echo" +ARG ADDITIONAL_DEV_APT_ENV="" ENV DEV_APT_DEPS=${DEV_APT_DEPS} \ ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} \ DEV_APT_COMMAND=${DEV_APT_COMMAND} \ ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND} \ - ADDITIONAL_DEV_APT_ENV="" + ADDITIONAL_DEV_APT_ENV=${ADDITIONAL_DEV_APT_ENV} # Note missing man directories on debian-buster # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 @@ -216,7 +218,7 @@ ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \ RUN bash /scripts/docker/install_pip_version.sh; \ if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \ ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ - bash /scripts/docker/install_airflow_from_branch_tip.sh; \ + bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi COPY ${AIRFLOW_SOURCES_FROM} ${AIRFLOW_SOURCES_TO} @@ -236,14 +238,11 @@ ARG INSTALL_FROM_PYPI="true" # * pyjwt<2.0.0: flask-jwt-extended requires it # * dill<0.3.3 required by apache-beam ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="pyjwt<2.0.0 dill<0.3.3 certifi<2021.0.0" -ARG CONTINUE_ON_PIP_CHECK_FAILURE="false" - ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \ INSTALL_FROM_DOCKER_CONTEXT_FILES=${INSTALL_FROM_DOCKER_CONTEXT_FILES} \ INSTALL_FROM_PYPI=${INSTALL_FROM_PYPI} \ - EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \ - CONTINUE_ON_PIP_CHECK_FAILURE=${CONTINUE_ON_PIP_CHECK_FAILURE} + EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} WORKDIR /opt/airflow @@ -276,7 +275,7 @@ RUN if [[ -f /docker-context-files/requirements.txt ]]; then \ ARG BUILD_ID ARG COMMIT_SHA -ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" +ARG AIRFLOW_IMAGE_REPOSITORY ARG AIRFLOW_IMAGE_DATE_CREATED ENV BUILD_ID=${BUILD_ID} COMMIT_SHA=${COMMIT_SHA} @@ -293,15 +292,14 @@ LABEL org.apache.airflow.distro="debian" \ org.opencontainers.image.created=${AIRFLOW_IMAGE_DATE_CREATED} \ org.opencontainers.image.authors="dev@airflow.apache.org" \ org.opencontainers.image.url="https://airflow.apache.org" \ - org.opencontainers.image.documentation="https://airflow.apache.org/docs/apache-airflow/stable/production-deployment.html" \ - org.opencontainers.image.source="https://github.com/apache/airflow" \ + org.opencontainers.image.documentation="https://airflow.apache.org/docs/docker-stack/index.html" \ org.opencontainers.image.version="${AIRFLOW_VERSION}" \ org.opencontainers.image.revision="${COMMIT_SHA}" \ org.opencontainers.image.vendor="Apache Software Foundation" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.ref.name="airflow-build-image" \ org.opencontainers.image.title="Build Image Segment for Production Airflow Image" \ - org.opencontainers.image.description="Installed Apache Airflow with build-time dependencies" + org.opencontainers.image.description="Reference build-time dependencies image for production-ready Apache Airflow image" ############################################################################################## # This is the actual Airflow image - much smaller than the build one. We copy @@ -379,7 +377,7 @@ ARG AIRFLOW_HOME ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow" ARG BUILD_ID ARG COMMIT_SHA -ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" +ARG AIRFLOW_IMAGE_REPOSITORY ARG AIRFLOW_IMAGE_DATE_CREATED # By default PIP will install everything in ~/.local ARG PIP_USER="true" @@ -468,15 +466,14 @@ LABEL org.apache.airflow.distro="debian" \ org.opencontainers.image.created=${AIRFLOW_IMAGE_DATE_CREATED} \ org.opencontainers.image.authors="dev@airflow.apache.org" \ org.opencontainers.image.url="https://airflow.apache.org" \ - org.opencontainers.image.documentation="https://airflow.apache.org/docs/apache-airflow/stable/production-deployment.html" \ - org.opencontainers.image.source="https://github.com/apache/airflow" \ + org.opencontainers.image.documentation="https://airflow.apache.org/docs/docker-stack/index.html" \ org.opencontainers.image.version="${AIRFLOW_VERSION}" \ org.opencontainers.image.revision="${COMMIT_SHA}" \ org.opencontainers.image.vendor="Apache Software Foundation" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.ref.name="airflow" \ org.opencontainers.image.title="Production Airflow Image" \ - org.opencontainers.image.description="Installed Apache Airflow" + org.opencontainers.image.description="Reference, production-ready Apache Airflow image" ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"] diff --git a/Dockerfile.ci b/Dockerfile.ci index 7b1dbf14ba95b..cca868ecb0cd4 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -21,7 +21,8 @@ FROM ${PYTHON_BASE_IMAGE} as main SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster" -ARG AIRFLOW_VERSION="2.1.0.dev0" +ARG AIRFLOW_VERSION="2.1.3.dev0" +ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" # By increasing this number we can do force build of all dependencies ARG DEPENDENCIES_EPOCH_NUMBER="6" @@ -98,6 +99,7 @@ RUN mkdir -pv /usr/share/man/man1 \ COPY scripts/docker/*.sh /scripts/docker/ RUN bash /scripts/docker/install_mysql.sh dev \ && adduser airflow \ + && echo "airflow:airflow" | chpasswd \ && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ && chmod 0440 /etc/sudoers.d/airflow @@ -125,6 +127,15 @@ ARG RUNTIME_APT_DEPS="\ unzip \ vim \ xxd" + +# Install Helm +ARG HELM_VERSION="v3.6.3" + +RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \ + && HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz" \ + && curl --location "${HELM_URL}" | tar -xvz -O "${SYSTEM}"-amd64/helm > /usr/local/bin/helm \ + && chmod +x /usr/local/bin/helm + ARG ADDITIONAL_RUNTIME_APT_DEPS="" ARG RUNTIME_APT_COMMAND="" ARG ADDITIONAL_RUNTIME_APT_COMMAND="" @@ -207,7 +218,7 @@ ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" # By default in the image, we are installing all providers when installing from sources ARG INSTALL_PROVIDERS_FROM_SOURCES="true" ARG INSTALL_FROM_PYPI="true" -ARG AIRFLOW_PIP_VERSION=21.1.2 +ARG AIRFLOW_PIP_VERSION=21.2.2 # Setup PIP # By default PIP install run without cache to make image smaller ARG PIP_NO_CACHE_DIR="true" @@ -270,7 +281,7 @@ ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENT RUN bash /scripts/docker/install_pip_version.sh; \ if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \ ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ - bash /scripts/docker/install_airflow_from_branch_tip.sh; \ + bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi # Generate random hex dump file so that we can determine whether it's faster to rebuild the image @@ -299,8 +310,6 @@ COPY setup.cfg ${AIRFLOW_SOURCES}/setup.cfg COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/__init__.py -ARG CONTINUE_ON_PIP_CHECK_FAILURE="false" - # The goal of this line is to install the dependencies from the most current setup.py from sources # This will be usually incremental small set of packages in CI optimized build, so it will be very fast # In non-CI optimized build this will install all dependencies before installing sources. @@ -325,11 +334,13 @@ RUN chmod a+x /entrypoint COPY scripts/docker/load.bash /opt/bats/lib/ -# We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not -# copying over stuff that is accidentally generated or that we do not need (such as egg-info) -# if you want to add something that is missing and you expect to see it in the image you can -# add it with ! in .dockerignore next to the airflow, test etc. directories there -COPY . ${AIRFLOW_SOURCES}/ +# Additional python deps to install +ARG ADDITIONAL_PYTHON_DEPS="" + +RUN bash /scripts/docker/install_pip_version.sh; \ + if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ + bash /scripts/docker/install_additional_dependencies.sh; \ + fi # Install autocomplete for airflow RUN if command -v airflow; then \ @@ -339,27 +350,16 @@ RUN if command -v airflow; then \ # Install autocomplete for Kubectl RUN echo "source /etc/bash_completion" >> ~/.bashrc -WORKDIR ${AIRFLOW_SOURCES} - -# Install Helm -ARG HELM_VERSION="v3.2.4" - -RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \ - && HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz" \ - && curl --location "${HELM_URL}" | tar -xvz -O "${SYSTEM}"-amd64/helm > /usr/local/bin/helm \ - && chmod +x /usr/local/bin/helm - -# Additional python deps to install -ARG ADDITIONAL_PYTHON_DEPS="" +# We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not +# copying over stuff that is accidentally generated or that we do not need (such as egg-info) +# if you want to add something that is missing and you expect to see it in the image you can +# add it with ! in .dockerignore next to the airflow, test etc. directories there +COPY . ${AIRFLOW_SOURCES}/ -RUN bash /scripts/docker/install_pip_version.sh; \ - if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ - bash /scripts/docker/install_additional_dependencies.sh; \ - fi +WORKDIR ${AIRFLOW_SOURCES} ARG BUILD_ID ARG COMMIT_SHA -ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" ARG AIRFLOW_IMAGE_DATE_CREATED ENV PATH="/files/bin/:/opt/airflow/scripts/in_container/bin/:${HOME}:${PATH}" \ diff --git a/IMAGES.rst b/IMAGES.rst index 82e69895f3842..a1b1ace08491a 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -24,7 +24,7 @@ Airflow has two main images (build from Dockerfiles): * Production image (Dockerfile) - that can be used to build your own production-ready Airflow installation You can read more about building and using the production image in the - `Production Deployments `_ document. + `Docker stack `_ documentation. The image is built using `Dockerfile `_ * CI image (Dockerfile.ci) - used for running tests and local development. The image is built using @@ -246,19 +246,21 @@ Images with a commit SHA (built for pull requests and pushes) .. code-block:: bash - ghcr.io/apache/airflow--pythonX.Y-ci-v2: - for CI images - ghcr.io/apache/airflow--pythonX.Y-v2: - for production images - ghcr.io/apache/airflow--pythonX.Y-build-v2: - for production build stage - ghcr.io/apache/airflow-python-v2:X.Y-slim-buster- - for base Python images + ghcr.io/apache/airflow//ci/python: - for CI images + ghcr.io/apache/airflow//prod/python: - for production images + +We do not push Base Python images and prod-build images when we prepare COMMIT builds, because those +images are never rebuilt locally, so there is no need to store base images specific for those builds. Latest images (pushed when main merge succeeds): .. code-block:: bash - ghcr.io/apache/airflow--pythonX.Y-ci-v2:latest - for CI images - ghcr.io/apache/airflow--pythonX.Y-v2:latest - for production images - ghcr.io/apache/airflow--pythonX.Y-build-v2:latest - for production build stage - ghcr.io/apache/airflow-python-v2:X.Y-slim-buster - for base Python images + ghcr.io/apache/airflow//python:-slim-buster - for base Python images + ghcr.io/apache/airflow//ci/python:latest - for CI images + ghcr.io/apache/airflow//ci-manifest/python:latest - for CI Manifest images + ghcr.io/apache/airflow//prod/python:latest - for production images + ghcr.io/apache/airflow//prod-build/python:latest - for production build stage You can see all the current GitHub images at ``_ @@ -445,12 +447,6 @@ The following build arguments (``--build-arg`` in docker build command) can be u | | | upgraded to newer versions matching | | | | setup.py before installation. | +------------------------------------------+------------------------------------------+------------------------------------------+ -| ``CONTINUE_ON_PIP_CHECK_FAILURE`` | ``false`` | By default the image will fail if pip | -| | | check fails for it. This is good for | -| | | interactive building but on CI the | -| | | image should be built regardless - we | -| | | have a separate step to verify image. | -+------------------------------------------+------------------------------------------+------------------------------------------+ | ``AIRFLOW_PRE_CACHED_PIP_PACKAGES`` | ``true`` | Allows to pre-cache airflow PIP packages | | | | from the GitHub of Apache Airflow | | | | This allows to optimize iterations for | @@ -558,8 +554,8 @@ way of querying image details via API. You really need to download the image to We workaround it in the way that always when we build the image we build a very small image manifest containing randomly generated UUID and push it to registry together with the main CI image. The tag for the manifest image reflects the image it refers to with added ``-manifest`` suffix. -The manifest image for ``ghcr.io/apache/airflow-main-python3.6-ci-v2`` is named -``ghcr.io/apache/airflow-main-python3.6-ci-v2-manifest``. +The manifest image for ``ghcr.io/apache/airflow/main/ci/python3.6`` is named +``ghcr.io/apache/airflow/main/ci-manifest/python3.6``. The image is quickly pulled (it is really, really small) when important files change and the content of the randomly generated UUID is compared with the one in our image. If the contents are different diff --git a/README.md b/README.md index 97981362320c5..d58a39fea59a6 100644 --- a/README.md +++ b/README.md @@ -209,8 +209,8 @@ Those are - in the order of most common ways people install Airflow: - [Docker Images](https://hub.docker.com/r/apache/airflow) to install airflow via `docker` tool, use them in Kubernetes, Helm Charts, `docker-compose`, `docker swarm` etc. You can read more about using, customising, and extending the images in the - [Latest docs](https://airflow.apache.org/docs/apache-airflow/stable/production-deployment.html), and - learn details on the internals in the [IMAGES.rst](IMAGES.rst) document. + [Latest docs](https://airflow.apache.org/docs/docker-stack/index.html), and + learn details on the internals in the [IMAGES.rst](https://github.com/apache/airflow/blob/main/IMAGES.rst) document. - [Tags in GitHub](https://github.com/apache/airflow/tags) to retrieve the git project sources that were used to generate official source packages via git diff --git a/breeze b/breeze index 7decdf67d6704..2fe14582fef3a 100755 --- a/breeze +++ b/breeze @@ -164,6 +164,9 @@ function breeze::setup_default_breeze_constants() { # Can be overridden by '--force-build-images' flag. export FORCE_BUILD_IMAGES="false" + # When we push from breeze we always want to push base python images + export PUSH_PYTHON_BASE_IMAGE="true" + # Determines whether to reinstall airflow at entering the image. export USE_AIRFLOW_VERSION="" # if set to true, the ci image will look for wheel packages in dist folder and will install them @@ -569,8 +572,7 @@ EOF # AIRFLOW_SOURCES # AIRFLOW_CI_IMAGE # AIRFLOW_PROD_IMAGE -# AIRFLOW_PROD_IMAGE_KUBERNETES -# AIRFLOW_PROD_BASE_TAG +# AIRFLOW_IMAGE_KUBERNETES # SQLITE_URL # # Arguments: @@ -633,8 +635,7 @@ export MYSQL_VERSION="${MYSQL_VERSION}" export AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" export AIRFLOW_CI_IMAGE="${AIRFLOW_CI_IMAGE}" export AIRFLOW_PROD_IMAGE="${AIRFLOW_PROD_IMAGE}" -export AIRFLOW_PROD_IMAGE_KUBERNETES="${AIRFLOW_PROD_IMAGE_KUBERNETES}" -export AIRFLOW_PROD_BASE_TAG="${AIRFLOW_PROD_BASE_TAG}" +export AIRFLOW_IMAGE_KUBERNETES="${AIRFLOW_IMAGE_KUBERNETES}" export SQLITE_URL="${SQLITE_URL}" export USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION}" export USE_PACKAGES_FROM_DIST="${USE_PACKAGES_FROM_DIST}" @@ -650,7 +651,6 @@ EOF # # Global constants set: # -# PYTHON_BASE_IMAGE_VERSION # PYTHON_BASE_IMAGE # AIRFLOW_CI_IMAGE # BUILT_CI_IMAGE_FLAG_FILE @@ -934,7 +934,6 @@ function breeze::parse_arguments() { echo export DOCKER_CACHE="disabled" # if not set here, docker cached is determined later, depending on type of image to be build - readonly DOCKER_CACHE export FORCE_BUILD_IMAGES="true" shift ;; @@ -950,7 +949,6 @@ function breeze::parse_arguments() { echo export DOCKER_CACHE="local" # if not set here, docker cached is determined later, depending on type of image to be build - readonly DOCKER_CACHE shift ;; -U | --build-cache-pulled) @@ -958,14 +956,12 @@ function breeze::parse_arguments() { echo export DOCKER_CACHE="pulled" # if not set here, docker cached is determined later, depending on type of image to be build - readonly DOCKER_CACHE shift ;; -X | --build-cache-disabled) echo "Use disabled cache to build images" echo export DOCKER_CACHE="disabled" - readonly DOCKER_CACHE # if not set here, docker cached is determined later, depending on type of image to be build shift ;; @@ -1126,7 +1122,6 @@ function breeze::parse_arguments() { export CHECK_IMAGE_FOR_REBUILD="false" export SKIP_BUILDING_PROD_IMAGE="true" export SKIP_CHECK_REMOTE_IMAGE="true" - export FAIL_ON_GITHUB_DOCKER_PULL_ERROR="true" shift 2 ;; --init-script) @@ -1177,12 +1172,6 @@ function breeze::parse_arguments() { echo shift ;; - --continue-on-pip-check-failure) - export CONTINUE_ON_PIP_CHECK_FAILURE="true" - echo "Skip PIP check failure." - echo - shift - ;; --package-format) export PACKAGE_FORMAT="${2}" echo "Selected package type: ${PACKAGE_FORMAT}" @@ -3571,7 +3560,7 @@ breeze::check_and_save_all_params build_images::determine_docker_cache_strategy -build_images::get_docker_image_names +build_images::get_docker_cache_image_names initialization::make_constants_read_only diff --git a/breeze-complete b/breeze-complete index db9e42a494cbc..045bb65ade6b6 100644 --- a/breeze-complete +++ b/breeze-complete @@ -29,7 +29,7 @@ _breeze_allowed_integrations="cassandra kerberos mongo openldap pinot rabbitmq r _breeze_allowed_generate_constraints_modes="source-providers pypi-providers no-providers" _breeze_allowed_kubernetes_modes="image" _breeze_allowed_kubernetes_versions="v1.20.2 v1.19.7 v1.18.15" -_breeze_allowed_helm_versions="v3.2.4" +_breeze_allowed_helm_versions="v3.6.3" _breeze_allowed_kind_versions="v0.11.1" _breeze_allowed_mysql_versions="5.7 8" _breeze_allowed_postgres_versions="9.6 10 11 12 13" diff --git a/dev/retag_docker_images.py b/dev/retag_docker_images.py index 5eeda8efb78f7..f29ce1b5a13db 100755 --- a/dev/retag_docker_images.py +++ b/dev/retag_docker_images.py @@ -36,10 +36,11 @@ GHCR_IO_PREFIX = "ghcr.io/apache/airflow" GHCR_IO_IMAGES = [ - "{prefix}-{branch}-python{python_version}-ci-v2-manifest:latest", - "{prefix}-{branch}-python{python_version}-ci-v2:latest", - "{prefix}-{branch}-python{python_version}-v2:latest", - "{prefix}-{branch}-python{python_version}-build-v2:latest", + "{prefix}/{branch}/ci-manifest/python{python_version}:latest", + "{prefix}/{branch}/ci/python{python_version}:latest", + "{prefix}/{branch}/prod-build/python{python_version}-build-v2:latest", + "{prefix}/{branch}/prod/python{python_version}-build-v2:latest", + "{prefix}/{branch}/python:{python_version}-slim-buster", ] diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index 8780970f613c5..f2507e05db423 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -79,12 +79,6 @@ for examples of using those arguments. +------------------------------------------+------------------------------------------+------------------------------------------+ | Build argument | Default value | Description | +==========================================+==========================================+==========================================+ -| ``CONTINUE_ON_PIP_CHECK_FAILURE`` | ``false`` | By default the image build fails if pip | -| | | check fails for it. This is good for | -| | | interactive building but on CI the | -| | | image should be built regardless - we | -| | | have a separate step to verify image. | -+------------------------------------------+------------------------------------------+------------------------------------------+ | ``UPGRADE_TO_NEWER_DEPENDENCIES`` | ``false`` | If set to true, the dependencies are | | | | upgraded to newer versions matching | | | | setup.py before installation. | diff --git a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh index a550038ab85dc..8a89a30c4861c 100755 --- a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh +++ b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh @@ -29,24 +29,13 @@ function build_ci_image_on_ci() { if [[ ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == "true" ]]; then # Pretend that the image was build. We already have image with the right sources baked in! + # so all the checksums are assumed to be correct md5sum::calculate_md5sum_for_all_files - # Tries to wait for the images indefinitely - # skips further image checks - since we already have the target image - - local python_tag_suffix="" - if [[ ${GITHUB_REGISTRY_PULL_IMAGE_TAG} != "latest" ]]; then - python_tag_suffix="-${GITHUB_REGISTRY_PULL_IMAGE_TAG}" - fi - # first we pull base python image. We will need it to re-push it after main build - # Becoming the new "latest" image for other builds - build_images::wait_for_image_tag "${AIRFLOW_PYTHON_BASE_IMAGE}" \ - "${python_tag_suffix}" - - # And then the actual image - build_images::wait_for_image_tag "${AIRFLOW_CI_IMAGE}" \ - ":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + # Remove me on 7th August 2021 after all users had chance to rebase + legacy_ci_image="ghcr.io/${GITHUB_REPOSITORY}-${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci-v2:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + build_images::wait_for_image_tag "${AIRFLOW_CI_IMAGE}" ":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" "${legacy_ci_image}" md5sum::update_all_md5_with_group else build_images::rebuild_ci_image_if_needed diff --git a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh index dbcb07d00a91c..14bd71f14a478 100755 --- a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh +++ b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh @@ -33,36 +33,13 @@ function build_prod_images_on_ci() { if [[ ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == "true" ]]; then # Tries to wait for the images indefinitely - # skips further image checks - since we already have the target image - - local python_tag_suffix="" - if [[ ${GITHUB_REGISTRY_PULL_IMAGE_TAG} != "latest" ]]; then - python_tag_suffix="-${GITHUB_REGISTRY_PULL_IMAGE_TAG}" - fi - - if [[ "${WAIT_FOR_PYTHON_BASE_IMAGE=}" == "true" ]]; then - # first we pull base python image. We will need it to re-push it after main build - # Becoming the new "latest" image for other builds - build_images::wait_for_image_tag "${AIRFLOW_PYTHON_BASE_IMAGE}" \ - "${python_tag_suffix}" - fi - - # And then the actual image - build_images::wait_for_image_tag "${AIRFLOW_PROD_IMAGE}" \ - ":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" - - # And the prod build image - if [[ "${WAIT_FOR_PROD_BUILD_IMAGE=}" == "true" ]]; then - # If specified in variable - also waits for the build image - build_images::wait_for_image_tag "${AIRFLOW_PROD_BUILD_IMAGE}" \ - ":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" - fi - + # Remove me on 7th August 2021 after all users had chance to rebase + legacy_prod_image="ghcr.io/${GITHUB_REPOSITORY}-${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-v2:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + build_images::wait_for_image_tag "${AIRFLOW_PROD_IMAGE}" ":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" "${legacy_prod_image}" else build_images::build_prod_images_from_locally_built_airflow_packages fi - # Disable force pulling forced above this is needed for the subsequent scripts so that # They do not try to pull/build images again unset FORCE_PULL_IMAGES diff --git a/scripts/ci/images/ci_push_legacy_ci_images.sh b/scripts/ci/images/ci_push_legacy_ci_images.sh new file mode 100755 index 0000000000000..aa6696b346ccf --- /dev/null +++ b/scripts/ci/images/ci_push_legacy_ci_images.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" + +# This script pushes legacy images to old-naming-convention images +# It should be removed ~ 7th of August, giving users time to rebase their old pull requests +build_images::prepare_ci_build + +legacy_ci_image="ghcr.io/${GITHUB_REPOSITORY}-${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci-v2:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}" +docker tag "${AIRFLOW_CI_IMAGE}" "${legacy_ci_image}" +docker push "${legacy_ci_image}" diff --git a/scripts/ci/images/ci_push_legacy_prod_images.sh b/scripts/ci/images/ci_push_legacy_prod_images.sh new file mode 100755 index 0000000000000..3f74874da81a9 --- /dev/null +++ b/scripts/ci/images/ci_push_legacy_prod_images.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" + +# This script pushes legacy images to old-naming-convention images which will keep old PRs working before +# Rebasing to main. It should be removed ~7th of August 2021, giving users time to rebase their old pull requests +build_images::prepare_ci_build + +legacy_prod_image="ghcr.io/${GITHUB_REPOSITORY}-${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-v2:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}" +docker tag "${AIRFLOW_PROD_IMAGE}" "${legacy_prod_image}" +docker push "${legacy_prod_image}" diff --git a/scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh b/scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh index 4255374309fbe..39dfe8f5db72c 100755 --- a/scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh +++ b/scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh @@ -25,6 +25,8 @@ source "${LIBRARIES_DIR}/_all_libs.sh" initialization::set_output_color_variables +PARALLEL_TAIL_LENGTH=5 + parallel::make_sure_gnu_parallel_is_installed parallel::make_sure_python_versions_are_specified diff --git a/scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh b/scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh index 08ed54b323da2..6786a3189b460 100755 --- a/scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh +++ b/scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh @@ -25,6 +25,8 @@ source "${LIBRARIES_DIR}/_all_libs.sh" initialization::set_output_color_variables +PARALLEL_TAIL_LENGTH=5 + parallel::make_sure_gnu_parallel_is_installed parallel::make_sure_python_versions_are_specified diff --git a/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh b/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh index 105bcfb3b803b..bb181005e177d 100755 --- a/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh +++ b/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh @@ -28,25 +28,34 @@ shift # shellcheck source=scripts/ci/libraries/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" +image_name_with_tag="${AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + +# Remove me on 7th August 2021 after all users had chance to rebase +legacy_ci_image="ghcr.io/${GITHUB_REPOSITORY}-${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci-v2:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + function pull_ci_image() { - local image_name_with_tag="${AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" - start_end::group_start "Pulling ${image_name_with_tag} image" - push_pull_remove_images::pull_image_if_not_present_or_forced "${image_name_with_tag}" + start_end::group_start "Pulling image: ${IMAGE_AVAILABLE}" + push_pull_remove_images::pull_image_if_not_present_or_forced "${IMAGE_AVAILABLE}" + # Remove me on 7th August 2021 after all users had chance to rebase + if [[ ${IMAGE_AVAILABLE} != "${image_name_with_tag}" ]]; then + verbosity::print_info "Tagging the legacy ${IMAGE_AVAILABLE} with ${image_name_with_tag}" + docker tag "${IMAGE_AVAILABLE}" "${image_name_with_tag}" + fi start_end::group_end - } start_end::group_start "Configure Docker Registry" build_images::configure_docker_registry start_end::group_end -start_end::group_start "Waiting for ${AIRFLOW_CI_IMAGE}" - -push_pull_remove_images::wait_for_image "${AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" +start_end::group_start "Waiting for ${image_name_with_tag}" +# Remove me on 7th August 2021 after all users had chance to rebase +push_pull_remove_images::wait_for_image "${image_name_with_tag}" "${legacy_ci_image}" build_images::prepare_ci_build -pull_ci_image start_end::group_end +pull_ci_image + if [[ ${VERIFY_IMAGE=} != "false" ]]; then - verify_image::verify_ci_image "${AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + verify_image::verify_ci_image "${image_name_with_tag}" fi diff --git a/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh b/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh index 482b0a558289d..d93da72e2ee22 100755 --- a/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh +++ b/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh @@ -28,22 +28,34 @@ shift # shellcheck source=scripts/ci/libraries/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" -start_end::group_start "Configure Docker Registry" -build_images::configure_docker_registry -start_end::group_end +image_name_with_tag="${AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + +# Remove me on 7th August 2021 after all users had chance to rebase +legacy_prod_image="ghcr.io/${GITHUB_REPOSITORY}-${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-v2:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" -start_end::group_start "Waiting for ${AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" +function pull_prod_image() { + start_end::group_start "Pulling image: ${IMAGE_AVAILABLE}" + push_pull_remove_images::pull_image_if_not_present_or_forced "${IMAGE_AVAILABLE}" + # Remove me on 7th August 2021 after all users had chance to rebase + if [[ ${IMAGE_AVAILABLE} != "${image_name_with_tag}" ]]; then + verbosity::print_info "Tagging the legacy ${IMAGE_AVAILABLE} with ${image_name_with_tag}" + docker tag "${IMAGE_AVAILABLE}" "${image_name_with_tag}" + fi + start_end::group_end +} -push_pull_remove_images::wait_for_image "${AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" +start_end::group_start "Configure Docker Registry" +build_images::configure_docker_registry start_end::group_end -start_end::group_start "Pulling the PROD Image" +start_end::group_start "Waiting for ${image_name_with_tag}" +# Remove me on 7th August 2021 after all users had chance to rebase +push_pull_remove_images::wait_for_image "${image_name_with_tag}" "${legacy_prod_image}" build_images::prepare_prod_build -image_name_with_tag="${AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" -verbosity::print_info "Pulling the ${image_name_with_tag} image and tagging with ${AIRFLOW_PROD_IMAGE}" -push_pull_remove_images::pull_image_if_not_present_or_forced "${image_name_with_tag}" start_end::group_end +pull_prod_image + if [[ ${VERIFY_IMAGE=} != "false" ]]; then - verify_image::verify_prod_image "${AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + verify_image::verify_prod_image "${image_name_with_tag}" fi diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index ca94e4cf316b6..dfbfe1ae73632 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -251,7 +251,6 @@ EOF # Retrieves information about build cache hash random file from the local image # function build_images::get_local_build_cache_hash() { - set +e # Remove the container just in case docker_v rm --force "local-airflow-ci-container" 2>/dev/null >/dev/null @@ -262,6 +261,7 @@ function build_images::get_local_build_cache_hash() { LOCAL_MANIFEST_IMAGE_UNAVAILABLE="true" export LOCAL_MANIFEST_IMAGE_UNAVAILABLE touch "${LOCAL_IMAGE_BUILD_CACHE_HASH_FILE}" + set -e return fi @@ -296,6 +296,7 @@ function build_images::get_remote_image_build_cache_hash() { REMOTE_DOCKER_REGISTRY_UNREACHABLE="true" export REMOTE_DOCKER_REGISTRY_UNREACHABLE touch "${REMOTE_IMAGE_BUILD_CACHE_HASH_FILE}" + set -e return fi set -e @@ -358,49 +359,44 @@ function build_images::get_github_container_registry_image_prefix() { echo "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]' } -function build_images::get_docker_image_names() { - # python image version to use - export PYTHON_BASE_IMAGE_VERSION=${PYTHON_BASE_IMAGE_VERSION:=${PYTHON_MAJOR_MINOR_VERSION}} - +function build_images::get_docker_cache_image_names() { # Python base image to use - export PYTHON_BASE_IMAGE="python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster" + export PYTHON_BASE_IMAGE="python:${PYTHON_MAJOR_MINOR_VERSION}-slim-buster" local image_name image_name="${GITHUB_REGISTRY}/$(build_images::get_github_container_registry_image_prefix)" - # CI image base tag - export AIRFLOW_CI_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci" + # Example: + # ghcr.io/apache/airflow/main/python:3.8-slim-buster + export AIRFLOW_PYTHON_BASE_IMAGE="${image_name}/${BRANCH_NAME}/python:${PYTHON_MAJOR_MINOR_VERSION}-slim-buster" # Example: - # ghcr.io/apache/airflow-main-python3.8-ci-v2 - export AIRFLOW_CI_IMAGE="${image_name}-${AIRFLOW_CI_BASE_TAG}${GITHUB_REGISTRY_IMAGE_SUFFIX}" + # ghcr.io/apache/airflow/main/ci/python3.8 + export AIRFLOW_CI_IMAGE="${image_name}/${BRANCH_NAME}/ci/python${PYTHON_MAJOR_MINOR_VERSION}" - export AIRFLOW_CI_LOCAL_MANIFEST_IMAGE="local-airflow-ci-manifest:${AIRFLOW_CI_BASE_TAG}" + # Example: + # local-airflow-ci-manifest/main/python3.8 + export AIRFLOW_CI_LOCAL_MANIFEST_IMAGE="local-airflow-ci-manifest/${BRANCH_NAME}/python${PYTHON_MAJOR_MINOR_VERSION}" # Example: - # ghcr.io/apache/airflow-main-python3.8-ci-v2-manifest - export AIRFLOW_CI_REMOTE_MANIFEST_IMAGE="${image_name}-${AIRFLOW_CI_BASE_TAG}${GITHUB_REGISTRY_IMAGE_SUFFIX}-manifest" + # ghcr.io/apache/airflow/main/ci-manifest/python3.8 + export AIRFLOW_CI_REMOTE_MANIFEST_IMAGE="${image_name}/${BRANCH_NAME}/ci-manifest/python${PYTHON_MAJOR_MINOR_VERSION}" # File that is touched when the CI image is built for the first time locally export BUILT_CI_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}" - # PROD image to build - export AIRFLOW_PROD_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}" - # Example: - # ghcr.io/apache/airflow-v2-1-test-python-v2:3.6-slim-buster - export AIRFLOW_PROD_IMAGE="${image_name}-${AIRFLOW_PROD_BASE_TAG}${GITHUB_REGISTRY_IMAGE_SUFFIX}" - - # PROD Kubernetes image to build - export AIRFLOW_PROD_IMAGE_KUBERNETES="${AIRFLOW_PROD_IMAGE}-kubernetes" + # ghcr.io/apache/airflow/main/prod/python3.8 + export AIRFLOW_PROD_IMAGE="${image_name}/${BRANCH_NAME}/prod/python${PYTHON_MAJOR_MINOR_VERSION}" # Example: - # ghcr.io/apache/airflow-main-python3.6-build-v2 - export AIRFLOW_PROD_BUILD_IMAGE="${image_name}-${AIRFLOW_PROD_BASE_TAG}-build${GITHUB_REGISTRY_IMAGE_SUFFIX}" + # ghcr.io/apache/airflow/main/prod-build/python3.8 + export AIRFLOW_PROD_BUILD_IMAGE="${image_name}/${BRANCH_NAME}/prod-build/python${PYTHON_MAJOR_MINOR_VERSION}" + + # Kubernetes image to build + # ghcr.io/apache/airflow/main/kubernetes/python3.8 + export AIRFLOW_IMAGE_KUBERNETES="${image_name}/${BRANCH_NAME}/kubernetes/python${PYTHON_MAJOR_MINOR_VERSION}" - # Example: - # ghcr.io/apache/airflow-python-v2:3.6-slim-buster - export AIRFLOW_PYTHON_BASE_IMAGE="${image_name}-python${GITHUB_REGISTRY_IMAGE_SUFFIX}:${PYTHON_BASE_IMAGE_VERSION}-slim-buster" } @@ -669,7 +665,6 @@ Docker building ${AIRFLOW_CI_IMAGE}. --build-arg ADDITIONAL_RUNTIME_APT_DEPS="${ADDITIONAL_RUNTIME_APT_DEPS}" \ --build-arg ADDITIONAL_RUNTIME_APT_ENV="${ADDITIONAL_RUNTIME_APT_ENV}" \ --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \ - --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \ --build-arg CONSTRAINTS_GITHUB_REPOSITORY="${CONSTRAINTS_GITHUB_REPOSITORY}" \ --build-arg AIRFLOW_CONSTRAINTS_REFERENCE="${DEFAULT_CONSTRAINTS_BRANCH}" \ --build-arg AIRFLOW_CONSTRAINTS="${AIRFLOW_CONSTRAINTS}" \ @@ -810,7 +805,6 @@ function build_images::build_prod_images() { --build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \ --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \ --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \ - --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \ --build-arg BUILD_ID="${CI_BUILD_ID}" \ --build-arg COMMIT_SHA="${COMMIT_SHA}" \ --build-arg CONSTRAINTS_GITHUB_REPOSITORY="${CONSTRAINTS_GITHUB_REPOSITORY}" \ @@ -845,7 +839,6 @@ function build_images::build_prod_images() { --build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \ --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \ --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \ - --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ @@ -868,17 +861,37 @@ function build_images::build_prod_images() { fi } +# Tags source image with names provided +# $1 source image +# $2, $3 - target image names +function build_images::tag_image() { + local source_image_name="$1" + shift + local target_image_name + for target_image_name in "${@}"; do + echo + echo "Tagging ${source_image_name} as ${target_image_name}." + echo + docker_v tag "${source_image_name}" "${target_image_name}" + done +} + # Waits for image tag to appear in GitHub Registry, pulls it and tags with the target tag # Parameters: # $1 - image name to wait for -# $2 - suffix of the image to wait for -# $3, $4, ... - target tags to tag the image with +# $2 - fallback image to wait for +# $3 - suffix of the image to wait for (Remove me on 7th August 2021 after all users had chance to rebase) +# $4, $5, ... - target tags to tag the image with function build_images::wait_for_image_tag() { local image_name="${1}" local image_suffix="${2}" shift 2 + # Remove me 7th of August 2021 + local legacy_image_to_pull="${1}" + shift + local image_to_wait_for="${image_name}${image_suffix}" start_end::group_start "Wait for image tag ${image_to_wait_for}" while true; do @@ -891,26 +904,34 @@ function build_images::wait_for_image_tag() { image_hash="$(docker images -q "${image_to_wait_for}" 2>>"${OUTPUT_LOG}" || true)" if [[ -z "${image_hash}" ]]; then echo - echo "The image ${image_to_wait_for} is not yet available. No local hash for the image. Waiting." + echo "The image ${image_to_wait_for} is not yet available. No local hash for the image. Falling bacl to legacy." echo echo "Last log:" cat "${OUTPUT_LOG}" || true echo - sleep 10 - else - echo - echo "The image ${image_to_wait_for} with '${image_name}' tag" - echo - echo - echo "Tagging ${image_to_wait_for} as ${image_name}." - echo - docker_v tag "${image_to_wait_for}" "${image_name}" - for TARGET_TAG in "${@}"; do + echo "Checking Legacy image!" + # Legacy - Remove me 7th of August 2021 + set +e + echo "${COLOR_BLUE}Docker pull ${legacy_image_to_pull} ${COLOR_RESET}" >"${OUTPUT_LOG}" + docker_v pull "${legacy_image_to_pull}" >>"${OUTPUT_LOG}" 2>&1 + set -e + echo "${COLOR_BLUE} Docker images -q ${legacy_image_to_pull}${COLOR_RESET}" >>"${OUTPUT_LOG}" + image_hash="$(docker images -q "${legacy_image_to_pull}" 2>>"${OUTPUT_LOG}" || true)" + if [[ -z "${image_hash}" ]]; then echo - echo "Tagging ${image_to_wait_for} as ${TARGET_TAG}." + echo "The image ${legacy_image_to_pull} is not yet available. No local hash for the image. Waiting." echo - docker_v tag "${image_to_wait_for}" "${TARGET_TAG}" - done + echo "Last log:" + cat "${OUTPUT_LOG}" || true + sleep 10 + else + # Legacy - Rremove me 7th of August 2021 + # Pretend that the image we waited for was downloaded :) + build_images::tag_image "${legacy_image_to_pull}" "${image_to_wait_for}" "${image_name}:latest" "${@}" + break + fi + else + build_images::tag_image "${image_to_wait_for}" "${image_name}:latest" "${@}" break fi done @@ -927,7 +948,6 @@ function build_images::determine_docker_cache_strategy() { export DOCKER_CACHE="pulled" fi fi - readonly DOCKER_CACHE verbosity::print_info verbosity::print_info "Using ${DOCKER_CACHE} cache strategy for the build." verbosity::print_info diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index cbeb93268a2a7..a51cdd08aac90 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -153,6 +153,10 @@ function initialization::initialize_base_variables() { # Dry run - only show docker-compose and docker commands but do not execute them export DRY_RUN_DOCKER=${DRY_RUN_DOCKER:="false"} + + # By default we only push built ci/prod images - base python images are only pushed + # When requested + export PUSH_PYTHON_BASE_IMAGE=${PUSH_PYTHON_BASE_IMAGE:="false"} } # Determine current branch @@ -200,7 +204,7 @@ function initialization::initialize_files_for_rebuild_check() { "scripts/docker/common.sh" "scripts/docker/install_additional_dependencies.sh" "scripts/docker/install_airflow.sh" - "scripts/docker/install_airflow_from_branch_tip.sh" + "scripts/docker/install_airflow_dependencies_from_branch_tip.sh" "scripts/docker/install_from_docker_context_files.sh" "scripts/docker/install_mysql.sh" "airflow/www/package.json" @@ -282,9 +286,6 @@ function initialization::initialize_force_variables() { # Can be set to true to skip if the image is newer in registry export SKIP_CHECK_REMOTE_IMAGE=${SKIP_CHECK_REMOTE_IMAGE:="false"} - - # Should be set to true if you expect image frm GitHub to be present and downloaded - export FAIL_ON_GITHUB_DOCKER_PULL_ERROR=${FAIL_ON_GITHUB_DOCKER_PULL_ERROR:="false"} } # Determine information about the host @@ -401,7 +402,7 @@ function initialization::initialize_image_build_variables() { export INSTALLED_PROVIDERS export INSTALLED_EXTRAS="async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv" - AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.1"} + AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.2.2"} export AIRFLOW_PIP_VERSION # We also pin version of wheel used to get consistent builds @@ -425,9 +426,6 @@ function initialization::initialize_image_build_variables() { # Installs different airflow version than current from the sources export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""} - # Continue on PIP CHECK failure - export CONTINUE_ON_PIP_CHECK_FAILURE=${CONTINUE_ON_PIP_CHECK_FAILURE:="false"} - # Determines if airflow should be installed from a specified reference in GitHub export INSTALL_AIRFLOW_REFERENCE=${INSTALL_AIRFLOW_REFERENCE:=""} @@ -482,7 +480,7 @@ function initialization::initialize_kubernetes_variables() { CURRENT_KIND_VERSIONS+=("v0.11.1") export CURRENT_KIND_VERSIONS # Currently supported versions of Helm - CURRENT_HELM_VERSIONS+=("v3.2.4") + CURRENT_HELM_VERSIONS+=("v3.6.3") export CURRENT_HELM_VERSIONS # Current executor in chart CURRENT_EXECUTOR+=("KubernetesExecutor") @@ -535,7 +533,6 @@ function initialization::initialize_git_variables() { function initialization::initialize_github_variables() { # Defaults for interacting with GitHub export GITHUB_REGISTRY="ghcr.io" - export GITHUB_REGISTRY_IMAGE_SUFFIX=${GITHUB_REGISTRY_IMAGE_SUFFIX:="-v2"} export GITHUB_REGISTRY_WAIT_FOR_IMAGE=${GITHUB_REGISTRY_WAIT_FOR_IMAGE:="false"} export GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG:="latest"} export GITHUB_REGISTRY_PUSH_IMAGE_TAG=${GITHUB_REGISTRY_PUSH_IMAGE_TAG:="latest"} @@ -634,7 +631,6 @@ Force variables: FORCE_BUILD_IMAGES: ${FORCE_BUILD_IMAGES} FORCE_ANSWER_TO_QUESTIONS: ${FORCE_ANSWER_TO_QUESTIONS} SKIP_CHECK_REMOTE_IMAGE: ${SKIP_CHECK_REMOTE_IMAGE} - FAIL_ON_GITHUB_DOCKER_PULL_ERROR: ${FAIL_ON_GITHUB_DOCKER_PULL_ERROR} Host variables: @@ -663,7 +659,6 @@ Common image build variables: INSTALL_FROM_PYPI: '${INSTALL_FROM_PYPI}' AIRFLOW_PRE_CACHED_PIP_PACKAGES: '${AIRFLOW_PRE_CACHED_PIP_PACKAGES}' UPGRADE_TO_NEWER_DEPENDENCIES: '${UPGRADE_TO_NEWER_DEPENDENCIES}' - CONTINUE_ON_PIP_CHECK_FAILURE: '${CONTINUE_ON_PIP_CHECK_FAILURE}' CHECK_IMAGE_FOR_REBUILD: '${CHECK_IMAGE_FOR_REBUILD}' AIRFLOW_CONSTRAINTS_LOCATION: '${AIRFLOW_CONSTRAINTS_LOCATION}' AIRFLOW_CONSTRAINTS_REFERENCE: '${AIRFLOW_CONSTRAINTS_REFERENCE}' @@ -830,8 +825,6 @@ function initialization::make_constants_read_only() { readonly ADDITIONAL_RUNTIME_APT_DEPS readonly ADDITIONAL_RUNTIME_APT_ENV - readonly DOCKER_CACHE - readonly GITHUB_REGISTRY readonly GITHUB_REGISTRY_WAIT_FOR_IMAGE readonly GITHUB_REGISTRY_PULL_IMAGE_TAG @@ -847,11 +840,8 @@ function initialization::make_constants_read_only() { readonly VERSION_SUFFIX_FOR_PYPI - readonly PYTHON_BASE_IMAGE_VERSION readonly PYTHON_BASE_IMAGE - readonly AIRFLOW_CI_BASE_TAG - readonly AIRFLOW_PROD_BASE_TAG - readonly AIRFLOW_PROD_IMAGE_KUBERNETES + readonly AIRFLOW_IMAGE_KUBERNETES readonly BUILT_CI_IMAGE_FLAG_FILE readonly INIT_SCRIPT_FILE diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh index d4910d92c8e07..1fb77ebe2e1e1 100644 --- a/scripts/ci/libraries/_kind.sh +++ b/scripts/ci/libraries/_kind.sh @@ -262,8 +262,8 @@ function kind::build_image_for_kubernetes_tests() { if [[ -n ${GITHUB_REGISTRY_PULL_IMAGE_TAG=} ]]; then image_tag="${GITHUB_REGISTRY_PULL_IMAGE_TAG}" fi - echo "Building ${AIRFLOW_PROD_IMAGE_KUBERNETES}:latest from ${AIRFLOW_PROD_IMAGE}:${image_tag}" - docker_v build --tag "${AIRFLOW_PROD_IMAGE_KUBERNETES}:latest" . -f - < image to pull +# $2 - fallback image function push_pull_remove_images::pull_image_if_not_present_or_forced() { local image_to_pull="${1}" local image_hash @@ -62,25 +63,6 @@ function push_pull_remove_images::pull_image_if_not_present_or_forced() { echo "Pulling the image ${image_to_pull}" echo docker_v pull "${image_to_pull}" - local exit_value="$?" - if [[ ${exit_value} != "0" && ${FAIL_ON_GITHUB_DOCKER_PULL_ERROR} == "true" ]]; then - echo - echo """ -${COLOR_RED}ERROR: Exiting on docker pull error - -If you have authorisation problems, you might want to run: - -docker login ${image_to_pull%%\/*} - -You need to use generate token as the password, not your personal password. -You can generate one at https://github.com/settings/tokens -Make sure to choose 'read:packages' scope. -${COLOR_RESET} -""" - exit ${exit_value} - fi - echo - return ${exit_value} fi } @@ -90,7 +72,7 @@ function push_pull_remove_images::check_and_rebuild_python_base_image_if_needed( local dockerhub_python_version dockerhub_python_version=$(docker run "${PYTHON_BASE_IMAGE}" python -c 'import sys; print(sys.version)') local local_python_version - local_python_version=$(docker run "${AIRFLOW_PYTHON_BASE_IMAGE}" python -c 'import sys; print(sys.version)') + local_python_version=$(docker run "${AIRFLOW_PYTHON_BASE_IMAGE}" python -c 'import sys; print(sys.version)' || true) if [[ ${local_python_version} != "${dockerhub_python_version}" ]]; then echo echo "There is a new Python Base image updated!" @@ -102,6 +84,10 @@ function push_pull_remove_images::check_and_rebuild_python_base_image_if_needed( docker_v build \ --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ -t "${AIRFLOW_PYTHON_BASE_IMAGE}" - + else + echo + echo "Not rebuilding the base python image - the image has the same python version ${dockerhub_python_version}" + echo fi } @@ -116,10 +102,10 @@ function push_pull_remove_images::check_and_rebuild_python_base_image_if_needed( # it will pull the right image using the specified suffix function push_pull_remove_images::pull_base_python_image() { echo - echo "Docker pulling base python image. Upgrade to newer deps: ${UPGRADE_TO_NEWER_DEPENDENCIES}" + echo "Docker pull base python image. Upgrade to newer deps: ${UPGRADE_TO_NEWER_DEPENDENCIES}" echo if [[ -n ${DETECTED_TERMINAL=} ]]; then - echo -n "Docker pulling base python image. Upgrade to newer deps: ${UPGRADE_TO_NEWER_DEPENDENCIES} + echo -n "Docker pull base python image. Upgrade to newer deps: ${UPGRADE_TO_NEWER_DEPENDENCIES} " > "${DETECTED_TERMINAL}" fi if [[ ${GITHUB_REGISTRY_PULL_IMAGE_TAG} != "latest" ]]; then @@ -132,8 +118,14 @@ function push_pull_remove_images::pull_base_python_image() { return 1 fi else + set +e push_pull_remove_images::pull_image_if_not_present_or_forced "${AIRFLOW_PYTHON_BASE_IMAGE}" - if [[ ${CHECK_IF_BASE_PYTHON_IMAGE_UPDATED} == "true" ]] ; then + local res="$?" + set -e + if [[ ${CHECK_IF_BASE_PYTHON_IMAGE_UPDATED} == "true" || ${res} != "0" ]] ; then + # Rebuild the base python image using DockerHub - either when we explicitly want it + # or when there is no image available yet in ghcr.io (usually when you build it for the + # first time in your repository push_pull_remove_images::check_and_rebuild_python_base_image_if_needed fi fi @@ -151,8 +143,26 @@ function push_pull_remove_images::pull_ci_images_if_needed() { fi fi if [[ "${DOCKER_CACHE}" == "pulled" ]]; then + set +e push_pull_remove_images::pull_image_if_not_present_or_forced \ "${AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + local res="$?" + set -e + if [[ ${res} != "0" ]]; then + if [[ ${GITHUB_REGISTRY_PULL_IMAGE_TAG} == "latest" ]] ; then + echo + echo "The CI image cache does not exist. This is likely the first time you build the image" + echo "Switching to 'local' cache for docker images" + echo + DOCKER_CACHE="local" + else + echo + echo "The CI image cache does not exist and we want to pull tag ${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + echo "Failing as we have to pull the tagged image in order to continue" + echo + return "${res}" + fi + fi fi } @@ -169,12 +179,33 @@ function push_pull_remove_images::pull_prod_images_if_needed() { fi fi if [[ "${DOCKER_CACHE}" == "pulled" ]]; then + set +e # "Build" segment of production image push_pull_remove_images::pull_image_if_not_present_or_forced \ "${AIRFLOW_PROD_BUILD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" - # "Main" segment of production image - push_pull_remove_images::pull_image_if_not_present_or_forced \ - "${AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + local res="$?" + if [[ ${res} == "0" ]]; then + # "Main" segment of production image + push_pull_remove_images::pull_image_if_not_present_or_forced \ + "${AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + res="$?" + fi + set -e + if [[ ${res} != "0" ]]; then + if [[ ${GITHUB_REGISTRY_PULL_IMAGE_TAG} == "latest" ]] ; then + echo + echo "The PROD image cache does not exist. This is likely the first time you build the image" + echo "Switching to 'local' cache for docker images" + echo + DOCKER_CACHE="local" + else + echo + echo "The PROD image cache does not exist and we want to pull tag ${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + echo "Failing as we have to pull the tagged image in order to continue" + echo + return "${res}" + fi + fi fi } @@ -203,17 +234,12 @@ function push_pull_remove_images::push_ci_images_to_github() { local airflow_ci_tagged_image="${AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}" docker_v tag "${AIRFLOW_CI_IMAGE}" "${airflow_ci_tagged_image}" push_pull_remove_images::push_image_with_retries "${airflow_ci_tagged_image}" + # Also push ci manifest iumage image if GITHUB_REGISTRY_PUSH_IMAGE_TAG is "latest" if [[ ${GITHUB_REGISTRY_PUSH_IMAGE_TAG} == "latest" ]]; then - local airflow_ci_manifest_tagged_image="${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}" + local airflow_ci_manifest_tagged_image="${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}:latest" docker_v tag "${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" "${airflow_ci_manifest_tagged_image}" push_pull_remove_images::push_image_with_retries "${airflow_ci_manifest_tagged_image}" fi - if [[ -n ${GITHUB_SHA=} ]]; then - # Also push image to GitHub registry with commit SHA - local airflow_ci_sha_image="${AIRFLOW_CI_IMAGE}:${COMMIT_SHA}" - docker_v tag "${AIRFLOW_CI_IMAGE}" "${airflow_ci_sha_image}" - push_pull_remove_images::push_image_with_retries "${airflow_ci_sha_image}" - fi } # Pushes PROD image to registry in GitHub @@ -222,19 +248,18 @@ function push_pull_remove_images::push_ci_images_to_github() { # "${COMMIT_SHA}" - in case of pull-request triggered 'workflow_run' builds # "latest" - in case of push builds function push_pull_remove_images::push_prod_images_to_github () { + if [[ "${PUSH_PYTHON_BASE_IMAGE=}" != "false" ]]; then + push_pull_remove_images::push_python_image_to_github + fi local airflow_prod_tagged_image="${AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}" docker_v tag "${AIRFLOW_PROD_IMAGE}" "${airflow_prod_tagged_image}" push_pull_remove_images::push_image_with_retries "${airflow_prod_tagged_image}" - if [[ -n ${COMMIT_SHA=} ]]; then - # Also push image to GitHub registry with commit SHA - local airflow_prod_sha_image="${AIRFLOW_PROD_IMAGE}:${COMMIT_SHA}" - docker_v tag "${AIRFLOW_PROD_IMAGE}" "${airflow_prod_sha_image}" - push_pull_remove_images::push_image_with_retries "${airflow_prod_sha_image}" + # Also push prod build image if GITHUB_REGISTRY_PUSH_IMAGE_TAG is "latest" + if [[ ${GITHUB_REGISTRY_PUSH_IMAGE_TAG} == "latest" ]]; then + local airflow_prod_build_tagged_image="${AIRFLOW_PROD_BUILD_IMAGE}:latest" + docker_v tag "${AIRFLOW_PROD_BUILD_IMAGE}" "${airflow_prod_build_tagged_image}" + push_pull_remove_images::push_image_with_retries "${airflow_prod_build_tagged_image}" fi - # Also push prod build image - local airflow_prod_build_tagged_image="${AIRFLOW_PROD_BUILD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}" - docker_v tag "${AIRFLOW_PROD_BUILD_IMAGE}" "${airflow_prod_build_tagged_image}" - push_pull_remove_images::push_image_with_retries "${airflow_prod_build_tagged_image}" } # waits for an image to be available in GitHub Container Registry. Should be run with `set +e` @@ -253,12 +278,18 @@ function push_pull_remove_images::check_image_manifest() { } # waits for an image to be available in the GitHub registry +# Remove the fallback on 7th of August 2021 function push_pull_remove_images::wait_for_image() { set +e - echo " Waiting for github registry image: " "$1" + echo " Waiting for github registry image: $1 with $2 fallback" while true do if push_pull_remove_images::check_image_manifest "$1"; then + export IMAGE_AVAILABLE="$1" + break + fi + if push_pull_remove_images::check_image_manifest "$2"; then + export IMAGE_AVAILABLE="$2" break fi sleep 30 diff --git a/scripts/ci/libraries/_script_init.sh b/scripts/ci/libraries/_script_init.sh index 0f3c862a44154..dc79fd506002c 100755 --- a/scripts/ci/libraries/_script_init.sh +++ b/scripts/ci/libraries/_script_init.sh @@ -41,7 +41,7 @@ build_images::determine_docker_cache_strategy initialization::get_environment_for_builds_on_ci -build_images::get_docker_image_names +build_images::get_docker_cache_image_names initialization::make_constants_read_only diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh index e7a9144941633..ee3b5069366d0 100755 --- a/scripts/ci/selective_ci_checks.sh +++ b/scripts/ci/selective_ci_checks.sh @@ -46,10 +46,12 @@ function check_upgrade_to_newer_dependencies_needed() { # shellcheck disable=SC2153 if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" || ${GITHUB_EVENT_NAME=} == 'push' || ${GITHUB_EVENT_NAME=} == "scheduled" ]]; then - # Trigger upgrading to latest constraints where label is set or when - # SHA of the merge commit triggers rebuilding layer in the docker image + # Trigger upgrading to latest constraints when we are in push or schedule event or when it is forced + # By UPGRADE_TO_NEWER_DEPENDENCIES set to non-false. The variable is set to + # SHA of the merge commit - so that it always triggers rebuilding layer in the docker image # Each build that upgrades to latest constraints will get truly latest constraints, not those - # Cached in the image this way + # Cached in the image if we set it to "true". This upgrade_to_newer_dependencies variable + # can later be overridden in case we find that setup.* files changed (see below) upgrade_to_newer_dependencies="${INCOMING_COMMIT_SHA}" fi } @@ -338,6 +340,8 @@ function check_if_setup_files_changed() { show_changed_files if [[ $(count_changed_files) != "0" ]]; then + # In case the setup files changed, we automatically force upgrading to newer dependencies + # no matter what was set before upgrade_to_newer_dependencies="${INCOMING_COMMIT_SHA}" fi start_end::group_end diff --git a/scripts/ci/tools/fix_ownership.sh b/scripts/ci/tools/fix_ownership.sh index 6ed1161be747f..de1562122a779 100755 --- a/scripts/ci/tools/fix_ownership.sh +++ b/scripts/ci/tools/fix_ownership.sh @@ -33,8 +33,12 @@ sanity_checks::sanitize_mounted_files read -r -a EXTRA_DOCKER_FLAGS <<<"$(local_mounts::convert_local_mounts_to_docker_params)" -docker_v run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \ - --rm \ - --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \ - "${AIRFLOW_CI_IMAGE}" \ - -c /opt/airflow/scripts/in_container/run_fix_ownership.sh || true +if docker image inspect "${AIRFLOW_CI_IMAGE}" >/dev/null 2>&1; then + docker_v run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \ + --rm \ + --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \ + "${AIRFLOW_CI_IMAGE}" \ + -c /opt/airflow/scripts/in_container/run_fix_ownership.sh || true +else + echo "Skip fixing ownership as seems that you do not have the ${AIRFLOW_CI_IMAGE} image yet" +fi diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh index 6c035ae5def09..4f9c05f6b7680 100755 --- a/scripts/docker/install_additional_dependencies.sh +++ b/scripts/docker/install_additional_dependencies.sh @@ -23,7 +23,6 @@ test -v ADDITIONAL_PYTHON_DEPS test -v EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS test -v AIRFLOW_INSTALL_USER_FLAG test -v AIRFLOW_PIP_VERSION -test -v CONTINUE_ON_PIP_CHECK_FAILURE # shellcheck source=scripts/docker/common.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" @@ -41,7 +40,7 @@ function install_additional_dependencies() { ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check else echo echo Installing additional dependencies upgrading only if needed @@ -51,7 +50,7 @@ function install_additional_dependencies() { ${ADDITIONAL_PYTHON_DEPS} # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check fi } diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh index 49040275a04ac..e2bca4fc839a0 100755 --- a/scripts/docker/install_airflow.sh +++ b/scripts/docker/install_airflow.sh @@ -60,7 +60,7 @@ function install_airflow() { # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check else \ echo echo Installing all packages with constraints and upgrade if needed @@ -76,7 +76,7 @@ function install_airflow() { "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check fi } diff --git a/scripts/docker/install_airflow_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh similarity index 85% rename from scripts/docker/install_airflow_from_branch_tip.sh rename to scripts/docker/install_airflow_dependencies_from_branch_tip.sh index 925a872fa50ea..61aaa13ef467f 100755 --- a/scripts/docker/install_airflow_from_branch_tip.sh +++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -30,28 +30,29 @@ . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -function install_airflow_from_branch_tip() { +function install_airflow_dependencies_from_branch_tip() { echo echo "Installing airflow from ${AIRFLOW_BRANCH}. It is used to cache dependencies" echo if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,} fi - # Install latest set of dependencies using constraints + # Install latest set of dependencies using constraints. In case constraints were upgraded and there + # are conflicts, this might fail, but it should be fixed in the following installation steps pip install ${AIRFLOW_INSTALL_USER_FLAG} \ "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" + --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes || true echo echo Uninstalling just airflow. Dependencies remain. echo - pip uninstall --yes apache-airflow + pip uninstall --yes apache-airflow || true } common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -install_airflow_from_branch_tip +install_airflow_dependencies_from_branch_tip diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh index 813d1b02df7e0..d8ed6bc72bd9a 100755 --- a/scripts/docker/install_from_docker_context_files.sh +++ b/scripts/docker/install_from_docker_context_files.sh @@ -96,7 +96,7 @@ function install_airflow_and_providers_from_docker_context_files(){ # make sure correct PIP version is left installed pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check }