diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9586e38337b38..11816e61c40f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,9 +21,9 @@ on: # yamllint disable-line rule:truthy schedule: - cron: '28 0 * * *' push: - branches: ['master', 'v1-10-test', 'v1-10-stable', 'v2-0-test', 'v2-1-test'] + branches: ['master', 'v[0-9]+-[0-9]+-test'] pull_request: - branches: ['master', 'v1-10-test', 'v1-10-stable', 'v2-0-test', 'v2-1-test'] + branches: ['master', 'v[0-9]+-[0-9]+-test', 'v[0-9]+-[0-9]+-stable'] env: MOUNT_SELECTED_LOCAL_SOURCES: "false" @@ -608,7 +608,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} MOUNT_SELECTED_LOCAL_SOURCES: "true" TEST_TYPES: "Helm" - BACKEND: "sqlite" + BACKEND: "" + DB_RESET: "false" PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} if: > @@ -627,7 +628,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Tests: ${{needs.build-info.outputs.testTypes}}" + - name: "Tests: Helm" run: ./scripts/ci/testing/ci_run_airflow_testing.sh - name: "Upload airflow logs" uses: actions/upload-artifact@v2 @@ -967,7 +968,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" tests-kubernetes: timeout-minutes: 50 - name: K8s tests + name: Helm Chart runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} needs: [build-info, prod-images] env: @@ -1048,9 +1049,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - tests-kubernetes - prod-images - docs + # TODO: Generalize me (find a better way to select matching branches) if: > (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' || - github.ref == 'refs/heads/v2-0-test') && + github.ref == 'refs/heads/v2-0-test' || github.ref == 'refs/heads/v2-1-test') && github.event_name != 'schedule' strategy: matrix: @@ -1108,9 +1110,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - tests-kubernetes - ci-images - docs + # TODO: Generalize me (find a better way to select matching branches) if: > (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' || - github.ref == 'refs/heads/v2-0-test') && + github.ref == 'refs/heads/v2-0-test' || github.ref == 'refs/heads/v2-1-test') && github.event_name != 'schedule' strategy: matrix: @@ -1156,9 +1159,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: ${{needs.build-info.outputs.pythonVersionsListAsString}} # Only run it for direct pushes + # TODO: Generalize me (find a better way to select matching branches) if: > github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' || - github.ref == 'refs/heads/v2-0-test' + github.ref == 'refs/heads/v2-0-test' || github.ref == 'refs/heads/v2-1-test' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v2 diff --git a/BREEZE.rst b/BREEZE.rst index c7b1ac4b502ac..95db27cf53a1b 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -541,7 +541,7 @@ For all development tasks, unit tests, integration tests, and static code checks **CI image** maintained on the DockerHub in the ``apache/airflow`` repository. This Docker image contains a lot of test-related packages (size of ~1GB). Its tag follows the pattern of ``-python-ci`` -(for example, ``apache/airflow:master-python3.6-ci`` or ``apache/airflow:v2-0-test-python3.6-ci``). +(for example, ``apache/airflow:master-python3.6-ci`` or ``apache/airflow:v2-1-test-python3.6-ci``). The image is built using the ``_ Dockerfile. The CI image is built automatically as needed, however it can be rebuilt manually with @@ -638,7 +638,7 @@ The **Production image** is also maintained on the DockerHub in the ``apache/airflow`` repository. This Docker image (and Dockerfile) contains size-optimised Airflow installation with selected extras and dependencies. Its tag follows the pattern of ``-python`` (for example, ``apache/airflow:master-python3.6`` -or ``apache/airflow:v2-0-test-python3.6``). +or ``apache/airflow:v2-1-test-python3.6``). However in many cases you want to add your own custom version of the image - with added apt dependencies, python dependencies, additional Airflow extras. Breeze's ``build-image`` command helps to build your own, @@ -1279,7 +1279,7 @@ This is the current syntax for `./breeze <./breeze>`_: -t, --install-airflow-reference INSTALL_AIRFLOW_REFERENCE Installs Airflow directly from reference in GitHub when building PROD image. - This can be a GitHub branch like master or v2-0-test, or a tag like 2.0.0a1. + This can be a GitHub branch like master or v2-1-test, or a tag like 2.1.0a1. --installation-method INSTALLATION_METHOD Method of installing Airflow in PROD image - either from the sources ('.') @@ -2504,7 +2504,7 @@ This is the current syntax for `./breeze <./breeze>`_: -t, --install-airflow-reference INSTALL_AIRFLOW_REFERENCE Installs Airflow directly from reference in GitHub when building PROD image. - This can be a GitHub branch like master or v2-0-test, or a tag like 2.0.0a1. + This can be a GitHub branch like master or v2-1-test, or a tag like 2.1.0a1. --installation-method INSTALLATION_METHOD Method of installing Airflow in PROD image - either from the sources ('.') diff --git a/CI.rst b/CI.rst index 7bbbaff96b417..e7ed6af6e7538 100644 --- a/CI.rst +++ b/CI.rst @@ -21,7 +21,7 @@ CI Environment ============== Continuous Integration is important component of making Apache Airflow robust and stable. We are running -a lot of tests for every pull request, for master and v2-0-test branches and regularly as CRON jobs. +a lot of tests for every pull request, for master and v2-*-test branches and regularly as CRON jobs. Our execution environment for CI is `GitHub Actions `_. GitHub Actions (GA) are very well integrated with GitHub code and Workflow and it has evolved fast in 2019/202 to become @@ -812,15 +812,15 @@ The image names follow the patterns: | | | | It contains only compiled libraries and minimal set of dependencies to run Airflow. | +--------------+----------------------------+--------------------------------+--------------------------------------------------------------------------------------------+ -* might be either "master" or "v1-10-test" or "v2-0-test" -* - Python version (Major + Minor). For "master" and "v2-0-test" should be in ["3.6", "3.7", "3.8"]. For +* might be either "master" or "v1-10-test" or "v2-*-test" +* - Python version (Major + Minor). For "master" and "v2-*-test" should be in ["3.6", "3.7", "3.8"]. For v1-10-test it should be in ["2.7", "3.5", "3.6". "3.7", "3.8"]. * - GitHub Actions RUN_ID. You can get it from CI action job outputs (run id is printed in logs and displayed as part of the step name. All PRs belong to some RUN_ID and this way you can pull the very exact version of image used in that RUN_ID -* - for images that get merged to "master", "v2-0-test" of "v1-10-test" the images are also tagged +* - for images that get merged to "master", "v2-*-test" of "v1-10-test" the images are also tagged with the commit SHA of that particular commit. This way you can easily find the image that was used - for testing for that "master", "v2-0-test" or "v1-10-test" test run. + for testing for that "master", "v2-*-test" or "v1-10-test" test run. Reproducing CI Runs locally =========================== @@ -921,7 +921,7 @@ In order to add a new version the following operations should be done (example u +=============+================+=======================+=====================+===============+===========+===============+========================================================================+ | Tag | nightly-master | master-python3.9 | Dockerfile | / | x | - | Nightly CI/PROD images from successful scheduled master nightly builds | +-------------+----------------+-----------------------+---------------------+---------------+-----------+---------------+------------------------------------------------------------------------+ -| Branch | v2-0-stable | v2-0-stable-python3.9 | Dockerfile | / | x | | CI/PROD images automatically built pushed stable branch | +| Branch | v2-*-stable | v2-*-stable-python3.9 | Dockerfile | / | x | | CI/PROD images automatically built pushed stable branch | +-------------+----------------+-----------------------+---------------------+---------------+-----------+---------------+------------------------------------------------------------------------+ | Tag | /^([1-2].*)$/ | {\1}-python3.9 | Dockerfile | / | x | | CI/PROD images automatically built from pushed release tags | +-------------+----------------+-----------------------+---------------------+---------------+-----------+---------------+------------------------------------------------------------------------+ diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 94f325f96ad3d..1a480e2c0ade2 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -398,34 +398,34 @@ Airflow Git Branches All new development in Airflow happens in the ``master`` branch. All PRs should target that branch. -We also have a ``v2-0-test`` branch that is used to test ``2.0.x`` series of Airflow and where committers +We also have a ``v2-*-test`` branches that are used to test ``2.*.x`` series of Airflow and where committers cherry-pick selected commits from the master branch. Cherry-picking is done with the ``-x`` flag. -The ``v2-0-test`` branch might be broken at times during testing. Expect force-pushes there so -committers should coordinate between themselves on who is working on the ``v2-0-test`` branch - +The ``v2-*-test`` branch might be broken at times during testing. Expect force-pushes there so +committers should coordinate between themselves on who is working on the ``v2-*-test`` branch - usually these are developers with the release manager permissions. -The ``v2-0-stable`` branch is rather stable - there are minimum changes coming from approved PRs that +The ``v2-*-stable`` branch is rather stable - there are minimum changes coming from approved PRs that passed the tests. This means that the branch is rather, well, "stable". -Once the ``v2-0-test`` branch stabilises, the ``v2-0-stable`` branch is synchronized with ``v2-0-test``. -The ``v2-0-stable`` branch is used to release ``2.0.x`` releases. +Once the ``v2-*-test`` branch stabilises, the ``v2-*-stable`` branch is synchronized with ``v2-*-test``. +The ``v2-*-stable`` branches are used to release ``2.*.x`` releases. The general approach is that cherry-picking a commit that has already had a PR and unit tests run -against main is done to ``v2-0-test`` branch, but PRs from contributors towards 2.0 should target -``v2-0-stable`` branch. +against main is done to ``v2-*-test`` branches, but PRs from contributors towards 2.0 should target +``v2-*-stable`` branches. -The ``v2-0-test`` branch and ``v2-0-stable`` ones are merged just before the release and that's the +The ``v2-*-test`` branches and ``v2-*-stable`` ones are merged just before the release and that's the time when they converge. The production images are build in DockerHub from: * master branch for development -* v2-0-test branch for testing 2.0.x release -* ``2.0.*``, ``2.0.*rc*`` releases from the ``v2-0-stable`` branch when we prepare release candidates and - final releases. There are no production images prepared from v2-0-stable branch. +* v2-*-test branches for testing 2.*.x release +* ``2.*.*``, ``2.*.*rc*`` releases from the ``v2-*-stable`` branch when we prepare release candidates and + final releases. There are no production images prepared from v2-*-stable branch. Similar rules apply to ``1.10.x`` releases until June 2021. We have ``v1-10-test`` and ``v1-10-stable`` branches there. diff --git a/IMAGES.rst b/IMAGES.rst index 7215871b25cd7..51b29342a7cfe 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -69,8 +69,8 @@ The images are named as follows: where: * ``BRANCH_OR_TAG`` - branch or tag used when creating the image. Examples: ``master``, - ``v2-0-test``, ``2.0.0``. The ``master``, ``v2-0-test`` labels are - built from branches so they change over time. The ````2.*`` labels are built from git tags + ``v2-1-test``, ``2.1.0``. The ``master``, ``v2-*-test`` labels are + built from branches so they change over time. The ``2.*.*`` labels are built from git tags and they are "fixed" once built. * ``PYTHON_MAJOR_MINOR_VERSION`` - version of Python used to build the image. Examples: ``3.6``, ``3.7``, ``3.8`` @@ -284,8 +284,8 @@ For example: apache/airflow:master-python3.6 - production "latest" image from current master apache/airflow:master-python3.6-ci - CI "latest" image from current master - apache/airflow:v2-0-test-python3.6-ci - CI "latest" image from current v2-0-test branch - apache/airflow:2.0.0-python3.6 - production image for 2.0.0 release + apache/airflow:v2-1-test-python3.6-ci - CI "latest" image from current v2-1-test branch + apache/airflow:2.1.0-python3.6 - production image for 2.1.0 release apache/airflow:python3.6-master - base Python image for the master branch You can see DockerHub images at ``_ @@ -320,10 +320,10 @@ Images built as "Run ID snapshot": .. code-block:: bash - docker.pkg.github.com.io/apache-airflow/-pythonX.Y-ci-v2: - for CI images - docker.pkg.github.com/apache-airflow/-pythonX.Y-v2: - for production images - docker.pkg.github.com/apache-airflow/-pythonX.Y-build-v2: - for production build stage - docker.pkg.github.com/apache-airflow/pythonX.Y--v2:X.Y-slim-buster- - for base Python images + docker.pkg.github.com.io/apache-airflow/-pythonX.Y-ci-v2: - for CI images + docker.pkg.github.com/apache-airflow/-pythonX.Y-v2: - for production images + docker.pkg.github.com/apache-airflow/-pythonX.Y-build-v2: - for production build stage + docker.pkg.github.com/apache-airflow/python-v2:X.Y-slim-buster- - for base Python images Latest images (pushed when master merge succeeds): @@ -332,7 +332,7 @@ Latest images (pushed when master merge succeeds): docker.pkg.github.com/apache/airflow/-pythonX.Y-ci-v2:latest - for CI images docker.pkg.github.com/apache/airflow/-pythonX.Y-v2:latest - for production images docker.pkg.github.com/apache/airflow/-pythonX.Y-build-v2:latest - for production build stage - docker.pkg.github.com/apache/airflow/python--v1:X.Y-slim-buster - for base Python images + docker.pkg.github.com/apache/airflow/python-v2:X.Y-slim-buster - for base Python images Naming convention for GitHub Container Registry @@ -342,10 +342,10 @@ Images built as "Run ID snapshot": .. code-block:: bash - ghcr.io/apache/airflow--pythonX.Y-ci-v2: - for CI images - ghcr.io/apache/airflow--pythonX.Y-v2: - for production images - ghcr.io/apache/airflow--pythonX.Y-build-v2: - for production build stage - ghcr.io/apache/airflow-pythonX.Y--v2:X.Y-slim-buster- - for base Python images + ghcr.io/apache/airflow--pythonX.Y-ci-v2: - for CI images + ghcr.io/apache/airflow--pythonX.Y-v2: - for production images + ghcr.io/apache/airflow--pythonX.Y-build-v2: - for production build stage + ghcr.io/apache/airflow-python-v2:X.Y-slim-buster- - for base Python images Latest images (pushed when master merge succeeds): @@ -354,7 +354,7 @@ Latest images (pushed when master merge succeeds): ghcr.io/apache/airflow--pythonX.Y-ci-v2:latest - for CI images ghcr.io/apache/airflow--pythonX.Y-v2:latest - for production images ghcr.io/apache/airflow--pythonX.Y-build-v2:latest - for production build stage - ghcr.io/apache/airflow-python--v2:X.Y-slim-buster - for base Python images + ghcr.io/apache/airflow-python-v2:X.Y-slim-buster - for base Python images Note that we never push or pull "release" images to GitHub registry. It is only used for CI builds diff --git a/airflow/kubernetes/pod_template_file_examples/git_sync_template.yaml b/airflow/kubernetes/pod_template_file_examples/git_sync_template.yaml index 445f509139fd2..cf40a6f614611 100644 --- a/airflow/kubernetes/pod_template_file_examples/git_sync_template.yaml +++ b/airflow/kubernetes/pod_template_file_examples/git_sync_template.yaml @@ -31,7 +31,7 @@ spec: - name: GIT_SYNC_REV value: "HEAD" - name: GIT_SYNC_BRANCH - value: "v2-0-stable" + value: "v2-1-stable" - name: GIT_SYNC_REPO value: "https://github.com/apache/airflow.git" - name: GIT_SYNC_DEPTH diff --git a/breeze b/breeze index 115c77c4daa07..69dbf29d91782 100755 --- a/breeze +++ b/breeze @@ -2426,7 +2426,7 @@ ${FORMATTED_INSTALL_AIRFLOW_VERSIONS} -t, --install-airflow-reference INSTALL_AIRFLOW_REFERENCE Installs Airflow directly from reference in GitHub when building PROD image. - This can be a GitHub branch like master or v2-0-test, or a tag like 2.0.0a1. + This can be a GitHub branch like master or v2-1-test, or a tag like 2.1.0a1. --installation-method INSTALLATION_METHOD Method of installing Airflow in PROD image - either from the sources ('.') @@ -3099,14 +3099,18 @@ function breeze::read_saved_environment_variables() { ####################################################################################################### function breeze::check_and_save_all_params() { parameters::check_and_save_allowed_param "PYTHON_MAJOR_MINOR_VERSION" "Python version" "--python" - + # TODO: Generalize me (make it automatically use matching versions) if [[ -n "${INSTALL_AIRFLOW_REFERENCE=}" ]]; then if [[ ${INSTALL_AIRFLOW_REFERENCE=} == *2_0* ]]; then export BRANCH_NAME="v2-0-test" + elif [[ ${INSTALL_AIRFLOW_REFERENCE=} == *2_1* ]]; then + export BRANCH_NAME="v2-1-test" fi elif [[ -n "${INSTALL_AIRFLOW_VERSION=}" ]]; then if [[ ${INSTALL_AIRFLOW_VERSION=} == *2.0* ]]; then export BRANCH_NAME="v2-0-test" + elif [[ ${INSTALL_AIRFLOW_VERSION=} == *2.1* ]]; then + export BRANCH_NAME="v2-1-test" fi fi diff --git a/chart/values.schema.json b/chart/values.schema.json index e5b4a114cb59f..488a59ef19912 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -2337,7 +2337,7 @@ "branch": { "description": "Git branch", "type": "string", - "default": "v2-0-stable" + "default": "v2-1-stable" }, "rev": { "description": "Git revision.", diff --git a/chart/values.yaml b/chart/values.yaml index e7e55331ae978..de661924acfc8 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -963,7 +963,7 @@ dags: # git@github.com:apache/airflow.git # https example: https://github.com/apache/airflow.git repo: https://github.com/apache/airflow.git - branch: v2-0-stable + branch: v2-1-stable rev: HEAD root: "/git" dest: "repo" diff --git a/codecov.yml b/codecov.yml index d68d7a8f644c2..16ef1f68a90b7 100644 --- a/codecov.yml +++ b/codecov.yml @@ -41,6 +41,7 @@ coverage: - v1-10-stable - v1-10-test - v2-0-test + - v2-1-test if_not_found: success if_ci_failed: error informational: true @@ -58,6 +59,7 @@ coverage: - v1-10-stable - v1-10-test - v2-0-test + - v2-1-test if_no_uploads: error if_not_found: success if_ci_failed: error diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 8ff7603f34298..2206663668452 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -22,6 +22,7 @@ - [Prepare the Apache Airflow Package RC](#prepare-the-apache-airflow-package-rc) - [Build RC artifacts](#build-rc-artifacts) + - [[\Optional\] Create new release branch](#%5Coptional%5C-create-new-release-branch) - [Prepare PyPI convenience "snapshot" packages](#prepare-pypi-convenience-snapshot-packages) - [\[Optional\] - Manually prepare production Docker Image](#%5Coptional%5C---manually-prepare-production-docker-image) - [Prepare Vote email on the Apache Airflow release candidate](#prepare-vote-email-on-the-apache-airflow-release-candidate) @@ -126,6 +127,30 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag svn commit -m "Add artifacts for Airflow ${VERSION}" ``` +## [\Optional\] Create new release branch + +When you just released the `X.Y.0` version (first release of new minor version) you need to create release +branches: `vX-Y-test` and `vX-Y-stable` (for example with `2.1.0rc1` release you need to create v2-1-test and +`v2-1-stable` branches): + + + ```shell script + # First clone the repo + BRANCH_PREFIX=v2-1 + git branch ${BRANCH_PREFIX}-test + git branch ${BRANCH_PREFIX}-stable + git push origin ${BRANCH_PREFIX}-test ${BRANCH_PREFIX}-stable + ``` + +Search and replace all the vX-Y for previous branches (TODO: we should likely automate this a bit more) + +Run script to re-tag images from the ``master`` branch to the ``vX-Y-test`` branch: + + ```shell script + ./dev/retag_docker_images.py --source-branch master --target-branch ${BRANCH_PREFIX}-test + ``` + + ## Prepare PyPI convenience "snapshot" packages At this point we have the artefact that we vote on, but as a convenience to developers we also want to diff --git a/dev/retag_docker_images.py b/dev/retag_docker_images.py new file mode 100755 index 0000000000000..061d610110696 --- /dev/null +++ b/dev/retag_docker_images.py @@ -0,0 +1,105 @@ +#!/usr/bin/python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# This scripts re-tags images from one branch to another. Since we keep +# images "per-branch" we sometimes need to "clone" the current +# images to provide a starting cache image to build images in the +# new branch. This can be usful in a few situations: +# +# * when starting new release branch (for example `v2-1-test`) +# * when renaming a branch (for example `master->main`) +# +# Docker registries we are using: +# +# * DockerHub - we keep `apache/airflow` image with distinct tags +# that determine type of the image, because in DockerHub we only +# have access to `apache/airflow` image +# +# * GitHub Docker Registries: (depends on the type of registry) we have +# more flexibility: +# * In the old GitHub docker registry - docker.pkg.github.com - +# (current but already deprecated) we can use +# "apache/airflow/IMAGE:tag" i +# * in the new package registry (ghcr.io) - we can submitg anything +# under apache/airflow-* but then we link it to the +# project via docker image label. +# +# The script helps to keep all the registries in-sync - copies +# `master` to `main` so that we can run it to test the rename and +# re-run it just before we switch the branches. + +import subprocess +from typing import List + +import click + +PYTHON_VERSIONS = ["3.6", "3.7", "3.8"] + +DOCKERHUB_PREFIX = "apache/airflow" + +DOCKERHUB_IMAGES = [ + "{prefix}:python{python_version}-{branch}", + "{prefix}:{branch}-python{python_version}-ci", + "{prefix}:{branch}-python{python_version}-ci-manifest", + "{prefix}:{branch}-python{python_version}", + "{prefix}:{branch}-python{python_version}-build", +] + +GITHUB_DOCKER_REGISTRY_PREFIX = "docker.pkg.github.com/apache/airflow" + +GITHUB_REGISTRY_IMAGES = [ + "{prefix}/{branch}-python{python_version}-ci-v2:latest", + "{prefix}/{branch}-python{python_version}-v2:latest", + "{prefix}/{branch}-python{python_version}-build-v2:latest", +] + + +GHCR_IO_PREFIX = "ghcr.io/apache/airflow" + +GHCR_IO_IMAGES = [ + "{prefix}-{branch}-python{python_version}-ci-v2:latest", + "{prefix}-{branch}-python{python_version}-v2:latest", + "{prefix}-{branch}-python{python_version}-build-v2:latest", +] + + +# noinspection StrFormat +def pull_push_all_images(prefix: str, images: List[str], source_branch: str, target_branch: str): + for python_version in PYTHON_VERSIONS: + for image in images: + source_image = image.format(prefix=prefix, branch=source_branch, python_version=python_version) + target_image = image.format(prefix=prefix, branch=target_branch, python_version=python_version) + print(f"Copying image: {source_image} -> {target_image}") + subprocess.run(["docker", "pull", source_image], check=True) + subprocess.run(["docker", "tag", source_image, target_image], check=True) + subprocess.run(["docker", "push", target_image], check=True) + + +@click.group(invoke_without_command=True) +@click.option("--source-branch", type=str, default="master", help="Source branch name [master]") +@click.option("--target-branch", type=str, default="main", help="Target branch name [main]") +def main(source_branch: str, target_branch: str): + pull_push_all_images(DOCKERHUB_PREFIX, DOCKERHUB_IMAGES, source_branch, target_branch) + pull_push_all_images(GITHUB_DOCKER_REGISTRY_PREFIX, GITHUB_REGISTRY_IMAGES, source_branch, target_branch) + pull_push_all_images(GHCR_IO_PREFIX, GHCR_IO_IMAGES, source_branch, target_branch) + + +if __name__ == "__main__": + main() # noqa diff --git a/docs/docker-stack/build.rst b/docs/docker-stack/build.rst index b461737e546c6..354c88da79bb4 100644 --- a/docs/docker-stack/build.rst +++ b/docs/docker-stack/build.rst @@ -353,12 +353,12 @@ constraints are taken from latest version of the constraints-master branch in Gi :end-before: [END build] The following example builds the production image with default extras from the -latest ``v2-0-test`` version and constraints are taken from the latest version of -the ``constraints-2-0`` branch in GitHub. Note that this command might fail occasionally as only -the "released version" constraints when building a version and "master" constraints when building -master are guaranteed to work. +latest ``v2-*-test`` version and constraints are taken from the latest version of +the ``constraints-2-*`` branch in GitHub (for example ``v2-1-test`` branch matches ``constraints-2-1``). +Note that this command might fail occasionally as only the "released version" constraints when building a +version and "master" constraints when building master are guaranteed to work. -.. exampleinclude:: docker-examples/customizing/github-v2-0-test.sh +.. exampleinclude:: docker-examples/customizing/github-v2-1-test.sh :language: bash :start-after: [START build] :end-before: [END build] diff --git a/docs/docker-stack/docker-examples/customizing/github-v2-0-test.sh b/docs/docker-stack/docker-examples/customizing/github-v2-1-test.sh similarity index 95% rename from docs/docker-stack/docker-examples/customizing/github-v2-0-test.sh rename to docs/docker-stack/docker-examples/customizing/github-v2-1-test.sh index b893618a66834..6ec055843907e 100755 --- a/docs/docker-stack/docker-examples/customizing/github-v2-0-test.sh +++ b/docs/docker-stack/docker-examples/customizing/github-v2-1-test.sh @@ -24,7 +24,7 @@ cd "${AIRFLOW_SOURCES}" # [START build] docker build . \ --build-arg PYTHON_BASE_IMAGE="python:3.8-slim-buster" \ - --build-arg AIRFLOW_INSTALLATION_METHOD="https://github.com/apache/airflow/archive/v2-0-test.tar.gz#egg=apache-airflow" \ + --build-arg AIRFLOW_INSTALLATION_METHOD="https://github.com/apache/airflow/archive/v2-1-test.tar.gz#egg=apache-airflow" \ --build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-2-0" \ --tag "$(basename "$0")" # [END build] diff --git a/scripts/ci/constraints/ci_branch_constraints.sh b/scripts/ci/constraints/ci_branch_constraints.sh index b8041b169b07b..35dfa07b92890 100755 --- a/scripts/ci/constraints/ci_branch_constraints.sh +++ b/scripts/ci/constraints/ci_branch_constraints.sh @@ -24,6 +24,8 @@ elif [[ ${GITHUB_REF} == 'refs/heads/master' ]]; then echo "::set-output name=branch::constraints-master" elif [[ ${GITHUB_REF} == 'refs/heads/v2-0-test' ]]; then echo "::set-output name=branch::constraints-2-0" +elif [[ ${GITHUB_REF} == 'refs/heads/v2-1-test' ]]; then + echo "::set-output name=branch::constraints-2-1" else echo echo "Unexpected ref ${GITHUB_REF}. Exiting!" diff --git a/scripts/ci/images/ci_build_dockerhub.sh b/scripts/ci/images/ci_build_dockerhub.sh index fe4cc27a609f7..e1279bce4e163 100755 --- a/scripts/ci/images/ci_build_dockerhub.sh +++ b/scripts/ci/images/ci_build_dockerhub.sh @@ -56,7 +56,7 @@ if [[ ! "${DOCKER_TAG}" =~ ^[0-9].* ]]; then # is built from non-release tag. If this is not set, then building images from locally build # packages fails, because the packages with non-dev version are skipped (as they are already released) export VERSION_SUFFIX_FOR_PYPI=".dev0" - # Only build and push CI image for the nightly-master, v2-0-test branches + # Only build and push CI image for the nightly-master, v2-*-test branches # for tagged releases we build everything from PyPI, so we do not need CI images # For development images, we have to build all packages from current sources because we want to produce # `Latest and greatest` image from those branches. We need to build and push CI image as well as PROD diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 51763df6dbcdf..6f7f8a10cb9ba 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -54,13 +54,18 @@ function build_images::add_build_args_for_remote_install() { ) fi # Depending on the version built, we choose the right branch for preloading the packages from - # For v2-0-test we choose v2-0-test - # all other builds when you choose a specific version (1.0 or 2.0 series) should choose stable branch + # For v2-*-test we choose v2-*-test + # all other builds when you choose a specific version (1.0, 2.0, 2.1. series) should choose stable branch # to preload. For all other builds we use the default branch defined in _initialization.sh + # TODO: Generalize me if [[ ${AIRFLOW_VERSION} == 'v2-0-test' ]]; then AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="v2-0-test" - elif [[ ${AIRFLOW_VERSION} =~ v?2.* ]]; then + elif [[ ${AIRFLOW_VERSION} == 'v2-1-test' ]]; then + AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="v2-1-test" + elif [[ ${AIRFLOW_VERSION} =~ v?2\.0* ]]; then AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="v2-0-stable" + elif [[ ${AIRFLOW_VERSION} =~ v?2\.1* ]]; then + AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="v2-1-stable" else AIRFLOW_BRANCH_FOR_PYPI_PRELOADING=${DEFAULT_BRANCH} fi