Revert "Cleaner output of docker image building scripts (#20679)" (#2…

…0721) This reverts commit fb87800.
apache · Jan 6, 2022 · 64cba10 · 64cba10
1 parent fa802ed
commit 64cba10
Show file tree

Hide file tree

Showing 19 changed files with 65 additions and 84 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -43,7 +43,7 @@ ARG AIRFLOW_UID="50000"
 
 ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
 
-ARG AIRFLOW_PIP_VERSION=21.3.1
+ARG AIRFLOW_PIP_VERSION=21.2.4
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
 
 # By default PIP has progress bar but you can disable it.
@@ -186,6 +186,9 @@ ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \
     PATH=${PATH}:/root/.local/bin \
     AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
     PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \
+    # Install Airflow with "--user" flag, so that we can copy the whole .local folder to the final image
+    # from the build image and always in non-editable mode
+    AIRFLOW_INSTALL_USER_FLAG="--user" \
     AIRFLOW_INSTALL_EDITABLE_FLAG="" \
     UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}
 

diff --git a/Dockerfile.ci b/Dockerfile.ci
@@ -223,7 +223,7 @@ ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
 # By default in the image, we are installing all providers when installing from sources
 ARG INSTALL_PROVIDERS_FROM_SOURCES="true"
 ARG INSTALL_FROM_PYPI="true"
-ARG AIRFLOW_PIP_VERSION=21.3.1
+ARG AIRFLOW_PIP_VERSION=21.2.4
 # Setup PIP
 # By default PIP install run without cache to make image smaller
 ARG PIP_NO_CACHE_DIR="true"
@@ -258,6 +258,7 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\
     INSTALL_MYSQL_CLIENT="true" \
     INSTALL_MSSQL_CLIENT="true" \
     AIRFLOW_INSTALLATION_METHOD="." \
+    AIRFLOW_INSTALL_USER_FLAG="" \
     AIRFLOW_INSTALL_EDITABLE_FLAG="--editable" \
     AIRFLOW_VERSION_SPECIFICATION="" \
     PIP_NO_CACHE_DIR=${PIP_NO_CACHE_DIR} \

diff --git a/IMAGES.rst b/IMAGES.rst
@@ -41,13 +41,14 @@ to run Kubernetes tests. See below for the list of arguments that should be prov
 production image from the local sources.
 
 The image is primarily optimised for size of the final image, but also for speed of rebuilds - the
-'airflow-build-image' segment uses the same technique as the CI jobs for pre-installing dependencies.
+'airflow-build-image' segment uses the same technique as the CI jobs for pre-installing PIP dependencies.
 It first pre-installs them from the right GitHub branch and only after that final airflow installation is
-done from either local sources or remote location (PyPI or GitHub repository).
+done from either local sources or remote location (PIP or GitHub repository).
 
 You can read more details about building, extending and customizing the PROD image in the
 `Latest documentation <https://airflow.apache.org/docs/docker-stack/index.html>`_
 
+
 CI image
 --------
 
@@ -491,7 +492,7 @@ The following build arguments (``--build-arg`` in docker build command) can be u
 | ``ADDITIONAL_RUNTIME_APT_ENV``           |                                          | Additional env variables defined         |
 |                                          |                                          | when installing runtime deps             |
 +------------------------------------------+------------------------------------------+------------------------------------------+
-| ``AIRFLOW_PIP_VERSION``                  | ``21.3.1``                               | PIP version used.                        |
+| ``AIRFLOW_PIP_VERSION``                  | ``21.2.4``                               | PIP version used.                        |
 +------------------------------------------+------------------------------------------+------------------------------------------+
 | ``PIP_PROGRESS_BAR``                     | ``on``                                   | Progress bar for PIP installation        |
 +------------------------------------------+------------------------------------------+------------------------------------------+

diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md
@@ -611,9 +611,9 @@ provider packages. This is especially helpful when you want to test integrations
 additional tools. Below is an example Dockerfile, which installs providers for Google/
 
 ```dockerfile
-FROM apache/airflow:2.2.3
+FROM apache/airflow:2.0.0
 
-RUN pip install --user apache-airflow-providers-google==2.2.2.rc1
+RUN pip install --upgrade --user apache-airflow-providers-google==2.0.0.rc1
 
 USER ${AIRFLOW_UID}
 ```

diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst
@@ -45,7 +45,7 @@ Those are the most common arguments that you use when you want to build a custom
 +------------------------------------------+------------------------------------------+---------------------------------------------+
 | ``AIRFLOW_USER_HOME_DIR``                | ``/home/airflow``                        | Home directory of the Airflow user.         |
 +------------------------------------------+------------------------------------------+---------------------------------------------+
-| ``AIRFLOW_PIP_VERSION``                  | ``21.3.1``                               |  PIP version used.                          |
+| ``AIRFLOW_PIP_VERSION``                  | ``21.2.4``                                | PIP version used.                          |
 +------------------------------------------+------------------------------------------+---------------------------------------------+
 | ``PIP_PROGRESS_BAR``                     | ``on``                                   | Progress bar for PIP installation           |
 +------------------------------------------+------------------------------------------+---------------------------------------------+

diff --git a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
@@ -82,7 +82,7 @@ function create_virtualenv() {
 
     . "${virtualenv_path}/bin/activate"
 
-    pip install "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}"
+    pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}"
 
     local constraints=(
         --constraint

diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
@@ -421,7 +421,7 @@ function initialization::initialize_image_build_variables() {
 
     export INSTALLED_EXTRAS="async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv"
 
-    AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.3.1"}
+    AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.2.4"}
     export AIRFLOW_PIP_VERSION
 
     # We also pin version of wheel used to get consistent builds

diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh
@@ -17,11 +17,14 @@
 # under the License.
 set -euo pipefail
 
-: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
-: "${INSTALL_MSSQL_CLIENT:?Should be true or false}"
-: "${AIRFLOW_REPO:?Should be set}"
-: "${AIRFLOW_BRANCH:?Should be set}"
-: "${AIRFLOW_PIP_VERSION:?Should be set}"
+test -v INSTALL_MYSQL_CLIENT
+test -v INSTALL_MSSQL_CLIENT
+test -v AIRFLOW_INSTALL_USER_FLAG
+test -v AIRFLOW_REPO
+test -v AIRFLOW_BRANCH
+test -v AIRFLOW_PIP_VERSION
+
+set -x
 
 function common::get_airflow_version_specification() {
     if [[ -z ${AIRFLOW_VERSION_SPECIFICATION}
@@ -39,6 +42,7 @@ function common::override_pip_version_if_needed() {
     fi
 }
 
+
 function common::get_constraints_location() {
     # auto-detect Airflow-constraint reference and location
     if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE}" ]]; then
@@ -56,9 +60,3 @@ function common::get_constraints_location() {
         AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS}-${python_version}.txt"
     fi
 }
-
-function common::show_pip_version_and_location() {
-   echo "PATH=${PATH}"
-   echo "pip on path: $(which pip)"
-   echo "Using pip: $(pip --version)"
-}
diff --git a/scripts/docker/compile_www_assets.sh b/scripts/docker/compile_www_assets.sh
@@ -17,6 +17,7 @@
 # under the License.
 # shellcheck disable=SC2086
 set -euo pipefail
+set -x
 
 # Installs additional dependencies passed as Argument to the Docker build command
 function compile_www_assets() {
@@ -34,27 +35,8 @@ function compile_www_assets() {
         www_dir="$(python -m site --user-site)/airflow/www"
     fi
     pushd ${www_dir} || exit 1
-    set +e
-    yarn install --frozen-lockfile --no-cache 2>/tmp/out-yarn-install.txt
-    local res=$?
-    if [[ ${res} != 0 ]]; then
-        >&2 echo
-        >&2 echo "Error when running yarn install:"
-        >&2 echo
-        >&2 cat /tmp/out-yarn-install.txt && rm -f /tmp/out-yarn-install.txt
-        exit 1
-    fi
-    yarn run prod 2>/tmp/out-yarn-run.txt
-    res=$?
-    if [[ ${res} != 0 ]]; then
-        >&2 echo
-        >&2 echo "Error when running yarn install:"
-        >&2 echo
-        >&2 cat /tmp/out-yarn-run.txt && rm -f /tmp/out-yarn-run.txt
-        exit 1
-    fi
-    rm -f /tmp/out-yarn-run.txt
-    set -e
+    yarn install --frozen-lockfile --no-cache
+    yarn run prod
     find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > "${md5sum_file}"
     rm -rf "${www_dir}/node_modules"
     rm -vf "${www_dir}"/{package.json,yarn.lock,.eslintignore,.eslintrc,.stylelintignore,.stylelintrc,compile_assets.sh,webpack.config.js}

diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh
@@ -18,10 +18,11 @@
 # shellcheck disable=SC2086
 set -euo pipefail
 
-: "${UPGRADE_TO_NEWER_DEPENDENCIES:?Should be true or false}"
-: "${ADDITIONAL_PYTHON_DEPS:?Should be set}"
-: "${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS:?Should be set}"
-: "${AIRFLOW_PIP_VERSION:?Should be set}"
+test -v UPGRADE_TO_NEWER_DEPENDENCIES
+test -v ADDITIONAL_PYTHON_DEPS
+test -v EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS
+test -v AIRFLOW_INSTALL_USER_FLAG
+test -v AIRFLOW_PIP_VERSION
 
 # shellcheck source=scripts/docker/common.sh
 . "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
@@ -35,26 +36,26 @@ function install_additional_dependencies() {
         echo
         echo Installing additional dependencies while upgrading to newer dependencies
         echo
-        pip install --upgrade --upgrade-strategy eager \
+        pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy eager \
             ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
         # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
+        pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
         pip check
     else
         echo
         echo Installing additional dependencies upgrading only if needed
         echo
-        pip install --upgrade --upgrade-strategy only-if-needed \
+        pip install ${AIRFLOW_INSTALL_USER_FLAG} \
+            --upgrade --upgrade-strategy only-if-needed \
             ${ADDITIONAL_PYTHON_DEPS}
         # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
+        pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
         pip check
     fi
 }
 
 common::get_airflow_version_specification
 common::override_pip_version_if_needed
 common::get_constraints_location
-common::show_pip_version_and_location
 
 install_additional_dependencies
diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh
@@ -47,7 +47,7 @@ function install_airflow() {
         echo Installing all packages with eager upgrade
         echo
         # eager upgrade
-        pip install --upgrade --upgrade-strategy eager \
+        pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy eager \
             "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
             ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
         if [[ -n "${AIRFLOW_INSTALL_EDITABLE_FLAG}" ]]; then
@@ -59,23 +59,23 @@ function install_airflow() {
         fi
 
         # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
+        pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
         pip check
     else \
         echo
         echo Installing all packages with constraints and upgrade if needed
         echo
-        pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
+        pip install ${AIRFLOW_INSTALL_USER_FLAG} ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
             "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
             --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
         # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
+        pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
         # then upgrade if needed without using constraints to account for new limits in setup.py
-        pip install --upgrade --upgrade-strategy only-if-needed \
+        pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy only-if-needed \
             ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
-            "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
+            "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
         # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
+        pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
         pip check
     fi
 
@@ -84,6 +84,5 @@ function install_airflow() {
 common::get_airflow_version_specification
 common::override_pip_version_if_needed
 common::get_constraints_location
-common::show_pip_version_and_location
 
 install_airflow
diff --git a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
@@ -39,12 +39,12 @@ function install_airflow_dependencies_from_branch_tip() {
     fi
     # Install latest set of dependencies using constraints. In case constraints were upgraded and there
     # are conflicts, this might fail, but it should be fixed in the following installation steps
-    pip install \
+    pip install ${AIRFLOW_INSTALL_USER_FLAG} \
       "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
       --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
     # make sure correct PIP version is used
-    pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
-    pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true
+    pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+    pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes || true
     echo
     echo Uninstalling just airflow. Dependencies remain.
     echo
@@ -54,6 +54,5 @@ function install_airflow_dependencies_from_branch_tip() {
 common::get_airflow_version_specification
 common::override_pip_version_if_needed
 common::get_constraints_location
-common::show_pip_version_and_location
 
 install_airflow_dependencies_from_branch_tip
diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh
@@ -34,6 +34,7 @@ function install_airflow_and_providers_from_docker_context_files(){
     local pip_flags=(
         # Don't quote this -- if it is empty we don't want it to create an
         # empty array element
+        ${AIRFLOW_INSTALL_USER_FLAG}
         --find-links="file:///docker-context-files"
     )
 
@@ -87,14 +88,14 @@ function install_airflow_and_providers_from_docker_context_files(){
             --constraint /tmp/constraints.txt
         rm /tmp/constraints.txt
         # make sure correct PIP version is used \
-        pip install "pip==${AIRFLOW_PIP_VERSION}"
+        pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
         # then upgrade if needed without using constraints to account for new limits in setup.py
-        pip install --upgrade --upgrade-strategy only-if-needed \
+        pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy only-if-needed \
              ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages}
     fi
 
     # make sure correct PIP version is left installed
-    pip install "pip==${AIRFLOW_PIP_VERSION}"
+    pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
     pip check
 
 }
@@ -103,7 +104,7 @@ function install_airflow_and_providers_from_docker_context_files(){
 # without dependencies. This is extremely useful in case you want to install via pip-download
 # method on air-gaped system where you do not want to download any dependencies from remote hosts
 # which is a requirement for serious installations
-function install_all_other_packages_from_docker_context_files() {
+install_all_other_packages_from_docker_context_files() {
     echo
     echo Force re-installing all other package from local files without dependencies
     echo
@@ -112,18 +113,15 @@ function install_all_other_packages_from_docker_context_files() {
     reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz} 2>/dev/null | \
         grep -v apache_airflow | grep -v apache-airflow || true)
     if [[ -n "${reinstalling_other_packages}" ]]; then \
-        pip install --force-reinstall --no-deps --no-index ${reinstalling_other_packages}
+        pip install ${AIRFLOW_INSTALL_USER_FLAG} --force-reinstall --no-deps --no-index ${reinstalling_other_packages}
         # make sure correct PIP version is used
-        pip install "pip==${AIRFLOW_PIP_VERSION}"
+        pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
     fi
 }
 
 common::get_airflow_version_specification
 common::override_pip_version_if_needed
 common::get_constraints_location
-common::show_pip_version_and_location
 
 install_airflow_and_providers_from_docker_context_files
-
-common::show_pip_version_and_location
 install_all_other_packages_from_docker_context_files
diff --git a/scripts/docker/install_mssql.sh b/scripts/docker/install_mssql.sh
@@ -15,13 +15,13 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-set -euo pipefail
+set -exuo pipefail
 function install_mssql_client() {
     echo
     echo Installing mssql client
     echo
-    curl --silent https://packages.microsoft.com/keys/microsoft.asc | apt-key add - >/dev/null 2>&1
-    curl --silent https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list
+    curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
+    curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list
     apt-get update -yqq
     apt-get upgrade -yqq
     ACCEPT_EULA=Y apt-get -yqq install -y --no-install-recommends msodbcsql17 mssql-tools

diff --git a/scripts/docker/install_mysql.sh b/scripts/docker/install_mysql.sh
@@ -15,7 +15,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-set -euo pipefail
+set -exuo pipefail
 declare -a packages
 
 MYSQL_VERSION="8.0"
@@ -46,13 +46,14 @@ install_mysql_client() {
     for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \
                                keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80)
     do
-        gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break
+        gpg --keyserver "${keyserver}" --recv-keys "${key}" && break
     done
     set -e
     gpg --export "${key}" > /etc/apt/trusted.gpg.d/mysql.gpg
     gpgconf --kill all
     rm -rf "${GNUPGHOME}"
     unset GNUPGHOME
+    apt-key list > /dev/null 2>&1
     echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list
     apt-get update
     apt-get install --no-install-recommends -y "${packages[@]}"