Skip to content

Commit

Permalink
#867: Fix builds and updated dependencies (#396)
Browse files Browse the repository at this point in the history
* Update packages
* Add simple help to scripts/update_apt_package_in_package_list.sh
* Update poetry and exclude docker-py 7.0.0 because of docker/docker-py#3223
* Add script to update multiple apt packages from stdin
* Ignore Kernel CVEs
* Add modules to exclude list in import_modules.py test
* Fix bug in scripts/update_apt_package_in_package_list.sh for packages with same prefix
* Fix missing crypt.h in conda-based SLC
* Replace hardcoded /opt/conda with environment variable
* Disable package mode in pyproject.toml
* Update poetry in the Github action
  • Loading branch information
tkilias committed Mar 4, 2024
1 parent 8690910 commit b9c5335
Show file tree
Hide file tree
Showing 39 changed files with 600 additions and 692 deletions.
2 changes: 1 addition & 1 deletion .github/actions/prepare_poetry_env/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ runs:
python-version: ${{ inputs.python-version }}
- uses: abatilo/actions-poetry@v2
with:
poetry-version: 1.4.0
poetry-version: 1.8.2
- name: Poetry install
run: poetry install
shell: bash
Expand Down
2 changes: 1 addition & 1 deletion exaslct_scripts/exaslct.sh

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
gdb|9.2-0ubuntu1~20.04.1
valgrind|1:3.15.0-1ubuntu9.1
gdbserver|9.2-0ubuntu1~20.04.1
binutils|2.34-6ubuntu1.7
binutils|2.34-6ubuntu1.9
patchelf|0.10-2build1
strace|5.5-3ubuntu1
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
unzip|6.0-25ubuntu1.1
unzip|6.0-25ubuntu1.2
git|1:2.25.1-1ubuntu3.11
libcurl4-openssl-dev|7.68.0-1ubuntu2.21
build-essential|12.8ubuntu1.1
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
pandas|2.0.2
numpy|1.24.3
pyarrow|14.0.1
pyarrow|14.0.2
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ CVE-2023-4622
CVE-2023-4623
CVE-2023-4921
CVE-2023-4244
CVE-2024-1086
CVE-2024-26597
# CVE-2023-20569 does not have a fix and is a problem of host system (amd cpus)
CVE-2023-20569
#CVE-2022-23648 is a bug in containerd, not issue for containers
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@ locales|2.31-0ubuntu9.14
libnss-db|2.2.3pre1-6build6
libzmq3-dev|4.3.2-2ubuntu1
libprotobuf-dev|3.6.1.3-2ubuntu5.2
libssl-dev|1.1.1f-1ubuntu2.20
libssl-dev|1.1.1f-1ubuntu2.22
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
gdb|12.1-0ubuntu1~22.04
valgrind|1:3.18.1-1ubuntu2
gdbserver|12.1-0ubuntu1~22.04
binutils|2.38-4ubuntu2.4
binutils|2.38-4ubuntu2.6
patchelf|0.14.3-1
strace|5.16-0ubuntu3
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
unzip|6.0-26ubuntu3.1
unzip|6.0-26ubuntu3.2
git|1:2.34.1-1ubuntu1.10
libcurl4-openssl-dev|7.81.0-1ubuntu1.15
build-essential|12.9ubuntu3
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
pandas|2.0.2
numpy|1.24.3
pyarrow|14.0.1
pyarrow|14.0.2
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ CVE-2023-4622
CVE-2023-4623
CVE-2023-4921
CVE-2023-6176
CVE-2024-0646
CVE-2024-1085
CVE-2024-1086
CVE-2024-26597
# CVE-2023-20569 does not have a fix and is a problem of host system (amd cpus)
CVE-2023-20569
#CVE-2022-23648 is a bug in containerd, not issue for containers
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
coreutils|8.32-4.1ubuntu1
locales|2.35-0ubuntu3.5
coreutils|8.32-4.1ubuntu1.1
locales|2.35-0ubuntu3.6
libnss-db|2.2.3pre1-6ubuntu3
libzmq3-dev|4.3.4-2
libprotobuf-dev|3.12.4-1ubuntu7.22.04.1
libssl-dev|3.0.2-0ubuntu1.12
libssl-dev|3.0.2-0ubuntu1.15
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ COPY /exaudfclient/base /exaudfclient/base

WORKDIR /exaudfclient/base
# We need to build with optimize here, because the debug build fails to run with a runtime linking error regarding protobuf
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries-py3
RUN cp -r -L bazel-bin/* /exaudf
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ FROM {{language_deps}}
RUN mkdir -p /build_info/packages
COPY build_deps/packages /build_info/packages/build_deps

RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/build_deps/conda_packages --channel-file /build_info/packages/build_deps/conda_channels --with-versions --conda-binary /opt/conda/bin/mamba
RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/build_deps/conda_packages --channel-file /build_info/packages/build_deps/conda_channels --with-versions --conda-binary "$MAMBA_ROOT_PREFIX/bin/mamba"

RUN curl -L -o swig-2.0.4.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-2.0.4.tar.gz && \
tar zxf swig-2.0.4.tar.gz && \
(cd swig-2.0.4 && ./configure --prefix=/usr && make && make install) && \
rm -rf swig-2.0.4 swig-2.0.4.tar.gz

ENV PROTOBUF_BIN=/opt/conda/bin/protoc
ENV PROTOBUF_BIN=$MAMBA_ROOT_PREFIX/bin/protoc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
protobuf|4.24.4
protobuf|4.25.1
pcre|8.45
chrpath|0.16
openjdk|11.0.15
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ RUN mkdir /exaudfclient /exaudf
COPY exaudfclient/base /exaudfclient/base

WORKDIR /exaudfclient/base
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary-py3
RUN cp -r -L bazel-bin/* /exaudf

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@ FROM {{ language_deps }}
RUN mkdir -p /build_info/packages
COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps

RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/flavor_base_deps/conda_packages --channel-file /build_info/packages/flavor_base_deps/conda_channels --with-versions --conda-binary /opt/conda/bin/mamba
RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/flavor_base_deps/conda_packages --channel-file /build_info/packages/flavor_base_deps/conda_channels --with-versions --conda-binary "$MAMBA_ROOT_PREFIX/bin/mamba"

RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary /opt/conda/bin/python3.8 --with-versions --allow-no-version
RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary "$MAMBA_ROOT_PREFIX/bin/python3.8" --with-versions --allow-no-version
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@ COPY language_deps/packages/ /build_info/packages/language_deps

RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/language_deps/conda_packages --channel-file /build_info/packages/language_deps/conda_channels --with-versions --conda-binary /bin/micromamba

ENV PYTHON3_PREFIX /opt/conda
ENV PYTHON3_VERSION python3.8
ENV PYTHON3_PREFIX="$MAMBA_ROOT_PREFIX"
ENV PYTHON3_VERSION=python3.8

Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
libxcrypt|4.4.36 # Needed for building the udfclient and because libcrypt got removed from conda sysroot, see https://github.com/conda-forge/linux-sysroot-feedstock/issues/52
python|3.8.13
numpy|1.24.3
pandas|2.0.2
pyarrow|14.0.1
pyarrow|14.0.2
libblas|3.9.0=15_linux64_mkl
mamba|1.3.1
ld_impl_linux-64|2.36.1
cryptography|41.0.3
cryptography|42.0.4
libxml2|2.10.3=hca2bb57_4 # Pinned for OpenJDK in build_deps
icu|70.1=h27087fc_0 # Pinned for OpenJDK in build_deps
Original file line number Diff line number Diff line change
Expand Up @@ -66,5 +66,5 @@ RUN true # workaround for https://github.com/moby/moby/issues/37965

RUN mkdir -p /build_info/actual_installed_packages/release && \
/scripts/list_installed_scripts/list_installed_apt.sh > /build_info/actual_installed_packages/release/apt_get_packages && \
/scripts/list_installed_scripts/list_installed_pip.sh /opt/conda/bin/python3.8 > /build_info/actual_installed_packages/release/python3_pip_packages && \
/scripts/list_installed_scripts/list_installed_pip.sh "$MAMBA_ROOT_PREFIX/bin/python3.8" > /build_info/actual_installed_packages/release/python3_pip_packages && \
/scripts/list_installed_scripts/list_installed_conda.sh > /build_info/actual_installed_packages/release/conda_packages
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ COPY udfclient_deps/packages /build_info/packages/udfclient_deps

RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/udfclient_deps/conda_packages --channel-file /build_info/packages/udfclient_deps/conda_channels --with-versions --conda-binary /bin/micromamba

ENV PROTOBUF_LIBRARY_PREFIX=/opt/conda/lib
ENV PROTOBUF_INCLUDE_PREFIX=/opt/conda/include
ENV ZMQ_LIBRARY_PREFIX=/opt/conda/lib
ENV ZMQ_INCLUDE_PREFIX=/opt/conda/include
ENV PROTOBUF_LIBRARY_PREFIX=$MAMBA_ROOT_PREFIX/lib
ENV PROTOBUF_INCLUDE_PREFIX=$MAMBA_ROOT_PREFIX/include
ENV ZMQ_LIBRARY_PREFIX=$MAMBA_ROOT_PREFIX/lib
ENV ZMQ_INCLUDE_PREFIX=$MAMBA_ROOT_PREFIX/include
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
libprotobuf|4.24.4
libprotobuf|4.25.1
zeromq|4.3.4
openssl|3.1.4
openssl|3.2.1
nss|3.77
cppzmq|4.8.1
_openmp_mutex|4.5=2_kmp_llvm # We need this, such that we can later install libblas with mkl, which is for example needed by pytorch
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ FROM {{flavor_base_deps}}
RUN mkdir -p /build_info/packages/flavor_customization

COPY flavor_customization/packages /build_info/packages/flavor_customization
RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/flavor_customization/conda_packages --channel-file /build_info/packages/flavor_customization/conda_channels --with-versions --allow-no-version --conda-binary /opt/conda/bin/mamba
RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/flavor_customization/conda_packages --channel-file /build_info/packages/flavor_customization/conda_channels --with-versions --allow-no-version --conda-binary "$MAMBA_ROOT_PREFIX/bin/mamba"

COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization
RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_customization/python3_pip_packages --python-binary /opt/conda/bin/python3.8 --with-versions --allow-no-version
RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_customization/python3_pip_packages --python-binary "$MAMBA_ROOT_PREFIX/bin/python3.8" --with-versions --allow-no-version

##########################################################################
##########################################################################
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ RUN mkdir /exaudfclient /exaudf
COPY /exaudfclient/base /exaudfclient/base

WORKDIR /exaudfclient/base
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
# We need to build with optimize here, because the debug build fails to run with a runtime linking error regarding protobuf
RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries-py3
RUN cp -r -L bazel-bin/* /exaudf
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ FROM {{language_deps}}
RUN mkdir -p /build_info/packages
COPY build_deps/packages /build_info/packages/build_deps

RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/build_deps/conda_packages --channel-file /build_info/packages/build_deps/conda_channels --with-versions --conda-binary /opt/conda/bin/mamba
RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/build_deps/conda_packages --channel-file /build_info/packages/build_deps/conda_channels --with-versions --conda-binary "$MAMBA_ROOT_PREFIX/bin/mamba"

RUN curl -L -o swig-2.0.4.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-2.0.4.tar.gz && \
tar zxf swig-2.0.4.tar.gz && \
(cd swig-2.0.4 && ./configure --prefix=/usr && make && make install) && \
rm -rf swig-2.0.4 swig-2.0.4.tar.gz

ENV PROTOBUF_BIN=/opt/conda/bin/protoc
ENV PROTOBUF_BIN=$MAMBA_ROOT_PREFIX/bin/protoc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
protobuf|4.24.4
protobuf|4.25.1
pcre|8.45
chrpath|0.16
openjdk|11.0.15
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ RUN mkdir /exaudfclient /exaudf
COPY exaudfclient/base /exaudfclient/base

WORKDIR /exaudfclient/base
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary-py3
RUN cp -r -L bazel-bin/* /exaudf

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@ COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps

ENV CONDA_OVERRIDE_CUDA=11.3

RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/flavor_base_deps/conda_packages --channel-file /build_info/packages/flavor_base_deps/conda_channels --with-versions --conda-binary /opt/conda/bin/mamba
RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/flavor_base_deps/conda_packages --channel-file /build_info/packages/flavor_base_deps/conda_channels --with-versions --conda-binary "$MAMBA_ROOT_PREFIX/bin/mamba"

RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary /opt/conda/bin/python3.8 --with-versions --allow-no-version
RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary "$MAMBA_ROOT_PREFIX/bin/python3.8" --with-versions --allow-no-version
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@ COPY language_deps/packages/ /build_info/packages/language_deps

RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/language_deps/conda_packages --channel-file /build_info/packages/language_deps/conda_channels --with-versions --conda-binary /bin/micromamba

ENV PYTHON3_PREFIX /opt/conda
ENV PYTHON3_VERSION python3.8
ENV PYTHON3_PREFIX=$MAMBA_ROOT_PREFIX
ENV PYTHON3_VERSION=python3.8

Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
libxcrypt|4.4.36 # Needed for building the udfclient and because libcrypt got removed from conda sysroot, see https://github.com/conda-forge/linux-sysroot-feedstock/issues/52
python|3.8.13
numpy|1.24.3
pandas|2.0.2
pyarrow|14.0.1=py38h296dbf9_0_cuda
pyarrow|14.0.2=py38h296dbf9_9_cuda
libblas|3.9.0=15_linux64_mkl
mamba|1.3.1
ld_impl_linux-64|2.36.1
cryptography|41.0.3
cryptography|42.0.4
libxml2|2.10.3=hca2bb57_4 # Pinned for OpenJDK in build_deps
icu|70.1=h27087fc_0 # Pinned for OpenJDK in build_deps
Original file line number Diff line number Diff line change
Expand Up @@ -66,5 +66,5 @@ RUN true # workaround for https://github.com/moby/moby/issues/37965

RUN mkdir -p /build_info/actual_installed_packages/release && \
/scripts/list_installed_scripts/list_installed_apt.sh > /build_info/actual_installed_packages/release/apt_get_packages && \
/scripts/list_installed_scripts/list_installed_pip.sh /opt/conda/bin/python3.8 > /build_info/actual_installed_packages/release/python3_pip_packages && \
/scripts/list_installed_scripts/list_installed_pip.sh "$MAMBA_ROOT_PREFIX/bin/python3.8" > /build_info/actual_installed_packages/release/python3_pip_packages && \
/scripts/list_installed_scripts/list_installed_conda.sh > /build_info/actual_installed_packages/release/conda_packages
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ COPY udfclient_deps/packages /build_info/packages/udfclient_deps

RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/udfclient_deps/conda_packages --channel-file /build_info/packages/udfclient_deps/conda_channels --with-versions --conda-binary /bin/micromamba

ENV PROTOBUF_LIBRARY_PREFIX=/opt/conda/lib
ENV PROTOBUF_INCLUDE_PREFIX=/opt/conda/include
ENV ZMQ_LIBRARY_PREFIX=/opt/conda/lib
ENV ZMQ_INCLUDE_PREFIX=/opt/conda/include
ENV PROTOBUF_LIBRARY_PREFIX=$MAMBA_ROOT_PREFIX/lib
ENV PROTOBUF_INCLUDE_PREFIX=$MAMBA_ROOT_PREFIX/include
ENV ZMQ_LIBRARY_PREFIX=$MAMBA_ROOT_PREFIX/lib
ENV ZMQ_INCLUDE_PREFIX=$MAMBA_ROOT_PREFIX/include
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
libprotobuf|4.24.4
libprotobuf|4.25.1
zeromq|4.3.4
openssl|3.1.4
openssl|3.2.1
nss|3.77
cppzmq|4.8.1
_openmp_mutex|4.5=2_kmp_llvm # We need this, such that we can later install libblas with mkl, which is for example needed by pytorch
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ FROM {{flavor_base_deps}}
RUN mkdir -p /build_info/packages/flavor_customization

COPY flavor_customization/packages /build_info/packages/flavor_customization
RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/flavor_customization/conda_packages --channel-file /build_info/packages/flavor_customization/conda_channels --with-versions --allow-no-version --conda-binary /opt/conda/bin/mamba
RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/flavor_customization/conda_packages --channel-file /build_info/packages/flavor_customization/conda_channels --with-versions --allow-no-version --conda-binary "$MAMBA_ROOT_PREFIX/bin/mamba"

COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization
RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_customization/python3_pip_packages --python-binary /opt/conda/bin/python3.8 --with-versions --allow-no-version
RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_customization/python3_pip_packages --python-binary "$MAMBA_ROOT_PREFIX/bin/python3.8" --with-versions --allow-no-version

##########################################################################
##########################################################################
Expand Down

0 comments on commit b9c5335

Please sign in to comment.