From f2cd9e7425b739cd3381bd4d6594d5111ab2d722 Mon Sep 17 00:00:00 2001 From: Nelson Liu Date: Thu, 26 May 2022 10:29:06 -0700 Subject: [PATCH] Update python from 3.6 to 3.7 in codalab docker containers (#4109) * Update python from 3.6 to 3.7 in codalab docker container * replace 3.6 with 3.7 in github actions * Pin jinja2 to fix mkdocs (mkdocs/mkdocs/issues/2794) * Try bumping mkdocs instead * Pin pygments for mkdocs, click for black (psf/black/issues/2964) --- .github/workflows/build_worker_images.yml | 2 +- .github/workflows/postcommit.yml | 4 ++-- .github/workflows/release.yml | 2 +- .github/workflows/stress.yml | 2 +- .github/workflows/test.yml | 22 ++++++++++----------- docker_config/dockerfiles/Dockerfile.server | 18 ++++++++--------- docker_config/dockerfiles/Dockerfile.worker | 20 +++++++++---------- requirements.docs.txt | 2 +- requirements.txt | 3 ++- 9 files changed, 37 insertions(+), 38 deletions(-) diff --git a/.github/workflows/build_worker_images.yml b/.github/workflows/build_worker_images.yml index 9ff152e22..a579fbc9c 100644 --- a/.github/workflows/build_worker_images.yml +++ b/.github/workflows/build_worker_images.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.7 - uses: actions/cache@v2 with: path: ~/.cache/pip diff --git a/.github/workflows/postcommit.yml b/.github/workflows/postcommit.yml index c7b43c411..d72e065c3 100644 --- a/.github/workflows/postcommit.yml +++ b/.github/workflows/postcommit.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.7 - uses: actions/cache@v2 with: path: ~/.cache/pip @@ -26,4 +26,4 @@ jobs: restore-keys: | pip- - run: pip install -r requirements.txt - - run: python3 codalab_service.py build \ No newline at end of file + - run: python3 codalab_service.py build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f6bb794af..4c79409ff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.7 - uses: actions/cache@v2 with: path: ~/.cache/pip diff --git a/.github/workflows/stress.yml b/.github/workflows/stress.yml index 24ecadd0e..ed656ca25 100644 --- a/.github/workflows/stress.yml +++ b/.github/workflows/stress.yml @@ -10,7 +10,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.7 - run: pip install -e . - name: Run tests run: python tests/stress/stress_test.py --instance https://worksheets-dev.codalab.org --heavy diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b302a2861..6f5c553a9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,10 +13,10 @@ jobs: - uses: actions/setup-node@v1 with: node-version: 14.x - - name: Set up Python 3.6 + - name: Set up Python 3.7 uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.7 - run: npm ci working-directory: ./frontend - run: npm run check-ci @@ -77,7 +77,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.7 - uses: actions/cache@v2 with: path: ~/.cache/pip @@ -122,7 +122,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.7 - uses: actions/cache@v2 with: path: ~/.cache/pip @@ -185,7 +185,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.7 - uses: actions/cache@v2 with: path: ~/.cache/pip @@ -234,7 +234,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.7 - uses: actions/cache@v2 with: path: ~/.cache/pip @@ -290,7 +290,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.7 - uses: actions/cache@v2 with: path: ~/.cache/pip @@ -339,7 +339,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.7 - uses: actions/cache@v2 with: path: ~/.cache/pip @@ -388,7 +388,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.7 - uses: actions/cache@v2 with: path: ~/.cache/pip @@ -454,7 +454,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.7 - uses: actions/cache@v2 with: path: ~/.cache/pip @@ -504,7 +504,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.7 - uses: actions/cache@v2 with: path: ~/.cache/pip diff --git a/docker_config/dockerfiles/Dockerfile.server b/docker_config/dockerfiles/Dockerfile.server index 36f1279fe..009147708 100644 --- a/docker_config/dockerfiles/Dockerfile.server +++ b/docker_config/dockerfiles/Dockerfile.server @@ -5,7 +5,6 @@ ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && \ apt-get install -y --no-install-recommends software-properties-common && \ - add-apt-repository ppa:fkrull/deadsnakes && \ apt-get update && \ apt-get install -y --no-install-recommends \ build-essential \ @@ -16,23 +15,22 @@ RUN apt-get update && \ libmysqlclient-dev \ libssl-dev \ mysql-client \ - python3.6 \ - python3.6-dev \ - python3-pip \ - python3-setuptools \ - python-virtualenv \ - python3-software-properties \ zip \ unzip \ rsync && \ rm -rf /var/lib/apt/lists/*; -# Set Python3.6 as the default python3 version -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 - ### Without this Python thinks we're ASCII and unicode chars fail ENV LANG C.UTF-8 +# Miniconda 4.5.11 installs Python 3.7 by default. +RUN curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \ + chmod +x ~/miniconda.sh && \ + ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh +ENV PATH /opt/conda/bin:$PATH +RUN conda --version + RUN python3 -m pip install --no-cache-dir --upgrade pip==20.3.4 RUN mkdir /opt/codalab-worksheets WORKDIR /opt/codalab-worksheets diff --git a/docker_config/dockerfiles/Dockerfile.worker b/docker_config/dockerfiles/Dockerfile.worker index d4c0b1243..095dc6ebd 100644 --- a/docker_config/dockerfiles/Dockerfile.worker +++ b/docker_config/dockerfiles/Dockerfile.worker @@ -25,14 +25,9 @@ ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && \ apt-get install -y --no-install-recommends software-properties-common curl && \ - add-apt-repository ppa:fkrull/deadsnakes && \ apt-get update && \ apt-get install -y --no-install-recommends \ build-essential \ - python3.6 \ - python3.6-dev \ - python3-pip \ - python3-setuptools \ libssl-dev \ uuid-dev \ libgpgme11-dev \ @@ -45,8 +40,13 @@ RUN apt-get update && \ unzip && \ rm -rf /var/lib/apt/lists/*; -# Set Python3.6 as the default python3 version -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 +# Miniconda 4.5.11 installs Python 3.7 by default. +RUN curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \ + chmod +x ~/miniconda.sh && \ + ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh +ENV PATH /opt/conda/bin:$PATH +RUN conda --version COPY --from=ecr-login-installation /root/go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/local /usr/local/bin COPY --from=ecr-login-installation /usr/local/bin/singularity /usr/local/bin @@ -62,9 +62,9 @@ RUN mkdir ${WORKDIR}/scripts # Install dependencies COPY requirements.txt requirements.txt -RUN python3.6 -m pip install --user --upgrade pip==20.3.4; \ - python3.6 -m pip install setuptools --upgrade; \ - python3.6 -m pip install --no-cache-dir -r requirements.txt; +RUN python3 -m pip install --user --upgrade pip==20.3.4; \ + python3 -m pip install setuptools --upgrade; \ + python3 -m pip install --no-cache-dir -r requirements.txt; # Install code COPY codalab/lib codalab/lib diff --git a/requirements.docs.txt b/requirements.docs.txt index 09fb6b444..4842d66f1 100644 --- a/requirements.docs.txt +++ b/requirements.docs.txt @@ -1,4 +1,4 @@ tornado<7.0 markdown==3.3.6 -mkdocs==1.2.3 +mkdocs==1.2.4 mkdocs-material==6.2.8 diff --git a/requirements.txt b/requirements.txt index 708b96183..7b78c27ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,8 @@ six==1.15.0 SQLAlchemy==1.3.19 watchdog>=2.0.0 fusepy==2.0.4 -pygments==2.11.2 +pygments>=2.12,<2.13 +click==8.0.2 python-dateutil==2.8.1 diffimg==0.2.3 selenium==3.141.0