Update python from 3.6 to 3.7 in codalab docker containers (#4109)

* Update python from 3.6 to 3.7 in codalab docker container

* replace 3.6 with 3.7 in github actions

* Pin jinja2 to fix mkdocs (mkdocs/mkdocs/issues/2794)

* Try bumping mkdocs instead

* Pin pygments for mkdocs, click for black (psf/black/issues/2964)

.github/workflows/build_worker_images.yml

@@ -23,7 +23,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.7
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip

.github/workflows/postcommit.yml

@@ -18,12 +18,12 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.7
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip
           key: pip-${{ hashFiles('requirements.txt') }}
           restore-keys: |
             pip-
       - run: pip install -r requirements.txt
-      - run: python3 codalab_service.py build
+      - run: python3 codalab_service.py build

.github/workflows/release.yml

@@ -11,7 +11,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.7
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip

.github/workflows/stress.yml

@@ -10,7 +10,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.7
       - run: pip install -e .
       - name: Run tests
         run: python tests/stress/stress_test.py --instance https://worksheets-dev.codalab.org --heavy

.github/workflows/test.yml

@@ -13,10 +13,10 @@ jobs:
       - uses: actions/setup-node@v1
         with:
           node-version: 14.x
-      - name: Set up Python 3.6
+      - name: Set up Python 3.7
         uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.7
       - run: npm ci
         working-directory: ./frontend
       - run: npm run check-ci
@@ -77,7 +77,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.7
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -122,7 +122,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.7
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -185,7 +185,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.7
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -234,7 +234,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.7
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -290,7 +290,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.7
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -339,7 +339,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.7
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -388,7 +388,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.7
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -454,7 +454,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.7
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip
@@ -504,7 +504,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.7
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip

docker_config/dockerfiles/Dockerfile.server

@@ -5,7 +5,6 @@ ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get update && \
   apt-get install -y --no-install-recommends software-properties-common && \
-  add-apt-repository ppa:fkrull/deadsnakes && \
   apt-get update && \
   apt-get install -y --no-install-recommends \
     build-essential \
@@ -16,23 +15,22 @@ RUN apt-get update && \
     libmysqlclient-dev \
     libssl-dev \
     mysql-client \
-    python3.6 \
-    python3.6-dev \
-    python3-pip \
-    python3-setuptools \
-    python-virtualenv \
-    python3-software-properties \
     zip \
     unzip \
     rsync && \
     rm -rf /var/lib/apt/lists/*;
 
-# Set Python3.6 as the default python3 version
-RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1
-
 ### Without this Python thinks we're ASCII and unicode chars fail
 ENV LANG C.UTF-8
 
+# Miniconda 4.5.11 installs Python 3.7 by default.
+RUN curl -o ~/miniconda.sh -O  https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \
+     chmod +x ~/miniconda.sh && \
+     ~/miniconda.sh -b -p /opt/conda && \
+     rm ~/miniconda.sh
+ENV PATH /opt/conda/bin:$PATH
+RUN conda --version
+
 RUN python3 -m pip install --no-cache-dir --upgrade pip==20.3.4
 RUN mkdir /opt/codalab-worksheets
 WORKDIR /opt/codalab-worksheets

docker_config/dockerfiles/Dockerfile.worker

@@ -25,14 +25,9 @@ ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get update && \
   apt-get install -y --no-install-recommends software-properties-common curl && \
-  add-apt-repository ppa:fkrull/deadsnakes && \
   apt-get update && \
   apt-get install -y --no-install-recommends \
     build-essential \
-    python3.6 \
-    python3.6-dev \
-    python3-pip \
-    python3-setuptools \
     libssl-dev \
     uuid-dev \
     libgpgme11-dev \
@@ -45,8 +40,13 @@ RUN apt-get update && \
     unzip && \
     rm -rf /var/lib/apt/lists/*;
 
-# Set Python3.6 as the default python3 version
-RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1
+# Miniconda 4.5.11 installs Python 3.7 by default.
+RUN curl -o ~/miniconda.sh -O  https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \
+     chmod +x ~/miniconda.sh && \
+     ~/miniconda.sh -b -p /opt/conda && \
+     rm ~/miniconda.sh
+ENV PATH /opt/conda/bin:$PATH
+RUN conda --version
 
 COPY --from=ecr-login-installation /root/go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/local /usr/local/bin
 COPY --from=ecr-login-installation /usr/local/bin/singularity /usr/local/bin
@@ -62,9 +62,9 @@ RUN mkdir ${WORKDIR}/scripts
 
 # Install dependencies
 COPY requirements.txt requirements.txt
-RUN python3.6 -m pip install --user --upgrade pip==20.3.4; \
-    python3.6 -m pip install setuptools --upgrade; \
-    python3.6 -m pip install --no-cache-dir -r requirements.txt;
+RUN python3 -m pip install --user --upgrade pip==20.3.4; \
+    python3 -m pip install setuptools --upgrade; \
+    python3 -m pip install --no-cache-dir -r requirements.txt;
 
 # Install code
 COPY codalab/lib codalab/lib

requirements.docs.txt

@@ -1,4 +1,4 @@
 tornado<7.0
 markdown==3.3.6
-mkdocs==1.2.3
+mkdocs==1.2.4
 mkdocs-material==6.2.8

requirements.txt

@@ -14,7 +14,8 @@ six==1.15.0
 SQLAlchemy==1.3.19
 watchdog>=2.0.0
 fusepy==2.0.4
-pygments==2.11.2
+pygments>=2.12,<2.13
+click==8.0.2
 python-dateutil==2.8.1
 diffimg==0.2.3
 selenium==3.141.0