From e1ce6e3115eb3941ade27ef009d1245e4b0b74f1 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Mon, 19 Sep 2022 09:48:46 +0200 Subject: [PATCH 1/7] python-setup: Fix venv creation in Ubuntu 22.04 Fixes https://github.com/github/codeql-action/issues/1249 --- python-setup/install_tools.ps1 | 8 ++++++-- python-setup/install_tools.sh | 10 ++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/python-setup/install_tools.ps1 b/python-setup/install_tools.ps1 index 3c78378a3d..796c0ecb9d 100644 --- a/python-setup/install_tools.ps1 +++ b/python-setup/install_tools.ps1 @@ -1,7 +1,11 @@ #! /usr/bin/pwsh -py -2 -m pip install --user --upgrade pip setuptools wheel -py -3 -m pip install --user --upgrade pip setuptools wheel +# while waiting for the next release of `virtualenv` after v20.16.5, we install an older +# version of `setuptools` to ensure that binaries are always put under +# `/bin`, which wouldn't always happen with the GitHub actions version of +# Ubuntu 22.04. See https://github.com/github/codeql-action/issues/1249 +py -2 -m pip install --user --upgrade pip 'setuptools<60' wheel +py -3 -m pip install --user --upgrade pip 'setuptools<60' wheel # virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of # pip/setuptools/wheel which basic `python3 -m venv venv` won't diff --git a/python-setup/install_tools.sh b/python-setup/install_tools.sh index 7acb33f146..e2e5be36b6 100755 --- a/python-setup/install_tools.sh +++ b/python-setup/install_tools.sh @@ -11,7 +11,13 @@ set -e export PATH="$HOME/.local/bin:$PATH" # Setup Python 3 dependency installation tools. -python3 -m pip install --user --upgrade pip setuptools wheel + +# we install an older version of `setuptools` to ensure that binaries are always put +# under `/bin`, which wouldn't always happen with the GitHub actions version +# of Ubuntu 22.04. See https://github.com/github/codeql-action/issues/1249. The the next +# release of `virtualenv` after v20.16.5 will include a fix for this, so we can remove +# this bit of the logic again. +python3 -m pip install --user --upgrade pip 'setuptools<60' wheel # virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of # pip/setuptools/wheel which basic `python3 -m venv venv` won't @@ -40,7 +46,7 @@ if command -v python2 >/dev/null 2>&1; then curl --location --fail https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2 fi - python2 -m pip install --user --upgrade pip setuptools wheel + python2 -m pip install --user --upgrade pip 'setuptools<60' wheel python2 -m pip install --user 'virtualenv<20.11' fi From 3f97671248b6456a0573d7033f812d6521f4ba52 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 09:08:04 +0200 Subject: [PATCH 2/7] python-setup: run tests on Ubuntu 22.04 --- .github/workflows/python-deps.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml index 3d64f171a3..a17d6723df 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps.yml @@ -26,7 +26,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, ubuntu-22.04, macos-latest] python_deps_type: [pipenv, poetry, requirements, setup_py] python_version: [2, 3] exclude: @@ -63,6 +63,7 @@ jobs: case ${{ matrix.os }} in ubuntu-latest*) basePath="/opt";; + ubuntu-22.04*) basePath="/opt";; macos-latest*) basePath="/Users/runner";; esac echo ${basePath} @@ -86,7 +87,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, ubuntu-22.04, macos-latest] steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it @@ -109,6 +110,7 @@ jobs: case ${{ matrix.os }} in ubuntu-latest*) basePath="/opt";; + ubuntu-22.04*) basePath="/opt";; macos-latest*) basePath="/Users/runner";; esac echo ${basePath} From 1fa5d728463866f63a4442b14b8b4ea5dfd1e0cc Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 14:50:13 +0200 Subject: [PATCH 3/7] python-setup: Fail early if installing for Python 2, and `python2` not available --- python-setup/auto_install_packages.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index b0a623735c..9731052716 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -5,6 +5,7 @@ import subprocess from tempfile import mkdtemp from typing import Optional +import shutil import extractor_version @@ -154,6 +155,17 @@ def install_packages(codeql_base_dir) -> Optional[str]: # get_extractor_version returns the Python version the extractor thinks this repo is using version = extractor_version.get_extractor_version(codeql_base_dir, quiet=False) + if version == 2 and not sys.platform.startswith('win32'): + # On Ubuntu 22.04 'python2' is not available by default. We want to give a slightly better + # error message than a traceback + `No such file or directory: 'python2'` + if shutil.which("python2") is None: + sys.exit( + "package installation failed: we detected this code as Python 2, but 'python2' executable was not available." + "To enable automatic package installation, please install 'python2' before the 'github/codeql-action/init' step, " + "such as running 'sudo apt install python2' (Ubuntu 22.04)." + "If your code is not Python 2, but actually Python 3, please file a bug report at https://github.com/github/codeql-action/issues/new" + ) + if os.path.exists('requirements.txt'): print('Found requirements.txt, will install packages with pip', flush=True) return install_requirements_txt_packages(version) From 93ba53f2de80a13ee5ef48ee7a6f68de121da298 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 15:34:36 +0200 Subject: [PATCH 4/7] add missing spaces --- python-setup/auto_install_packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index 9731052716..edb9aa93a2 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -160,9 +160,9 @@ def install_packages(codeql_base_dir) -> Optional[str]: # error message than a traceback + `No such file or directory: 'python2'` if shutil.which("python2") is None: sys.exit( - "package installation failed: we detected this code as Python 2, but 'python2' executable was not available." + "package installation failed: we detected this code as Python 2, but 'python2' executable was not available. " "To enable automatic package installation, please install 'python2' before the 'github/codeql-action/init' step, " - "such as running 'sudo apt install python2' (Ubuntu 22.04)." + "such as running 'sudo apt install python2' (Ubuntu 22.04). " "If your code is not Python 2, but actually Python 3, please file a bug report at https://github.com/github/codeql-action/issues/new" ) From 8a893ddf189a2414cad54a3cdc31c8db322b2b27 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 15:34:44 +0200 Subject: [PATCH 5/7] python-setup: Flush even more --- python-setup/auto_install_packages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index edb9aa93a2..f58900b90f 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -154,6 +154,8 @@ def install_packages(codeql_base_dir) -> Optional[str]: # get_extractor_version returns the Python version the extractor thinks this repo is using version = extractor_version.get_extractor_version(codeql_base_dir, quiet=False) + sys.stdout.flush() + sys.stderr.flush() if version == 2 and not sys.platform.startswith('win32'): # On Ubuntu 22.04 'python2' is not available by default. We want to give a slightly better From b2fc1e178e0d4698fbfe13a3dacbd46e1f0dbe23 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 14:53:59 +0200 Subject: [PATCH 6/7] python-setup: Disable python2 tests on ubuntu-22.04 --- .github/workflows/python-deps.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml index a17d6723df..9f43459dac 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps.yml @@ -36,6 +36,9 @@ jobs: # Python2 and pipenv are not supported since pipenv v2021.11.5 - python_version: 2 python_deps_type: pipenv + # Python2 is not available on ubuntu-22.04 by default -- see https://github.com/github/codeql-action/pull/1257 + - python_version: 2 + os: ubuntu-22.04 env: From 32ca2cf500269f3c253386e63fb24f1ca7de5164 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 16:06:07 +0200 Subject: [PATCH 7/7] Apply suggestions from code review Co-authored-by: Henry Mercer --- python-setup/auto_install_packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index f58900b90f..3deb5ad75a 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -162,9 +162,9 @@ def install_packages(codeql_base_dir) -> Optional[str]: # error message than a traceback + `No such file or directory: 'python2'` if shutil.which("python2") is None: sys.exit( - "package installation failed: we detected this code as Python 2, but 'python2' executable was not available. " + "Python package installation failed: we detected this code as Python 2, but the 'python2' executable was not available. " "To enable automatic package installation, please install 'python2' before the 'github/codeql-action/init' step, " - "such as running 'sudo apt install python2' (Ubuntu 22.04). " + "for example by running 'sudo apt install python2' (Ubuntu 22.04). " "If your code is not Python 2, but actually Python 3, please file a bug report at https://github.com/github/codeql-action/issues/new" )