Merge main into releases/v2

.github/update-release-branch.py

@@ -67,40 +67,48 @@ def open_pr(
   body.append('Merging ' + source_branch_short_sha + ' into ' + target_branch)
 
   body.append('')
-  body.append('Conductor for this PR is @' + conductor)
+  body.append(f'Conductor for this PR is @{conductor}.')
 
   # List all PRs merged
   if len(pull_requests) > 0:
     body.append('')
     body.append('Contains the following pull requests:')
     for pr in pull_requests:
       merger = get_merger_of_pr(repo, pr)
-      body.append('- #' + str(pr.number) + ' - ' + pr.title +' (@' + merger + ')')
+      body.append(f'- #{pr.number} (@{merger})')
 
   # List all commits not part of a PR
   if len(commits_without_pull_requests) > 0:
     body.append('')
     body.append('Contains the following commits not from a pull request:')
     for commit in commits_without_pull_requests:
-      author_description = ' (@' + commit.author.login + ')' if commit.author is not None else ''
-      body.append('- ' + commit.sha + ' - ' + get_truncated_commit_message(commit) + author_description)
+      author_description = f' (@{commit.author.login})' if commit.author is not None else ''
+      body.append(f'- {commit.sha} - {get_truncated_commit_message(commit)}{author_description}')
 
   body.append('')
-  body.append('Please review the following:')
+  body.append('Please do the following:')
   if len(conflicted_files) > 0:
-    body.append(' - [ ] The `package.json` file contains the correct version.')
-    body.append(' - [ ] You have added commits to this branch that resolve the merge conflicts ' +
+    body.append(' - [ ] Ensure `package.json` file contains the correct version.')
+    body.append(' - [ ] Add commits to this branch to resolve the merge conflicts ' +
       'in the following files:')
     body.extend([f'    - [ ] `{file}`' for file in conflicted_files])
-    body.append(' - [ ] Another maintainer has reviewed the additional commits you added to this ' +
+    body.append(' - [ ] Ensure another maintainer has reviewed the additional commits you added to this ' +
       'branch to resolve the merge conflicts.')
-  body.append(' - [ ] The CHANGELOG displays the correct version and date.')
-  body.append(' - [ ] The CHANGELOG includes all relevant, user-facing changes since the last release.')
-  body.append(' - [ ] There are no unexpected commits being merged into the ' + target_branch + ' branch.')
-  body.append(' - [ ] The docs team is aware of any documentation changes that need to be released.')
+  body.append(' - [ ] Ensure the CHANGELOG displays the correct version and date.')
+  body.append(' - [ ] Ensure the CHANGELOG includes all relevant, user-facing changes since the last release.')
+  body.append(' - [ ] Check that there are not any unexpected commits being merged into the ' + target_branch + ' branch.')
+  body.append(' - [ ] Ensure the docs team is aware of any documentation changes that need to be released.')
+
+  if not is_v2_release:
+    body.append(' - [ ] Remove and re-add the "Update dependencies" label to the PR to trigger just this workflow.')
+    body.append(' - [ ] Wait for the "Update dependencies" workflow to push a commit updating the dependencies.')
+    body.append(' - [ ] Mark the PR as ready for review to trigger the full set of PR checks.')
+
+  body.append(' - [ ] Approve and merge this PR.')
+
   if is_v2_release:
-    body.append(' - [ ] The mergeback PR is merged back into ' + source_branch + ' after this PR is merged.')
-    body.append(' - [ ] The v1 release PR is merged after this PR is merged.')
+    body.append(' - [ ] Merge the mergeback PR that will automatically be created once this PR is merged.')
+    body.append(' - [ ] Merge the v1 release PR that will automatically be created once this PR is merged.')
 
   title = 'Merge ' + source_branch + ' into ' + target_branch
 

.github/workflows/post-release-mergeback.yml

@@ -114,7 +114,17 @@ jobs:
         run: |
           set -exu
           pr_title="Mergeback ${VERSION} ${HEAD_BRANCH} into ${BASE_BRANCH}"
-          pr_body="Updates version and changelog."
+          pr_body=$(cat << EOF
+            This PR bumps the version number and updates the changelog after the ${VERSION} release.
+
+            Please do the following:
+
+            - [ ] Remove and re-add the "Update dependencies" label to the PR to trigger just this workflow.
+            - [ ] Wait for the "Update dependencies" workflow to push a commit updating the dependencies.
+            - [ ] Mark the PR as ready for review to trigger the full set of PR checks.
+            - [ ] Approve and merge the PR.
+          EOF
+          )
 
           # Update the version number ready for the next release
           npm version patch --no-git-tag-version
@@ -134,4 +144,5 @@ jobs:
             --title "${pr_title}" \
             --label "Update dependencies" \
             --body "${pr_body}" \
+            --assignee "${GITHUB_ACTOR}" \
             --draft

.github/workflows/python-deps.yml

@@ -26,7 +26,7 @@ jobs:
     strategy:
         fail-fast: false
         matrix:
-          os: [ubuntu-latest, macos-latest]
+          os: [ubuntu-latest, ubuntu-22.04, macos-latest]
           python_deps_type: [pipenv, poetry, requirements, setup_py]
           python_version: [2, 3]
           exclude:
@@ -36,6 +36,9 @@ jobs:
             # Python2 and pipenv are not supported since pipenv v2021.11.5
             - python_version: 2
               python_deps_type: pipenv
+            # Python2 is not available on ubuntu-22.04 by default -- see https://github.com/github/codeql-action/pull/1257
+            - python_version: 2
+              os: ubuntu-22.04
 
 
     env:
@@ -63,6 +66,7 @@ jobs:
 
         case ${{ matrix.os }} in
             ubuntu-latest*)     basePath="/opt";;
+            ubuntu-22.04*)     basePath="/opt";;
             macos-latest*)    basePath="/Users/runner";;
         esac
         echo ${basePath}
@@ -86,7 +90,7 @@ jobs:
     strategy:
         fail-fast: false
         matrix:
-          os: [ubuntu-latest, macos-latest]
+          os: [ubuntu-latest, ubuntu-22.04, macos-latest]
 
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
@@ -109,6 +113,7 @@ jobs:
 
         case ${{ matrix.os }} in
             ubuntu-latest*)     basePath="/opt";;
+            ubuntu-22.04*)     basePath="/opt";;
             macos-latest*)    basePath="/Users/runner";;
         esac
         echo ${basePath}

CHANGELOG.md

@@ -1,5 +1,10 @@
 # CodeQL Action Changelog
 
+## 2.1.25 - 21 Sep 2022
+
+- We will soon be rolling out a feature of the CodeQL Action that stores some information used to make future runs faster in the GitHub Actions cache. Initially, this will only be enabled on JavaScript repositories, but we plan to add more languages to this soon. The new feature can be disabled by passing the `trap-caching: false` option to your workflow's `init` step, for example if you are already using the GitHub Actions cache for a different purpose and are near the storage limit for it.
+- Add support for Python automatic dependency installation with Poetry 1.2 [#1258](https://github.com/github/codeql-action/pull/1258).
+
 ## 2.1.24 - 16 Sep 2022
 
 No user facing changes.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "2.1.24",
+  "version": "2.1.25",
   "private": true,
   "description": "CodeQL action",
   "scripts": {

python-setup/auto_install_packages.py

@@ -5,31 +5,46 @@
 import subprocess
 from tempfile import mkdtemp
 from typing import Optional
+import shutil
 
 import extractor_version
 
 
-def _check_call(command):
+def _check_call(command, extra_env={}):
     print('+ {}'.format(' '.join(command)), flush=True)
-    subprocess.check_call(command, stdin=subprocess.DEVNULL)
+
+    env = os.environ.copy()
+    env.update(extra_env)
+    subprocess.check_call(command, stdin=subprocess.DEVNULL, env=env)
+    sys.stdout.flush()
+    sys.stderr.flush()
 
 
-def _check_output(command):
+def _check_output(command, extra_env={}):
     print('+ {}'.format(' '.join(command)), flush=True)
-    out = subprocess.check_output(command, stdin=subprocess.DEVNULL)
+
+    env = os.environ.copy()
+    env.update(extra_env)
+    out = subprocess.check_output(command, stdin=subprocess.DEVNULL, env=env)
     print(out, flush=True)
     sys.stderr.flush()
     return out
 
 
 def install_packages_with_poetry():
+
+    # To handle poetry 1.2, which started to use keyring interaction MUCH more, we need
+    # add a workaround. See
+    # https://github.com/python-poetry/poetry/issues/2692#issuecomment-1235683370
+    extra_poetry_env = {"PYTHON_KEYRING_BACKEND": "keyring.backends.null.Keyring"}
+
     command = [sys.executable, '-m', 'poetry']
     if sys.platform.startswith('win32'):
         # In windows the default path were the deps are installed gets wiped out between steps,
         # so we have to set it up to a folder that will be kept
         os.environ['POETRY_VIRTUALENVS_PATH'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs')
     try:
-        _check_call(command + ['install', '--no-root'])
+        _check_call(command + ['install', '--no-root'], extra_env=extra_poetry_env)
     except subprocess.CalledProcessError:
         sys.exit('package installation with poetry failed, see error above')
 
@@ -38,7 +53,7 @@ def install_packages_with_poetry():
     # virtualenv for the package, which was the case for using poetry for Python 2 when
     # default system interpreter was Python 3 :/
 
-    poetry_out = _check_output(command + ['run', 'which', 'python'])
+    poetry_out = _check_output(command + ['run', 'which', 'python'], extra_env=extra_poetry_env)
     python_executable_path = poetry_out.decode('utf-8').splitlines()[-1]
 
     if sys.platform.startswith('win32'):
@@ -153,6 +168,19 @@ def install_packages(codeql_base_dir) -> Optional[str]:
 
     # get_extractor_version returns the Python version the extractor thinks this repo is using
     version = extractor_version.get_extractor_version(codeql_base_dir, quiet=False)
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    if version == 2 and not sys.platform.startswith('win32'):
+        # On Ubuntu 22.04 'python2' is not available by default. We want to give a slightly better
+        # error message than a traceback + `No such file or directory: 'python2'`
+        if shutil.which("python2") is None:
+            sys.exit(
+                "Python package installation failed: we detected this code as Python 2, but the 'python2' executable was not available. "
+                "To enable automatic package installation, please install 'python2' before the 'github/codeql-action/init' step, "
+                "for example by running 'sudo apt install python2' (Ubuntu 22.04). "
+                "If your code is not Python 2, but actually Python 3, please file a bug report at https://github.com/github/codeql-action/issues/new"
+            )
 
     if os.path.exists('requirements.txt'):
         print('Found requirements.txt, will install packages with pip', flush=True)

python-setup/install_tools.ps1

@@ -1,13 +1,16 @@
 #! /usr/bin/pwsh
 
-py -2 -m pip install --user --upgrade pip setuptools wheel
-py -3 -m pip install --user --upgrade pip setuptools wheel
+# while waiting for the next release of `virtualenv` after v20.16.5, we install an older
+# version of `setuptools` to ensure that binaries are always put under
+# `<venv-path>/bin`, which wouldn't always happen with the GitHub actions version of
+# Ubuntu 22.04. See https://github.com/github/codeql-action/issues/1249
+py -2 -m pip install --user --upgrade pip 'setuptools<60' wheel
+py -3 -m pip install --user --upgrade pip 'setuptools<60' wheel
 
 # virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of
 # pip/setuptools/wheel which basic `python3 -m venv venv` won't
-py -2 -m pip install --user 'virtualenv<20.11'
-py -3 -m pip install --user 'virtualenv<20.11'
+py -2 -m pip install --user 'virtualenv!=20.12.0'
+py -3 -m pip install --user virtualenv
 
-# We aren't compatible with poetry 1.2
-py -3 -m pip install --user "poetry>=1.1,<1.2"
+py -3 -m pip install --user "poetry>=1.1"
 py -3 -m pip install --user pipenv

python-setup/install_tools.sh

@@ -11,11 +11,17 @@ set -e
 export PATH="$HOME/.local/bin:$PATH"
 
 # Setup Python 3 dependency installation tools.
-python3 -m pip install --user --upgrade pip setuptools wheel
+
+# we install an older version of `setuptools` to ensure that binaries are always put
+# under `<venv-path>/bin`, which wouldn't always happen with the GitHub actions version
+# of Ubuntu 22.04. See https://github.com/github/codeql-action/issues/1249. The the next
+# release of `virtualenv` after v20.16.5 will include a fix for this, so we can remove
+# this bit of the logic again.
+python3 -m pip install --user --upgrade pip 'setuptools<60' wheel
 
 # virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of
 # pip/setuptools/wheel which basic `python3 -m venv venv` won't
-python3 -m pip install --user 'virtualenv<20.11'
+python3 -m pip install --user virtualenv
 
 # We install poetry with pip instead of the recommended way, since the recommended way
 # caused some problem since `poetry run` gives output like:
@@ -24,8 +30,7 @@ python3 -m pip install --user 'virtualenv<20.11'
 #       "program uses threads.", RuntimeWarning)
 #     LGTM_PYTHON_SETUP_VERSION=The currently activated Python version 2.7.18 is not supported by the project (^3.5). Trying to find and use a compatible version. Using python3 (3.8.2) 3
 
-# We aren't compatible with poetry 1.2
-python3 -m pip install --user "poetry>=1.1,<1.2"
+python3 -m pip install --user "poetry>=1.1"
 python3 -m pip install --user pipenv
 
 if command -v python2 >/dev/null 2>&1; then
@@ -40,7 +45,7 @@ if command -v python2 >/dev/null 2>&1; then
 		curl --location --fail https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2
 	fi
 
-	python2 -m pip install --user --upgrade pip setuptools wheel
+	python2 -m pip install --user --upgrade pip 'setuptools<60' wheel
 
-	python2 -m pip install --user 'virtualenv<20.11'
+	python2 -m pip install --user 'virtualenv!=20.12.0'
 fi