diff --git a/.github/update-release-branch.py b/.github/update-release-branch.py index 7697e59430..9d20e38ebf 100644 --- a/.github/update-release-branch.py +++ b/.github/update-release-branch.py @@ -67,7 +67,7 @@ def open_pr( body.append('Merging ' + source_branch_short_sha + ' into ' + target_branch) body.append('') - body.append('Conductor for this PR is @' + conductor) + body.append(f'Conductor for this PR is @{conductor}.') # List all PRs merged if len(pull_requests) > 0: @@ -75,32 +75,40 @@ def open_pr( body.append('Contains the following pull requests:') for pr in pull_requests: merger = get_merger_of_pr(repo, pr) - body.append('- #' + str(pr.number) + ' - ' + pr.title +' (@' + merger + ')') + body.append(f'- #{pr.number} (@{merger})') # List all commits not part of a PR if len(commits_without_pull_requests) > 0: body.append('') body.append('Contains the following commits not from a pull request:') for commit in commits_without_pull_requests: - author_description = ' (@' + commit.author.login + ')' if commit.author is not None else '' - body.append('- ' + commit.sha + ' - ' + get_truncated_commit_message(commit) + author_description) + author_description = f' (@{commit.author.login})' if commit.author is not None else '' + body.append(f'- {commit.sha} - {get_truncated_commit_message(commit)}{author_description}') body.append('') - body.append('Please review the following:') + body.append('Please do the following:') if len(conflicted_files) > 0: - body.append(' - [ ] The `package.json` file contains the correct version.') - body.append(' - [ ] You have added commits to this branch that resolve the merge conflicts ' + + body.append(' - [ ] Ensure `package.json` file contains the correct version.') + body.append(' - [ ] Add commits to this branch to resolve the merge conflicts ' + 'in the following files:') body.extend([f' - [ ] `{file}`' for file in conflicted_files]) - body.append(' - [ ] Another maintainer has reviewed the additional commits you added to this ' + + body.append(' - [ ] Ensure another maintainer has reviewed the additional commits you added to this ' + 'branch to resolve the merge conflicts.') - body.append(' - [ ] The CHANGELOG displays the correct version and date.') - body.append(' - [ ] The CHANGELOG includes all relevant, user-facing changes since the last release.') - body.append(' - [ ] There are no unexpected commits being merged into the ' + target_branch + ' branch.') - body.append(' - [ ] The docs team is aware of any documentation changes that need to be released.') + body.append(' - [ ] Ensure the CHANGELOG displays the correct version and date.') + body.append(' - [ ] Ensure the CHANGELOG includes all relevant, user-facing changes since the last release.') + body.append(' - [ ] Check that there are not any unexpected commits being merged into the ' + target_branch + ' branch.') + body.append(' - [ ] Ensure the docs team is aware of any documentation changes that need to be released.') + + if not is_v2_release: + body.append(' - [ ] Remove and re-add the "Update dependencies" label to the PR to trigger just this workflow.') + body.append(' - [ ] Wait for the "Update dependencies" workflow to push a commit updating the dependencies.') + body.append(' - [ ] Mark the PR as ready for review to trigger the full set of PR checks.') + + body.append(' - [ ] Approve and merge this PR.') + if is_v2_release: - body.append(' - [ ] The mergeback PR is merged back into ' + source_branch + ' after this PR is merged.') - body.append(' - [ ] The v1 release PR is merged after this PR is merged.') + body.append(' - [ ] Merge the mergeback PR that will automatically be created once this PR is merged.') + body.append(' - [ ] Merge the v1 release PR that will automatically be created once this PR is merged.') title = 'Merge ' + source_branch + ' into ' + target_branch diff --git a/.github/workflows/post-release-mergeback.yml b/.github/workflows/post-release-mergeback.yml index f7a58dce21..7b8b8a7768 100644 --- a/.github/workflows/post-release-mergeback.yml +++ b/.github/workflows/post-release-mergeback.yml @@ -114,7 +114,17 @@ jobs: run: | set -exu pr_title="Mergeback ${VERSION} ${HEAD_BRANCH} into ${BASE_BRANCH}" - pr_body="Updates version and changelog." + pr_body=$(cat << EOF + This PR bumps the version number and updates the changelog after the ${VERSION} release. + + Please do the following: + + - [ ] Remove and re-add the "Update dependencies" label to the PR to trigger just this workflow. + - [ ] Wait for the "Update dependencies" workflow to push a commit updating the dependencies. + - [ ] Mark the PR as ready for review to trigger the full set of PR checks. + - [ ] Approve and merge the PR. + EOF + ) # Update the version number ready for the next release npm version patch --no-git-tag-version @@ -134,4 +144,5 @@ jobs: --title "${pr_title}" \ --label "Update dependencies" \ --body "${pr_body}" \ + --assignee "${GITHUB_ACTOR}" \ --draft diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml index 3d64f171a3..9f43459dac 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps.yml @@ -26,7 +26,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, ubuntu-22.04, macos-latest] python_deps_type: [pipenv, poetry, requirements, setup_py] python_version: [2, 3] exclude: @@ -36,6 +36,9 @@ jobs: # Python2 and pipenv are not supported since pipenv v2021.11.5 - python_version: 2 python_deps_type: pipenv + # Python2 is not available on ubuntu-22.04 by default -- see https://github.com/github/codeql-action/pull/1257 + - python_version: 2 + os: ubuntu-22.04 env: @@ -63,6 +66,7 @@ jobs: case ${{ matrix.os }} in ubuntu-latest*) basePath="/opt";; + ubuntu-22.04*) basePath="/opt";; macos-latest*) basePath="/Users/runner";; esac echo ${basePath} @@ -86,7 +90,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, ubuntu-22.04, macos-latest] steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it @@ -109,6 +113,7 @@ jobs: case ${{ matrix.os }} in ubuntu-latest*) basePath="/opt";; + ubuntu-22.04*) basePath="/opt";; macos-latest*) basePath="/Users/runner";; esac echo ${basePath} diff --git a/CHANGELOG.md b/CHANGELOG.md index c2eea74753..1e605219b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # CodeQL Action Changelog +## 2.1.25 - 21 Sep 2022 + +- We will soon be rolling out a feature of the CodeQL Action that stores some information used to make future runs faster in the GitHub Actions cache. Initially, this will only be enabled on JavaScript repositories, but we plan to add more languages to this soon. The new feature can be disabled by passing the `trap-caching: false` option to your workflow's `init` step, for example if you are already using the GitHub Actions cache for a different purpose and are near the storage limit for it. +- Add support for Python automatic dependency installation with Poetry 1.2 [#1258](https://github.com/github/codeql-action/pull/1258). + ## 2.1.24 - 16 Sep 2022 No user facing changes. diff --git a/node_modules/.package-lock.json b/node_modules/.package-lock.json index 68e009d438..5fd07b39cb 100644 --- a/node_modules/.package-lock.json +++ b/node_modules/.package-lock.json @@ -1,6 +1,6 @@ { "name": "codeql", - "version": "2.1.24", + "version": "2.1.25", "lockfileVersion": 2, "requires": true, "packages": { diff --git a/package-lock.json b/package-lock.json index 068c0b88a2..578467a0a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "codeql", - "version": "2.1.24", + "version": "2.1.25", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "codeql", - "version": "2.1.24", + "version": "2.1.25", "license": "MIT", "dependencies": { "@actions/artifact": "^1.0.0", diff --git a/package.json b/package.json index 749671cc3e..ce0bb499f1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codeql", - "version": "2.1.24", + "version": "2.1.25", "private": true, "description": "CodeQL action", "scripts": { diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index b0a623735c..0e3a5fa1fd 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -5,31 +5,46 @@ import subprocess from tempfile import mkdtemp from typing import Optional +import shutil import extractor_version -def _check_call(command): +def _check_call(command, extra_env={}): print('+ {}'.format(' '.join(command)), flush=True) - subprocess.check_call(command, stdin=subprocess.DEVNULL) + + env = os.environ.copy() + env.update(extra_env) + subprocess.check_call(command, stdin=subprocess.DEVNULL, env=env) + sys.stdout.flush() + sys.stderr.flush() -def _check_output(command): +def _check_output(command, extra_env={}): print('+ {}'.format(' '.join(command)), flush=True) - out = subprocess.check_output(command, stdin=subprocess.DEVNULL) + + env = os.environ.copy() + env.update(extra_env) + out = subprocess.check_output(command, stdin=subprocess.DEVNULL, env=env) print(out, flush=True) sys.stderr.flush() return out def install_packages_with_poetry(): + + # To handle poetry 1.2, which started to use keyring interaction MUCH more, we need + # add a workaround. See + # https://github.com/python-poetry/poetry/issues/2692#issuecomment-1235683370 + extra_poetry_env = {"PYTHON_KEYRING_BACKEND": "keyring.backends.null.Keyring"} + command = [sys.executable, '-m', 'poetry'] if sys.platform.startswith('win32'): # In windows the default path were the deps are installed gets wiped out between steps, # so we have to set it up to a folder that will be kept os.environ['POETRY_VIRTUALENVS_PATH'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs') try: - _check_call(command + ['install', '--no-root']) + _check_call(command + ['install', '--no-root'], extra_env=extra_poetry_env) except subprocess.CalledProcessError: sys.exit('package installation with poetry failed, see error above') @@ -38,7 +53,7 @@ def install_packages_with_poetry(): # virtualenv for the package, which was the case for using poetry for Python 2 when # default system interpreter was Python 3 :/ - poetry_out = _check_output(command + ['run', 'which', 'python']) + poetry_out = _check_output(command + ['run', 'which', 'python'], extra_env=extra_poetry_env) python_executable_path = poetry_out.decode('utf-8').splitlines()[-1] if sys.platform.startswith('win32'): @@ -153,6 +168,19 @@ def install_packages(codeql_base_dir) -> Optional[str]: # get_extractor_version returns the Python version the extractor thinks this repo is using version = extractor_version.get_extractor_version(codeql_base_dir, quiet=False) + sys.stdout.flush() + sys.stderr.flush() + + if version == 2 and not sys.platform.startswith('win32'): + # On Ubuntu 22.04 'python2' is not available by default. We want to give a slightly better + # error message than a traceback + `No such file or directory: 'python2'` + if shutil.which("python2") is None: + sys.exit( + "Python package installation failed: we detected this code as Python 2, but the 'python2' executable was not available. " + "To enable automatic package installation, please install 'python2' before the 'github/codeql-action/init' step, " + "for example by running 'sudo apt install python2' (Ubuntu 22.04). " + "If your code is not Python 2, but actually Python 3, please file a bug report at https://github.com/github/codeql-action/issues/new" + ) if os.path.exists('requirements.txt'): print('Found requirements.txt, will install packages with pip', flush=True) diff --git a/python-setup/install_tools.ps1 b/python-setup/install_tools.ps1 index 3c78378a3d..353edfe4fd 100644 --- a/python-setup/install_tools.ps1 +++ b/python-setup/install_tools.ps1 @@ -1,13 +1,16 @@ #! /usr/bin/pwsh -py -2 -m pip install --user --upgrade pip setuptools wheel -py -3 -m pip install --user --upgrade pip setuptools wheel +# while waiting for the next release of `virtualenv` after v20.16.5, we install an older +# version of `setuptools` to ensure that binaries are always put under +# `/bin`, which wouldn't always happen with the GitHub actions version of +# Ubuntu 22.04. See https://github.com/github/codeql-action/issues/1249 +py -2 -m pip install --user --upgrade pip 'setuptools<60' wheel +py -3 -m pip install --user --upgrade pip 'setuptools<60' wheel # virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of # pip/setuptools/wheel which basic `python3 -m venv venv` won't -py -2 -m pip install --user 'virtualenv<20.11' -py -3 -m pip install --user 'virtualenv<20.11' +py -2 -m pip install --user 'virtualenv!=20.12.0' +py -3 -m pip install --user virtualenv -# We aren't compatible with poetry 1.2 -py -3 -m pip install --user "poetry>=1.1,<1.2" +py -3 -m pip install --user "poetry>=1.1" py -3 -m pip install --user pipenv diff --git a/python-setup/install_tools.sh b/python-setup/install_tools.sh index 7acb33f146..a247da4ead 100755 --- a/python-setup/install_tools.sh +++ b/python-setup/install_tools.sh @@ -11,11 +11,17 @@ set -e export PATH="$HOME/.local/bin:$PATH" # Setup Python 3 dependency installation tools. -python3 -m pip install --user --upgrade pip setuptools wheel + +# we install an older version of `setuptools` to ensure that binaries are always put +# under `/bin`, which wouldn't always happen with the GitHub actions version +# of Ubuntu 22.04. See https://github.com/github/codeql-action/issues/1249. The the next +# release of `virtualenv` after v20.16.5 will include a fix for this, so we can remove +# this bit of the logic again. +python3 -m pip install --user --upgrade pip 'setuptools<60' wheel # virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of # pip/setuptools/wheel which basic `python3 -m venv venv` won't -python3 -m pip install --user 'virtualenv<20.11' +python3 -m pip install --user virtualenv # We install poetry with pip instead of the recommended way, since the recommended way # caused some problem since `poetry run` gives output like: @@ -24,8 +30,7 @@ python3 -m pip install --user 'virtualenv<20.11' # "program uses threads.", RuntimeWarning) # LGTM_PYTHON_SETUP_VERSION=The currently activated Python version 2.7.18 is not supported by the project (^3.5). Trying to find and use a compatible version. Using python3 (3.8.2) 3 -# We aren't compatible with poetry 1.2 -python3 -m pip install --user "poetry>=1.1,<1.2" +python3 -m pip install --user "poetry>=1.1" python3 -m pip install --user pipenv if command -v python2 >/dev/null 2>&1; then @@ -40,7 +45,7 @@ if command -v python2 >/dev/null 2>&1; then curl --location --fail https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2 fi - python2 -m pip install --user --upgrade pip setuptools wheel + python2 -m pip install --user --upgrade pip 'setuptools<60' wheel - python2 -m pip install --user 'virtualenv<20.11' + python2 -m pip install --user 'virtualenv!=20.12.0' fi