/
auto_install_packages.py
executable file
·196 lines (146 loc) · 8.25 KB
/
auto_install_packages.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
#!/usr/bin/env python3
import sys
import os
import subprocess
from tempfile import mkdtemp
from typing import Optional
import extractor_version
def _check_call(command, extra_env={}):
print('+ {}'.format(' '.join(command)), flush=True)
env = os.environ.copy()
env.update(extra_env)
subprocess.check_call(command, stdin=subprocess.DEVNULL, env=env)
sys.stdout.flush()
sys.stderr.flush()
def _check_output(command, extra_env={}):
print('+ {}'.format(' '.join(command)), flush=True)
env = os.environ.copy()
env.update(extra_env)
out = subprocess.check_output(command, stdin=subprocess.DEVNULL, env=env)
print(out, flush=True)
sys.stderr.flush()
return out
def install_packages_with_poetry():
# To handle poetry 1.2, which started to use keyring interaction MUCH more, we need
# add a workaround. See
# https://github.com/python-poetry/poetry/issues/2692#issuecomment-1235683370
extra_poetry_env = {"PYTHON_KEYRING_BACKEND": "keyring.backends.null.Keyring"}
command = [sys.executable, '-m', 'poetry']
if sys.platform.startswith('win32'):
# In windows the default path were the deps are installed gets wiped out between steps,
# so we have to set it up to a folder that will be kept
os.environ['POETRY_VIRTUALENVS_PATH'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs')
try:
_check_call(command + ['install', '--no-root'], extra_env=extra_poetry_env)
except subprocess.CalledProcessError:
sys.exit('package installation with poetry failed, see error above')
# poetry is super annoying with `poetry run`, since it will put lots of output on
# STDOUT if the current global python interpreter is not matching the one in the
# virtualenv for the package, which was the case for using poetry for Python 2 when
# default system interpreter was Python 3 :/
poetry_out = _check_output(command + ['run', 'which', 'python'], extra_env=extra_poetry_env)
python_executable_path = poetry_out.decode('utf-8').splitlines()[-1]
if sys.platform.startswith('win32'):
# Poetry produces a path that starts by /d instead of D:\ and Windows doesn't like that way of specifying the drive letter.
# We completely remove it because it is not needed as everything is in the same drive (We are installing the dependencies in the RUNNER_WORKSPACE)
python_executable_path = python_executable_path[2:]
return python_executable_path
def install_packages_with_pipenv(has_lockfile):
command = [sys.executable, '-m', 'pipenv']
if sys.platform.startswith('win32'):
# In windows the default path were the deps are installed gets wiped out between steps,
# so we have to set it up to a folder that will be kept
os.environ['WORKON_HOME'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs')
lock_args = ['--keep-outdated', '--ignore-pipfile'] if has_lockfile else ['--skip-lock']
try:
_check_call(command + ['install'] + lock_args)
except subprocess.CalledProcessError:
sys.exit('package installation with pipenv failed, see error above')
pipenv_out = _check_output(command + ['run', 'which', 'python'])
python_executable_path = pipenv_out.decode('utf-8').splitlines()[-1]
if sys.platform.startswith('win32'):
# Pipenv produces a path that starts by /d instead of D:\ and Windows doesn't like that way of specifying the drive letter.
# We completely remove it because it is not needed as everything is in the same drive (We are installing the dependencies in the RUNNER_WORKSPACE)
python_executable_path = python_executable_path[2:]
return python_executable_path
def _create_venv(version: int):
# create temporary directory ... that just lives "forever"
venv_path = os.path.join(os.environ['RUNNER_WORKSPACE'], 'codeql-action-python-autoinstall')
print ("Creating venv in " + venv_path, flush = True)
# virtualenv is a bit nicer for setting up virtual environment, since it will provide
# up-to-date versions of pip/setuptools/wheel which basic `python3 -m venv venv` won't
if sys.platform.startswith('win32'):
if version == 2:
_check_call(['py', '-2', '-m', 'virtualenv', venv_path])
elif version == 3:
_check_call(['py', '-3', '-m', 'virtualenv', venv_path])
else:
if version == 2:
_check_call(['python2', '-m', 'virtualenv', venv_path])
elif version == 3:
_check_call(['python3', '-m', 'virtualenv', venv_path])
return venv_path
def install_requirements_txt_packages(version: int):
venv_path = _create_venv(version)
venv_pip = os.path.join(venv_path, 'bin', 'pip')
venv_python = os.path.join(venv_path, 'bin', 'python')
if sys.platform.startswith('win32'):
venv_pip = os.path.join(venv_path, 'Scripts', 'pip')
venv_python = os.path.join(venv_path, 'Scripts', 'python')
try:
_check_call([venv_pip, 'install', '-r', 'requirements.txt'])
except subprocess.CalledProcessError:
sys.exit('package installation with `pip install -r requirements.txt` failed, see error above')
return venv_python
def install_with_setup_py(version: int):
venv_path = _create_venv(version)
venv_pip = os.path.join(venv_path, 'bin', 'pip')
venv_python = os.path.join(venv_path, 'bin', 'python')
if sys.platform.startswith('win32'):
venv_pip = os.path.join(venv_path, 'Scripts', 'pip')
venv_python = os.path.join(venv_path, 'Scripts', 'python')
try:
# We have to choose between `python setup.py develop` and `pip install -e .`.
# Modern projects use `pip install -e .` and I wasn't able to see any downsides
# to doing so. However, `python setup.py develop` has some downsides -- from
# https://stackoverflow.com/a/19048754 :
# > Note that it is highly recommended to use pip install . (install) and pip
# > install -e . (developer install) to install packages, as invoking setup.py
# > directly will do the wrong things for many dependencies, such as pull
# > prereleases and incompatible package versions, or make the package hard to
# > uninstall with pip.
_check_call([venv_pip, 'install', '-e', '.'])
except subprocess.CalledProcessError:
sys.exit('package installation with `pip install -e .` failed, see error above')
return venv_python
def install_packages(codeql_base_dir) -> Optional[str]:
if os.path.exists('poetry.lock'):
print('Found poetry.lock, will install packages with poetry', flush=True)
return install_packages_with_poetry()
if os.path.exists('Pipfile') or os.path.exists('Pipfile.lock'):
if os.path.exists('Pipfile.lock'):
print('Found Pipfile.lock, will install packages with Pipenv', flush=True)
return install_packages_with_pipenv(has_lockfile=True)
else:
print('Found Pipfile, will install packages with Pipenv', flush=True)
return install_packages_with_pipenv(has_lockfile=False)
# get_extractor_version returns the Python version the extractor thinks this repo is using
version = extractor_version.get_extractor_version(codeql_base_dir, quiet=False)
if os.path.exists('requirements.txt'):
print('Found requirements.txt, will install packages with pip', flush=True)
return install_requirements_txt_packages(version)
if os.path.exists('setup.py'):
print('Found setup.py, will install package with pip in editable mode', flush=True)
return install_with_setup_py(version)
print("was not able to install packages automatically", flush=True)
return None
if __name__ == "__main__":
if len(sys.argv) != 2:
sys.exit('Must provide base directory for codeql tool as only argument')
codeql_base_dir = sys.argv[1]
python_executable_path = install_packages(codeql_base_dir)
if python_executable_path is not None:
# see https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
env_file = open(os.environ["GITHUB_ENV"], mode="at")
print("Setting CODEQL_PYTHON={}".format(python_executable_path))
print("CODEQL_PYTHON={}".format(python_executable_path), file=env_file)