Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Install Python Dev Dependencies to Virtual Environment #15602

Merged
merged 33 commits into from
Jul 21, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
f5a5dbe
Rename install_backend_python_libs.py
U8NWXD Jun 19, 2022
17ad8fe
Install python dev dependencies into venv
U8NWXD Jun 19, 2022
1f1b0ca
Add header and fix syntax
U8NWXD Jun 21, 2022
43197a8
Merge branch 'develop' into python-dev-deps-venv
U8NWXD Jun 21, 2022
faef94b
Make no venv ok on CI
U8NWXD Jun 22, 2022
9f95216
Fix lint, type issues
U8NWXD Jul 10, 2022
3cfa760
Add backend tests for install_python_dev_dependencies.py
U8NWXD Jul 10, 2022
81091d3
Remove PYTHONPATH editing for dev dependencies
U8NWXD Jul 11, 2022
07d1526
Fix lint errors
U8NWXD Jul 11, 2022
59ee71f
Fix mypy checks script
U8NWXD Jul 11, 2022
63eb483
Fix backend tests script
U8NWXD Jul 11, 2022
4dd03c6
Assert dev requirements file stays compiled
U8NWXD Jul 11, 2022
9fcd170
Add backend test to shards
U8NWXD Jul 11, 2022
8c25efb
Removed unused pip installation code
U8NWXD Jul 11, 2022
cb647c0
Fix backend tests
U8NWXD Jul 12, 2022
d3f1e1f
Fix more backend tests
U8NWXD Jul 12, 2022
3aebc2c
Fix lint checks
U8NWXD Jul 12, 2022
29139cb
Formatting fixes
U8NWXD Jul 14, 2022
aaa4375
Update testing self.swap for nonexistent attributes
U8NWXD Jul 15, 2022
69dd997
Make virtualenv check more rigorous
U8NWXD Jul 15, 2022
815ebe6
Merge branch 'develop' into python-dev-deps-venv
U8NWXD Jul 15, 2022
19daf7f
Fix requirements_dev.in comment
U8NWXD Jul 15, 2022
46234ca
Load-balance backend test shards
U8NWXD Jul 15, 2022
97216df
Merge branch 'develop' into python-dev-deps-venv
U8NWXD Jul 15, 2022
b54f7bd
Merge branch 'develop' into python-dev-deps-venv
U8NWXD Jul 19, 2022
fd0436f
Remove scripts/regenerate_requirements.py
U8NWXD Jul 19, 2022
d147a67
Address review comments
U8NWXD Jul 19, 2022
dff8c02
Fix backend tests
U8NWXD Jul 19, 2022
30ac5fc
Fix backend test
U8NWXD Jul 19, 2022
d6de237
Address review comments
U8NWXD Jul 20, 2022
cded151
Avoid modifying GenericTestBase.swap()
U8NWXD Jul 20, 2022
0125b7b
Clarify check_python_env name
U8NWXD Jul 20, 2022
af82499
Fix typo
U8NWXD Jul 20, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 5 additions & 5 deletions scripts/common_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,11 @@
from . import common


# The pool_size argument is required by Requester.__init__(), but it is
# missing from the typing definition in Requester.pyi. We therefore
# disable type checking here. A PR was open to PyGithub to fix this, but it was
# closed due to inactivity (the project does not seem very active). Here is the
# PR: https://github.com/PyGithub/PyGithub/pull/2151.
# The pool_size argument is required by Requester.__init__(), but it is missing
# from the typing definition in Requester.pyi. We therefore disable type
# checking here. A PR was opened to PyGithub to fix this, but it was closed due
# to inactivity (the project does not seem very active). Here is the PR:
# https://github.com/PyGithub/PyGithub/pull/2151.
_MOCK_REQUESTER = github.Requester.Requester( # type: ignore
login_or_token=None,
password=None,
Expand Down
13 changes: 10 additions & 3 deletions scripts/install_python_dev_dependencies.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,15 @@
help='Assert that the dev requirements file is already compiled.')


def check_python_env() -> None:
"""Raise an error if we are not in a virtual environment or on CI."""
def check_python_env_is_suitable() -> None:
"""Raise an error if we are not in a virtual environment or on CI.

We want developers to use a virtual environment when developing locally so
that our scripts don't change their global Python environments. On CI
however, it's okay to change the global environment since the checks are
running in an epehmeral virtual machine. Therefore, a "suitable" Python
U8NWXD marked this conversation as resolved.
Show resolved Hide resolved
environment is one that either is on CI or is a virtual environment.
"""
if 'GITHUB_ACTION' in os.environ:
U8NWXD marked this conversation as resolved.
Show resolved Hide resolved
# The GITHUB_ACTION environment variable indicates we are running on
# GitHub Actions according to
Expand Down Expand Up @@ -116,7 +123,7 @@ def compile_pip_requirements(
def main(cli_args: Optional[List[str]] = None) -> None:
"""Install all dev dependencies."""
args = _PARSER.parse_args(cli_args)
check_python_env()
check_python_env_is_suitable()
install_installation_tools()
not_compiled = compile_pip_requirements(
REQUIREMENTS_DEV_FILE_PATH, COMPILED_REQUIREMENTS_DEV_FILE_PATH)
Expand Down
36 changes: 23 additions & 13 deletions scripts/install_python_dev_dependencies_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,17 +57,19 @@ def sys_real_prefix_context(
else:
delattr(sys, 'real_prefix')

def test_check_python_env_passes_when_in_venv(self) -> None:
def test_check_python_env_is_suitable_passes_when_in_venv(self) -> None:
prefix_swap = self.swap(
sys, 'prefix', '/home/user/.pyenv/versions/3.7.10')
base_prefix_swap = self.swap(
sys, 'base_prefix', '/home/user/.pyenv/versions/oppia')
real_prefix_manager = self.sys_real_prefix_context('')
environ_swap = self.swap(os, 'environ', {})
with prefix_swap, base_prefix_swap, real_prefix_manager, environ_swap:
install_python_dev_dependencies.check_python_env()
install_python_dev_dependencies.check_python_env_is_suitable()

def test_check_python_env_passes_when_in_venv_real_prefix(self) -> None:
def test_check_python_env_is_suitable_passes_when_in_venv_real_prefix(
self
) -> None:
prefix_swap = self.swap(
sys, 'prefix', '/home/user/.pyenv/versions/3.7.10')
base_prefix_swap = self.swap(
Expand All @@ -76,9 +78,9 @@ def test_check_python_env_passes_when_in_venv_real_prefix(self) -> None:
'/home/user/.pyenv/versions/oppia')
environ_swap = self.swap(os, 'environ', {})
with prefix_swap, base_prefix_swap, real_prefix_manager, environ_swap:
install_python_dev_dependencies.check_python_env()
install_python_dev_dependencies.check_python_env_is_suitable()

def test_check_python_env_fails_when_out_of_venv(self) -> None:
def test_check_python_env_is_suitable_fails_when_out_of_venv(self) -> None:
prefix_swap = self.swap(
sys, 'prefix', '/home/user/.pyenv/versions/3.7.10')
base_prefix_swap = self.swap(
Expand All @@ -88,20 +90,24 @@ def test_check_python_env_fails_when_out_of_venv(self) -> None:
expected_error = (
'Oppia must be developed within a virtual environment.')
with self.assertRaisesRegex( # type: ignore[no-untyped-call]
AssertionError, expected_error):
AssertionError, expected_error
):
with prefix_swap, base_prefix_swap, real_prefix_manager:
with environ_swap:
install_python_dev_dependencies.check_python_env()
(
install_python_dev_dependencies
.check_python_env_is_suitable()
)

def test_check_python_env_passes_when_on_ci(self) -> None:
def test_check_python_env_is_suitable_passes_when_on_ci(self) -> None:
prefix_swap = self.swap(
sys, 'prefix', '/home/user/.pyenv/versions/3.7.10')
base_prefix_swap = self.swap(
sys, 'base_prefix', '/home/user/.pyenv/versions/3.7.10')
real_prefix_manager = self.sys_real_prefix_context('')
environ_swap = self.swap(os, 'environ', {'GITHUB_ACTION': '1'})
with prefix_swap, base_prefix_swap, real_prefix_manager, environ_swap:
install_python_dev_dependencies.check_python_env()
install_python_dev_dependencies.check_python_env_is_suitable()

def test_install_installation_tools(self) -> None:
expected_tools = {
Expand Down Expand Up @@ -244,7 +250,8 @@ def mock_compile(*_args: Any) -> bool:
return False

assert_swap = self.swap_with_checks(
install_python_dev_dependencies, 'check_python_env', mock_func)
install_python_dev_dependencies, 'check_python_env_is_suitable',
mock_func)
install_tools_swap = self.swap_with_checks(
install_python_dev_dependencies,
'install_installation_tools', mock_func)
Expand All @@ -267,7 +274,8 @@ def mock_compile(*_args: Any) -> bool:
return False

assert_swap = self.swap_with_checks(
install_python_dev_dependencies, 'check_python_env', mock_func)
install_python_dev_dependencies, 'check_python_env_is_suitable',
mock_func)
install_tools_swap = self.swap_with_checks(
install_python_dev_dependencies,
'install_installation_tools', mock_func)
Expand All @@ -291,7 +299,8 @@ def mock_compile(*_args: Any) -> bool:
return True

assert_swap = self.swap_with_checks(
install_python_dev_dependencies, 'check_python_env', mock_func)
install_python_dev_dependencies, 'check_python_env_is_suitable',
mock_func)
install_tools_swap = self.swap_with_checks(
install_python_dev_dependencies,
'install_installation_tools', mock_func)
Expand All @@ -314,7 +323,8 @@ def mock_compile(*_args: Any) -> bool:
return True

assert_swap = self.swap_with_checks(
install_python_dev_dependencies, 'check_python_env', mock_func)
install_python_dev_dependencies, 'check_python_env_is_suitable',
mock_func)
install_tools_swap = self.swap_with_checks(
install_python_dev_dependencies,
'install_installation_tools', mock_func)
Expand Down