Install Python Dev Dependencies to Virtual Environment (#15602)

.github/CODEOWNERS

@@ -742,6 +742,8 @@
 /core/domain/user*.py @seanlip
 /requirements.txt @vojtechjelinek
 /requirements.in @vojtechjelinek
+/requirements_dev.txt @vojtechjelinek
+/requirements_dev.in @vojtechjelinek
 /dependencies.json @vojtechjelinek
 /package.json @vojtechjelinek
 /yarn.lock @vojtechjelinek

core/utils.py

@@ -27,8 +27,8 @@
 import os
 import random
 import re
+import ssl
 import string
-import sys
 import time
 import unicodedata
 import urllib.parse
@@ -38,21 +38,14 @@
 from core import feconf
 from core.constants import constants
 
+import certifi
+import yaml
+
 from typing import ( # isort:skip
     IO, Any, BinaryIO, Callable, Dict, Iterable, Iterator, List, Optional,
     TextIO, Tuple, TypeVar, Union, overload)
 from typing_extensions import Literal # isort:skip
 
-_YAML_PATH = os.path.join(os.getcwd(), '..', 'oppia_tools', 'pyyaml-6.0')
-sys.path.insert(0, _YAML_PATH)
-
-_CERTIFI_PATH = os.path.join(
-    os.getcwd(), '..', 'oppia_tools', 'certifi-2021.10.8')
-sys.path.insert(0, _CERTIFI_PATH)
-
-import yaml  # isort:skip  # pylint: disable=wrong-import-position
-import certifi  # isort:skip  pylint: disable=wrong-import-position, wrong-import-order
-import ssl  # isort:skip  pylint: disable=wrong-import-position, wrong-import-order
 
 DATETIME_FORMAT = '%m/%d/%Y, %H:%M:%S:%f'
 ISO_8601_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fz'

requirements.txt

@@ -6,7 +6,7 @@
 # This file is autogenerated by pip-compile with python 3.7
 # To update, run:
 #
-#    pip-compile --no-emit-index-url
+#    pip-compile --no-emit-index-url --output-file=requirements.txt requirements.in
 #
 apache-beam[gcp]==2.38.0
     # via -r requirements.in

requirements_dev.in

@@ -0,0 +1,22 @@
+# NOTE TO DEVELOPERS: If you make changes to this file, please run
+#
+#    python -m scripts.install_python_dev_dependencies
+#
+# to update `requirements_dev.txt` and your current Python environment.
+Pillow==9.0.1
+PyGithub==1.55
+certifi==2021.10.8
+coverage==6.1.2
+esprima==4.0.1
+future==0.18.2
+grpcio==1.41.1
+isort==5.10.1
+protobuf==3.13.0
+psutil==5.8.0
+pycodestyle==2.8.0
+pylint==2.11.1
+git+https://github.com/oppia/pylint-quotes.git
+pyyaml==6.0
+six==1.16.0
+typing-extensions==4.0.1
+webtest==3.0.0

requirements_dev.txt

@@ -0,0 +1,97 @@
+#
+# This file is autogenerated by pip-compile with python 3.7
+# To update, run:
+#
+#    pip-compile --no-emit-index-url --output-file=requirements_dev.txt requirements_dev.in
+#
+astroid==2.8.6
+    # via pylint
+beautifulsoup4==4.11.1
+    # via webtest
+certifi==2021.10.8
+    # via
+    #   -r requirements_dev.in
+    #   requests
+cffi==1.15.0
+    # via pynacl
+charset-normalizer==2.0.12
+    # via requests
+coverage==6.1.2
+    # via -r requirements_dev.in
+deprecated==1.2.13
+    # via pygithub
+esprima==4.0.1
+    # via -r requirements_dev.in
+future==0.18.2
+    # via -r requirements_dev.in
+grpcio==1.41.1
+    # via -r requirements_dev.in
+idna==3.3
+    # via requests
+isort==5.10.1
+    # via
+    #   -r requirements_dev.in
+    #   pylint
+lazy-object-proxy==1.7.1
+    # via astroid
+mccabe==0.6.1
+    # via pylint
+pillow==9.0.1
+    # via -r requirements_dev.in
+platformdirs==2.5.2
+    # via pylint
+protobuf==3.13.0
+    # via -r requirements_dev.in
+psutil==5.8.0
+    # via -r requirements_dev.in
+pycodestyle==2.8.0
+    # via -r requirements_dev.in
+pycparser==2.21
+    # via cffi
+pygithub==1.55
+    # via -r requirements_dev.in
+pyjwt==2.4.0
+    # via pygithub
+pylint==2.11.1
+    # via
+    #   -r requirements_dev.in
+    #   pylint-quotes
+pylint-quotes @ git+https://github.com/oppia/pylint-quotes.git
+    # via -r requirements_dev.in
+pynacl==1.5.0
+    # via pygithub
+pyyaml==6.0
+    # via -r requirements_dev.in
+requests==2.27.1
+    # via pygithub
+six==1.16.0
+    # via
+    #   -r requirements_dev.in
+    #   grpcio
+    #   protobuf
+soupsieve==2.3.2.post1
+    # via beautifulsoup4
+toml==0.10.2
+    # via pylint
+typed-ast==1.5.4
+    # via astroid
+typing-extensions==4.0.1
+    # via
+    #   -r requirements_dev.in
+    #   astroid
+    #   pylint
+urllib3==1.26.9
+    # via requests
+waitress==2.1.1
+    # via webtest
+webob==1.8.7
+    # via webtest
+webtest==3.0.0
+    # via -r requirements_dev.in
+wrapt==1.13.3
+    # via
+    #   astroid
+    #   deprecated
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

scripts/backend_test_shards.json

@@ -102,7 +102,8 @@
         "scripts.linters.linter_utils_test",
         "scripts.linters.python_linter_test",
         "core.platform.auth.firebase_auth_services_test",
-        "scripts.install_backend_python_libs_test",
+        "scripts.install_python_prod_dependencies_test",
+        "scripts.install_python_dev_dependencies_test",
         "scripts.common_test",
         "core.controllers.beam_jobs_test",
         "core.controllers.contributor_dashboard_admin_test",
@@ -297,7 +298,6 @@
         "core.domain.translation_services_test",
         "core.storage.recommendations.gae_models_test",
         "scripts.linters.other_files_linter_test",
-        "scripts.regenerate_requirements_test",
         "scripts.install_third_party_test",
         "scripts.check_frontend_test_coverage_test",
         "core.platform.models_test",

scripts/backend_tests_incomplete_coverage.txt

@@ -12,7 +12,7 @@ core.domain.auth_services_test
 core.domain.opportunity_services_test
 core.domain.feedback_domain_test
 core.domain.blog_domain_test
-scripts.install_backend_python_libs_test
+scripts.install_python_prod_dependencies_test
 core.domain.subtopic_page_domain_test
 core.storage.skill.gae_models_test
 core.storage.topic.gae_models_test

scripts/common.py

@@ -44,21 +44,6 @@
 
 CURRENT_PYTHON_BIN = sys.executable
 
-# Versions of libraries used in devflow.
-COVERAGE_VERSION = '6.1.2'
-ESPRIMA_VERSION = '4.0.1'
-ISORT_VERSION = '5.10.1'
-PYCODESTYLE_VERSION = '2.8.0'
-PSUTIL_VERSION = '5.8.0'
-PYLINT_VERSION = '2.11.1'
-PYLINT_QUOTES_VERSION = '0.2.4'
-PYGITHUB_VERSION = '1.55'
-WEBTEST_VERSION = '3.0.0'
-PIP_TOOLS_VERSION = '6.6.2'
-GRPCIO_VERSION = '1.41.1'
-PROTOBUF_VERSION = '3.13.0'
-SETUPTOOLS_VERSION = '58.5.3'
-
 # Node version.
 NODE_VERSION = '16.13.0'
 
@@ -70,9 +55,9 @@
 
 # Buf version.
 BUF_VERSION = '0.29.0'
-# Protoc is the compiler for protobuf files and the version must be same as
-# the version of protobuf library being used.
-PROTOC_VERSION = PROTOBUF_VERSION
+
+# Must match the version of protobuf in requirements_dev.in.
+PROTOC_VERSION = '3.13.0'
 
 # IMPORTANT STEPS FOR DEVELOPERS TO UPGRADE REDIS:
 # 1. Download the new version of the redis cli.
@@ -103,28 +88,19 @@
 GOOGLE_APP_ENGINE_SDK_HOME = os.path.join(
     GOOGLE_CLOUD_SDK_HOME, 'platform', 'google_appengine')
 GOOGLE_CLOUD_SDK_BIN = os.path.join(GOOGLE_CLOUD_SDK_HOME, 'bin')
-ISORT_PATH = os.path.join(OPPIA_TOOLS_DIR, 'isort-%s' % ISORT_VERSION)
 WEBPACK_BIN_PATH = (
     os.path.join(CURR_DIR, 'node_modules', 'webpack', 'bin', 'webpack.js'))
 DEV_APPSERVER_PATH = (
     os.path.join(GOOGLE_CLOUD_SDK_BIN, 'dev_appserver.py'))
 GCLOUD_PATH = os.path.join(GOOGLE_CLOUD_SDK_BIN, 'gcloud')
 NODE_PATH = os.path.join(OPPIA_TOOLS_DIR, 'node-%s' % NODE_VERSION)
-PYLINT_PATH = os.path.join(OPPIA_TOOLS_DIR, 'pylint-%s' % PYLINT_VERSION)
-PYCODESTYLE_PATH = os.path.join(
-    OPPIA_TOOLS_DIR, 'pycodestyle-%s' % PYCODESTYLE_VERSION)
-PYLINT_QUOTES_PATH = os.path.join(
-    OPPIA_TOOLS_DIR, 'pylint-quotes-%s' % PYLINT_QUOTES_VERSION)
-PY_GITHUB_PATH = os.path.join(
-    OPPIA_TOOLS_DIR, 'PyGithub-%s' % PYGITHUB_VERSION)
 NODE_MODULES_PATH = os.path.join(CURR_DIR, 'node_modules')
 FRONTEND_DIR = os.path.join(CURR_DIR, 'core', 'templates')
 YARN_PATH = os.path.join(OPPIA_TOOLS_DIR, 'yarn-%s' % YARN_VERSION)
 FIREBASE_PATH = os.path.join(
     NODE_MODULES_PATH, 'firebase-tools', 'lib', 'bin', 'firebase.js')
 OS_NAME = platform.system()
 ARCHITECTURE = platform.machine()
-PSUTIL_DIR = os.path.join(OPPIA_TOOLS_DIR, 'psutil-%s' % PSUTIL_VERSION)
 REDIS_SERVER_PATH = os.path.join(
     OPPIA_TOOLS_DIR, 'redis-cli-%s' % REDIS_CLI_VERSION,
     'src', 'redis-server')
@@ -138,7 +114,6 @@
 FIREBASE_EMULATOR_CACHE_DIR = (
     os.path.join(CURR_DIR, os.pardir, 'firebase_emulator_cache'))
 
-sys.path.insert(0, PY_GITHUB_PATH)
 # By specifying this condition, we are importing the below module only while
 # type checking, not in runtime.
 MYPY = False
@@ -206,15 +181,7 @@
 
 DIRS_TO_ADD_TO_SYS_PATH = [
     GOOGLE_APP_ENGINE_SDK_HOME,
-    PYLINT_PATH,
-    os.path.join(OPPIA_TOOLS_DIR, 'webtest-%s' % WEBTEST_VERSION),
-    os.path.join(OPPIA_TOOLS_DIR, 'Pillow-%s' % PILLOW_VERSION),
-    os.path.join(OPPIA_TOOLS_DIR, 'protobuf-%s' % PROTOBUF_VERSION),
-    PSUTIL_DIR,
     os.path.join(CURR_DIR, 'proto_files'),
-    os.path.join(OPPIA_TOOLS_DIR, 'grpcio-%s' % GRPCIO_VERSION),
-    os.path.join(OPPIA_TOOLS_DIR, 'setuptools-%s' % '36.6.0'),
-    PY_GITHUB_PATH,
     CURR_DIR,
     THIRD_PARTY_PYTHON_LIBS_DIR,
 ]