Consolidate all requirements

Fix gh-2588.

Signed-off-by: Juan Luis Cano Rodríguez <juan_luis_cano@mckinsey.com>

.circleci/continue_config.yml

@@ -61,7 +61,7 @@ commands:
           command: conda install -c conda-forge pytables -y
       - run:
           name: Install requirements and test requirements
-          command: pip install --upgrade -r test_requirements.txt
+          command: pip install --upgrade .[test]
       - run:
           # this is needed to fix java cacerts so
           # spark can automatically download packages from mvn
@@ -146,7 +146,7 @@ commands:
     steps:
       - restore_cache:
           name: Restore package cache
-          key: kedro-deps-v1-win-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
+          key: kedro-deps-v1-win-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }}
         # We don't restore the conda environment cache for python 3.10 as it conflicts with the
         # 'Install GDAL, Fiona and pytables' step breaking the conda environment (missing zlib.dll).
       - unless:
@@ -155,7 +155,7 @@ commands:
           steps:
             - restore_cache:
                 name: Restore conda environment cache
-                key: kedro-deps-v1-win-<<parameters.python_version>>-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
+                key: kedro-deps-v1-win-<<parameters.python_version>>-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }}
         # pytables and Fiona have a series of binary dependencies under Windows that
         # are best handled by conda-installing instead of pip-installing them.
         # Dependency resolution works best when installing these altogether in one
@@ -168,7 +168,7 @@ commands:
           command: conda activate kedro_builder; pip debug --verbose
       - run:
           name: Install all requirements
-          command: conda activate kedro_builder; pip install -v -r test_requirements.txt -U
+          command: conda activate kedro_builder; pip install -v -U .[test]
       - run:
           name: Print Python environment
           command: conda activate kedro_builder; make print-python-env
@@ -337,7 +337,7 @@ jobs:
           steps:
           - save_cache:
               name: Save Python package cache
-              key: kedro-deps-v1-win-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
+              key: kedro-deps-v1-win-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }}
               paths:
                 # Cache pip cache and conda packages directories
                 - c:\tools\miniconda3\pkgs
@@ -350,7 +350,7 @@ jobs:
           steps:
             - save_cache:
                 name: Save conda environment cache
-                key: kedro-deps-v1-win-<<parameters.python_version>>-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
+                key: kedro-deps-v1-win-<<parameters.python_version>>-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }}
                 paths:
                   - c:\tools\miniconda3\envs\kedro_builder
       - run:

.gitpod.yml

@@ -5,10 +5,9 @@ tasks:
 
     init: |
       make sign-off
-      pip install -e /workspace/kedro
+      pip install -e /workspace/kedro[test]
       cd /workspace
       yes project | kedro new -s pandas-iris --checkout main
-      pip install -r /workspace/kedro/test_requirements.txt
       cd /workspace/kedro
       pre-commit install --install-hooks
 

.readthedocs.yml

@@ -36,4 +36,4 @@ python:
       path: .
       extra_requirements:
         - docs
-    - requirements: test_requirements.txt
+        - test

MANIFEST.in

@@ -1,7 +1,5 @@
 include README.md
 include LICENSE.md
-include dependency/requirements.txt
-include test_requirements.txt
 include kedro/framework/project/default_logging.yml
 include kedro/ipython/*.png
 include kedro/ipython/*.svg

Makefile

@@ -48,7 +48,7 @@ package: clean install
 	python -m pip install build && python -m build
 
 install-test-requirements:
-	pip install -r test_requirements.txt
+	pip install .[test]
 
 install-pre-commit: install-test-requirements
 	pre-commit install --install-hooks

features/steps/cli_steps.py

@@ -408,7 +408,7 @@ def update_pyproject_toml(context: behave.runner.Context, new_source_dir):
 @given("I have updated kedro requirements")
 def update_kedro_req(context: behave.runner.Context):
     """Replace kedro as a standalone requirement with a line
-    that includes all of kedro's dependencies (-r kedro/requirements.txt)
+    that includes all of kedro's dependencies (equivalent to `pip install .`).
     """
     reqs_path = context.root_project_dir / "src" / "requirements.txt"
 

features/windows_reqs.txt

@@ -1,4 +1,4 @@
-# same versions as `test_requirements`
+# same versions as [test] optional requirements
 # e2e tests on Windows are slow but we don't need to install
 # everything, so just this subset will be enough for CI
 behave==1.2.6

pyproject.toml

@@ -11,6 +11,32 @@ authors = [
 ]
 description = "Kedro helps you build production-ready data and analytics pipelines"
 requires-python = ">=3.7, <3.11"
+dependencies = [
+    "anyconfig~=0.10.0",
+    "attrs>=21.3",
+    "build",
+    "cachetools~=5.3",
+    "click<9.0",
+    "cookiecutter>=2.1.1, <3.0",
+    "dynaconf>=3.1.2, <4.0",
+    "fsspec>=2021.4, <2024.1",  # Upper bound set arbitrarily, to be reassessed in early 2024
+    "gitpython~=3.0",
+    "importlib-metadata>=3.6; python_version >= '3.8'",
+    "importlib_metadata>=3.6, <5.0; python_version < '3.8'",  # The "selectable" entry points were introduced in `importlib_metadata` 3.6 and Python 3.10. Bandit on Python 3.7 relies on a library with `importlib_metadata` < 5.0
+    "importlib_resources>=1.3",  # The `files()` API was introduced in `importlib_resources` 1.3 and Python 3.9.
+    "jmespath>=0.9.5, <1.0",
+    "more_itertools~=9.0",
+    "omegaconf~=2.3",
+    "parse~=1.19.0",
+    "pip-tools~=6.5",
+    "pluggy~=1.0",
+    "PyYAML>=4.2, <7.0",
+    "rich>=12.0, <14.0",
+    "rope>=0.21, <2.0",  # subject to LGPLv3 license
+    "setuptools>=65.5.1",
+    "toml~=0.10",
+    "toposort~=1.5",  # Needs to be at least 1.5 to be able to raise CircularDependencyError
+]
 keywords = [
     "pipelines",
     "machine learning",
@@ -26,7 +52,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
 ]
-dynamic = ["readme", "version", "dependencies", "optional-dependencies"]
+dynamic = ["readme", "version", "optional-dependencies"]
 
 [project.urls]
 Homepage = "https://kedro.org"
@@ -46,7 +72,6 @@ include = ["kedro*"]
 [tool.setuptools.dynamic]
 readme = {file = "README.md", content-type = "text/markdown"}
 version = {attr = "kedro.__version__"}
-dependencies = {file = "dependency/requirements.txt"}
 
 [tool.black]
 exclude = "/templates/|^features/steps/test_starter"

setup.py

@@ -1,23 +1,14 @@
-from codecs import open
 from glob import glob
 from itertools import chain
-from os import path
 
 from setuptools import setup
 
-name = "kedro"
-here = path.abspath(path.dirname(__file__))
-
 # at least 1.3 to be able to use XMLDataSet and pandas integration with fsspec
 PANDAS = "pandas~=1.3"
 SPARK = "pyspark>=2.2, <4.0"
 HDFS = "hdfs>=2.5.8, <3.0"
 S3FS = "s3fs>=0.3.0, <0.5"
 
-# get the dependencies and installs
-with open("dependency/requirements.txt", encoding="utf-8") as f:
-    requires = [x.strip() for x in f if x.strip()]
-
 template_files = []
 for pattern in ["**/*", "**/.*", "**/.*/**", "**/.*/.**"]:
     template_files.extend(
@@ -59,7 +50,7 @@ def _collect_requirements(requires):
     "pandas.XMLDataSet": [PANDAS, "lxml~=4.6"],
     "pandas.GenericDataSet": [PANDAS],
 }
-pickle_require = {"pickle.PickleDataSet": ["compress-pickle[lz4]~=2.1.0"]}
+pickle_require = {"pickle.PickleDataSet": ["compress-pickle[lz4]~=1.2.0"]}
 pillow_require = {"pillow.ImageDataSet": ["Pillow~=9.0"]}
 video_require = {
     "video.VideoDataSet": ["opencv-python~=4.5.5.64"]
@@ -80,7 +71,9 @@ def _collect_requirements(requires):
     "tensorflow.TensorflowModelDataset": [
         # currently only TensorFlow V2 supported for saving and loading.
         # V1 requires HDF5 and serialises differently
-        "tensorflow~=2.0"
+        "tensorflow~=2.0; platform_system != 'Darwin' or platform_machine != 'arm64'",
+        # https://developer.apple.com/metal/tensorflow-plugin/
+        "tensorflow-macos~=2.0; platform_system == 'Darwin' and platform_machine == 'arm64'",
     ]
 }
 yaml_require = {"yaml.YAMLDataSet": [PANDAS, "PyYAML>=4.2, <7.0"]}
@@ -139,10 +132,77 @@ def _collect_requirements(requires):
 }
 
 extras_require["all"] = _collect_requirements(extras_require)
+extras_require["test"] = [
+    "adlfs>=2021.7.1, <=2022.2; python_version == '3.7'",
+    "adlfs~=2023.1; python_version >= '3.8'",
+    "bandit>=1.6.2, <2.0",
+    "behave==1.2.6",
+    "biopython~=1.73",
+    "blacken-docs==1.9.2",
+    "black~=22.0",
+    "compress-pickle[lz4]~=1.2.0",
+    "coverage[toml]",
+    "dask[complete]~=2021.10",  # pinned by Snyk to avoid a vulnerability
+    "delta-spark~=1.2.1",  # 1.2.0 has a bug that breaks some of our tests: https://github.com/delta-io/delta/issues/1070
+    "dill~=0.3.1",
+    "filelock>=3.4.0, <4.0",
+    "gcsfs>=2021.4, <=2023.1; python_version == '3.7'",
+    "gcsfs>=2023.1, <2023.3; python_version >= '3.8'",
+    "geopandas>=0.6.0, <1.0",
+    "hdfs>=2.5.8, <3.0",
+    "holoviews~=1.13.0",
+    "import-linter[toml]==1.8.0",
+    "ipython>=7.31.1, <8.0; python_version < '3.8'",
+    "ipython~=8.10; python_version >= '3.8'",
+    "isort~=5.0",
+    "Jinja2<3.1.0",
+    "joblib>=0.14",
+    "jupyterlab_server>=2.11.1, <2.16.0",  # 2.16.0 requires importlib_metedata >= 4.8.3 which conflicts with flake8 requirement
+    "jupyterlab~=3.0, <3.6.0",  # 3.6.0 requires jupyterlab_server~=2.19
+    "jupyter~=1.0",
+    "lxml~=4.6",
+    "matplotlib>=3.0.3, <3.4; python_version < '3.10'",  # 3.4.0 breaks holoviews
+    "matplotlib>=3.5, <3.6; python_version == '3.10'",
+    "memory_profiler>=0.50.0, <1.0",
+    "moto==1.3.7; python_version < '3.10'",
+    "moto==3.0.4; python_version == '3.10'",
+    "networkx~=2.4",
+    "opencv-python~=4.5.5.64",
+    "openpyxl>=3.0.3, <4.0",
+    "pandas-gbq>=0.12.0, <0.18.0",
+    "pandas~=1.3  # 1.3 for read_xml/to_xml",
+    "Pillow~=9.0",
+    "plotly>=4.8.0, <6.0",
+    "pre-commit>=2.9.2, <3.0",  # The hook `mypy` requires pre-commit version 2.9.2.
+    "psutil~=5.8",
+    "pyarrow>=1.0, <7.0",
+    "pylint>=2.17.0, <3.0",
+    "pyproj~=3.0",
+    "pyspark>=2.2, <4.0",
+    "pytest-cov~=3.0",
+    "pytest-mock>=1.7.1, <2.0",
+    "pytest-xdist[psutil]~=2.2.1",
+    "pytest~=7.2",
+    "redis~=4.1",
+    "requests-mock~=1.6",
+    "requests~=2.20",
+    "s3fs>=0.3.0, <0.5",  # Needs to be at least 0.3.0 to make use of `cachable` attribute on S3FileSystem.
+    "scikit-learn~=1.0.2",
+    "scipy~=1.7.3",
+    "SQLAlchemy~=1.2",
+    "tables~=3.6.0; platform_system == 'Windows' and python_version<'3.9'",
+    "tables~=3.6; platform_system != 'Windows'",
+    "tensorflow~=2.0; platform_system != 'Darwin' or platform_machine != 'arm64'",
+    # https://developer.apple.com/metal/tensorflow-plugin/
+    "tensorflow-macos~=2.0; platform_system == 'Darwin' and platform_machine == 'arm64'",
+    "triad>=0.6.7, <1.0",
+    "trufflehog~=2.1",
+    "xlsxwriter~=1.0",
+]
 
 setup(
     package_data={
-        name: ["py.typed", "test_requirements.txt"] + template_files
+        "kedro": ["py.typed"] + template_files
     },
     extras_require=extras_require,
 )