use PEP 420: implicit namespace package (#990)

* use PEP 420: implicit namespace package

* update api docs config for namespace package

* allow internationalization data to be included as a namespace 'package'

* missing import

* sphinx apidoc use implicit namespaces option #967

* restore setup package_data

* update docs requirements to keep up with other requirements files

* set language to en

* remove unused imports, refactor excluded modules

* Update doc/api-docs/conf.py

Co-authored-by: James Douglass <jamesdouglassusa@gmail.com>

Co-authored-by: James Douglass <jamesdouglassusa@gmail.com>

doc/api-docs/conf.py

@@ -12,33 +12,19 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
+from datetime import datetime
 import importlib
-import itertools
 import os
 import pkgutil
-import subprocess
 import sys
-
-from datetime import datetime
-from sphinx.ext import apidoc
 from unittest.mock import MagicMock
 
+import natcap.invest
+from sphinx.ext import apidoc
 
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
 DOCS_SOURCE_DIR = os.path.dirname(__file__)
-INVEST_ROOT_DIR = os.path.join(DOCS_SOURCE_DIR, '..', '..')
-INVEST_BUILD_DIR = os.path.join(INVEST_ROOT_DIR, 'build')
-
-# get name of build/lib directory; this depends on the OS and python version
-# e.g. lib.macosx-10.9-x86_64-3.8
-for path in os.listdir(INVEST_BUILD_DIR):
-    if path.startswith('lib'):
-        INVEST_LIB_DIR = os.path.join(INVEST_BUILD_DIR, path)
-        break
-else:
-    raise ValueError(f'lib directory not found in {INVEST_BUILD_DIR}')
+# get the directory that the natcap package lives in
+INVEST_LIB_DIR = os.path.dirname(os.path.dirname(natcap.invest.__file__))
 
 # -- General configuration ------------------------------------------------
 
@@ -67,7 +53,6 @@
 project = 'InVEST'
 copyright = f'{datetime.now().year}, The Natural Capital Project'
 
-import natcap.invest
 # The full version, including alpha/beta/rc tags.
 release = natcap.invest.__version__
 version = release.split('+')[0]
@@ -77,7 +62,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -186,6 +171,7 @@ class MockQGroupBox:
     '--templatedir', os.path.join(DOCS_SOURCE_DIR, 'templates'),  # use custom templates
     '--separate',  # make a separate page for each module
     '--no-toc',  # table of contents page is redundant
+    '--implicit-namespaces',
     INVEST_LIB_DIR,
     '/*.so'  # must be absolute path, see https://github.com/sphinx-doc/sphinx/issues/10200
 ])
@@ -202,78 +188,58 @@ class MockQGroupBox:
 
 All InVEST models share a consistent python API:
 
-    1) The model has a function called ``execute`` that takes a single python
-       dict (``"args"``) as its argument.
-    2) This arguments dict contains an entry, ``'workspace_dir'``, which
-       points to the folder on disk where all files created by the model
-       should be saved.
+    - Every InVEST model has a corresponding module or subpackage in the 
+      ``natcap.invest`` package
+    - The model modules contain a function called ``execute``
+    - The ``execute`` function takes a single argument (``args``), a dictionary
+      that stores all data inputs and configuration options for the model
+    - This dictionary contains an entry, ``'workspace_dir'``, which is a path
+      to the folder where the model will create its output files
+    - Other required and optional entries in the dictionary are specific to
+      each model
 
-Calling a model requires importing the model's execute function and then
-calling the model with the correct parameters.  For example, if you were
-to call the Carbon Storage and Sequestration model, your script might
-include
+To run a model, import the model's ``execute`` function and then call it with
+the correct parameters. For example, a script for the Carbon model might look
+like
 
 .. code-block:: python
 
-    import natcap.invest.carbon.carbon_combined
+    import natcap.invest.carbon
     args = {
         'workspace_dir': 'path/to/workspace'
         # Other arguments, as needed for Carbon.
     }
 
-    natcap.invest.carbon.carbon_combined.execute(args)
+    natcap.invest.carbon.execute(args)
 
-For examples of scripts that could be created around a model run,
-or multiple successive model runs, see :ref:`CreatingSamplePythonScripts`.
+For examples of scripting a model run, or multiple successive model runs,
+see :ref:`CreatingSamplePythonScripts`.
 
 
 .. contents:: Available Models and Tools:
     :local:
 
 """
 
-EXCLUDED_MODULES = [
-    '_core',  # anything ending in '_core'
-    'recmodel_server',
-    'recmodel_workspace_fetcher'
-]
 MODEL_ENTRYPOINTS_FILE = os.path.join(DOCS_SOURCE_DIR, 'models.rst')
-
 # Find all importable modules with an execute function
 # write out to a file models.rst in the source directory
-all_modules = {}
-for _loader, name, _is_pkg in itertools.chain(
-        pkgutil.walk_packages(path=[INVEST_LIB_DIR]),  # catch packages
-        pkgutil.iter_modules(path=[INVEST_LIB_DIR])):  # catch modules
-
-    if (any([name.endswith(x) for x in EXCLUDED_MODULES]) or
-        name.startswith('natcap.invest.ui')):
-        continue
-
-    try:
-        module = importlib.import_module(name)
-    except Exception as ex:
-        print(ex)
-        continue
-
-    if not hasattr(module, 'execute'):
-        continue
-
-    try:
-        module_title = module.execute.__doc__.strip().split('\n')[0]
-        if module_title.endswith('.'):
-            module_title = module_title[:-1]
-    except AttributeError:
-        module_title = None
-    all_modules[name] = module_title
+invest_model_modules = {}
+for _, name, _ in pkgutil.walk_packages(path=[INVEST_LIB_DIR],
+                                        prefix='natcap.'):
+    module = importlib.import_module(name)
+    # any module with an ARGS_SPEC is an invest model
+    if hasattr(module, 'ARGS_SPEC'):
+        model_title = module.ARGS_SPEC['model_name']
+        invest_model_modules[model_title] = name
 
 # Write sphinx autodoc function for each entrypoint
 with open(MODEL_ENTRYPOINTS_FILE, 'w') as models_rst:
     models_rst.write(MODEL_RST_TEMPLATE)
-    for name, module_title in sorted(all_modules.items(), key=lambda x: x[1]):
-        underline = ''.join(['=']*len(module_title))
+    for model_title, name in sorted(invest_model_modules.items()):
+        underline = ''.join(['=']*len(model_title))
         models_rst.write(
-            f'{module_title}\n'
+            f'{model_title}\n'
             f'{underline}\n'
             f'.. autofunction:: {name}.execute\n'
             '   :noindex:\n\n')

requirements-docs.yml

@@ -12,34 +12,32 @@ channels:
 dependencies:
 - chardet>=3.0.4
 - Cython
-- GDAL>=3.1.2,!=3.3.0
+- GDAL>=3.1.2,!=3.3.0,<3.4.1
 - Flask
 - flask_cors
 - numpy>=1.11.0,!=1.16.0
-- pandas>=1.0,<1.2.0
+- pandas>=1.2.1
 - pip
 - psutil>=5.6.6
 - qtpy>1.3
 - requests
-- Rtree==0.9.4
+- Rtree>=0.8.2,!=0.9.1
 - setuptools>=8.0
-- Shapely>=1.7.1,<2.0.0
-- sphinx==3.3.1
+- setuptools_scm>=6.4.0
+- shapely>=1.7.1,<1.8.2
+- sphinx
 - sphinx-intl
 - scipy>=1.6.0
-- taskgraph[niced_processes]>=0.10.2
+- taskgraph[niced_processes]>=0.11.0
 - virtualenv>=12.0.1
 - wheel>=0.27.0
-- xlrd>=1.2.0
-- jinja2<3.1
 - pint
 - xlwt
 - pip:
-  - pygeoprocessing>=2.3.2  # pip-only
-  - PyInstaller==3.5  # pip-only
-  - Pyro4==4.77  # pip-only
-  - PySide2!=5.15.0  # pip-only
-  - qtawesome  # pip-only
-  - setuptools_scm>=6.4.0
+  - pygeoprocessing>=2.3.2
+  - PyInstaller>=4.10
+  - Pyro4==4.77
+  - PySide2!=5.15.0
+  - qtawesome
   - sphinx-rtd-theme
   - sphinx-reredirects

setup.py

@@ -14,7 +14,7 @@
 
 import Cython.Build
 import numpy
-from setuptools import setup
+from setuptools import find_namespace_packages, setup
 from setuptools.command.build_py import build_py as _build_py
 from setuptools.extension import Extension
 
@@ -69,19 +69,7 @@ def run(self):
     maintainer='James Douglass',
     maintainer_email='jdouglass@stanford.edu',
     url='http://github.com/natcap/invest',
-    namespace_packages=['natcap'],
-    packages=[
-        'natcap',
-        'natcap.invest',
-        'natcap.invest.coastal_blue_carbon',
-        'natcap.invest.delineateit',
-        'natcap.invest.ui',
-        'natcap.invest.ndr',
-        'natcap.invest.sdr',
-        'natcap.invest.recreation',
-        'natcap.invest.scenic_quality',
-        'natcap.invest.seasonal_water_yield',
-    ],
+    packages=find_namespace_packages('src'),
     package_dir={
         'natcap': 'src/natcap'
     },