From 03bf4302facaf6e02324a0bd55f8e9b342ac57e2 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 10 Aug 2021 13:17:15 +0200 Subject: [PATCH 01/33] Moved /ewatercycle to /src/ewatercycle + adjust config to match Refs #228 --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- setup.cfg | 12 ++++++------ sonar-project.properties | 2 +- {ewatercycle => src/ewatercycle}/__init__.py | 0 .../ewatercycle}/analysis/__init__.py | 0 {ewatercycle => src/ewatercycle}/config/__init__.py | 0 .../ewatercycle}/config/_config_object.py | 0 .../ewatercycle}/config/_validated_config.py | 0 .../ewatercycle}/config/_validators.py | 0 .../ewatercycle}/config/ewatercycle.yaml | 0 {ewatercycle => src/ewatercycle}/forcing/__init__.py | 0 {ewatercycle => src/ewatercycle}/forcing/_default.py | 0 {ewatercycle => src/ewatercycle}/forcing/_hype.py | 0 .../ewatercycle}/forcing/_lisflood.py | 0 {ewatercycle => src/ewatercycle}/forcing/_marrmot.py | 0 .../ewatercycle}/forcing/_pcrglobwb.py | 0 {ewatercycle => src/ewatercycle}/forcing/_wflow.py | 0 {ewatercycle => src/ewatercycle}/forcing/datasets.py | 0 {ewatercycle => src/ewatercycle}/models/__init__.py | 0 {ewatercycle => src/ewatercycle}/models/abstract.py | 0 {ewatercycle => src/ewatercycle}/models/lisflood.py | 0 {ewatercycle => src/ewatercycle}/models/marrmot.py | 0 {ewatercycle => src/ewatercycle}/models/pcrglobwb.py | 0 {ewatercycle => src/ewatercycle}/models/wflow.py | 0 .../ewatercycle}/observation/__init__.py | 0 {ewatercycle => src/ewatercycle}/observation/grdc.py | 0 {ewatercycle => src/ewatercycle}/observation/usgs.py | 0 .../ewatercycle}/parameter_sets/__init__.py | 0 .../ewatercycle}/parameter_sets/_example.py | 0 .../ewatercycle}/parameter_sets/_lisflood.py | 0 .../ewatercycle}/parameter_sets/_pcrglobwb.py | 0 .../ewatercycle}/parameter_sets/_wflow.py | 0 .../ewatercycle}/parameter_sets/default.py | 0 .../ewatercycle}/parametersetdb/__init__.py | 0 .../ewatercycle}/parametersetdb/config.py | 0 .../ewatercycle}/parametersetdb/datafiles.py | 0 {ewatercycle => src/ewatercycle}/util.py | 0 {ewatercycle => src/ewatercycle}/version.py | 0 39 files changed, 10 insertions(+), 10 deletions(-) rename {ewatercycle => src/ewatercycle}/__init__.py (100%) rename {ewatercycle => src/ewatercycle}/analysis/__init__.py (100%) rename {ewatercycle => src/ewatercycle}/config/__init__.py (100%) rename {ewatercycle => src/ewatercycle}/config/_config_object.py (100%) rename {ewatercycle => src/ewatercycle}/config/_validated_config.py (100%) rename {ewatercycle => src/ewatercycle}/config/_validators.py (100%) rename {ewatercycle => src/ewatercycle}/config/ewatercycle.yaml (100%) rename {ewatercycle => src/ewatercycle}/forcing/__init__.py (100%) rename {ewatercycle => src/ewatercycle}/forcing/_default.py (100%) rename {ewatercycle => src/ewatercycle}/forcing/_hype.py (100%) rename {ewatercycle => src/ewatercycle}/forcing/_lisflood.py (100%) rename {ewatercycle => src/ewatercycle}/forcing/_marrmot.py (100%) rename {ewatercycle => src/ewatercycle}/forcing/_pcrglobwb.py (100%) rename {ewatercycle => src/ewatercycle}/forcing/_wflow.py (100%) rename {ewatercycle => src/ewatercycle}/forcing/datasets.py (100%) rename {ewatercycle => src/ewatercycle}/models/__init__.py (100%) rename {ewatercycle => src/ewatercycle}/models/abstract.py (100%) rename {ewatercycle => src/ewatercycle}/models/lisflood.py (100%) rename {ewatercycle => src/ewatercycle}/models/marrmot.py (100%) rename {ewatercycle => src/ewatercycle}/models/pcrglobwb.py (100%) rename {ewatercycle => src/ewatercycle}/models/wflow.py (100%) rename {ewatercycle => src/ewatercycle}/observation/__init__.py (100%) rename {ewatercycle => src/ewatercycle}/observation/grdc.py (100%) rename {ewatercycle => src/ewatercycle}/observation/usgs.py (100%) rename {ewatercycle => src/ewatercycle}/parameter_sets/__init__.py (100%) rename {ewatercycle => src/ewatercycle}/parameter_sets/_example.py (100%) rename {ewatercycle => src/ewatercycle}/parameter_sets/_lisflood.py (100%) rename {ewatercycle => src/ewatercycle}/parameter_sets/_pcrglobwb.py (100%) rename {ewatercycle => src/ewatercycle}/parameter_sets/_wflow.py (100%) rename {ewatercycle => src/ewatercycle}/parameter_sets/default.py (100%) rename {ewatercycle => src/ewatercycle}/parametersetdb/__init__.py (100%) rename {ewatercycle => src/ewatercycle}/parametersetdb/config.py (100%) rename {ewatercycle => src/ewatercycle}/parametersetdb/datafiles.py (100%) rename {ewatercycle => src/ewatercycle}/util.py (100%) rename {ewatercycle => src/ewatercycle}/version.py (100%) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 6e117ff3..b806a011 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,7 @@ [bumpversion] current_version = 1.1.1 -[bumpversion:file:ewatercycle/version.py] +[bumpversion:file:src/ewatercycle/version.py] search = __version__ = '{current_version}' replace = __version__ = '{new_version}' diff --git a/docs/conf.py b/docs/conf.py index b90ad262..c89a35c2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ import sys here = os.path.dirname(__file__) -sys.path.insert(0, os.path.abspath(os.path.join(here, '..'))) +sys.path.insert(0, os.path.abspath(os.path.join(here, '..', 'src'))) # -- General configuration ------------------------------------------------ @@ -90,7 +90,7 @@ def run_apidoc(_): here = os.path.dirname(__file__) out = os.path.abspath(os.path.join(here, 'apidocs')) - src = os.path.abspath(os.path.join(here, '..', 'ewatercycle')) + src = os.path.abspath(os.path.join(here, '..', 'src', 'ewatercycle')) ignore_paths = [] diff --git a/setup.cfg b/setup.cfg index 73d7856d..e653b2eb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,6 +34,8 @@ project_urls = [options] zip_safe = False python_requires = >=3.7 +package_dir = + = src packages = find: install_requires = basic_modeling_interface @@ -83,18 +85,16 @@ dev = * = *.yaml [options.packages.find] -include = - ewatercycle - ewatercycle.* +where = src [coverage:run] branch = True -source = ewatercycle +source = src [tool:pytest] testpaths = tests - ewatercycle + src addopts = --mypy --cov @@ -112,4 +112,4 @@ builder = html [mypy] ignore_missing_imports = True -files = ewatercycle, tests +files = src, tests diff --git a/sonar-project.properties b/sonar-project.properties index 73ce7c4b..d7e3bf61 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,6 +1,6 @@ sonar.organization=ewatercycle sonar.projectKey=eWaterCycle_ewatercycle -sonar.sources=ewatercycle/ +sonar.sources=src/ sonar.tests=tests sonar.language=py sonar.python.xunit.reportPath=xunit-result.xml diff --git a/ewatercycle/__init__.py b/src/ewatercycle/__init__.py similarity index 100% rename from ewatercycle/__init__.py rename to src/ewatercycle/__init__.py diff --git a/ewatercycle/analysis/__init__.py b/src/ewatercycle/analysis/__init__.py similarity index 100% rename from ewatercycle/analysis/__init__.py rename to src/ewatercycle/analysis/__init__.py diff --git a/ewatercycle/config/__init__.py b/src/ewatercycle/config/__init__.py similarity index 100% rename from ewatercycle/config/__init__.py rename to src/ewatercycle/config/__init__.py diff --git a/ewatercycle/config/_config_object.py b/src/ewatercycle/config/_config_object.py similarity index 100% rename from ewatercycle/config/_config_object.py rename to src/ewatercycle/config/_config_object.py diff --git a/ewatercycle/config/_validated_config.py b/src/ewatercycle/config/_validated_config.py similarity index 100% rename from ewatercycle/config/_validated_config.py rename to src/ewatercycle/config/_validated_config.py diff --git a/ewatercycle/config/_validators.py b/src/ewatercycle/config/_validators.py similarity index 100% rename from ewatercycle/config/_validators.py rename to src/ewatercycle/config/_validators.py diff --git a/ewatercycle/config/ewatercycle.yaml b/src/ewatercycle/config/ewatercycle.yaml similarity index 100% rename from ewatercycle/config/ewatercycle.yaml rename to src/ewatercycle/config/ewatercycle.yaml diff --git a/ewatercycle/forcing/__init__.py b/src/ewatercycle/forcing/__init__.py similarity index 100% rename from ewatercycle/forcing/__init__.py rename to src/ewatercycle/forcing/__init__.py diff --git a/ewatercycle/forcing/_default.py b/src/ewatercycle/forcing/_default.py similarity index 100% rename from ewatercycle/forcing/_default.py rename to src/ewatercycle/forcing/_default.py diff --git a/ewatercycle/forcing/_hype.py b/src/ewatercycle/forcing/_hype.py similarity index 100% rename from ewatercycle/forcing/_hype.py rename to src/ewatercycle/forcing/_hype.py diff --git a/ewatercycle/forcing/_lisflood.py b/src/ewatercycle/forcing/_lisflood.py similarity index 100% rename from ewatercycle/forcing/_lisflood.py rename to src/ewatercycle/forcing/_lisflood.py diff --git a/ewatercycle/forcing/_marrmot.py b/src/ewatercycle/forcing/_marrmot.py similarity index 100% rename from ewatercycle/forcing/_marrmot.py rename to src/ewatercycle/forcing/_marrmot.py diff --git a/ewatercycle/forcing/_pcrglobwb.py b/src/ewatercycle/forcing/_pcrglobwb.py similarity index 100% rename from ewatercycle/forcing/_pcrglobwb.py rename to src/ewatercycle/forcing/_pcrglobwb.py diff --git a/ewatercycle/forcing/_wflow.py b/src/ewatercycle/forcing/_wflow.py similarity index 100% rename from ewatercycle/forcing/_wflow.py rename to src/ewatercycle/forcing/_wflow.py diff --git a/ewatercycle/forcing/datasets.py b/src/ewatercycle/forcing/datasets.py similarity index 100% rename from ewatercycle/forcing/datasets.py rename to src/ewatercycle/forcing/datasets.py diff --git a/ewatercycle/models/__init__.py b/src/ewatercycle/models/__init__.py similarity index 100% rename from ewatercycle/models/__init__.py rename to src/ewatercycle/models/__init__.py diff --git a/ewatercycle/models/abstract.py b/src/ewatercycle/models/abstract.py similarity index 100% rename from ewatercycle/models/abstract.py rename to src/ewatercycle/models/abstract.py diff --git a/ewatercycle/models/lisflood.py b/src/ewatercycle/models/lisflood.py similarity index 100% rename from ewatercycle/models/lisflood.py rename to src/ewatercycle/models/lisflood.py diff --git a/ewatercycle/models/marrmot.py b/src/ewatercycle/models/marrmot.py similarity index 100% rename from ewatercycle/models/marrmot.py rename to src/ewatercycle/models/marrmot.py diff --git a/ewatercycle/models/pcrglobwb.py b/src/ewatercycle/models/pcrglobwb.py similarity index 100% rename from ewatercycle/models/pcrglobwb.py rename to src/ewatercycle/models/pcrglobwb.py diff --git a/ewatercycle/models/wflow.py b/src/ewatercycle/models/wflow.py similarity index 100% rename from ewatercycle/models/wflow.py rename to src/ewatercycle/models/wflow.py diff --git a/ewatercycle/observation/__init__.py b/src/ewatercycle/observation/__init__.py similarity index 100% rename from ewatercycle/observation/__init__.py rename to src/ewatercycle/observation/__init__.py diff --git a/ewatercycle/observation/grdc.py b/src/ewatercycle/observation/grdc.py similarity index 100% rename from ewatercycle/observation/grdc.py rename to src/ewatercycle/observation/grdc.py diff --git a/ewatercycle/observation/usgs.py b/src/ewatercycle/observation/usgs.py similarity index 100% rename from ewatercycle/observation/usgs.py rename to src/ewatercycle/observation/usgs.py diff --git a/ewatercycle/parameter_sets/__init__.py b/src/ewatercycle/parameter_sets/__init__.py similarity index 100% rename from ewatercycle/parameter_sets/__init__.py rename to src/ewatercycle/parameter_sets/__init__.py diff --git a/ewatercycle/parameter_sets/_example.py b/src/ewatercycle/parameter_sets/_example.py similarity index 100% rename from ewatercycle/parameter_sets/_example.py rename to src/ewatercycle/parameter_sets/_example.py diff --git a/ewatercycle/parameter_sets/_lisflood.py b/src/ewatercycle/parameter_sets/_lisflood.py similarity index 100% rename from ewatercycle/parameter_sets/_lisflood.py rename to src/ewatercycle/parameter_sets/_lisflood.py diff --git a/ewatercycle/parameter_sets/_pcrglobwb.py b/src/ewatercycle/parameter_sets/_pcrglobwb.py similarity index 100% rename from ewatercycle/parameter_sets/_pcrglobwb.py rename to src/ewatercycle/parameter_sets/_pcrglobwb.py diff --git a/ewatercycle/parameter_sets/_wflow.py b/src/ewatercycle/parameter_sets/_wflow.py similarity index 100% rename from ewatercycle/parameter_sets/_wflow.py rename to src/ewatercycle/parameter_sets/_wflow.py diff --git a/ewatercycle/parameter_sets/default.py b/src/ewatercycle/parameter_sets/default.py similarity index 100% rename from ewatercycle/parameter_sets/default.py rename to src/ewatercycle/parameter_sets/default.py diff --git a/ewatercycle/parametersetdb/__init__.py b/src/ewatercycle/parametersetdb/__init__.py similarity index 100% rename from ewatercycle/parametersetdb/__init__.py rename to src/ewatercycle/parametersetdb/__init__.py diff --git a/ewatercycle/parametersetdb/config.py b/src/ewatercycle/parametersetdb/config.py similarity index 100% rename from ewatercycle/parametersetdb/config.py rename to src/ewatercycle/parametersetdb/config.py diff --git a/ewatercycle/parametersetdb/datafiles.py b/src/ewatercycle/parametersetdb/datafiles.py similarity index 100% rename from ewatercycle/parametersetdb/datafiles.py rename to src/ewatercycle/parametersetdb/datafiles.py diff --git a/ewatercycle/util.py b/src/ewatercycle/util.py similarity index 100% rename from ewatercycle/util.py rename to src/ewatercycle/util.py diff --git a/ewatercycle/version.py b/src/ewatercycle/version.py similarity index 100% rename from ewatercycle/version.py rename to src/ewatercycle/version.py From 910c075c497ada3e448c55e4d5c1012588fcdb02 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 10 Aug 2021 13:34:25 +0200 Subject: [PATCH 02/33] Add black to dev deps Replaces yapf as formatter --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index e653b2eb..b4e3a018 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,7 +68,8 @@ dev = isort prospector[with_pyroma,with_mypy] pycodestyle - yapf + # Formatters + black # Dependencies for documentation generation nbsphinx recommonmark From 9607f3e45650c37926cd67b38c3eeeba81897da1 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 10 Aug 2021 13:34:46 +0200 Subject: [PATCH 03/33] Configure black to also format notebooks --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index de11f324..a1805406 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,3 +5,5 @@ requires = [ ] build-backend = "setuptools.build_meta" +[tool.black] +include = '(\.pyi?|\.ipynb)$' From bd3d472df9efeb28dd53bbf3755cb0153be82ff8 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 10 Aug 2021 14:55:44 +0200 Subject: [PATCH 04/33] Added black and friends as pre-commit hooks and to ci --- .github/workflows/ci.yml | 35 +++---- .github/workflows/sonar.yml | 56 +++++----- .pre-commit-config.yaml | 49 +++++++++ docs/conf.py | 200 +++++++++++++++++++----------------- pyproject.toml | 4 + setup.cfg | 64 ++++++------ 6 files changed, 232 insertions(+), 176 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb70b2ac..6cfa0861 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,10 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - name: Python package on: push: branches: [main] pull_request: - types: [opened, synchronize, reopened] + types: [opened, synchronize, reopened] jobs: build: @@ -19,18 +16,18 @@ jobs: fail-fast: false name: Run tests in conda environment ${{ matrix.python-version }} steps: - - uses: actions/checkout@v2 - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: ewatercycle - environment-file: environment.yml - python-version: ${{ matrix.python-version }} - miniconda-version: "latest" - channels: conda-forge - - name: Install dependencies - shell: bash -l {0} - run: | - pip3 install -e .[dev] - - name: Test with pytest - run: pytest - shell: bash -l {0} + - uses: actions/checkout@v2 + - uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: ewatercycle + environment-file: environment.yml + python-version: ${{ matrix.python-version }} + miniconda-version: "latest" + channels: conda-forge + - name: Install dependencies + shell: bash -l {0} + run: | + pip3 install -e .[dev] + - name: Test with pytest + run: pytest + shell: bash -l {0} diff --git a/.github/workflows/sonar.yml b/.github/workflows/sonar.yml index 48f49546..2cd235bb 100644 --- a/.github/workflows/sonar.yml +++ b/.github/workflows/sonar.yml @@ -4,34 +4,38 @@ on: push: branches: [main] pull_request: - types: [opened, synchronize, reopened] + types: [opened, synchronize, reopened] jobs: sonarcloud: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up Python - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: ewatercycle - environment-file: environment.yml - python-version: 3.8 - miniconda-version: "latest" - channels: conda-forge - - name: Install dependencies - shell: bash -l {0} - run: | - pip3 install -e .[dev] - - name: Tests with coverage - run: pytest --cov --cov-report term --cov-report xml --junitxml=xunit-result.xml - shell: bash -l {0} - - name: Correct coverage paths - run: sed -i "s+$PWD/++g" coverage.xml - - name: SonarCloud Scan - uses: sonarsource/sonarcloud-github-action@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python + uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: ewatercycle + environment-file: environment.yml + python-version: 3.8 + miniconda-version: "latest" + channels: conda-forge + - name: Install dependencies + shell: bash -l {0} + run: | + pip3 install -e .[dev] + - name: Run pre commit hooks like black formatter + uses: pre-commit/action@v2.0.3 + - name: Tests with coverage + run: | + pytest --cov --cov-report term --cov-report xml \ + --junitxml=xunit-result.xml + shell: bash -l {0} + - name: Correct coverage paths + run: sed -i "s+$PWD/++g" coverage.xml + - name: SonarCloud Scan + uses: sonarsource/sonarcloud-github-action@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..ee101a12 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,49 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.4.0 + hooks: + - id: check-added-large-files + - id: check-ast + - id: check-case-conflict + - id: check-merge-conflict + - id: debug-statements + - id: end-of-file-fixer + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - repo: https://github.com/adrienverge/yamllint + rev: 'v1.26.0' + hooks: + - id: yamllint + - repo: https://github.com/asottile/setup-cfg-fmt + rev: v1.17.0 + hooks: + - id: setup-cfg-fmt + - repo: https://github.com/psf/black + rev: 21.7b0 + hooks: + - id: black + - repo: https://github.com/PyCQA/isort + rev: '5.9.3' + hooks: + - id: isort + - repo: https://gitlab.com/pycqa/flake8 + rev: '3.9.2' + hooks: + - id: flake8 + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.910 + hooks: + - id: mypy + additional_dependencies: [types-python-dateutil] + - repo: https://github.com/nbQA-dev/nbQA + rev: 1.1.0 + hooks: + - id: nbqa-black + # Match version of black used for .py and .pynb + additional_dependencies: [black==21.7b0] + - id: nbqa-isort + additional_dependencies: [isort==5.9.3] + - id: nbqa-mypy + additional_dependencies: [mypy==0.910] + - id: nbqa-flake8 + additional_dependencies: [flake8==3.9.2] diff --git a/docs/conf.py b/docs/conf.py index c89a35c2..a1bdf3e8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ import sys here = os.path.dirname(__file__) -sys.path.insert(0, os.path.abspath(os.path.join(here, '..', 'src'))) +sys.path.insert(0, os.path.abspath(os.path.join(here, "..", "src"))) # -- General configuration ------------------------------------------------ @@ -33,29 +33,29 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.napoleon', - 'nbsphinx', - 'sphinx.ext.intersphinx', - 'sphinx_copybutton', + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "nbsphinx", + "sphinx.ext.intersphinx", + "sphinx_copybutton", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'ewatercycle' -copyright = u'2018, Netherlands eScience Center & Delft University of Technology' -author = u'Stefan Verhoeven' +project = "ewatercycle" +copyright = "2018, Netherlands eScience Center & Delft University of Technology" +author = "Stefan Verhoeven" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -63,7 +63,7 @@ # # The short X.Y version. # The full version, including alpha/beta/rc tags. -version = '1.1.1' +version = "1.1.1" release = version # The language for content autogenerated by Sphinx. Refer to documentation @@ -76,10 +76,10 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -89,8 +89,8 @@ # See https://github.com/rtfd/readthedocs.org/issues/1139 def run_apidoc(_): here = os.path.dirname(__file__) - out = os.path.abspath(os.path.join(here, 'apidocs')) - src = os.path.abspath(os.path.join(here, '..', 'src', 'ewatercycle')) + out = os.path.abspath(os.path.join(here, "apidocs")) + src = os.path.abspath(os.path.join(here, "..", "src", "ewatercycle")) ignore_paths = [] @@ -100,23 +100,26 @@ def run_apidoc(_): "-e", "-M", "--implicit-namespaces", - "-o", out, - src + "-o", + out, + src, ] + ignore_paths try: # Sphinx 1.7+ from sphinx.ext import apidoc + apidoc.main(argv) except ImportError: # Sphinx 1.6 (and earlier) from sphinx import apidoc + argv.insert(0, apidoc.__file__) apidoc.main(argv) def setup(app): - app.connect('builder-inited', run_apidoc) + app.connect("builder-inited", run_apidoc) # -- Options for HTML output ---------------------------------------------- @@ -124,9 +127,9 @@ def setup(app): # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -#html_theme = 'alabaster' -html_theme = 'sphinx_rtd_theme' -html_logo = 'examples/logo.png' +# html_theme = 'alabaster' +html_theme = "sphinx_rtd_theme" +html_logo = "examples/logo.png" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -137,7 +140,7 @@ def setup(app): # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -145,9 +148,9 @@ def setup(app): # This is required for the alabaster theme # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars html_sidebars = { - '**': [ - 'relations.html', # needs 'show_related': True theme option to display - 'searchbox.html', + "**": [ + "relations.html", # needs 'show_related': True theme option to display + "searchbox.html", ] } @@ -155,35 +158,37 @@ def setup(app): # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. -htmlhelp_basename = 'ewatercycle_doc' +htmlhelp_basename = "ewatercycle_doc" # -- Options for LaTeX output --------------------------------------------- -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} +# latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +# +# 'papersize': 'letterpaper', +# The font size ('10pt', '11pt' or '12pt'). +# +# 'pointsize': '10pt', +# Additional stuff for the LaTeX preamble. +# +# 'preamble': '', +# Latex figure (float) alignment +# +# 'figure_align': 'htbp', +# } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'ewatercycle.tex', u'ewatercycle Documentation', - u'Stefan Verhoeven', 'manual'), + ( + master_doc, + "ewatercycle.tex", + "ewatercycle Documentation", + "Stefan Verhoeven", + "manual", + ), ] @@ -191,10 +196,7 @@ def setup(app): # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'ewatercycle', u'ewatercycle Documentation', - [author], 1) -] +man_pages = [(master_doc, "ewatercycle", "ewatercycle Documentation", [author], 1)] # -- Options for Texinfo output ------------------------------------------- @@ -203,67 +205,71 @@ def setup(app): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'ewatercycle', u'ewatercycle Documentation', - author, 'ewatercycle', 'Python utilities to gather input files for running a hydrology model', - 'Miscellaneous'), + ( + master_doc, + "ewatercycle", + "ewatercycle Documentation", + author, + "ewatercycle", + "Python utilities to gather input files for running a hydrology model", + "Miscellaneous", + ), ] autodoc_mock_imports = [ - 'basic_modeling_interface', - 'cftime', - 'dask', - 'esmvalcore', - 'fiona', - 'dateutil', - 'shapely', - 'hydrostats', - 'matplotlib', - 'numpy', - 'pandas', - 'pyoos', - 'grpc4bmi', - 'grpc', - 'ruamel.yaml', - 'scipy', - 'xarray', + "basic_modeling_interface", + "cftime", + "dask", + "esmvalcore", + "fiona", + "dateutil", + "shapely", + "hydrostats", + "matplotlib", + "numpy", + "pandas", + "pyoos", + "grpc4bmi", + "grpc", + "ruamel.yaml", + "scipy", + "xarray", ] # Prevent alphabetic sorting of (@data)class attributes/methods -autodoc_member_order = 'bysource' +autodoc_member_order = "bysource" # nbsphinx configuration values cf. https://nbsphinx.readthedocs.io/en/0.8.6/usage.html -nbsphinx_execute = 'never' +nbsphinx_execute = "never" # Hacky way to 'remove' the cell count from the prompt. # Inspired by https://github.com/spatialaudio/nbsphinx/issues/126 -nbsphinx_prompt_width = '0' -nbsphinx_input_prompt = '%s In:' -nbsphinx_output_prompt = '%s Out:' +nbsphinx_prompt_width = "0" +nbsphinx_input_prompt = "%s In:" +nbsphinx_output_prompt = "%s Out:" # Nice formatting of model-specific input parameters napoleon_custom_sections = [ - ('hype', 'params_style'), - ('lisflood', 'params_style'), - ('marrmot', 'params_style'), - ('pcrglobwb', 'params_style'), - ('wflow', 'params_style'), + ("hype", "params_style"), + ("lisflood", "params_style"), + ("marrmot", "params_style"), + ("pcrglobwb", "params_style"), + ("wflow", "params_style"), ] intersphinx_mapping = { - 'cf_units': ('https://scitools.org.uk/cf-units/docs/latest/', None), - 'esmvalcore': - (f'https://docs.esmvaltool.org/projects/esmvalcore/en/latest/', - None), - 'esmvaltool': (f'https://docs.esmvaltool.org/en/latest/', None), - 'grpc4bmi': (f'https://grpc4bmi.readthedocs.io/en/latest/', None), - 'iris': ('https://scitools-iris.readthedocs.io/en/latest/', None), - 'lime': ('https://lime-ml.readthedocs.io/en/latest/', None), - 'basic_modeling_interface': ('https://bmi.readthedocs.io/en/latest/', None), - 'matplotlib': ('https://matplotlib.org/', None), - 'numpy': ('https://numpy.org/doc/stable/', None), - 'pandas': ('https://pandas.pydata.org/pandas-docs/dev', None), - 'python': ('https://docs.python.org/3/', None), - 'scipy': ('https://docs.scipy.org/doc/scipy/reference/', None), - 'seaborn': ('https://seaborn.pydata.org/', None), - 'sklearn': ('https://scikit-learn.org/stable', None), + "cf_units": ("https://scitools.org.uk/cf-units/docs/latest/", None), + "esmvalcore": (f"https://docs.esmvaltool.org/projects/esmvalcore/en/latest/", None), + "esmvaltool": (f"https://docs.esmvaltool.org/en/latest/", None), + "grpc4bmi": (f"https://grpc4bmi.readthedocs.io/en/latest/", None), + "iris": ("https://scitools-iris.readthedocs.io/en/latest/", None), + "lime": ("https://lime-ml.readthedocs.io/en/latest/", None), + "basic_modeling_interface": ("https://bmi.readthedocs.io/en/latest/", None), + "matplotlib": ("https://matplotlib.org/", None), + "numpy": ("https://numpy.org/doc/stable/", None), + "pandas": ("https://pandas.pydata.org/pandas-docs/dev", None), + "python": ("https://docs.python.org/3/", None), + "scipy": ("https://docs.scipy.org/doc/scipy/reference/", None), + "seaborn": ("https://seaborn.pydata.org/", None), + "sklearn": ("https://scikit-learn.org/stable", None), } diff --git a/pyproject.toml b/pyproject.toml index a1805406..6bbe54f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,3 +7,7 @@ build-backend = "setuptools.build_meta" [tool.black] include = '(\.pyi?|\.ipynb)$' + +[tool.isort] +profile = "black" +multi_line_output = 3 diff --git a/setup.cfg b/setup.cfg index b4e3a018..974df757 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,23 +2,20 @@ name = ewatercycle version = 1.1.1 description = A Python package for running and validating a hydrology model +long_description = file: README.md +long_description_content_type = text/markdown +url = https://www.ewatercycle.org/ author = Stefan Verhoeven author_email = s.verhoeven@esciencecenter.nl -url = https://www.ewatercycle.org/ license = Apache Software License 2.0 -long_description = file: README.md -long_description_content_type = text/markdown -keywords = - ewatercycle - FAIR - BMI - Geoscience +license_file = LICENSE classifiers = Development Status :: 2 - Pre-Alpha Intended Audience :: Developers License :: OSI Approved :: Apache Software License Natural Language :: English Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 @@ -26,22 +23,26 @@ classifiers = Topic :: Scientific/Engineering :: GIS Topic :: Scientific/Engineering :: Hydrology Typing :: Typed +keywords = + ewatercycle + FAIR + BMI + Geoscience project_urls = Bug Tracker = https://github.com/eWaterCycle/ewatercycle/issues Documentation = https://ewatercycle.readthedocs.io/ Source Code = https://github.com/eWaterCycle/ewatercycle [options] -zip_safe = False -python_requires = >=3.7 -package_dir = - = src packages = find: install_requires = + Fiona + Shapely basic_modeling_interface cftime esmvaltool>=2.3.0 grpc4bmi>=0.2.12,<0.3 + grpcio hydrostats matplotlib numpy @@ -51,43 +52,39 @@ install_requires = ruamel.yaml scipy xarray - Fiona - Shapely - grpcio +python_requires = >=3.7 +package_dir = + = src +zip_safe = False + +[options.packages.find] +where = src [options.extras_require] dev = - # Test + black + build + bump2version deepdiff + ipython + isort + nbsphinx + pre-commit + prospector[with_pyroma,with_mypy] + pycodestyle pytest pytest-cov pytest-mypy pytest-runner - types-python-dateutil - # Linters - isort - prospector[with_pyroma,with_mypy] - pycodestyle - # Formatters - black - # Dependencies for documentation generation - nbsphinx recommonmark sphinx sphinx_rtd_theme - # ipython syntax highlighting is required in doc notebooks - ipython - # release - bump2version - build twine + types-python-dateutil [options.package_data] * = *.yaml -[options.packages.find] -where = src - [coverage:run] branch = True source = src @@ -104,7 +101,6 @@ addopts = --cov-report html --junit-xml=xunit-result.xml -# Define `python setup.py build_sphinx` [build_sphinx] source-dir = docs build-dir = docs/_build From d9a6fa55e03ca414deb9357edcc1699d8f8c1ee5 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 10 Aug 2021 14:56:41 +0200 Subject: [PATCH 05/33] Files changes by `pre-commit run --all-files` command --- .prospector.yml | 20 +- CONTRIBUTING.md | 1 + LICENSE | 1 - docs/examples/data/Rhine/Rhine.prj | 2 +- docs/examples/pcrglobwb_merrimack.ini | 93 +++-- docs/observations.rst | 2 +- docs/user_guide.ipynb | 4 +- environment.yml | 3 +- src/ewatercycle/analysis/__init__.py | 46 +-- src/ewatercycle/config/__init__.py | 12 +- src/ewatercycle/config/_config_object.py | 51 +-- src/ewatercycle/config/_validated_config.py | 15 +- src/ewatercycle/config/_validators.py | 18 +- src/ewatercycle/forcing/__init__.py | 59 +-- src/ewatercycle/forcing/_default.py | 25 +- src/ewatercycle/forcing/_hype.py | 40 +- src/ewatercycle/forcing/_lisflood.py | 94 +++-- src/ewatercycle/forcing/_marrmot.py | 48 +-- src/ewatercycle/forcing/_pcrglobwb.py | 13 +- src/ewatercycle/forcing/_wflow.py | 95 ++--- src/ewatercycle/forcing/datasets.py | 24 +- src/ewatercycle/models/__init__.py | 5 +- src/ewatercycle/models/abstract.py | 96 +++-- src/ewatercycle/models/lisflood.py | 8 +- src/ewatercycle/models/marrmot.py | 229 +++++------ src/ewatercycle/models/pcrglobwb.py | 6 +- src/ewatercycle/models/wflow.py | 2 +- src/ewatercycle/observation/grdc.py | 125 +++--- src/ewatercycle/observation/usgs.py | 67 ++-- src/ewatercycle/parameter_sets/__init__.py | 28 +- src/ewatercycle/parameter_sets/_example.py | 19 +- src/ewatercycle/parameter_sets/_lisflood.py | 2 +- src/ewatercycle/parameter_sets/_pcrglobwb.py | 2 +- src/ewatercycle/parameter_sets/_wflow.py | 2 +- src/ewatercycle/parameter_sets/default.py | 15 +- src/ewatercycle/parametersetdb/__init__.py | 8 +- src/ewatercycle/parametersetdb/config.py | 18 +- src/ewatercycle/parametersetdb/datafiles.py | 20 +- src/ewatercycle/util.py | 20 +- src/ewatercycle/version.py | 2 +- tests/config/test_config.py | 165 ++++---- tests/conftest.py | 59 ++- tests/forcing/test_default.py | 102 +++-- tests/forcing/test_lisflood.py | 368 +++++++++--------- tests/forcing/test_marrmot.py | 186 +++++---- tests/forcing/test_pcrglobwb.py | 49 ++- tests/forcing/test_wflow.py | 168 ++++---- tests/models/test_abstract.py | 135 ++++--- tests/models/test_lisflood.py | 40 +- tests/models/test_marrmotm01.py | 132 +++---- tests/models/test_marrmotm14.py | 160 ++++---- tests/models/test_pcrglobwb.py | 31 +- tests/models/test_wflow.py | 25 +- tests/observation/test_grdc.py | 99 ++--- tests/parameter_sets/__init__.py | 4 +- .../test_default_parameterset.py | 24 +- tests/parameter_sets/test_example.py | 73 ++-- tests/test_analysis.py | 27 +- tests/test_config.py | 8 +- tests/test_datafiles.py | 12 +- tests/test_parameter_sets.py | 101 ++--- tests/test_parameterset.py | 18 +- 62 files changed, 1750 insertions(+), 1576 deletions(-) diff --git a/.prospector.yml b/.prospector.yml index 94063ef9..0ec2121e 100644 --- a/.prospector.yml +++ b/.prospector.yml @@ -10,20 +10,20 @@ test-warnings: true member-warnings: false pyroma: - run: true + run: true pep8: - full: true + full: true mypy: run: true pep257: - disable: [ - # Disable because not part of PEP257 official convention: - # see http://pep257.readthedocs.io/en/latest/error_codes.html - D203, # 1 blank line required before class docstring - D212, # Multi-line docstring summary should start at the first line - D213, # Multi-line docstring summary should start at the second line - D404, # First word of the docstring should not be This - ] + disable: [ + # Disable because not part of PEP257 official convention: + # see http://pep257.readthedocs.io/en/latest/error_codes.html + D203, # 1 blank line required before class docstring + D212, # Multi-line docstring summary should start at the first line + D213, # Multi-line docstring summary should start at the second line + D404, # First word of the docstring should not be This + ] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 187a6c4e..e3da3a9d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,6 +55,7 @@ The sections below outline the steps in each case. and [here](https://help.github.com/articles/syncing-a-fork/)); 4. install the package in editable mode and its dependencies with `pip3 install -e .[dev]`; +4. make sure pre commit hook is installed by running `pre-commit install`, causes linting and formatting to be applied during commit; 5. make sure the existing tests still work by running `pytest`; 6. make sure the existing documentation can still by generated without warnings by running `cd docs && make html`. [Pandoc](https://pandoc.org/) is required to generate docs, it can be installed with ``conda install -c conda-forge pandoc`` ; diff --git a/LICENSE b/LICENSE index baecc0b5..e5599134 100644 --- a/LICENSE +++ b/LICENSE @@ -14,4 +14,3 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/docs/examples/data/Rhine/Rhine.prj b/docs/examples/data/Rhine/Rhine.prj index a30c00a5..8f73f480 100644 --- a/docs/examples/data/Rhine/Rhine.prj +++ b/docs/examples/data/Rhine/Rhine.prj @@ -1 +1 @@ -GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]] \ No newline at end of file +GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]] diff --git a/docs/examples/pcrglobwb_merrimack.ini b/docs/examples/pcrglobwb_merrimack.ini index d1177570..a6158e07 100644 --- a/docs/examples/pcrglobwb_merrimack.ini +++ b/docs/examples/pcrglobwb_merrimack.ini @@ -2,7 +2,7 @@ # Set the pcrglobwb output directory in an absolute path. outputDir = /data/output -# Set the input directory map in an absolute path. +# Set the input directory map in an absolute path. # - The input forcing and parameter directories and files will be relative to this. inputDir = /data/input @@ -10,14 +10,14 @@ inputDir = /data/input # - Spatial resolution and coverage are based on this map: cloneMap = global_05min/cloneMaps/merrimack_05min.map # The area/landmask of interest: -# If None, area/landmask is limited for cells with ldd value. +# If None, area/landmask is limited for cells with ldd value. landmask = global_05min/cloneMaps/merrimack_05min.map # netcdf attributes for output files: institution = Department of Physical Geography, Utrecht University title = PCR-GLOBWB 2 output (not coupled to MODFLOW) -description = by Edwin H. Sutanudjaja (contact: e.h.sutanudjaja@uu.nl) +description = by Edwin H. Sutanudjaja (contact: e.h.sutanudjaja@uu.nl) startTime = 2002-01-01 endTime = 2002-12-31 @@ -44,12 +44,12 @@ temperatureNC = global_05min/meteo/pcrglobwb_OBS6_ERA5_reanaly_1_day_tas_2002- referenceETPotMethod = Hamon refETPotFileNC = 'test' -# variable names in the forcing files (optional) +# variable names in the forcing files (optional) precipitationVariableName = pr temperatureVariableName = tas -referenceEPotVariableName = potentialEvaporation +referenceEPotVariableName = potentialEvaporation -# conversion constants and factors to correct forcing values (optional) so that the units are in m.day-1 and degree Celcius +# conversion constants and factors to correct forcing values (optional) so that the units are in m.day-1 and degree Celcius precipitationConstant = 0.0 precipitationFactor = 1.0 temperatureConstant = -273.15 @@ -59,26 +59,26 @@ referenceEPotFactor = 1.0 [meteoDownscalingOptions] -# This section is for a 5 arcmin run, for downscaling meteorological forcing at 30 arcmin to 5 arcmin. - -downscalePrecipitation = True -downscaleTemperature = True -downscaleReferenceETPot = True - -# downscaling (based on the digital elevation model): -# The downscaling will be performed by providing the "cellIds" (meteoDownscaleIds) of lower resolution cells. -meteoDownscaleIds = global_05min/meteo/downscaling_from_30min/uniqueIds_30min.map -highResolutionDEM = global_05min/meteo/downscaling_from_30min/gtopo05min.map - -# lapse rates: -temperLapseRateNC = global_05min/meteo/downscaling_from_30min/temperature_slope.nc -precipLapseRateNC = global_05min/meteo/downscaling_from_30min/precipitation_slope.nc - -# downscaling criteria (TODO: remove these): -temperatCorrelNC = global_05min/meteo/downscaling_from_30min/temperature_correl.nc -precipitCorrelNC = global_05min/meteo/downscaling_from_30min/precipitation_correl.nc - -# windows length (unit: arc-degree) for smoothing/averaging forcing data (not recommended): +# This section is for a 5 arcmin run, for downscaling meteorological forcing at 30 arcmin to 5 arcmin. + +downscalePrecipitation = True +downscaleTemperature = True +downscaleReferenceETPot = True + +# downscaling (based on the digital elevation model): +# The downscaling will be performed by providing the "cellIds" (meteoDownscaleIds) of lower resolution cells. +meteoDownscaleIds = global_05min/meteo/downscaling_from_30min/uniqueIds_30min.map +highResolutionDEM = global_05min/meteo/downscaling_from_30min/gtopo05min.map + +# lapse rates: +temperLapseRateNC = global_05min/meteo/downscaling_from_30min/temperature_slope.nc +precipLapseRateNC = global_05min/meteo/downscaling_from_30min/precipitation_slope.nc + +# downscaling criteria (TODO: remove these): +temperatCorrelNC = global_05min/meteo/downscaling_from_30min/temperature_correl.nc +precipitCorrelNC = global_05min/meteo/downscaling_from_30min/precipitation_correl.nc + +# windows length (unit: arc-degree) for smoothing/averaging forcing data (not recommended): smoothingWindowsLength = 0 @@ -88,16 +88,16 @@ debugWaterBalance = True numberOfUpperSoilLayers = 2 # soil and parameters -# - they are used for all land cover types, unless they are are defined in certain land cover type options -# (e.g. different/various soil types for agriculture areas) +# - they are used for all land cover types, unless they are are defined in certain land cover type options +# (e.g. different/various soil types for agriculture areas) topographyNC = global_05min/landSurface/topography/topography_parameters_5_arcmin_october_2015.nc soilPropertiesNC = global_05min/landSurface/soil/soilProperties5ArcMin.nc includeIrrigation = True -# netcdf time series for historical expansion of irrigation areas (unit: hectares). -# Note: The resolution of this map must be consisten with the resolution of cellArea. +# netcdf time series for historical expansion of irrigation areas (unit: hectares). +# Note: The resolution of this map must be consisten with the resolution of cellArea. historicalIrrigationArea = global_05min/waterUse/irrigation/irrigated_areas/irrigationArea05ArcMin.nc # a pcraster map/value defining irrigation efficiency (dimensionless) - optional @@ -118,10 +118,10 @@ livestockWaterDemandFile = global_05min/waterUse/waterDemand/livestock/livestock desalinationWater = global_05min/waterUse/desalination/desalination_water_version_april_2015.nc -# zone IDs (scale) at which allocations of groundwater and surface water (as well as desalinated water) are performed +# zone IDs (scale) at which allocations of groundwater and surface water (as well as desalinated water) are performed allocationSegmentsForGroundSurfaceWater = global_05min/waterUse/abstraction_zones/abstraction_zones_60min_05min.map -# pcraster maps defining the partitioning of groundwater - surface water source +# pcraster maps defining the partitioning of groundwater - surface water source # # - predefined surface water - groundwater partitioning for irrigation demand (e.g. based on Siebert, Global Map of Irrigation Areas version 5) irrigationSurfaceWaterAbstractionFractionData = global_05min/waterUse/source_partitioning/surface_water_fraction_for_irrigation/AEI_SWFRAC.map @@ -164,9 +164,9 @@ fracVegCover = global_05min/landSurface/landCover/naturalTall/vegf_tall.map minSoilDepthFrac = global_30min/landSurface/landCover/naturalTall/minf_tall_permafrost.map maxSoilDepthFrac = global_30min/landSurface/landCover/naturalTall/maxf_tall.map rootFraction1 = global_05min/landSurface/landCover/naturalTall/rfrac1_tall.map -rootFraction2 = global_05min/landSurface/landCover/naturalTall/rfrac2_tall.map +rootFraction2 = global_05min/landSurface/landCover/naturalTall/rfrac2_tall.map maxRootDepth = 1.0 -# Note: The maxRootDepth is not used for non irrigated land cover type. +# Note: The maxRootDepth is not used for non irrigated land cover type. # # Parameters for the Arno's scheme: arnoBeta = None @@ -209,9 +209,9 @@ fracVegCover = global_05min/landSurface/landCover/naturalShort/vegf_short.m minSoilDepthFrac = global_30min/landSurface/landCover/naturalShort/minf_short_permafrost.map maxSoilDepthFrac = global_30min/landSurface/landCover/naturalShort/maxf_short.map rootFraction1 = global_05min/landSurface/landCover/naturalShort/rfrac1_short.map -rootFraction2 = global_05min/landSurface/landCover/naturalShort/rfrac2_short.map +rootFraction2 = global_05min/landSurface/landCover/naturalShort/rfrac2_short.map maxRootDepth = 0.5 -# Note: The maxRootDepth is not used for non irrigated land cover type. +# Note: The maxRootDepth is not used for non irrigated land cover type. # # Parameters for the Arno's scheme: arnoBeta = None @@ -325,14 +325,14 @@ debugWaterBalance = True groundwaterPropertiesNC = global_05min/groundwater/properties/groundwaterProperties5ArcMin.nc # The file will containspecificYield (m3.m-3), kSatAquifer, recessionCoeff (day-1) # -# - minimum value for groundwater recession coefficient (day-1) +# - minimum value for groundwater recession coefficient (day-1) minRecessionCoeff = 1.0e-4 # some options for constraining groundwater abstraction limitFossilGroundWaterAbstraction = True estimateOfRenewableGroundwaterCapacity = 0.0 estimateOfTotalGroundwaterThickness = global_05min/groundwater/aquifer_thickness_estimate/thickness_05min.map -# minimum and maximum total groundwater thickness +# minimum and maximum total groundwater thickness minimumTotalGroundwaterThickness = 100. maximumTotalGroundwaterThickness = None @@ -350,11 +350,11 @@ avgTotalGroundwaterAbstractionIni = global_05min/initialConditions/avgTo avgTotalGroundwaterAllocationLongIni = global_05min/initialConditions/avgTotalGroundwaterAllocationLong_1999-12-31.map avgTotalGroundwaterAllocationShortIni = global_05min/initialConditions/avgTotalGroundwaterAllocationShort_1999-12-31.map # -# additional initial conditions (needed only for MODFLOW run) +# additional initial conditions (needed only for MODFLOW run) relativeGroundwaterHeadIni = global_05min/initialConditions/relativeGroundwaterHead_1999-12-31.map baseflowIni = global_05min/initialConditions/baseflow_1999-12-31.map -# zonal IDs (scale) at which zonal allocation of groundwater is performed +# zonal IDs (scale) at which zonal allocation of groundwater is performed allocationSegmentsForGroundwater = global_05min/waterUse/abstraction_zones/abstraction_zones_30min_05min.map # assumption for the minimum transmissivity value (unit: m2/day) that can be extracted (via capillary rise and/or groundwater abstraction) - optional @@ -389,7 +389,7 @@ floodplainManningsN = 0.07 # channel gradient gradient = global_05min/routing/channel_properties/channel_gradient.map -# constant channel depth +# constant channel depth constantChannelDepth = global_05min/routing/channel_properties/bankfull_depth.map # constant channel width (optional) @@ -400,15 +400,15 @@ minimumChannelWidth = global_05min/routing/channel_properties/bankfull_width.ma # channel properties for flooding bankfullCapacity = None -# - If None, it will be estimated from (bankfull) channel depth (m) and width (m) +# - If None, it will be estimated from (bankfull) channel depth (m) and width (m) -# files for relative elevation (above minimum dem) +# files for relative elevation (above minimum dem) relativeElevationFiles = global_05min/routing/channel_properties/dzRel%04d.map relativeElevationLevels = 0.0, 0.01, 0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 1.00 -# composite crop factors for WaterBodies: +# composite crop factors for WaterBodies: cropCoefficientWaterNC = global_30min/routing/kc_surface_water/cropCoefficientForOpenWater.nc minCropWaterKC = 1.00 @@ -436,7 +436,7 @@ avgLakeReservoirOutflowLongIni = global_05min/initialConditions/avgLakeReservoir # # number of days (timesteps) that have been performed for spinning up initial conditions in the routing module (i.e. channelStorageIni, avgDischargeLongIni, avgDischargeShortIni, etc.) timestepsToAvgDischargeIni = global_05min/initialConditions/timestepsToAvgDischarge_1999-12-31.map -# Note that: +# Note that: # - maximum number of days (timesteps) to calculate long term average flow values (default: 5 years = 5 * 365 days = 1825) # - maximum number of days (timesteps) to calculate short term average values (default: 1 month = 1 * 30 days = 30) @@ -464,6 +464,3 @@ outAnnuaMaxNC = None # netcdf format and zlib setup formatNetCDF = NETCDF4 zlib = True - - - diff --git a/docs/observations.rst b/docs/observations.rst index 960518eb..cb943a1d 100644 --- a/docs/observations.rst +++ b/docs/observations.rst @@ -7,7 +7,7 @@ USGS ---- The `U.S. Geological Survey Water Services `_ provides public discharge data for a large number of US based stations. In eWaterCycle we make use of the `USGS web service `_ to automatically retrieve this data. -The Discharge timestamp is corrected to the UTC timezone. Units are converted from cubic feet per second to cubic meter per second. +The Discharge timestamp is corrected to the UTC timezone. Units are converted from cubic feet per second to cubic meter per second. GRDC ---- diff --git a/docs/user_guide.ipynb b/docs/user_guide.ipynb index 5fa31a6f..9de9101e 100644 --- a/docs/user_guide.ipynb +++ b/docs/user_guide.ipynb @@ -809,7 +809,7 @@ " station_id=grdc_station_id,\n", " start_time=\"1990-01-01T00:00:00Z\", # or: model_instance.start_time_as_isostr\n", " end_time=\"1990-12-15T00:00:00Z\",\n", - " column=\"GRDC\"\n", + " column=\"GRDC\",\n", ")\n", "\n", "observations.head()" @@ -1012,4 +1012,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/environment.yml b/environment.yml index ccfd8f99..1da148ff 100644 --- a/environment.yml +++ b/environment.yml @@ -9,5 +9,6 @@ dependencies: - esmvaltool-python>=2.3.0 - subversion # Pin esmpy so we dont get forced to run all parallel tasks on single cpu - # Can be removed once https://github.com/ESMValGroup/ESMValCore/issues/1208 is resolved. + # Can be removed once + # https://github.com/ESMValGroup/ESMValCore/issues/1208 is resolved. - esmpy!=8.1.0 diff --git a/src/ewatercycle/analysis/__init__.py b/src/ewatercycle/analysis/__init__.py index 3c661230..5c62dd9f 100644 --- a/src/ewatercycle/analysis/__init__.py +++ b/src/ewatercycle/analysis/__init__.py @@ -1,8 +1,9 @@ -import pandas as pd import os -from typing import Union, Tuple -from hydrostats import metrics +from typing import Tuple, Union + import matplotlib.pyplot as plt +import pandas as pd +from hydrostats import metrics from matplotlib.dates import DateFormatter @@ -12,9 +13,9 @@ def hydrograph( reference: str, precipitation: pd.DataFrame = None, dpi: int = None, - title: str = 'Hydrograph', - discharge_units: str = 'm$^3$ s$^{-1}$', - precipitation_units: str = 'mm day$^{-1}$', + title: str = "Hydrograph", + discharge_units: str = "m$^3$ s$^{-1}$", + precipitation_units: str = "mm day$^{-1}$", figsize: Tuple[float, float] = (10, 10), filename: Union[os.PathLike, str] = None, **kwargs, @@ -68,11 +69,11 @@ def hydrograph( ncols=1, dpi=dpi, figsize=figsize, - gridspec_kw={'height_ratios': [3, 1]}, + gridspec_kw={"height_ratios": [3, 1]}, ) ax.set_title(title) - ax.set_ylabel(f'Discharge ({discharge_units})') + ax.set_ylabel(f"Discharge ({discharge_units})") y_obs.plot(ax=ax, **kwargs) y_sim.plot(ax=ax, **kwargs) @@ -83,11 +84,11 @@ def hydrograph( if precipitation is not None: ax_pr = ax.twinx() ax_pr.invert_yaxis() - ax_pr.set_ylabel(f'Precipitation ({precipitation_units})') + ax_pr.set_ylabel(f"Precipitation ({precipitation_units})") prop_cycler = ax._get_lines.prop_cycler for pr_label, pr_timeseries in precipitation.iteritems(): - color = next(prop_cycler)['color'] + color = next(prop_cycler)["color"] ax_pr.bar( pr_timeseries.index.values, pr_timeseries.values, @@ -106,33 +107,34 @@ def hydrograph( labels = labels_pr + labels # Put the legend outside the plot - ax.legend(handles, labels, bbox_to_anchor=(1.10, 1), loc='upper left') + ax.legend(handles, labels, bbox_to_anchor=(1.10, 1), loc="upper left") # set formatting for xticks date_fmt = DateFormatter("%Y-%m") ax.xaxis.set_major_formatter(date_fmt) - ax.tick_params(axis='x', rotation=30) + ax.tick_params(axis="x", rotation=30) # calculate metrics for data table underneath plot def calc_metric(metric) -> float: return y_sim.apply(metric, observed_array=y_obs) - metrs = pd.DataFrame({ - 'nse': calc_metric(metrics.nse), - 'kge_2009': calc_metric(metrics.kge_2009), - 'sa': calc_metric(metrics.sa), - 'me': calc_metric(metrics.me), - }) + metrs = pd.DataFrame( + { + "nse": calc_metric(metrics.nse), + "kge_2009": calc_metric(metrics.kge_2009), + "sa": calc_metric(metrics.sa), + "me": calc_metric(metrics.me), + } + ) # convert data in dataframe to strings - cell_text = [[f'{item:.2f}' for item in row[1]] - for row in metrs.iterrows()] + cell_text = [[f"{item:.2f}" for item in row[1]] for row in metrs.iterrows()] table = ax_tbl.table( cellText=cell_text, rowLabels=metrs.index, colLabels=metrs.columns, - loc='center', + loc="center", ) ax_tbl.set_axis_off() @@ -140,6 +142,6 @@ def calc_metric(metric) -> float: table.scale(1, 1.5) if filename is not None: - fig.savefig(filename, bbox_inches='tight', dpi=dpi) + fig.savefig(filename, bbox_inches="tight", dpi=dpi) return fig, (ax, ax_tbl) diff --git a/src/ewatercycle/config/__init__.py b/src/ewatercycle/config/__init__.py index ed5d1a3a..53b0b9ff 100644 --- a/src/ewatercycle/config/__init__.py +++ b/src/ewatercycle/config/__init__.py @@ -79,12 +79,6 @@ wflow.docker_images: ewatercycle/wflow-grpc4bmi:2020.1.1 """ -from ._config_object import CFG, Config, SYSTEM_CONFIG, USER_HOME_CONFIG, DEFAULT_CONFIG - -__all__ = [ - 'CFG', - 'Config', - 'DEFAULT_CONFIG', - 'SYSTEM_CONFIG', - 'USER_HOME_CONFIG' -] +from ._config_object import CFG, DEFAULT_CONFIG, SYSTEM_CONFIG, USER_HOME_CONFIG, Config + +__all__ = ["CFG", "Config", "DEFAULT_CONFIG", "SYSTEM_CONFIG", "USER_HOME_CONFIG"] diff --git a/src/ewatercycle/config/_config_object.py b/src/ewatercycle/config/_config_object.py index c5020e84..81b64faf 100644 --- a/src/ewatercycle/config/_config_object.py +++ b/src/ewatercycle/config/_config_object.py @@ -4,15 +4,15 @@ from io import StringIO from logging import getLogger from pathlib import Path -from typing import Union, Optional, TextIO +from typing import Optional, TextIO, Union from ruamel.yaml import YAML -from ._validators import _validators -from ._validated_config import ValidatedConfig - from ewatercycle.util import to_absolute_path +from ._validated_config import ValidatedConfig +from ._validators import _validators + logger = getLogger(__name__) @@ -26,7 +26,7 @@ class Config(ValidatedConfig): _validate = _validators @classmethod - def _load_user_config(cls, filename: Union[os.PathLike, str]) -> 'Config': + def _load_user_config(cls, filename: Union[os.PathLike, str]) -> "Config": """Load user configuration from the given file. The config is cleared and updated in-place. @@ -38,7 +38,7 @@ def _load_user_config(cls, filename: Union[os.PathLike, str]) -> 'Config': """ new = cls() mapping = read_config_file(filename) - mapping['ewatercycle_config'] = filename + mapping["ewatercycle_config"] = filename new.update(CFG_DEFAULT) new.update(mapping) @@ -46,7 +46,7 @@ def _load_user_config(cls, filename: Union[os.PathLike, str]) -> 'Config': return new @classmethod - def _load_default_config(cls, filename: Union[os.PathLike, str]) -> 'Config': + def _load_default_config(cls, filename: Union[os.PathLike, str]) -> "Config": """Load the default configuration.""" new = cls() mapping = read_config_file(filename) @@ -58,7 +58,7 @@ def load_from_file(self, filename: Union[os.PathLike, str]) -> None: """Load user configuration from the given file.""" path = to_absolute_path(str(filename)) if not path.exists(): - raise FileNotFoundError(f'Cannot find: `{filename}') + raise FileNotFoundError(f"Cannot find: `{filename}") self.clear() self.update(CFG_DEFAULT) @@ -66,12 +66,11 @@ def load_from_file(self, filename: Union[os.PathLike, str]) -> None: def reload(self) -> None: """Reload the config file.""" - filename = self.get('ewatercycle_config', DEFAULT_CONFIG) + filename = self.get("ewatercycle_config", DEFAULT_CONFIG) self.load_from_file(filename) def dump_to_yaml(self) -> str: - """Dumps YAML formatted string of Config object - """ + """Dumps YAML formatted string of Config object""" stream = StringIO() self._save_to_stream(stream) return stream.getvalue() @@ -87,7 +86,7 @@ def _save_to_stream(self, stream: TextIO): cp["output_dir"] = str(cp["output_dir"]) cp["parameterset_dir"] = str(cp["parameterset_dir"]) - yaml = YAML(typ='safe') + yaml = YAML(typ="safe") yaml.dump(cp, stream) def save_to_file(self, config_file: Optional[Union[os.PathLike, str]] = None): @@ -102,10 +101,12 @@ def save_to_file(self, config_file: Optional[Union[os.PathLike, str]] = None): old_config_file = self.get("ewatercycle_config", None) if config_file is None: - config_file = USER_HOME_CONFIG if old_config_file is None else old_config_file + config_file = ( + USER_HOME_CONFIG if old_config_file is None else old_config_file + ) if config_file == DEFAULT_CONFIG: - raise PermissionError(f'Not allowed to write to {config_file}', config_file) + raise PermissionError(f"Not allowed to write to {config_file}", config_file) with open(config_file, "w") as f: self._save_to_stream(f) @@ -119,10 +120,10 @@ def read_config_file(config_file: Union[os.PathLike, str]) -> dict: """Read config user file and store settings in a dictionary.""" config_file = to_absolute_path(str(config_file)) if not config_file.exists(): - raise IOError(f'Config file `{config_file}` does not exist.') + raise IOError(f"Config file `{config_file}` does not exist.") - with open(config_file, 'r') as file: - yaml = YAML(typ='safe') + with open(config_file, "r") as file: + yaml = YAML(typ="safe") cfg = yaml.load(file) return cfg @@ -137,15 +138,17 @@ def find_user_config(sources: tuple) -> Optional[os.PathLike]: return None -FILENAME = 'ewatercycle.yaml' +FILENAME = "ewatercycle.yaml" -USER_HOME_CONFIG = Path.home() / os.environ.get('XDG_CONFIG_HOME', '.config') / 'ewatercycle' / FILENAME -SYSTEM_CONFIG = Path('/etc') / FILENAME - -SOURCES = ( - USER_HOME_CONFIG, - SYSTEM_CONFIG +USER_HOME_CONFIG = ( + Path.home() + / os.environ.get("XDG_CONFIG_HOME", ".config") + / "ewatercycle" + / FILENAME ) +SYSTEM_CONFIG = Path("/etc") / FILENAME + +SOURCES = (USER_HOME_CONFIG, SYSTEM_CONFIG) USER_CONFIG = find_user_config(SOURCES) DEFAULT_CONFIG = Path(__file__).parent / FILENAME diff --git a/src/ewatercycle/config/_validated_config.py b/src/ewatercycle/config/_validated_config.py index b527f410..ae02089f 100644 --- a/src/ewatercycle/config/_validated_config.py +++ b/src/ewatercycle/config/_validated_config.py @@ -38,7 +38,8 @@ def __setitem__(self, key, val): raise InvalidConfigParameter(f"Key `{key}`: {verr}") from None except KeyError: raise InvalidConfigParameter( - f"`{key}` is not a valid config parameter.") from None + f"`{key}` is not a valid config parameter." + ) from None self._mapping[key] = cval @@ -50,15 +51,15 @@ def __repr__(self): """Return canonical string representation.""" class_name = self.__class__.__name__ indent = len(class_name) + 1 - repr_split = pprint.pformat(self._mapping, indent=1, - width=80 - indent).split('\n') - repr_indented = ('\n' + ' ' * indent).join(repr_split) - return '{}({})'.format(class_name, repr_indented) + repr_split = pprint.pformat(self._mapping, indent=1, width=80 - indent).split( + "\n" + ) + repr_indented = ("\n" + " " * indent).join(repr_split) + return "{}({})".format(class_name, repr_indented) def __str__(self): """Return string representation.""" - return '\n'.join( - map('{0[0]}: {0[1]}'.format, sorted(self._mapping.items()))) + return "\n".join(map("{0[0]}: {0[1]}".format, sorted(self._mapping.items()))) def __iter__(self): """Yield sorted list of keys.""" diff --git a/src/ewatercycle/config/_validators.py b/src/ewatercycle/config/_validators.py index c176e4c4..5b45f3ea 100644 --- a/src/ewatercycle/config/_validators.py +++ b/src/ewatercycle/config/_validators.py @@ -65,26 +65,18 @@ def func(inp): if allow_stringlist: # Sometimes, a list of colors might be a single string # of single-letter colornames. So give that a shot. - inp = [ - scalar_validator(val.strip()) - for val in inp - if val.strip() - ] + inp = [scalar_validator(val.strip()) for val in inp if val.strip()] else: raise # Allow any ordered sequence type -- generators, np.ndarray, pd.Series # -- but not sets, whose iteration order is non-deterministic. - elif isinstance(inp, Iterable) and not isinstance( - inp, (set, frozenset) - ): + elif isinstance(inp, Iterable) and not isinstance(inp, (set, frozenset)): # The condition on this list comprehension will preserve the # behavior of filtering out any empty strings (behavior was # from the original validate_stringlist()), while allowing # any non-string/text scalar values such as numbers and arrays. inp = [ - scalar_validator(val) - for val in inp - if not isinstance(val, str) or val + scalar_validator(val) for val in inp if not isinstance(val, str) or val ] else: raise ValidationError( @@ -101,9 +93,7 @@ def func(inp): func.__name__ = "{}list".format(scalar_validator.__name__) except AttributeError: # class instance. func.__name__ = "{}List".format(type(scalar_validator).__name__) - func.__qualname__ = ( - func.__qualname__.rsplit(".", 1)[0] + "." + func.__name__ - ) + func.__qualname__ = func.__qualname__.rsplit(".", 1)[0] + "." + func.__name__ if docstring is not None: docstring = scalar_validator.__doc__ func.__doc__ = docstring diff --git a/src/ewatercycle/forcing/__init__.py b/src/ewatercycle/forcing/__init__.py index 84d51ea9..ab8495c8 100644 --- a/src/ewatercycle/forcing/__init__.py +++ b/src/ewatercycle/forcing/__init__.py @@ -1,13 +1,13 @@ from pathlib import Path -from typing import Optional, Dict, Type +from typing import Dict, Optional, Type from ruamel.yaml import YAML -from ._default import DefaultForcing, FORCING_YAML -from . import _hype, _lisflood, _marrmot, _pcrglobwb, _wflow - from ewatercycle.util import to_absolute_path +from . import _hype, _lisflood, _marrmot, _pcrglobwb, _wflow +from ._default import FORCING_YAML, DefaultForcing + FORCING_CLASSES: Dict[str, Type[DefaultForcing]] = { "hype": _hype.HypeForcing, "lisflood": _lisflood.LisfloodForcing, @@ -37,18 +37,20 @@ def load(directory: str) -> DefaultForcing: forcing_info = yaml.load(source / FORCING_YAML) forcing_info.directory = source if forcing_info.shape: - forcing_info.shape = to_absolute_path(forcing_info.shape, parent = source) + forcing_info.shape = to_absolute_path(forcing_info.shape, parent=source) return forcing_info # Or load_custom , load_external, load_???., from_external, import_forcing, -def load_foreign(target_model, - start_time: str, - end_time: str, - directory: str = '.', - shape: str = None, - forcing_info: Optional[Dict] = None) -> DefaultForcing: +def load_foreign( + target_model, + start_time: str, + end_time: str, + directory: str = ".", + shape: str = None, + forcing_info: Optional[Dict] = None, +) -> DefaultForcing: """Load existing forcing data generated from an external source. Args: @@ -105,8 +107,9 @@ def load_foreign(target_model, constructor = FORCING_CLASSES.get(target_model, None) if constructor is None: raise NotImplementedError( - f'Target model `{target_model}` is not supported by the ' - 'eWatercycle forcing generator.') + f"Target model `{target_model}` is not supported by the " + "eWatercycle forcing generator." + ) if forcing_info is None: forcing_info = {} return constructor( # type: ignore # each subclass can have different forcing_info @@ -118,12 +121,14 @@ def load_foreign(target_model, ) -def generate(target_model: str, - dataset: str, - start_time: str, - end_time: str, - shape: str, - model_specific_options: Optional[Dict] = None) -> DefaultForcing: +def generate( + target_model: str, + dataset: str, + start_time: str, + end_time: str, + shape: str, + model_specific_options: Optional[Dict] = None, +) -> DefaultForcing: """Generate forcing data with ESMValTool. Args: @@ -146,19 +151,23 @@ def generate(target_model: str, constructor = FORCING_CLASSES.get(target_model, None) if constructor is None: raise NotImplementedError( - f'Target model `{target_model}` is not supported by the ' - 'eWatercycle forcing generator') + f"Target model `{target_model}` is not supported by the " + "eWatercycle forcing generator" + ) if model_specific_options is None: model_specific_options = {} - forcing_info = constructor.generate(dataset, start_time, end_time, shape, - **model_specific_options) + forcing_info = constructor.generate( + dataset, start_time, end_time, shape, **model_specific_options + ) forcing_info.save() return forcing_info # Append docstrings of with model-specific options to existing docstring load_foreign.__doc__ += "".join( # type:ignore - [f"\n {k}: {v.__init__.__doc__}" for k, v in FORCING_CLASSES.items()]) + [f"\n {k}: {v.__init__.__doc__}" for k, v in FORCING_CLASSES.items()] +) generate.__doc__ += "".join( # type:ignore - [f"\n {k}: {v.generate.__doc__}" for k, v in FORCING_CLASSES.items()]) + [f"\n {k}: {v.generate.__doc__}" for k, v in FORCING_CLASSES.items()] +) diff --git a/src/ewatercycle/forcing/_default.py b/src/ewatercycle/forcing/_default.py index e06b2199..64476dda 100644 --- a/src/ewatercycle/forcing/_default.py +++ b/src/ewatercycle/forcing/_default.py @@ -1,8 +1,8 @@ """Forcing related functionality for default models""" +import logging from copy import copy from pathlib import Path from typing import Optional -import logging from ruamel.yaml import YAML @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) -FORCING_YAML = 'ewatercycle_forcing.yaml' +FORCING_YAML = "ewatercycle_forcing.yaml" class DefaultForcing: @@ -24,11 +24,14 @@ class DefaultForcing: 'YYYY-MM-DDTHH:MM:SSZ'. shape: Path to a shape file. Used for spatial selection. """ - def __init__(self, - start_time: str, - end_time: str, - directory: str, - shape: Optional[str] = None): + + def __init__( + self, + start_time: str, + end_time: str, + directory: str, + shape: Optional[str] = None, + ): self.start_time = start_time self.end_time = end_time self.directory = to_absolute_path(directory) @@ -52,7 +55,7 @@ def generate( end_time: str, shape: str, **model_specific_options, - ) -> 'DefaultForcing': + ) -> "DefaultForcing": """Generate forcing data with ESMValTool.""" raise NotImplementedError("No default forcing generator available.") @@ -71,9 +74,11 @@ def save(self): clone.shape = str(clone.shape.relative_to(self.directory)) except ValueError: clone.shape = None - logger.info(f"Shapefile {self.shape} is not in forcing directory {self.directory}. So, it won't be saved in {target}.") + logger.info( + f"Shapefile {self.shape} is not in forcing directory {self.directory}. So, it won't be saved in {target}." + ) - with open(target, 'w') as f: + with open(target, "w") as f: yaml.dump(clone, f) return target diff --git a/src/ewatercycle/forcing/_hype.py b/src/ewatercycle/forcing/_hype.py index 5e2d91b0..cf80894e 100644 --- a/src/ewatercycle/forcing/_hype.py +++ b/src/ewatercycle/forcing/_hype.py @@ -12,6 +12,7 @@ class HypeForcing(DefaultForcing): """Container for hype forcing data.""" + def __init__( self, start_time: str, @@ -20,55 +21,58 @@ def __init__( shape: Optional[str] = None, ): """ - None: Hype does not have model-specific load options. + None: Hype does not have model-specific load options. """ super().__init__(start_time, end_time, directory, shape) @classmethod def generate( # type: ignore - cls, dataset: str, start_time: str, end_time: str, - shape: str) -> 'HypeForcing': + cls, dataset: str, start_time: str, end_time: str, shape: str + ) -> "HypeForcing": """ - None: Hype does not have model-specific generate options. + None: Hype does not have model-specific generate options. """ # load the ESMValTool recipe recipe_name = "hydrology/recipe_hype.yml" recipe = get_recipe(recipe_name) # model-specific updates to the recipe - preproc_names = ('preprocessor', 'temperature', 'water') + preproc_names = ("preprocessor", "temperature", "water") for preproc_name in preproc_names: - recipe.data['preprocessors'][preproc_name]['extract_shape'][ - 'shapefile'] = to_absolute_path(shape) + recipe.data["preprocessors"][preproc_name]["extract_shape"][ + "shapefile" + ] = to_absolute_path(shape) - recipe.data['datasets'] = [DATASETS[dataset]] + recipe.data["datasets"] = [DATASETS[dataset]] - variables = recipe.data['diagnostics']['hype']['variables'] - var_names = 'tas', 'tasmin', 'tasmax', 'pr' + variables = recipe.data["diagnostics"]["hype"]["variables"] + var_names = "tas", "tasmin", "tasmax", "pr" startyear = get_time(start_time).year for var_name in var_names: - variables[var_name]['start_year'] = startyear + variables[var_name]["start_year"] = startyear endyear = get_time(end_time).year for var_name in var_names: - variables[var_name]['end_year'] = endyear + variables[var_name]["end_year"] = endyear # generate forcing data and retreive useful information recipe_output = recipe.run() # TODO return files created by ESMValTOOL which are needed by Hype Model # forcing_path = list(recipe_output['...........']).data_files[0] - forcing_path = '/foobar.txt' + forcing_path = "/foobar.txt" forcing_file = Path(forcing_path).name directory = str(Path(forcing_file).parent) # instantiate forcing object based on generated data - return HypeForcing(directory=directory, - start_time=str(startyear), - end_time=str(endyear), - shape=shape) + return HypeForcing( + directory=directory, + start_time=str(startyear), + end_time=str(endyear), + shape=shape, + ) def plot(self): - raise NotImplementedError('Dont know how to plot') + raise NotImplementedError("Dont know how to plot") diff --git a/src/ewatercycle/forcing/_lisflood.py b/src/ewatercycle/forcing/_lisflood.py index 9ecc1b19..2185fff4 100644 --- a/src/ewatercycle/forcing/_lisflood.py +++ b/src/ewatercycle/forcing/_lisflood.py @@ -1,16 +1,22 @@ """Forcing related functionality for lisflood""" -from typing import Optional import logging +from typing import Optional from esmvalcore.experimental import get_recipe -from ..util import data_files_from_recipe_output, get_extents, get_time, to_absolute_path +from ..util import ( + data_files_from_recipe_output, + get_extents, + get_time, + to_absolute_path, +) from ._default import DefaultForcing from .datasets import DATASETS logger = logging.getLogger(__name__) + class LisfloodForcing(DefaultForcing): """Container for lisflood forcing data.""" @@ -21,23 +27,23 @@ def __init__( end_time: str, directory: str, shape: Optional[str] = None, - PrefixPrecipitation: str = 'pr.nc', - PrefixTavg: str = 'tas.nc', - PrefixE0: str = 'e0.nc', - PrefixES0: str = 'es0.nc', - PrefixET0: str = 'et0.nc', + PrefixPrecipitation: str = "pr.nc", + PrefixTavg: str = "tas.nc", + PrefixE0: str = "e0.nc", + PrefixES0: str = "es0.nc", + PrefixET0: str = "et0.nc", ): """ - PrefixPrecipitation: Path to a NetCDF or pcraster file with - precipitation data - PrefixTavg: Path to a NetCDF or pcraster file with average - temperature data - PrefixE0: Path to a NetCDF or pcraster file with potential - evaporation rate from open water surface data - PrefixES0: Path to a NetCDF or pcraster file with potential - evaporation rate from bare soil surface data - PrefixET0: Path to a NetCDF or pcraster file with potential - (reference) evapotranspiration rate data + PrefixPrecipitation: Path to a NetCDF or pcraster file with + precipitation data + PrefixTavg: Path to a NetCDF or pcraster file with average + temperature data + PrefixE0: Path to a NetCDF or pcraster file with potential + evaporation rate from open water surface data + PrefixES0: Path to a NetCDF or pcraster file with potential + evaporation rate from bare soil surface data + PrefixET0: Path to a NetCDF or pcraster file with potential + (reference) evapotranspiration rate data """ super().__init__(start_time, end_time, directory, shape) self.PrefixPrecipitation = PrefixPrecipitation @@ -55,47 +61,55 @@ def generate( # type: ignore shape: str, extract_region: dict = None, run_lisvap: bool = False, - ) -> 'LisfloodForcing': + ) -> "LisfloodForcing": """ - extract_region (dict): Region specification, dictionary must contain `start_longitude`, - `end_longitude`, `start_latitude`, `end_latitude` - run_lisvap (bool): if lisvap should be run. Default is False. Running lisvap is not supported yet. - TODO add regrid options so forcing can be generated for parameter set - TODO that is not on a 0.1x0.1 grid + extract_region (dict): Region specification, dictionary must contain `start_longitude`, + `end_longitude`, `start_latitude`, `end_latitude` + run_lisvap (bool): if lisvap should be run. Default is False. Running lisvap is not supported yet. + TODO add regrid options so forcing can be generated for parameter set + TODO that is not on a 0.1x0.1 grid """ # load the ESMValTool recipe recipe_name = "hydrology/recipe_lisflood.yml" recipe = get_recipe(recipe_name) # model-specific updates to the recipe - preproc_names = ('general', 'daily_water', 'daily_temperature', - 'daily_radiation', 'daily_windspeed') + preproc_names = ( + "general", + "daily_water", + "daily_temperature", + "daily_radiation", + "daily_windspeed", + ) basin = to_absolute_path(shape).stem for preproc_name in preproc_names: - recipe.data['preprocessors'][preproc_name]['extract_shape'][ - 'shapefile'] = shape - recipe.data['diagnostics']['diagnostic_daily']['scripts']['script'][ - 'catchment'] = basin + recipe.data["preprocessors"][preproc_name]["extract_shape"][ + "shapefile" + ] = shape + recipe.data["diagnostics"]["diagnostic_daily"]["scripts"]["script"][ + "catchment" + ] = basin if extract_region is None: extract_region = get_extents(shape) for preproc_name in preproc_names: - recipe.data['preprocessors'][preproc_name][ - 'extract_region'] = extract_region + recipe.data["preprocessors"][preproc_name][ + "extract_region" + ] = extract_region - recipe.data['datasets'] = [DATASETS[dataset]] + recipe.data["datasets"] = [DATASETS[dataset]] - variables = recipe.data['diagnostics']['diagnostic_daily']['variables'] - var_names = 'pr', 'tas', 'tasmax', 'tasmin', 'tdps', 'uas', 'vas', 'rsds' + variables = recipe.data["diagnostics"]["diagnostic_daily"]["variables"] + var_names = "pr", "tas", "tasmax", "tasmin", "tdps", "uas", "vas", "rsds" startyear = get_time(start_time).year for var_name in var_names: - variables[var_name]['start_year'] = startyear + variables[var_name]["start_year"] = startyear endyear = get_time(end_time).year for var_name in var_names: - variables[var_name]['end_year'] = endyear + variables[var_name]["end_year"] = endyear # generate forcing data and retrieve useful information recipe_output = recipe.run() @@ -106,11 +120,12 @@ def generate( # type: ignore # instantiate forcing object based on generated data if run_lisvap: - raise NotImplementedError('Dont know how to run LISVAP.') + raise NotImplementedError("Dont know how to run LISVAP.") else: message = ( f"Parameter `run_lisvap` is set to False. No forcing data will be generator for 'e0', 'es0' and 'et0'. " - f"However, the recipe creates LISVAP input data that can be found in {directory}.") + f"However, the recipe creates LISVAP input data that can be found in {directory}." + ) logger.warning("%s", message) return LisfloodForcing( directory=directory, @@ -121,6 +136,5 @@ def generate( # type: ignore PrefixTavg=forcing_files["tas"], ) - def plot(self): - raise NotImplementedError('Dont know how to plot') + raise NotImplementedError("Dont know how to plot") diff --git a/src/ewatercycle/forcing/_marrmot.py b/src/ewatercycle/forcing/_marrmot.py index 32f80f9c..2fa21cb2 100644 --- a/src/ewatercycle/forcing/_marrmot.py +++ b/src/ewatercycle/forcing/_marrmot.py @@ -12,18 +12,19 @@ class MarrmotForcing(DefaultForcing): """Container for marrmot forcing data.""" + def __init__( self, start_time: str, end_time: str, directory: str, shape: Optional[str] = None, - forcing_file: Optional[str] = 'marrmot.mat', + forcing_file: Optional[str] = "marrmot.mat", ): """ - forcing_file: Matlab file that contains forcings for Marrmot - models. See format forcing file in `model implementation - `_. + forcing_file: Matlab file that contains forcings for Marrmot + models. See format forcing file in `model implementation + `_. """ super().__init__(start_time, end_time, directory, shape) self.forcing_file = forcing_file @@ -35,9 +36,9 @@ def generate( # type: ignore start_time: str, end_time: str, shape: str, - ) -> 'MarrmotForcing': + ) -> "MarrmotForcing": """ - None: Marrmot does not have model-specific generate options. + None: Marrmot does not have model-specific generate options. """ # load the ESMValTool recipe @@ -46,24 +47,25 @@ def generate( # type: ignore # model-specific updates to the recipe basin = to_absolute_path(shape).stem - recipe.data['preprocessors']['daily']['extract_shape'][ - 'shapefile'] = shape - recipe.data['diagnostics']['diagnostic_daily']['scripts']['script'][ - 'basin'] = basin + recipe.data["preprocessors"]["daily"]["extract_shape"]["shapefile"] = shape + recipe.data["diagnostics"]["diagnostic_daily"]["scripts"]["script"][ + "basin" + ] = basin - recipe.data['diagnostics']['diagnostic_daily'][ - 'additional_datasets'] = [DATASETS[dataset]] + recipe.data["diagnostics"]["diagnostic_daily"]["additional_datasets"] = [ + DATASETS[dataset] + ] - variables = recipe.data['diagnostics']['diagnostic_daily']['variables'] - var_names = 'tas', 'pr', 'psl', 'rsds', 'rsdt' + variables = recipe.data["diagnostics"]["diagnostic_daily"]["variables"] + var_names = "tas", "pr", "psl", "rsds", "rsdt" startyear = get_time(start_time).year for var_name in var_names: - variables[var_name]['start_year'] = startyear + variables[var_name]["start_year"] = startyear endyear = get_time(end_time).year for var_name in var_names: - variables[var_name]['end_year'] = endyear + variables[var_name]["end_year"] = endyear # generate forcing data and retrieve useful information recipe_output = recipe.run() @@ -72,11 +74,13 @@ def generate( # type: ignore directory = str(Path(forcing_file).parent) # instantiate forcing object based on generated data - return MarrmotForcing(directory=directory, - start_time=start_time, - end_time=end_time, - shape=shape, - forcing_file=forcing_file.name) + return MarrmotForcing( + directory=directory, + start_time=start_time, + end_time=end_time, + shape=shape, + forcing_file=forcing_file.name, + ) def plot(self): - raise NotImplementedError('Dont know how to plot') + raise NotImplementedError("Dont know how to plot") diff --git a/src/ewatercycle/forcing/_pcrglobwb.py b/src/ewatercycle/forcing/_pcrglobwb.py index c3657f83..b29e87c9 100644 --- a/src/ewatercycle/forcing/_pcrglobwb.py +++ b/src/ewatercycle/forcing/_pcrglobwb.py @@ -5,7 +5,12 @@ from esmvalcore.experimental import get_recipe -from ..util import data_files_from_recipe_output, get_extents, get_time, to_absolute_path +from ..util import ( + data_files_from_recipe_output, + get_extents, + get_time, + to_absolute_path, +) from ._default import DefaultForcing from .datasets import DATASETS @@ -62,9 +67,9 @@ def generate( # type: ignore ) if dataset is not None: - recipe.data["diagnostics"]["diagnostic_daily"][ - "additional_datasets" - ] = [DATASETS[dataset]] + recipe.data["diagnostics"]["diagnostic_daily"]["additional_datasets"] = [ + DATASETS[dataset] + ] basin = to_absolute_path(shape).stem recipe.data["diagnostics"]["diagnostic_daily"]["scripts"]["script"][ diff --git a/src/ewatercycle/forcing/_wflow.py b/src/ewatercycle/forcing/_wflow.py index de23453c..75049536 100644 --- a/src/ewatercycle/forcing/_wflow.py +++ b/src/ewatercycle/forcing/_wflow.py @@ -11,6 +11,7 @@ class WflowForcing(DefaultForcing): """Container for wflow forcing data.""" + def __init__( self, start_time: str, @@ -24,14 +25,14 @@ def __init__( Inflow: Optional[str] = None, ): """ - netcdfinput (str) = "inmaps.nc": Path to forcing file." - Precipitation (str) = "/pr": Variable name of precipitation data in - input file. - EvapoTranspiration (str) = "/pet": Variable name of - evapotranspiration data in input file. - Temperature (str) = "/tas": Variable name of temperature data in - input file. - Inflow (str) = None: Variable name of inflow data in input file. + netcdfinput (str) = "inmaps.nc": Path to forcing file." + Precipitation (str) = "/pr": Variable name of precipitation data in + input file. + EvapoTranspiration (str) = "/pet": Variable name of + evapotranspiration data in input file. + Temperature (str) = "/tas": Variable name of temperature data in + input file. + Inflow (str) = None: Variable name of inflow data in input file. """ super().__init__(start_time, end_time, directory, shape) self.netcdfinput = netcdfinput @@ -49,76 +50,78 @@ def generate( # type: ignore shape: str, dem_file: str, extract_region: Dict[str, float] = None, - ) -> 'WflowForcing': + ) -> "WflowForcing": """ - dem_file (str): Name of the dem_file to use. Also defines the basin - param. - extract_region (dict): Region specification, dictionary must - contain `start_longitude`, `end_longitude`, `start_latitude`, - `end_latitude` + dem_file (str): Name of the dem_file to use. Also defines the basin + param. + extract_region (dict): Region specification, dictionary must + contain `start_longitude`, `end_longitude`, `start_latitude`, + `end_latitude` """ # load the ESMValTool recipe recipe_name = "hydrology/recipe_wflow.yml" recipe = get_recipe(recipe_name) basin = to_absolute_path(shape).stem - recipe.data['diagnostics']['wflow_daily']['scripts']['script'][ - 'basin'] = basin + recipe.data["diagnostics"]["wflow_daily"]["scripts"]["script"]["basin"] = basin # model-specific updates - script = recipe.data['diagnostics']['wflow_daily']['scripts']['script'] - script['dem_file'] = dem_file + script = recipe.data["diagnostics"]["wflow_daily"]["scripts"]["script"] + script["dem_file"] = dem_file if extract_region is None: extract_region = get_extents(shape) - recipe.data['preprocessors']['rough_cutout'][ - 'extract_region'] = extract_region + recipe.data["preprocessors"]["rough_cutout"]["extract_region"] = extract_region - recipe.data['diagnostics']['wflow_daily']['additional_datasets'] = [ + recipe.data["diagnostics"]["wflow_daily"]["additional_datasets"] = [ DATASETS[dataset] ] - variables = recipe.data['diagnostics']['wflow_daily']['variables'] - var_names = 'tas', 'pr', 'psl', 'rsds', 'rsdt' + variables = recipe.data["diagnostics"]["wflow_daily"]["variables"] + var_names = "tas", "pr", "psl", "rsds", "rsdt" startyear = get_time(start_time).year for var_name in var_names: - variables[var_name]['start_year'] = startyear + variables[var_name]["start_year"] = startyear endyear = get_time(end_time).year for var_name in var_names: - variables[var_name]['end_year'] = endyear + variables[var_name]["end_year"] = endyear # generate forcing data and retreive useful information recipe_output = recipe.run() - forcing_data = recipe_output['wflow_daily/script'].data_files[0] + forcing_data = recipe_output["wflow_daily/script"].data_files[0] forcing_file = forcing_data.path directory = str(forcing_file.parent) # instantiate forcing object based on generated data - return WflowForcing(directory=directory, - start_time=start_time, - end_time=end_time, - shape=shape, - netcdfinput=forcing_file.name) + return WflowForcing( + directory=directory, + start_time=start_time, + end_time=end_time, + shape=shape, + netcdfinput=forcing_file.name, + ) def __str__(self): """Nice formatting of the forcing data object.""" - return "\n".join([ - "Forcing data for Wflow", - "----------------------", - f"Directory: {self.directory}", - f"Start time: {self.start_time}", - f"End time: {self.end_time}", - f"Shapefile: {self.shape}", - f"Additional information for model config:", - f" - netcdfinput: {self.netcdfinput}", - f" - Precipitation: {self.Precipitation}", - f" - Temperature: {self.Temperature}", - f" - EvapoTranspiration: {self.EvapoTranspiration}", - f" - Inflow: {self.Inflow}", - ]) + return "\n".join( + [ + "Forcing data for Wflow", + "----------------------", + f"Directory: {self.directory}", + f"Start time: {self.start_time}", + f"End time: {self.end_time}", + f"Shapefile: {self.shape}", + f"Additional information for model config:", + f" - netcdfinput: {self.netcdfinput}", + f" - Precipitation: {self.Precipitation}", + f" - Temperature: {self.Temperature}", + f" - EvapoTranspiration: {self.EvapoTranspiration}", + f" - Inflow: {self.Inflow}", + ] + ) def plot(self): - raise NotImplementedError('Dont know how to plot') + raise NotImplementedError("Dont know how to plot") diff --git a/src/ewatercycle/forcing/datasets.py b/src/ewatercycle/forcing/datasets.py index 70d80858..1d87d1d6 100644 --- a/src/ewatercycle/forcing/datasets.py +++ b/src/ewatercycle/forcing/datasets.py @@ -4,18 +4,18 @@ """ DATASETS = { - 'ERA5': { - 'dataset': 'ERA5', - 'project': 'OBS6', - 'tier': 3, - 'type': 'reanaly', - 'version': 1 + "ERA5": { + "dataset": "ERA5", + "project": "OBS6", + "tier": 3, + "type": "reanaly", + "version": 1, }, - 'ERA-Interim': { - 'dataset': 'ERA-Interim', - 'project': 'OBS6', - 'tier': 3, - 'type': 'reanaly', - 'version': 1 + "ERA-Interim": { + "dataset": "ERA-Interim", + "project": "OBS6", + "tier": 3, + "type": "reanaly", + "version": 1, }, } diff --git a/src/ewatercycle/models/__init__.py b/src/ewatercycle/models/__init__.py index 0c72ea4b..9dd9f9de 100644 --- a/src/ewatercycle/models/__init__.py +++ b/src/ewatercycle/models/__init__.py @@ -1,5 +1,4 @@ from .lisflood import Lisflood -from .wflow import Wflow -from .marrmot import MarrmotM01 -from .marrmot import MarrmotM14 +from .marrmot import MarrmotM01, MarrmotM14 from .pcrglobwb import PCRGlobWB +from .wflow import Wflow diff --git a/src/ewatercycle/models/abstract.py b/src/ewatercycle/models/abstract.py index 78137ba1..0f11e6a2 100644 --- a/src/ewatercycle/models/abstract.py +++ b/src/ewatercycle/models/abstract.py @@ -2,33 +2,33 @@ import textwrap from abc import ABCMeta, abstractmethod from datetime import datetime -from typing import Tuple, Iterable, Any, TypeVar, Generic, Optional, ClassVar, Set +from typing import Any, ClassVar, Generic, Iterable, Optional, Set, Tuple, TypeVar import numpy as np import xarray as xr -from cftime import num2date from basic_modeling_interface import Bmi +from cftime import num2date from ewatercycle.forcing import DefaultForcing from ewatercycle.parameter_sets import ParameterSet logger = logging.getLogger(__name__) -ForcingT = TypeVar('ForcingT', bound=DefaultForcing) +ForcingT = TypeVar("ForcingT", bound=DefaultForcing) class AbstractModel(Generic[ForcingT], metaclass=ABCMeta): - """Abstract class of a eWaterCycle model. + """Abstract class of a eWaterCycle model.""" - """ available_versions: ClassVar[Tuple[str, ...]] = tuple() """Versions of model that are available in this class""" - def __init__(self, - version: str, - parameter_set: ParameterSet = None, - forcing: Optional[ForcingT] = None, - ): + def __init__( + self, + version: str, + parameter_set: ParameterSet = None, + forcing: Optional[ForcingT] = None, + ): self.version = version self.parameter_set = parameter_set self.forcing: Optional[ForcingT] = forcing @@ -51,9 +51,9 @@ def __str__(self): "-------------------", f"Version = {self.version}", "Parameter set = ", - textwrap.indent(str(self.parameter_set), ' '), + textwrap.indent(str(self.parameter_set), " "), "Forcing = ", - textwrap.indent(str(self.forcing), ' '), + textwrap.indent(str(self.forcing), " "), ] ) @@ -99,7 +99,9 @@ def get_value(self, name: str) -> np.ndarray: """ return self.bmi.get_value(name) - def get_value_at_coords(self, name, lat: Iterable[float], lon: Iterable[float]) -> np.ndarray: + def get_value_at_coords( + self, name, lat: Iterable[float], lon: Iterable[float] + ) -> np.ndarray: """Get a copy of values of the given variable at lat/lon coordinates. Args: @@ -122,7 +124,9 @@ def set_value(self, name: str, value: np.ndarray) -> None: """ self.bmi.set_value(name, value) - def set_value_at_coords(self, name: str, lat: Iterable[float], lon: Iterable[float], values: np.ndarray) -> None: + def set_value_at_coords( + self, name: str, lat: Iterable[float], lon: Iterable[float], values: np.ndarray + ) -> None: """Specify a new value for a model variable at at lat/lon coordinates. Args: @@ -136,7 +140,9 @@ def set_value_at_coords(self, name: str, lat: Iterable[float], lon: Iterable[flo indices = np.array(indices) self.bmi.set_value_at_indices(name, indices, values) - def _coords_to_indices(self, name: str, lat: Iterable[float], lon: Iterable[float]) -> Iterable[int]: + def _coords_to_indices( + self, name: str, lat: Iterable[float], lon: Iterable[float] + ) -> Iterable[int]: """Converts lat/lon values to index. Args: @@ -144,7 +150,9 @@ def _coords_to_indices(self, name: str, lat: Iterable[float], lon: Iterable[floa lon: Longitudinal value """ - raise NotImplementedError("Method to convert from coordinates to model indices not implemented for this model.") + raise NotImplementedError( + "Method to convert from coordinates to model indices not implemented for this model." + ) @abstractmethod def get_value_as_xarray(self, name: str) -> xr.DataArray: @@ -218,27 +226,30 @@ def time_as_isostr(self) -> str: @property def start_time_as_datetime(self) -> datetime: - """Start time of the model as a datetime object. - """ - return num2date(self.bmi.get_start_time(), - self.bmi.get_time_units(), - only_use_cftime_datetimes=False) + """Start time of the model as a datetime object.""" + return num2date( + self.bmi.get_start_time(), + self.bmi.get_time_units(), + only_use_cftime_datetimes=False, + ) @property def end_time_as_datetime(self) -> datetime: - """End time of the model as a datetime object'. - """ - return num2date(self.bmi.get_end_time(), - self.bmi.get_time_units(), - only_use_cftime_datetimes=False) + """End time of the model as a datetime object'.""" + return num2date( + self.bmi.get_end_time(), + self.bmi.get_time_units(), + only_use_cftime_datetimes=False, + ) @property def time_as_datetime(self) -> datetime: - """Current time of the model as a datetime object'. - """ - return num2date(self.bmi.get_current_time(), - self.bmi.get_time_units(), - only_use_cftime_datetimes=False) + """Current time of the model as a datetime object'.""" + return num2date( + self.bmi.get_current_time(), + self.bmi.get_time_units(), + only_use_cftime_datetimes=False, + ) def _check_parameter_set(self): if not self.parameter_set: @@ -246,17 +257,24 @@ def _check_parameter_set(self): return model_name = self.__class__.__name__.lower() if model_name != self.parameter_set.target_model: - raise ValueError(f'Parameter set has wrong target model, ' - f'expected {model_name} got {self.parameter_set.target_model}') + raise ValueError( + f"Parameter set has wrong target model, " + f"expected {model_name} got {self.parameter_set.target_model}" + ) if self.parameter_set.supported_model_versions == set(): - logger.info(f'Model version {self.version} is not explicitly listed in the supported model versions ' - f'of this parameter set. This can lead to compatibility issues.') + logger.info( + f"Model version {self.version} is not explicitly listed in the supported model versions " + f"of this parameter set. This can lead to compatibility issues." + ) elif self.version not in self.parameter_set.supported_model_versions: raise ValueError( - f'Parameter set is not compatible with version {self.version} of model, ' - f'parameter set only supports {self.parameter_set.supported_model_versions}') + f"Parameter set is not compatible with version {self.version} of model, " + f"parameter set only supports {self.parameter_set.supported_model_versions}" + ) def _check_version(self): if self.version not in self.available_versions: - raise ValueError(f'Supplied version {self.version} is not supported by this model. ' - f'Available versions are {self.available_versions}.') + raise ValueError( + f"Supplied version {self.version} is not supported by this model. " + f"Available versions are {self.available_versions}." + ) diff --git a/src/ewatercycle/models/lisflood.py b/src/ewatercycle/models/lisflood.py index ff9f041d..bf8e36bd 100644 --- a/src/ewatercycle/models/lisflood.py +++ b/src/ewatercycle/models/lisflood.py @@ -194,9 +194,7 @@ def _create_lisflood_config( # input for lisflood if "PrefixPrecipitation" in textvar_name: - textvar.set( - "value", Path(self.forcing.PrefixPrecipitation).stem - ) + textvar.set("value", Path(self.forcing.PrefixPrecipitation).stem) if "PrefixTavg" in textvar_name: textvar.set("value", Path(self.forcing.PrefixTavg).stem) @@ -339,9 +337,7 @@ def _generate_workdir(cfg_dir: Path = None) -> Path: timestamp = datetime.datetime.now(datetime.timezone.utc).strftime( "%Y%m%d_%H%M%S" ) - cfg_dir = to_absolute_path( - f"lisflood_{timestamp}", parent=Path(scratch_dir) - ) + cfg_dir = to_absolute_path(f"lisflood_{timestamp}", parent=Path(scratch_dir)) cfg_dir.mkdir(parents=True, exist_ok=True) return cfg_dir diff --git a/src/ewatercycle/models/marrmot.py b/src/ewatercycle/models/marrmot.py index dcce15c8..79ec15e6 100644 --- a/src/ewatercycle/models/marrmot.py +++ b/src/ewatercycle/models/marrmot.py @@ -1,8 +1,8 @@ -from dataclasses import asdict, dataclass import datetime +import logging +from dataclasses import asdict, dataclass from pathlib import Path from typing import Any, Iterable, List, Tuple -import logging import numpy as np import scipy.io as sio @@ -24,7 +24,8 @@ class Solver: """Solver, for current implementations see `here `_. """ - name: str = 'createOdeApprox_IE' + + name: str = "createOdeApprox_IE" resnorm_tolerance: float = 0.1 resnorm_maxiter: float = 6.0 @@ -35,10 +36,12 @@ def _generate_cfg_dir(cfg_dir: Path = None) -> Path: cfg_dir: If cfg dir is None or does not exist then create sub-directory in CFG['output_dir'] """ if cfg_dir is None: - scratch_dir = CFG['output_dir'] + scratch_dir = CFG["output_dir"] # TODO this timestamp isnot safe for parallel processing - timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S") - cfg_dir = to_absolute_path(f'marrmot_{timestamp}', parent=Path(scratch_dir)) + timestamp = datetime.datetime.now(datetime.timezone.utc).strftime( + "%Y%m%d_%H%M%S" + ) + cfg_dir = to_absolute_path(f"marrmot_{timestamp}", parent=Path(scratch_dir)) cfg_dir.mkdir(parents=True, exist_ok=True) return cfg_dir @@ -56,13 +59,14 @@ class MarrmotM01(AbstractModel[MarrmotForcing]): Example: See examples/marrmotM01.ipynb in `ewatercycle repository `_ """ + model_name = "m_01_collie1_1p_1s" """Name of model in Matlab code.""" - available_versions = ("2020.11", ) + available_versions = ("2020.11",) """Versions for which ewatercycle grpc4bmi docker images are available.""" def __init__(self, version: str, forcing: MarrmotForcing): - """Construct MarrmotM01 with initial values. """ + """Construct MarrmotM01 with initial values.""" super().__init__(version, forcing=forcing) self._parameters = [1000.0] self.store_ini = [900.0] @@ -73,26 +77,24 @@ def __init__(self, version: str, forcing: MarrmotForcing): self._set_docker_image() def _set_docker_image(self): - images = { - '2020.11': 'ewatercycle/marrmot-grpc4bmi:2020.11' - } + images = {"2020.11": "ewatercycle/marrmot-grpc4bmi:2020.11"} self.docker_image = images[self.version] def _set_singularity_image(self): - images = { - '2020.11': 'ewatercycle-marrmot-grpc4bmi_2020.11.sif' - } - if CFG.get('singularity_dir'): - self.singularity_image = CFG['singularity_dir'] / images[self.version] + images = {"2020.11": "ewatercycle-marrmot-grpc4bmi_2020.11.sif"} + if CFG.get("singularity_dir"): + self.singularity_image = CFG["singularity_dir"] / images[self.version] # unable to subclass with more specialized arguments so ignore type - def setup(self, # type: ignore - maximum_soil_moisture_storage: float = None, - initial_soil_moisture_storage: float = None, - start_time: str = None, - end_time: str = None, - solver: Solver = None, - cfg_dir: str = None) -> Tuple[str, str]: + def setup( # type: ignore + self, + maximum_soil_moisture_storage: float = None, + initial_soil_moisture_storage: float = None, + start_time: str = None, + end_time: str = None, + solver: Solver = None, + cfg_dir: str = None, + ) -> Tuple[str, str]: """Configure model run. 1. Creates config file and config directory based on the forcing variables and time range @@ -119,14 +121,14 @@ def setup(self, # type: ignore cfg_dir_as_path = _generate_cfg_dir(cfg_dir_as_path) config_file = self._create_marrmot_config(cfg_dir_as_path, start_time, end_time) - if CFG['container_engine'].lower() == 'singularity': + if CFG["container_engine"].lower() == "singularity": message = f"The singularity image {self.singularity_image} does not exist." assert self.singularity_image.exists(), message self.bmi = BmiClientSingularity( image=str(self.singularity_image), work_dir=str(cfg_dir_as_path), ) - elif CFG['container_engine'].lower() == 'docker': + elif CFG["container_engine"].lower() == "docker": self.bmi = BmiClientDocker( image=self.docker_image, image_port=55555, @@ -139,7 +141,7 @@ def setup(self, # type: ignore return str(config_file), str(cfg_dir_as_path) def _check_forcing(self, forcing): - """"Check forcing argument and get path, start and end time of forcing data.""" + """ "Check forcing argument and get path, start and end time of forcing data.""" if isinstance(forcing, MarrmotForcing): forcing_dir = to_absolute_path(forcing.directory) self.forcing_file = str(forcing_dir / forcing.forcing_file) @@ -152,17 +154,19 @@ def _check_forcing(self, forcing): ) # parse start/end time forcing_data = sio.loadmat(self.forcing_file, mat_dtype=True) - if 'parameters' in forcing_data: - self._parameters = forcing_data['parameters'][0] - if 'store_ini' in forcing_data: - self.store_ini = forcing_data['store_ini'][0] - if 'solver' in forcing_data: - forcing_solver = forcing_data['solver'] - self.solver.name = forcing_solver['name'][0][0][0] - self.solver.resnorm_tolerance = forcing_solver['resnorm_tolerance'][0][0][0] - self.solver.resnorm_maxiter = forcing_solver['resnorm_maxiter'][0][0][0] - - def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_time_iso: str = None) -> Path: + if "parameters" in forcing_data: + self._parameters = forcing_data["parameters"][0] + if "store_ini" in forcing_data: + self.store_ini = forcing_data["store_ini"][0] + if "solver" in forcing_data: + forcing_solver = forcing_data["solver"] + self.solver.name = forcing_solver["name"][0][0][0] + self.solver.resnorm_tolerance = forcing_solver["resnorm_tolerance"][0][0][0] + self.solver.resnorm_maxiter = forcing_solver["resnorm_maxiter"][0][0][0] + + def _create_marrmot_config( + self, cfg_dir: Path, start_time_iso: str = None, end_time_iso: str = None + ) -> Path: """Write model configuration file. Adds the model parameters to forcing file for the given period @@ -182,7 +186,7 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ if start_time_iso is not None: start_time = get_time(start_time_iso) if self.forcing_start_time <= start_time <= self.forcing_end_time: - forcing_data['time_start'][0][0:6] = [ + forcing_data["time_start"][0][0:6] = [ start_time.year, start_time.month, start_time.day, @@ -192,11 +196,11 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ ] self.forcing_start_time = start_time else: - raise ValueError('start_time outside forcing time range') + raise ValueError("start_time outside forcing time range") if end_time_iso is not None: end_time = get_time(end_time_iso) if self.forcing_start_time <= end_time <= self.forcing_end_time: - forcing_data['time_end'][0][0:6] = [ + forcing_data["time_end"][0][0:6] = [ end_time.year, end_time.month, end_time.day, @@ -206,7 +210,7 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ ] self.forcing_end_time = end_time else: - raise ValueError('end_time outside forcing time range') + raise ValueError("end_time outside forcing time range") # combine forcing and model parameters forcing_data.update( @@ -216,17 +220,18 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ store_ini=self.store_ini, ) - config_file = cfg_dir / 'marrmot-m01_config.mat' + config_file = cfg_dir / "marrmot-m01_config.mat" sio.savemat(config_file, forcing_data) return config_file def get_value_as_xarray(self, name: str) -> xr.DataArray: """Return the value as xarray object.""" - marrmot_vars = {'S(t)', 'flux_out_Q', 'flux_out_Ea', 'wb'} + marrmot_vars = {"S(t)", "flux_out_Q", "flux_out_Ea", "wb"} if name not in marrmot_vars: raise NotImplementedError( "Variable '{}' is not implemented. " - "Please choose one of {}.".format(name, marrmot_vars)) + "Please choose one of {}.".format(name, marrmot_vars) + ) # Get time information time_units = self.bmi.get_time_units() @@ -239,7 +244,7 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: coords={ "longitude": self.bmi.get_grid_x(grid), "latitude": self.bmi.get_grid_y(grid), - "time": num2date(self.bmi.get_current_time(), time_units) + "time": num2date(self.bmi.get_current_time(), time_units), }, dims=["latitude", "longitude"], name=name, @@ -250,22 +255,24 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: def parameters(self) -> Iterable[Tuple[str, Any]]: """List the parameters for this model.""" p = [ - ('maximum_soil_moisture_storage', self._parameters[0]), - ('initial_soil_moisture_storage', self.store_ini[0]), - ('solver', self.solver), - ('start time', self.forcing_start_time.strftime("%Y-%m-%dT%H:%M:%SZ")), - ('end time', self.forcing_end_time.strftime("%Y-%m-%dT%H:%M:%SZ")), + ("maximum_soil_moisture_storage", self._parameters[0]), + ("initial_soil_moisture_storage", self.store_ini[0]), + ("solver", self.solver), + ("start time", self.forcing_start_time.strftime("%Y-%m-%dT%H:%M:%SZ")), + ("end time", self.forcing_end_time.strftime("%Y-%m-%dT%H:%M:%SZ")), ] return p -M14_PARAMS = ('maximum_soil_moisture_storage', - 'threshold_flow_generation_evap_change', - 'leakage_saturated_zone_flow_coefficient', - 'zero_deficit_base_flow_speed', - 'baseflow_coefficient', - 'gamma_distribution_chi_parameter', - 'gamma_distribution_phi_parameter') +M14_PARAMS = ( + "maximum_soil_moisture_storage", + "threshold_flow_generation_evap_change", + "leakage_saturated_zone_flow_coefficient", + "zero_deficit_base_flow_speed", + "baseflow_coefficient", + "gamma_distribution_chi_parameter", + "gamma_distribution_phi_parameter", +) class MarrmotM14(AbstractModel[MarrmotForcing]): @@ -281,13 +288,14 @@ class MarrmotM14(AbstractModel[MarrmotForcing]): Example: See examples/marrmotM14.ipynb in `ewatercycle repository `_ """ + model_name = "m_14_topmodel_7p_2s" """Name of model in Matlab code.""" - available_versions = ("2020.11", ) + available_versions = ("2020.11",) """Versions for which ewatercycle grpc4bmi docker images are available.""" def __init__(self, version: str, forcing: MarrmotForcing): - """Construct MarrmotM14 with initial values. """ + """Construct MarrmotM14 with initial values.""" super().__init__(version, forcing=forcing) self._parameters = [1000.0, 0.5, 0.5, 100.0, 0.5, 4.25, 2.5] self.store_ini = [900.0, 900.0] @@ -298,33 +306,31 @@ def __init__(self, version: str, forcing: MarrmotForcing): self._set_docker_image() def _set_docker_image(self): - images = { - '2020.11': 'ewatercycle/marrmot-grpc4bmi:2020.11' - } + images = {"2020.11": "ewatercycle/marrmot-grpc4bmi:2020.11"} self.docker_image = images[self.version] def _set_singularity_image(self): - images = { - '2020.11': 'ewatercycle-marrmot-grpc4bmi_2020.11.sif' - } - if CFG.get('singularity_dir'): - self.singularity_image = CFG['singularity_dir'] / images[self.version] + images = {"2020.11": "ewatercycle-marrmot-grpc4bmi_2020.11.sif"} + if CFG.get("singularity_dir"): + self.singularity_image = CFG["singularity_dir"] / images[self.version] # unable to subclass with more specialized arguments so ignore type - def setup(self, # type: ignore - maximum_soil_moisture_storage: float = None, - threshold_flow_generation_evap_change: float = None, - leakage_saturated_zone_flow_coefficient: float = None, - zero_deficit_base_flow_speed: float = None, - baseflow_coefficient: float = None, - gamma_distribution_chi_parameter: float = None, - gamma_distribution_phi_parameter: float = None, - initial_upper_zone_storage: float = None, - initial_saturated_zone_storage: float = None, - start_time: str = None, - end_time: str = None, - solver: Solver = None, - cfg_dir: str = None) -> Tuple[str, str]: + def setup( # type: ignore + self, + maximum_soil_moisture_storage: float = None, + threshold_flow_generation_evap_change: float = None, + leakage_saturated_zone_flow_coefficient: float = None, + zero_deficit_base_flow_speed: float = None, + baseflow_coefficient: float = None, + gamma_distribution_chi_parameter: float = None, + gamma_distribution_phi_parameter: float = None, + initial_upper_zone_storage: float = None, + initial_saturated_zone_storage: float = None, + start_time: str = None, + end_time: str = None, + solver: Solver = None, + cfg_dir: str = None, + ) -> Tuple[str, str]: """Configure model run. 1. Creates config file and config directory based on the forcing variables and time range @@ -363,14 +369,14 @@ def setup(self, # type: ignore cfg_dir_as_path = _generate_cfg_dir(cfg_dir_as_path) config_file = self._create_marrmot_config(cfg_dir_as_path, start_time, end_time) - if CFG['container_engine'].lower() == 'singularity': + if CFG["container_engine"].lower() == "singularity": message = f"The singularity image {self.singularity_image} does not exist." assert self.singularity_image.exists(), message self.bmi = BmiClientSingularity( image=str(self.singularity_image), work_dir=str(cfg_dir_as_path), ) - elif CFG['container_engine'].lower() == 'docker': + elif CFG["container_engine"].lower() == "docker": self.bmi = BmiClientDocker( image=self.docker_image, image_port=55555, @@ -383,7 +389,7 @@ def setup(self, # type: ignore return str(config_file), str(cfg_dir_as_path) def _check_forcing(self, forcing): - """"Check forcing argument and get path, start and end time of forcing data.""" + """ "Check forcing argument and get path, start and end time of forcing data.""" if isinstance(forcing, MarrmotForcing): forcing_dir = to_absolute_path(forcing.directory) self.forcing_file = str(forcing_dir / forcing.forcing_file) @@ -396,25 +402,27 @@ def _check_forcing(self, forcing): ) # parse start/end time forcing_data = sio.loadmat(self.forcing_file, mat_dtype=True) - if 'parameters' in forcing_data: - if len(forcing_data['parameters']) == len(self._parameters): - self._parameters = forcing_data['parameters'] + if "parameters" in forcing_data: + if len(forcing_data["parameters"]) == len(self._parameters): + self._parameters = forcing_data["parameters"] else: message = f"The length of parameters in forcing {self.forcing_file} does not match the length of M14 parameters that is seven." logger.warning("%s", message) - if 'store_ini' in forcing_data: - if len(forcing_data['store_ini']) == len(self.store_ini): - self.store_ini = forcing_data['store_ini'] + if "store_ini" in forcing_data: + if len(forcing_data["store_ini"]) == len(self.store_ini): + self.store_ini = forcing_data["store_ini"] else: message = f"The length of initial stores in forcing {self.forcing_file} does not match the length of M14 iniatial stores that is two." logger.warning("%s", message) - if 'solver' in forcing_data: - forcing_solver = forcing_data['solver'] - self.solver.name = forcing_solver['name'][0][0][0] - self.solver.resnorm_tolerance = forcing_solver['resnorm_tolerance'][0][0][0] - self.solver.resnorm_maxiter = forcing_solver['resnorm_maxiter'][0][0][0] - - def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_time_iso: str = None) -> Path: + if "solver" in forcing_data: + forcing_solver = forcing_data["solver"] + self.solver.name = forcing_solver["name"][0][0][0] + self.solver.resnorm_tolerance = forcing_solver["resnorm_tolerance"][0][0][0] + self.solver.resnorm_maxiter = forcing_solver["resnorm_maxiter"][0][0][0] + + def _create_marrmot_config( + self, cfg_dir: Path, start_time_iso: str = None, end_time_iso: str = None + ) -> Path: """Write model configuration file. Adds the model parameters to forcing file for the given period @@ -434,7 +442,7 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ if start_time_iso is not None: start_time = get_time(start_time_iso) if self.forcing_start_time <= start_time <= self.forcing_end_time: - forcing_data['time_start'][0][0:6] = [ + forcing_data["time_start"][0][0:6] = [ start_time.year, start_time.month, start_time.day, @@ -444,11 +452,11 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ ] self.forcing_start_time = start_time else: - raise ValueError('start_time outside forcing time range') + raise ValueError("start_time outside forcing time range") if end_time_iso is not None: end_time = get_time(end_time_iso) if self.forcing_start_time <= end_time <= self.forcing_end_time: - forcing_data['time_end'][0][0:6] = [ + forcing_data["time_end"][0][0:6] = [ end_time.year, end_time.month, end_time.day, @@ -458,7 +466,7 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ ] self.forcing_end_time = end_time else: - raise ValueError('end_time outside forcing time range') + raise ValueError("end_time outside forcing time range") # combine forcing and model parameters forcing_data.update( @@ -468,17 +476,18 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ store_ini=self.store_ini, ) - config_file = cfg_dir / 'marrmot-m14_config.mat' + config_file = cfg_dir / "marrmot-m14_config.mat" sio.savemat(config_file, forcing_data) return config_file def get_value_as_xarray(self, name: str) -> xr.DataArray: """Return the value as xarray object.""" - marrmot_vars = {'S(t)', 'flux_out_Q', 'flux_out_Ea', 'wb'} + marrmot_vars = {"S(t)", "flux_out_Q", "flux_out_Ea", "wb"} if name not in marrmot_vars: raise NotImplementedError( "Variable '{}' is not implemented. " - "Please choose one of {}.".format(name, marrmot_vars)) + "Please choose one of {}.".format(name, marrmot_vars) + ) # Get time information time_units = self.bmi.get_time_units() @@ -491,7 +500,7 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: coords={ "longitude": self.bmi.get_grid_x(grid), "latitude": self.bmi.get_grid_y(grid), - "time": num2date(self.bmi.get_current_time(), time_units) + "time": num2date(self.bmi.get_current_time(), time_units), }, dims=["latitude", "longitude"], name=name, @@ -503,10 +512,10 @@ def parameters(self) -> Iterable[Tuple[str, Any]]: """List the parameters for this model.""" p: List[Tuple[str, Any]] = list(zip(M14_PARAMS, self._parameters)) p += [ - ('initial_upper_zone_storage', self.store_ini[0]), - ('initial_saturated_zone_storage', self.store_ini[1]), - ('solver', self.solver), - ('start time', self.forcing_start_time.strftime("%Y-%m-%dT%H:%M:%SZ")), - ('end time', self.forcing_end_time.strftime("%Y-%m-%dT%H:%M:%SZ")), + ("initial_upper_zone_storage", self.store_ini[0]), + ("initial_saturated_zone_storage", self.store_ini[1]), + ("solver", self.solver), + ("start time", self.forcing_start_time.strftime("%Y-%m-%dT%H:%M:%SZ")), + ("end time", self.forcing_end_time.strftime("%Y-%m-%dT%H:%M:%SZ")), ] return p diff --git a/src/ewatercycle/models/pcrglobwb.py b/src/ewatercycle/models/pcrglobwb.py index fe3ffcaf..3d75a464 100644 --- a/src/ewatercycle/models/pcrglobwb.py +++ b/src/ewatercycle/models/pcrglobwb.py @@ -159,9 +159,7 @@ def _update_config(self, **kwargs): ) if "routing_method" in kwargs: - cfg.set( - "routingOptions", "routingMethod", kwargs["routing_method"] - ) + cfg.set("routingOptions", "routingMethod", kwargs["routing_method"]) if "dynamic_flood_plain" in kwargs: cfg.set( @@ -203,7 +201,7 @@ def _start_container(self): ) elif CFG["container_engine"] == "singularity": self.bmi = BmiClientSingularity( - image=self._singularity_image(CFG['singularity_dir']), + image=self._singularity_image(CFG["singularity_dir"]), work_dir=str(self.work_dir), input_dirs=additional_input_dirs, timeout=15, diff --git a/src/ewatercycle/models/wflow.py b/src/ewatercycle/models/wflow.py index 37933207..68596738 100644 --- a/src/ewatercycle/models/wflow.py +++ b/src/ewatercycle/models/wflow.py @@ -166,7 +166,7 @@ def _start_container(self): ) elif CFG["container_engine"] == "singularity": self.bmi = BmiClientSingularity( - image=self._singularity_image(CFG['singularity_dir']), + image=self._singularity_image(CFG["singularity_dir"]), work_dir=str(self.work_dir), timeout=15, ) diff --git a/src/ewatercycle/observation/grdc.py b/src/ewatercycle/observation/grdc.py index f05344b7..6facb2db 100644 --- a/src/ewatercycle/observation/grdc.py +++ b/src/ewatercycle/observation/grdc.py @@ -1,20 +1,23 @@ +import logging import os from typing import Dict, Tuple, Union -import logging import pandas as pd + from ewatercycle import CFG from ewatercycle.util import get_time, to_absolute_path logger = logging.getLogger(__name__) -def get_grdc_data(station_id: str, - start_time: str, - end_time: str, - parameter: str = 'Q', - data_home: str = None, - column: str = 'streamflow') -> Tuple[pd.core.frame.DataFrame, Dict[str, Union[str, int, float]]]: +def get_grdc_data( + station_id: str, + start_time: str, + end_time: str, + parameter: str = "Q", + data_home: str = None, + column: str = "streamflow", +) -> Tuple[pd.core.frame.DataFrame, Dict[str, Union[str, int, float]]]: """Get river discharge data from Global Runoff Data Centre (GRDC). Requires the GRDC daily data files in a local directory. The GRDC daily data @@ -83,24 +86,25 @@ def get_grdc_data(station_id: str, data_path = to_absolute_path(CFG["grdc_location"]) else: raise ValueError( - f'Provide the grdc path using `data_home` argument ' - f'or using `grdc_location` in ewatercycle configuration file.' - ) + f"Provide the grdc path using `data_home` argument " + f"or using `grdc_location` in ewatercycle configuration file." + ) if not data_path.exists(): - raise ValueError(f'The grdc directory {data_path} does not exist!') + raise ValueError(f"The grdc directory {data_path} does not exist!") # Read the raw data raw_file = data_path / f"{station_id}_{parameter}_Day.Cmd.txt" if not raw_file.exists(): - raise ValueError(f'The grdc file {raw_file} does not exist!') + raise ValueError(f"The grdc file {raw_file} does not exist!") # Convert the raw data to an xarray metadata, df = _grdc_read( raw_file, start=get_time(start_time).date(), end=get_time(end_time).date(), - column=column) + column=column, + ) # Add start/end_time to metadata metadata["UserStartTime"] = start_time @@ -116,33 +120,32 @@ def get_grdc_data(station_id: str, def _grdc_read(grdc_station_path, start, end, column): - with open( - grdc_station_path, 'r', encoding='cp1252', - errors='ignore') as file: + with open(grdc_station_path, "r", encoding="cp1252", errors="ignore") as file: data = file.read() metadata = _grdc_metadata_reader(grdc_station_path, data) - allLines = data.split('\n') + allLines = data.split("\n") header = 0 for i, line in enumerate(allLines): - if line.startswith('# DATA'): + if line.startswith("# DATA"): header = i + 1 break # Import GRDC data into dataframe and modify dataframe format grdc_data = pd.read_csv( grdc_station_path, - encoding='cp1252', + encoding="cp1252", skiprows=header, - delimiter=';', - parse_dates=['YYYY-MM-DD'], - na_values='-999') + delimiter=";", + parse_dates=["YYYY-MM-DD"], + na_values="-999", + ) grdc_station_df = pd.DataFrame( - {column: grdc_data[' Value'].values}, - index = grdc_data['YYYY-MM-DD'].values, - ) - grdc_station_df.index.rename('time', inplace=True) + {column: grdc_data[" Value"].values}, + index=grdc_data["YYYY-MM-DD"].values, + ) + grdc_station_df.index.rename("time", inplace=True) # Select GRDC station data that matches the forecast results Date grdc_station_select = grdc_station_df.loc[start:end] @@ -172,13 +175,19 @@ def _grdc_metadata_reader(grdc_station_path, allLines): # get grdc ids (from files) and check their consistency with their # file names id_from_file_name = int( - os.path.basename(grdc_station_path).split(".")[0].split("_")[0]) + os.path.basename(grdc_station_path).split(".")[0].split("_")[0] + ) id_from_grdc = None if id_from_file_name == int(allLines[8].split(":")[1].strip()): id_from_grdc = int(allLines[8].split(":")[1].strip()) else: - print("GRDC station " + str(id_from_file_name) + " (" + - str(grdc_station_path) + ") is NOT used.") + print( + "GRDC station " + + str(id_from_file_name) + + " (" + + str(grdc_station_path) + + ") is NOT used." + ) if id_from_grdc is not None: @@ -186,58 +195,57 @@ def _grdc_metadata_reader(grdc_station_path, allLines): attributeGRDC["id_from_grdc"] = id_from_grdc try: - attributeGRDC["file_generation_date"] = \ - str(allLines[6].split(":")[1].strip()) + attributeGRDC["file_generation_date"] = str( + allLines[6].split(":")[1].strip() + ) except: attributeGRDC["file_generation_date"] = "NA" try: - attributeGRDC["river_name"] = \ - str(allLines[9].split(":")[1].strip()) + attributeGRDC["river_name"] = str(allLines[9].split(":")[1].strip()) except: attributeGRDC["river_name"] = "NA" try: - attributeGRDC["station_name"] = \ - str(allLines[10].split(":")[1].strip()) + attributeGRDC["station_name"] = str(allLines[10].split(":")[1].strip()) except: attributeGRDC["station_name"] = "NA" try: - attributeGRDC["country_code"] = \ - str(allLines[11].split(":")[1].strip()) + attributeGRDC["country_code"] = str(allLines[11].split(":")[1].strip()) except: attributeGRDC["country_code"] = "NA" try: - attributeGRDC["grdc_latitude_in_arc_degree"] = \ - float(allLines[12].split(":")[1].strip()) + attributeGRDC["grdc_latitude_in_arc_degree"] = float( + allLines[12].split(":")[1].strip() + ) except: attributeGRDC["grdc_latitude_in_arc_degree"] = "NA" try: - attributeGRDC["grdc_longitude_in_arc_degree"] = \ - float(allLines[13].split(":")[1].strip()) + attributeGRDC["grdc_longitude_in_arc_degree"] = float( + allLines[13].split(":")[1].strip() + ) except: attributeGRDC["grdc_longitude_in_arc_degree"] = "NA" try: - attributeGRDC["grdc_catchment_area_in_km2"] = \ - float(allLines[14].split(":")[1].strip()) + attributeGRDC["grdc_catchment_area_in_km2"] = float( + allLines[14].split(":")[1].strip() + ) if attributeGRDC["grdc_catchment_area_in_km2"] <= 0.0: attributeGRDC["grdc_catchment_area_in_km2"] = "NA" except: attributeGRDC["grdc_catchment_area_in_km2"] = "NA" try: - attributeGRDC["altitude_masl"] = \ - float(allLines[15].split(":")[1].strip()) + attributeGRDC["altitude_masl"] = float(allLines[15].split(":")[1].strip()) except: attributeGRDC["altitude_masl"] = "NA" try: - attributeGRDC["dataSetContent"] = \ - str(allLines[20].split(":")[1].strip()) + attributeGRDC["dataSetContent"] = str(allLines[20].split(":")[1].strip()) except: attributeGRDC["dataSetContent"] = "NA" @@ -247,26 +255,24 @@ def _grdc_metadata_reader(grdc_station_path, allLines): attributeGRDC["units"] = "NA" try: - attributeGRDC["time_series"] = \ - str(allLines[23].split(":")[1].strip()) + attributeGRDC["time_series"] = str(allLines[23].split(":")[1].strip()) except: attributeGRDC["time_series"] = "NA" try: - attributeGRDC["no_of_years"] = \ - int(allLines[24].split(":")[1].strip()) + attributeGRDC["no_of_years"] = int(allLines[24].split(":")[1].strip()) except: attributeGRDC["no_of_years"] = "NA" try: - attributeGRDC["last_update"] = \ - str(allLines[25].split(":")[1].strip()) + attributeGRDC["last_update"] = str(allLines[25].split(":")[1].strip()) except: attributeGRDC["last_update"] = "NA" try: - attributeGRDC["nrMeasurements"] = \ - int(str(allLines[38].split(":")[1].strip())) + attributeGRDC["nrMeasurements"] = int( + str(allLines[38].split(":")[1].strip()) + ) except: attributeGRDC["nrMeasurements"] = "NA" @@ -281,9 +287,9 @@ def _count_missing_data(df, column): def _log_metadata(metadata): """Print some information about data.""" coords = ( - metadata['grdc_latitude_in_arc_degree'], - metadata['grdc_longitude_in_arc_degree'] - ) + metadata["grdc_latitude_in_arc_degree"], + metadata["grdc_longitude_in_arc_degree"], + ) message = ( f"GRDC station {metadata['id_from_grdc']} is selected. " f"The river name is: {metadata['river_name']}." @@ -291,5 +297,6 @@ def _log_metadata(metadata): f"The catchment area in km2 is: {metadata['grdc_catchment_area_in_km2']}. " f"There are {metadata['nrMissingData']} missing values " f"during {metadata['UserStartTime']}_{metadata['UserEndTime']} at this station. " - f"See the metadata for more information.") + f"See the metadata for more information." + ) logger.info("%s", message) diff --git a/src/ewatercycle/observation/usgs.py b/src/ewatercycle/observation/usgs.py index 6a541963..d89db225 100644 --- a/src/ewatercycle/observation/usgs.py +++ b/src/ewatercycle/observation/usgs.py @@ -1,17 +1,13 @@ import os from datetime import datetime -import xarray as xr import numpy as np +import xarray as xr from pyoos.collectors.usgs.usgs_rest import UsgsRest from pyoos.parsers.waterml import WaterML11ToPaegan -def get_usgs_data(station_id, - start_date, - end_date, - parameter='00060', - cache_dir=None): +def get_usgs_data(station_id, start_date, end_date, parameter="00060", cache_dir=None): """ Get river discharge data from the `U.S. Geological Survey Water Services `_ (USGS) rest web service. @@ -48,32 +44,51 @@ def get_usgs_data(station_id, location: (40.6758974, -80.5406244) """ if cache_dir is None: - cache_dir = os.environ['USGS_DATA_HOME'] + cache_dir = os.environ["USGS_DATA_HOME"] # Check if we have the netcdf data netcdf = os.path.join( - cache_dir, "USGS_" + station_id + "_" + parameter + "_" + start_date + - "_" + end_date + ".nc") + cache_dir, + "USGS_" + + station_id + + "_" + + parameter + + "_" + + start_date + + "_" + + end_date + + ".nc", + ) if os.path.exists(netcdf): return xr.open_dataset(netcdf) # Download the data if needed out = os.path.join( - cache_dir, "USGS_" + station_id + "_" + parameter + "_" + start_date + - "_" + end_date + ".wml") + cache_dir, + "USGS_" + + station_id + + "_" + + parameter + + "_" + + start_date + + "_" + + end_date + + ".wml", + ) if not os.path.exists(out): collector = UsgsRest() collector.filter( start=datetime.strptime(start_date, "%Y-%m-%d"), end=datetime.strptime(end_date, "%Y-%m-%d"), variables=[parameter], - features=[station_id]) + features=[station_id], + ) data = collector.raw() - with open(out, 'w') as file: + with open(out, "w") as file: file.write(data) collector.clear() else: - with open(out, 'r') as file: + with open(out, "r") as file: data = file.read() # Convert the raw data to an xarray @@ -86,26 +101,26 @@ def get_usgs_data(station_id, station = data.elements[0] # Unit conversion from cubic feet to cubic meter per second - values = np.array([ - float(point.members[0]['value']) / 35.315 - for point in station.elements - ], - dtype=np.float32) + values = np.array( + [float(point.members[0]["value"]) / 35.315 for point in station.elements], + dtype=np.float32, + ) times = [point.time for point in station.elements] attrs = { - 'units': 'cubic meters per second', + "units": "cubic meters per second", } # Create the xarray dataset - ds = xr.Dataset({'streamflow': (['time'], values, attrs)}, - coords={'time': times}) + ds = xr.Dataset( + {"streamflow": (["time"], values, attrs)}, coords={"time": times} + ) # Set some nice attributes - ds.attrs['title'] = 'USGS Data from streamflow data' - ds.attrs['station'] = station.name - ds.attrs['stationid'] = station.get_uid() - ds.attrs['location'] = (station.location.y, station.location.x) + ds.attrs["title"] = "USGS Data from streamflow data" + ds.attrs["station"] = station.name + ds.attrs["stationid"] = station.get_uid() + ds.attrs["location"] = (station.location.y, station.location.x) ds.to_netcdf(netcdf) diff --git a/src/ewatercycle/parameter_sets/__init__.py b/src/ewatercycle/parameter_sets/__init__.py index d9d178ff..14d1aad3 100644 --- a/src/ewatercycle/parameter_sets/__init__.py +++ b/src/ewatercycle/parameter_sets/__init__.py @@ -4,10 +4,11 @@ from typing import Dict, Tuple from ewatercycle import CFG -from . import _pcrglobwb, _lisflood, _wflow + +from ..config import DEFAULT_CONFIG, SYSTEM_CONFIG, USER_HOME_CONFIG +from . import _lisflood, _pcrglobwb, _wflow from ._example import ExampleParameterSet from .default import ParameterSet -from ..config import DEFAULT_CONFIG, SYSTEM_CONFIG, USER_HOME_CONFIG logger = getLogger(__name__) @@ -34,11 +35,13 @@ def available_parameter_sets(target_model: str = None) -> Tuple[str, ...]: """ all_parameter_sets = _parse_parametersets() if not all_parameter_sets: - if CFG['ewatercycle_config'] == DEFAULT_CONFIG: - raise ValueError(f'No configuration file found.') - raise ValueError(f'No parameter sets defined in {CFG["ewatercycle_config"]}. ' - f'Use `ewatercycle.parareter_sets.download_example_parameter_sets` to download examples ' - f'or define your own or ask whoever setup the ewatercycle system to do it.') + if CFG["ewatercycle_config"] == DEFAULT_CONFIG: + raise ValueError(f"No configuration file found.") + raise ValueError( + f'No parameter sets defined in {CFG["ewatercycle_config"]}. ' + f"Use `ewatercycle.parareter_sets.download_example_parameter_sets` to download examples " + f"or define your own or ask whoever setup the ewatercycle system to do it." + ) # TODO explain somewhere how to add new parameter sets filtered = tuple( name @@ -46,9 +49,11 @@ def available_parameter_sets(target_model: str = None) -> Tuple[str, ...]: if ps.is_available and (target_model is None or ps.target_model == target_model) ) if not filtered: - raise ValueError(f'No parameter sets defined for {target_model} model in {CFG["ewatercycle_config"]}. ' - f'Use `ewatercycle.parareter_sets.download_example_parameter_sets` to download examples ' - f'or define your own or ask whoever setup the ewatercycle system to do it.') + raise ValueError( + f'No parameter sets defined for {target_model} model in {CFG["ewatercycle_config"]}. ' + f"Use `ewatercycle.parareter_sets.download_example_parameter_sets` to download examples " + f"or define your own or ask whoever setup the ewatercycle system to do it." + ) return filtered @@ -82,8 +87,7 @@ def download_parameter_sets(zenodo_doi: str, target_model: str, config: str): def example_parameter_sets() -> Dict[str, ExampleParameterSet]: - """Lists example parameter sets that can be downloaded with :py:func:`~download_example_parameter_sets`. - """ + """Lists example parameter sets that can be downloaded with :py:func:`~download_example_parameter_sets`.""" # TODO how to add a new model docs should be updated with this part examples = chain( _wflow.example_parameter_sets(), diff --git a/src/ewatercycle/parameter_sets/_example.py b/src/ewatercycle/parameter_sets/_example.py index c1df98e2..cbcefe52 100644 --- a/src/ewatercycle/parameter_sets/_example.py +++ b/src/ewatercycle/parameter_sets/_example.py @@ -5,6 +5,7 @@ from urllib import request from ewatercycle import CFG + from .default import ParameterSet logger = getLogger(__name__) @@ -22,7 +23,9 @@ def __init__( doi="N/A", target_model="generic", ): - super().__init__(name, directory, config, doi, target_model, supported_model_versions) + super().__init__( + name, directory, config, doi, target_model, supported_model_versions + ) self.config_url = config_url """URL where model configuration file can be downloaded""" self.datafiles_url = datafiles_url @@ -31,12 +34,16 @@ def __init__( def download(self, skip_existing=False): if self.directory.exists(): if not skip_existing: - raise ValueError(f"Directory {self.directory} for parameter set {self.name}" - f" already exists, will not overwrite. " - f"Try again with skip_existing=True or remove {self.directory} directory.") + raise ValueError( + f"Directory {self.directory} for parameter set {self.name}" + f" already exists, will not overwrite. " + f"Try again with skip_existing=True or remove {self.directory} directory." + ) - logger.info(f'Directory {self.directory} for parameter set {self.name}' - f' already exists, skipping download.') + logger.info( + f"Directory {self.directory} for parameter set {self.name}" + f" already exists, skipping download." + ) return logger.info( f"Downloading example parameter set {self.name} to {self.directory}..." diff --git a/src/ewatercycle/parameter_sets/_lisflood.py b/src/ewatercycle/parameter_sets/_lisflood.py index bfb37255..c8f54028 100644 --- a/src/ewatercycle/parameter_sets/_lisflood.py +++ b/src/ewatercycle/parameter_sets/_lisflood.py @@ -16,6 +16,6 @@ def example_parameter_sets() -> Iterable[ExampleParameterSet]: config_url="https://github.com/ec-jrc/lisflood-usecases/raw/master/LF_lat_lon_UseCase/settings_lat_lon-Run.xml", doi="N/A", target_model="lisflood", - supported_model_versions={"20.10"} + supported_model_versions={"20.10"}, ) ] diff --git a/src/ewatercycle/parameter_sets/_pcrglobwb.py b/src/ewatercycle/parameter_sets/_pcrglobwb.py index c70a7aeb..035352a7 100644 --- a/src/ewatercycle/parameter_sets/_pcrglobwb.py +++ b/src/ewatercycle/parameter_sets/_pcrglobwb.py @@ -16,6 +16,6 @@ def example_parameter_sets() -> Iterable[ExampleParameterSet]: config_url="https://raw.githubusercontent.com/UU-Hydro/PCR-GLOBWB_input_example/master/ini_and_batch_files_for_pcrglobwb_course/rhine_meuse_30min_using_input_example/setup_natural_test.ini", doi="https://doi.org/10.5281/zenodo.1045339", target_model="pcrglobwb", - supported_model_versions={"setters"} + supported_model_versions={"setters"}, ) ] diff --git a/src/ewatercycle/parameter_sets/_wflow.py b/src/ewatercycle/parameter_sets/_wflow.py index 8634b524..74f7758e 100644 --- a/src/ewatercycle/parameter_sets/_wflow.py +++ b/src/ewatercycle/parameter_sets/_wflow.py @@ -16,6 +16,6 @@ def example_parameter_sets() -> Iterable[ExampleParameterSet]: config_url="https://github.com/openstreams/wflow/raw/master/examples/wflow_rhine_sbm_nc/wflow_sbm_NC.ini", doi="N/A", target_model="wflow", - supported_model_versions={"2020.1.1"} + supported_model_versions={"2020.1.1"}, ) ] diff --git a/src/ewatercycle/parameter_sets/default.py b/src/ewatercycle/parameter_sets/default.py index cccce94a..f84da59a 100644 --- a/src/ewatercycle/parameter_sets/default.py +++ b/src/ewatercycle/parameter_sets/default.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Set, Optional +from typing import Optional, Set from ewatercycle import CFG from ewatercycle.util import to_absolute_path @@ -30,11 +30,17 @@ def __init__( supported_model_versions: Optional[Set[str]] = None, ): self.name = name - self.directory = to_absolute_path(directory, parent = CFG.get("parameterset_dir"), must_be_in_parent=False) - self.config = to_absolute_path(config, parent = CFG.get("parameterset_dir"), must_be_in_parent=False) + self.directory = to_absolute_path( + directory, parent=CFG.get("parameterset_dir"), must_be_in_parent=False + ) + self.config = to_absolute_path( + config, parent=CFG.get("parameterset_dir"), must_be_in_parent=False + ) self.doi = doi self.target_model = target_model - self.supported_model_versions = set() if supported_model_versions is None else supported_model_versions + self.supported_model_versions = ( + set() if supported_model_versions is None else supported_model_versions + ) def __repr__(self): options = ", ".join(f"{k}={v!s}" for k, v in self.__dict__.items()) @@ -54,4 +60,3 @@ def __str__(self): def is_available(self) -> bool: """Tests if directory and config file is available on this machine""" return self.directory.exists() and self.config.exists() - diff --git a/src/ewatercycle/parametersetdb/__init__.py b/src/ewatercycle/parametersetdb/__init__.py index c63a9bb2..ee48235c 100644 --- a/src/ewatercycle/parametersetdb/__init__.py +++ b/src/ewatercycle/parametersetdb/__init__.py @@ -2,8 +2,8 @@ """Documentation about ewatercycle_parametersetdb""" from typing import Any -from ewatercycle.parametersetdb.config import AbstractConfig, CONFIG_FORMATS -from ewatercycle.parametersetdb.datafiles import AbstractCopier, DATAFILES_FORMATS +from ewatercycle.parametersetdb.config import CONFIG_FORMATS, AbstractConfig +from ewatercycle.parametersetdb.datafiles import DATAFILES_FORMATS, AbstractCopier class ParameterSet: @@ -45,7 +45,9 @@ def config(self) -> Any: return self._cfg.config -def build_from_urls(config_format, config_url, datafiles_format, datafiles_url) -> ParameterSet: +def build_from_urls( + config_format, config_url, datafiles_format, datafiles_url +) -> ParameterSet: """Construct ParameterSet based on urls Args: diff --git a/src/ewatercycle/parametersetdb/config.py b/src/ewatercycle/parametersetdb/config.py index 105d0313..284160f8 100644 --- a/src/ewatercycle/parametersetdb/config.py +++ b/src/ewatercycle/parametersetdb/config.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -from configparser import ConfigParser from abc import ABC, abstractmethod +from configparser import ConfigParser +from typing import Any, Dict, Type from urllib.request import urlopen -from typing import Any, Type, Dict from ruamel.yaml import YAML @@ -11,6 +11,7 @@ class CaseConfigParser(ConfigParser): """Case sensitive config parser See https://stackoverflow.com/questions/1611799/preserve-case-in-configparser """ + def optionxform(self, optionstr): return optionstr @@ -47,8 +48,8 @@ def save(self, target: str): class IniConfig(AbstractConfig): - """Config container where config is read/saved in ini format. - """ + """Config container where config is read/saved in ini format.""" + def __init__(self, source): super().__init__(source) body = fetch(source) @@ -56,12 +57,13 @@ def __init__(self, source): self.config.read_string(body) def save(self, target): - with open(target, 'w') as f: + with open(target, "w") as f: self.config.write(f) class YamlConfig(AbstractConfig): """Config container where config is read/saved in yaml format""" + yaml = YAML() def __init__(self, source): @@ -70,11 +72,11 @@ def __init__(self, source): self.config = self.yaml.load(body) def save(self, target): - with open(target, 'w') as f: + with open(target, "w") as f: self.yaml.dump(self.config, f) CONFIG_FORMATS: Dict[str, Type[AbstractConfig]] = { - 'ini': IniConfig, - 'yaml': YamlConfig, + "ini": IniConfig, + "yaml": YamlConfig, } diff --git a/src/ewatercycle/parametersetdb/datafiles.py b/src/ewatercycle/parametersetdb/datafiles.py index 193c568f..cf705b8a 100644 --- a/src/ewatercycle/parametersetdb/datafiles.py +++ b/src/ewatercycle/parametersetdb/datafiles.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- import os import subprocess -from abc import abstractmethod, ABC -from typing import Type, Dict +from abc import ABC, abstractmethod +from typing import Dict, Type class AbstractCopier(ABC): @@ -28,22 +28,22 @@ def save(self, target: str): class SubversionCopier(AbstractCopier): - """Uses subversion export to copy files from source to target - """ + """Uses subversion export to copy files from source to target""" + def save(self, target): if os.path.exists(target): - raise Exception('Target directory already exists, will not overwrite') - subprocess.check_call(['svn', 'export', self.source, target]) + raise Exception("Target directory already exists, will not overwrite") + subprocess.check_call(["svn", "export", self.source, target]) class SymlinkCopier(AbstractCopier): - """Creates symlink from source to target - """ + """Creates symlink from source to target""" + def save(self, target): os.symlink(self.source, target) DATAFILES_FORMATS: Dict[str, Type[AbstractCopier]] = { - 'svn': SubversionCopier, - 'symlink': SymlinkCopier, + "svn": SubversionCopier, + "symlink": SymlinkCopier, } diff --git a/src/ewatercycle/util.py b/src/ewatercycle/util.py index 0eddcfc4..a1649a93 100644 --- a/src/ewatercycle/util.py +++ b/src/ewatercycle/util.py @@ -1,11 +1,10 @@ +from datetime import datetime from os import PathLike -from typing import Any, Iterable, Tuple, Dict from pathlib import Path +from typing import Any, Dict, Iterable, Tuple import fiona import numpy as np -from datetime import datetime - from dateutil.parser import parse from esmvalcore.experimental.recipe_output import RecipeOutput from shapely import geometry @@ -49,9 +48,7 @@ def find_closest_point( dx = np.diff(grid_longitudes).max() * 111 # (1 degree ~ 111km) dy = np.diff(grid_latitudes).max() * 111 # (1 degree ~ 111km) if distance > max(dx, dy) * 2: - raise ValueError( - f"Point {point_longitude, point_latitude} outside model grid." - ) + raise ValueError(f"Point {point_longitude, point_latitude} outside model grid.") return idx_lon, idx_lat @@ -121,7 +118,12 @@ def data_files_from_recipe_output( return directory, forcing_files -def to_absolute_path(input_path: str, parent: Path = None, must_exist: bool = False, must_be_in_parent=True) -> Path: +def to_absolute_path( + input_path: str, + parent: Path = None, + must_exist: bool = False, + must_be_in_parent=True, +) -> Path: """Parse input string as :py:class:`pathlib.Path` object. Args: @@ -140,6 +142,8 @@ def to_absolute_path(input_path: str, parent: Path = None, must_exist: bool = Fa try: pathlike.relative_to(parent) except ValueError: - raise ValueError(f"Input path {input_path} is not a subpath of parent {parent}") + raise ValueError( + f"Input path {input_path} is not a subpath of parent {parent}" + ) return pathlike.expanduser().resolve(strict=must_exist) diff --git a/src/ewatercycle/version.py b/src/ewatercycle/version.py index 2a283e93..2d53356f 100644 --- a/src/ewatercycle/version.py +++ b/src/ewatercycle/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -__version__ = '1.1.1' +__version__ = "1.1.1" diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 2da62aae..bef1d412 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -5,8 +5,8 @@ import pytest from ewatercycle import CFG - from ewatercycle.config._config_object import Config +from ewatercycle.config._validated_config import InvalidConfigParameter from ewatercycle.config._validators import ( _listify_validator, validate_float, @@ -18,9 +18,6 @@ validate_string_or_none, ) -from ewatercycle.config._validated_config import ( - InvalidConfigParameter, ) - def generate_validator_testcases(valid): # The code for this function was taken from matplotlib (v3.3) and modified @@ -30,45 +27,54 @@ def generate_validator_testcases(valid): validation_tests = ( { - 'validator': - _listify_validator(validate_float, n_items=2), - 'success': - ((_, [1.5, 2.5]) - for _ in ('1.5, 2.5', [1.5, 2.5], [1.5, 2.5], (1.5, 2.5), - np.array((1.5, 2.5)))), - 'fail': ((_, ValueError) for _ in ('fail', ('a', 1), (1, 2, 3))) + "validator": _listify_validator(validate_float, n_items=2), + "success": ( + (_, [1.5, 2.5]) + for _ in ( + "1.5, 2.5", + [1.5, 2.5], + [1.5, 2.5], + (1.5, 2.5), + np.array((1.5, 2.5)), + ) + ), + "fail": ((_, ValueError) for _ in ("fail", ("a", 1), (1, 2, 3))), }, { - 'validator': - _listify_validator(validate_float, n_items=2), - 'success': - ((_, [1.5, 2.5]) - for _ in ('1.5, 2.5', [1.5, 2.5], [1.5, 2.5], (1.5, 2.5), - np.array((1.5, 2.5)))), - 'fail': ((_, ValueError) for _ in ('fail', ('a', 1), (1, 2, 3))) + "validator": _listify_validator(validate_float, n_items=2), + "success": ( + (_, [1.5, 2.5]) + for _ in ( + "1.5, 2.5", + [1.5, 2.5], + [1.5, 2.5], + (1.5, 2.5), + np.array((1.5, 2.5)), + ) + ), + "fail": ((_, ValueError) for _ in ("fail", ("a", 1), (1, 2, 3))), }, { - 'validator': - _listify_validator(validate_int, n_items=2), - 'success': - ((_, [1, 2]) - for _ in ('1, 2', [1.5, 2.5], [1, 2], (1, 2), np.array((1, 2)))), - 'fail': ((_, ValueError) for _ in ('fail', ('a', 1), (1, 2, 3))) + "validator": _listify_validator(validate_int, n_items=2), + "success": ( + (_, [1, 2]) + for _ in ("1, 2", [1.5, 2.5], [1, 2], (1, 2), np.array((1, 2))) + ), + "fail": ((_, ValueError) for _ in ("fail", ("a", 1), (1, 2, 3))), }, { - 'validator': validate_int_or_none, - 'success': ((None, None), ), - 'fail': (), + "validator": validate_int_or_none, + "success": ((None, None),), + "fail": (), }, { - 'validator': - validate_path, - 'success': ( - ('a/b/c', Path.cwd() / 'a' / 'b' / 'c'), - ('/a/b/c/', Path('/', 'a', 'b', 'c')), - ('~/', Path.home()), + "validator": validate_path, + "success": ( + ("a/b/c", Path.cwd() / "a" / "b" / "c"), + ("/a/b/c/", Path("/", "a", "b", "c")), + ("~/", Path.home()), ), - 'fail': ( + "fail": ( (None, ValueError), (123, ValueError), (False, ValueError), @@ -76,58 +82,57 @@ def generate_validator_testcases(valid): ), }, { - 'validator': validate_path_or_none, - 'success': ((None, None), ), - 'fail': (), + "validator": validate_path_or_none, + "success": ((None, None),), + "fail": (), }, { - 'validator': - _listify_validator(validate_string), - 'success': ( - ('', []), - ('a,b', ['a', 'b']), - ('abc', ['abc']), - ('abc, ', ['abc']), - ('abc, ,', ['abc']), - (['a', 'b'], ['a', 'b']), - (('a', 'b'), ['a', 'b']), - (iter(['a', 'b']), ['a', 'b']), - (np.array(['a', 'b']), ['a', 'b']), - ((1, 2), ['1', '2']), - (np.array([1, 2]), ['1', '2']), + "validator": _listify_validator(validate_string), + "success": ( + ("", []), + ("a,b", ["a", "b"]), + ("abc", ["abc"]), + ("abc, ", ["abc"]), + ("abc, ,", ["abc"]), + (["a", "b"], ["a", "b"]), + (("a", "b"), ["a", "b"]), + (iter(["a", "b"]), ["a", "b"]), + (np.array(["a", "b"]), ["a", "b"]), + ((1, 2), ["1", "2"]), + (np.array([1, 2]), ["1", "2"]), ), - 'fail': ( + "fail": ( (set(), ValueError), (1, ValueError), - ) + ), }, { - 'validator': validate_string_or_none, - 'success': ((None, None), ), - 'fail': (), + "validator": validate_string_or_none, + "success": ((None, None),), + "fail": (), }, ) for validator_dict in validation_tests: - validator = validator_dict['validator'] + validator = validator_dict["validator"] if valid: - for arg, target in validator_dict['success']: + for arg, target in validator_dict["success"]: yield validator, arg, target else: - for arg, error_type in validator_dict['fail']: + for arg, error_type in validator_dict["fail"]: yield validator, arg, error_type -@pytest.mark.parametrize('validator, arg, target', - generate_validator_testcases(True)) +@pytest.mark.parametrize("validator, arg, target", generate_validator_testcases(True)) def test_validator_valid(validator, arg, target): """Test valid cases for the validators.""" res = validator(arg) assert res == target -@pytest.mark.parametrize('validator, arg, exception_type', - generate_validator_testcases(False)) +@pytest.mark.parametrize( + "validator, arg, exception_type", generate_validator_testcases(False) +) def test_validator_invalid(validator, arg, exception_type): """Test invalid cases for the validators.""" with pytest.raises(exception_type): @@ -138,17 +143,17 @@ def test_config_object(): """Test that the config is of the right type.""" assert isinstance(CFG, MutableMapping) - del CFG['output_dir'] - assert 'output_dir' not in CFG + del CFG["output_dir"] + assert "output_dir" not in CFG CFG.reload() - assert 'output_dir' in CFG + assert "output_dir" in CFG def test_config_update(): """Test whether `config.update` raises the correct exception.""" - config = Config({'output_dir': 'directory'}) - fail_dict = {'output_dir': 123} + config = Config({"output_dir": "directory"}) + fail_dict = {"output_dir": 123} with pytest.raises(InvalidConfigParameter): config.update(fail_dict) @@ -157,19 +162,19 @@ def test_config_update(): def test_config_class(): """Test that the validators turn strings into paths.""" config = { - 'container_engine': 'docker', - 'grdc_location': 'path/to/grdc_location', - 'output_dir': 'path/to/output_dir', - 'singularity_dir': 'path/to/singularity_dir', - 'parameterset_dir': 'path/to/parameter_sets', - 'parameter_sets': {} + "container_engine": "docker", + "grdc_location": "path/to/grdc_location", + "output_dir": "path/to/output_dir", + "singularity_dir": "path/to/singularity_dir", + "parameterset_dir": "path/to/parameter_sets", + "parameter_sets": {}, } cfg = Config(config) - assert isinstance(cfg['container_engine'], str) - assert isinstance(cfg['grdc_location'], Path) - assert isinstance(cfg['output_dir'], Path) - assert isinstance(cfg['singularity_dir'], Path) - assert isinstance(cfg['parameterset_dir'], Path) - assert isinstance(cfg['parameter_sets'], dict) + assert isinstance(cfg["container_engine"], str) + assert isinstance(cfg["grdc_location"], Path) + assert isinstance(cfg["output_dir"], Path) + assert isinstance(cfg["singularity_dir"], Path) + assert isinstance(cfg["parameterset_dir"], Path) + assert isinstance(cfg["parameter_sets"], dict) diff --git a/tests/conftest.py b/tests/conftest.py index 1cf25937..03c8330d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,39 +13,56 @@ def yaml_config_url(): @pytest.fixture def yaml_config(): - return OrderedDict([ - ('data', 'data/PEQ_Hupsel.dat'), - ('parameters', OrderedDict([ - ('cW', 200), - ('cV', 4), - ('cG', 5000000.0), - ('cQ', 10), - ('cS', 4), - ('dG0', 1250), - ('cD', 1500), - ('aS', 0.01), - ('st', 'loamy_sand') - ])), - ('start', 367416), - ('end', 368904), - ('step', 1) - ]) + return OrderedDict( + [ + ("data", "data/PEQ_Hupsel.dat"), + ( + "parameters", + OrderedDict( + [ + ("cW", 200), + ("cV", 4), + ("cG", 5000000.0), + ("cQ", 10), + ("cS", 4), + ("dG0", 1250), + ("cD", 1500), + ("aS", 0.01), + ("st", "loamy_sand"), + ] + ), + ), + ("start", 367416), + ("end", 368904), + ("step", 1), + ] + ) @pytest.fixture def sample_parameterset(yaml_config_url): return build_from_urls( - config_format='yaml', config_url=yaml_config_url, - datafiles_format='svn', datafiles_url='http://example.com', + config_format="yaml", + config_url=yaml_config_url, + datafiles_format="svn", + datafiles_url="http://example.com", ) @pytest.fixture def sample_shape(): - return str(Path(__file__).parents[1] / 'docs' / 'examples' / 'data' / 'Rhine' / 'Rhine.shp') + return str( + Path(__file__).parents[1] / "docs" / "examples" / "data" / "Rhine" / "Rhine.shp" + ) + @pytest.fixture def sample_marrmot_forcing_file(): # Downloaded from # https://github.com/wknoben/MARRMoT/blob/master/BMI/Config/BMI_testcase_m01_BuffaloRiver_TN_USA.mat - return str(Path(__file__).parent / 'models' / 'data' / 'BMI_testcase_m01_BuffaloRiver_TN_USA.mat') + return str( + Path(__file__).parent + / "models" + / "data" + / "BMI_testcase_m01_BuffaloRiver_TN_USA.mat" + ) diff --git a/tests/forcing/test_default.py b/tests/forcing/test_default.py index a635974a..c7eb9b90 100644 --- a/tests/forcing/test_default.py +++ b/tests/forcing/test_default.py @@ -1,55 +1,73 @@ import logging + import pytest -from ewatercycle.forcing import generate, load_foreign, DefaultForcing, load, FORCING_YAML +from ewatercycle.forcing import ( + FORCING_YAML, + DefaultForcing, + generate, + load, + load_foreign, +) def test_generate_unknown_model(sample_shape): with pytest.raises(NotImplementedError): generate( target_model="unknown", - dataset='ERA5', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - shape=sample_shape + dataset="ERA5", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + shape=sample_shape, ) def test_load_foreign_unknown(): with pytest.raises(NotImplementedError) as excinfo: load_foreign( - target_model='unknown', - directory='/data/unknown-forcings-case1', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z' + target_model="unknown", + directory="/data/unknown-forcings-case1", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", ) - assert 'Target model `unknown` is not supported by the eWatercycle forcing generator' in str(excinfo.value) + assert ( + "Target model `unknown` is not supported by the eWatercycle forcing generator" + in str(excinfo.value) + ) @pytest.fixture def sample_forcing_yaml_content(): - return ''.join([ - '!DefaultForcing\n', - "start_time: '1989-01-02T00:00:00Z'\n", - "end_time: '1999-01-02T00:00:00Z'\n", - "shape:\n" - ]) + return "".join( + [ + "!DefaultForcing\n", + "start_time: '1989-01-02T00:00:00Z'\n", + "end_time: '1999-01-02T00:00:00Z'\n", + "shape:\n", + ] + ) + @pytest.fixture def sample_forcing_yaml_content_with_shape(): - return ''.join([ - '!DefaultForcing\n', - "start_time: '1989-01-02T00:00:00Z'\n", - "end_time: '1999-01-02T00:00:00Z'\n", - "shape: myshape.shp\n" - ]) - -def test_save_with_shapefile_outside_forcing_dir(sample_shape, tmp_path, sample_forcing_yaml_content, caplog): + return "".join( + [ + "!DefaultForcing\n", + "start_time: '1989-01-02T00:00:00Z'\n", + "end_time: '1999-01-02T00:00:00Z'\n", + "shape: myshape.shp\n", + ] + ) + + +def test_save_with_shapefile_outside_forcing_dir( + sample_shape, tmp_path, sample_forcing_yaml_content, caplog +): forcing = DefaultForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - shape=sample_shape + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + shape=sample_shape, ) with caplog.at_level(logging.INFO): forcing.save() @@ -58,15 +76,18 @@ def test_save_with_shapefile_outside_forcing_dir(sample_shape, tmp_path, sample_ written = file.read_text() expected = sample_forcing_yaml_content assert written == expected - assert 'is not in forcing directory' in caplog.text + assert "is not in forcing directory" in caplog.text -def test_save_with_shapefile_inside_forcing_dir(tmp_path, sample_forcing_yaml_content_with_shape, caplog): + +def test_save_with_shapefile_inside_forcing_dir( + tmp_path, sample_forcing_yaml_content_with_shape, caplog +): forcing = DefaultForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - shape=str(tmp_path / "myshape.shp") + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + shape=str(tmp_path / "myshape.shp"), ) with caplog.at_level(logging.INFO): forcing.save() @@ -75,15 +96,15 @@ def test_save_with_shapefile_inside_forcing_dir(tmp_path, sample_forcing_yaml_co written = file.read_text() expected = sample_forcing_yaml_content_with_shape assert written == expected - assert 'is not in forcing directory' not in caplog.text + assert "is not in forcing directory" not in caplog.text def test_save_without_shapefile(tmp_path, sample_forcing_yaml_content): forcing = DefaultForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", ) forcing.save() @@ -99,8 +120,8 @@ def test_load(tmp_path, sample_forcing_yaml_content): result = load(tmp_path) expected = DefaultForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", ) assert result == expected @@ -111,10 +132,9 @@ def test_load_with_shape(tmp_path, sample_forcing_yaml_content_with_shape): result = load(tmp_path) expected = DefaultForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - shape=tmp_path / "myshape.shp" - + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + shape=tmp_path / "myshape.shp", ) assert result == expected diff --git a/tests/forcing/test_lisflood.py b/tests/forcing/test_lisflood.py index 6d1736dd..5292470f 100644 --- a/tests/forcing/test_lisflood.py +++ b/tests/forcing/test_lisflood.py @@ -1,9 +1,9 @@ import numpy as np import pandas as pd import pytest +import xarray as xr from esmvalcore.experimental import Recipe from esmvalcore.experimental.recipe_output import DataFile -import xarray as xr from ewatercycle.forcing import generate, load from ewatercycle.forcing._lisflood import LisfloodForcing @@ -11,9 +11,9 @@ def test_plot(): f = LisfloodForcing( - directory='.', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + directory=".", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", ) with pytest.raises(NotImplementedError): f.plot() @@ -23,12 +23,14 @@ def create_netcdf(var_name, filename): var = 15 + 8 * np.random.randn(2, 2, 3) lon = [[-99.83, -99.32], [-99.79, -99.23]] lat = [[42.25, 42.21], [42.63, 42.59]] - ds = xr.Dataset({var_name: (["longitude", "latitude", "time"], var)}, - coords={ - "lon": (["longitude", "latitude"], lon), - "lat": (["longitude", "latitude"], lat), - "time": pd.date_range("2014-09-06", periods=3), - }) + ds = xr.Dataset( + {var_name: (["longitude", "latitude", "time"], var)}, + coords={ + "lon": (["longitude", "latitude"], lon), + "lat": (["longitude", "latitude"], lat), + "time": pd.date_range("2014-09-06", periods=3), + }, + ) ds.to_netcdf(filename) return DataFile(filename) @@ -40,15 +42,15 @@ def mock_recipe_run(monkeypatch, tmp_path): # TODO add lisvap input files once implemented, see issue #96 class MockTaskOutput: data_files = ( - create_netcdf('pr', tmp_path / 'lisflood_pr.nc'), - create_netcdf('tas', tmp_path / 'lisflood_tas.nc'), + create_netcdf("pr", tmp_path / "lisflood_pr.nc"), + create_netcdf("tas", tmp_path / "lisflood_tas.nc"), ) def mock_run(self): """Store recipe for inspection and return dummy output.""" nonlocal data - data['data_during_run'] = self.data - return {'diagnostic_daily/script': MockTaskOutput()} + data["data_during_run"] = self.data + return {"diagnostic_daily/script": MockTaskOutput()} monkeypatch.setattr(Recipe, "run", mock_run) return data @@ -58,226 +60,208 @@ class TestGenerateRegionFromShapeFile: @pytest.fixture def forcing(self, mock_recipe_run, sample_shape): return generate( - target_model='lisflood', - dataset='ERA5', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + target_model="lisflood", + dataset="ERA5", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, ) @pytest.fixture def reference_recipe(self): return { - 'datasets': [{ - 'dataset': 'ERA5', - 'project': 'OBS6', - 'tier': 3, - 'type': 'reanaly', - 'version': 1 - }], - 'diagnostics': { - 'diagnostic_daily': { - 'description': - 'LISFLOOD input ' - 'preprocessor for ' - 'ERA-Interim and ERA5 ' - 'data', - 'scripts': { - 'script': { - 'catchment': 'Rhine', - 'script': 'hydrology/lisflood.py' + "datasets": [ + { + "dataset": "ERA5", + "project": "OBS6", + "tier": 3, + "type": "reanaly", + "version": 1, + } + ], + "diagnostics": { + "diagnostic_daily": { + "description": "LISFLOOD input " + "preprocessor for " + "ERA-Interim and ERA5 " + "data", + "scripts": { + "script": { + "catchment": "Rhine", + "script": "hydrology/lisflood.py", } }, - 'variables': { - 'pr': { - 'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily_water', - 'start_year': 1989 + "variables": { + "pr": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily_water", + "start_year": 1989, }, - 'rsds': { - 'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily_radiation', - 'start_year': 1989 + "rsds": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily_radiation", + "start_year": 1989, }, - 'tas': { - 'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily_temperature', - 'start_year': 1989 + "tas": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily_temperature", + "start_year": 1989, }, - 'tasmax': { - 'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily_temperature', - 'start_year': 1989 + "tasmax": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily_temperature", + "start_year": 1989, }, - 'tasmin': { - 'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily_temperature', - 'start_year': 1989 + "tasmin": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily_temperature", + "start_year": 1989, }, - 'tdps': { - 'end_year': 1999, - 'mip': 'Eday', - 'preprocessor': 'daily_temperature', - 'start_year': 1989 + "tdps": { + "end_year": 1999, + "mip": "Eday", + "preprocessor": "daily_temperature", + "start_year": 1989, }, - 'uas': { - 'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily_windspeed', - 'start_year': 1989 + "uas": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily_windspeed", + "start_year": 1989, }, - 'vas': { - 'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily_windspeed', - 'start_year': 1989 - } - } + "vas": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily_windspeed", + "start_year": 1989, + }, + }, } }, - 'documentation': { - 'authors': - ['verhoeven_stefan', 'kalverla_peter', 'andela_bouwe'], - 'projects': ['ewatercycle'], - 'references': ['acknow_project'] + "documentation": { + "authors": ["verhoeven_stefan", "kalverla_peter", "andela_bouwe"], + "projects": ["ewatercycle"], + "references": ["acknow_project"], }, - 'preprocessors': { - 'daily_radiation': { - 'convert_units': { - 'units': 'J m-2 ' - 'day-1' + "preprocessors": { + "daily_radiation": { + "convert_units": {"units": "J m-2 " "day-1"}, + "custom_order": True, + "extract_region": { + "end_latitude": 52.2, + "end_longitude": 11.9, + "start_latitude": 46.3, + "start_longitude": 4.1, }, - 'custom_order': True, - 'extract_region': { - 'end_latitude': 52.2, - 'end_longitude': 11.9, - 'start_latitude': 46.3, - 'start_longitude': 4.1 + "extract_shape": {"crop": True, "method": "contains"}, + "regrid": { + "lat_offset": True, + "lon_offset": True, + "scheme": "linear", + "target_grid": "0.1x0.1", }, - 'extract_shape': { - 'crop': True, - 'method': 'contains' - }, - 'regrid': { - 'lat_offset': True, - 'lon_offset': True, - 'scheme': 'linear', - 'target_grid': '0.1x0.1' - } }, - 'daily_temperature': { - 'convert_units': { - 'units': 'degC' - }, - 'custom_order': True, - 'extract_region': { - 'end_latitude': 52.2, - 'end_longitude': 11.9, - 'start_latitude': 46.3, - 'start_longitude': 4.1 + "daily_temperature": { + "convert_units": {"units": "degC"}, + "custom_order": True, + "extract_region": { + "end_latitude": 52.2, + "end_longitude": 11.9, + "start_latitude": 46.3, + "start_longitude": 4.1, }, - 'extract_shape': { - 'crop': True, - 'method': 'contains' + "extract_shape": {"crop": True, "method": "contains"}, + "regrid": { + "lat_offset": True, + "lon_offset": True, + "scheme": "linear", + "target_grid": "0.1x0.1", }, - 'regrid': { - 'lat_offset': True, - 'lon_offset': True, - 'scheme': 'linear', - 'target_grid': '0.1x0.1' - } }, - 'daily_water': { - 'convert_units': { - 'units': 'kg m-2 d-1' + "daily_water": { + "convert_units": {"units": "kg m-2 d-1"}, + "custom_order": True, + "extract_region": { + "end_latitude": 52.2, + "end_longitude": 11.9, + "start_latitude": 46.3, + "start_longitude": 4.1, }, - 'custom_order': True, - 'extract_region': { - 'end_latitude': 52.2, - 'end_longitude': 11.9, - 'start_latitude': 46.3, - 'start_longitude': 4.1 + "extract_shape": {"crop": True, "method": "contains"}, + "regrid": { + "lat_offset": True, + "lon_offset": True, + "scheme": "linear", + "target_grid": "0.1x0.1", }, - 'extract_shape': { - 'crop': True, - 'method': 'contains' - }, - 'regrid': { - 'lat_offset': True, - 'lon_offset': True, - 'scheme': 'linear', - 'target_grid': '0.1x0.1' - } }, - 'daily_windspeed': { - 'custom_order': True, - 'extract_region': { - 'end_latitude': 52.2, - 'end_longitude': 11.9, - 'start_latitude': 46.3, - 'start_longitude': 4.1 + "daily_windspeed": { + "custom_order": True, + "extract_region": { + "end_latitude": 52.2, + "end_longitude": 11.9, + "start_latitude": 46.3, + "start_longitude": 4.1, }, - 'extract_shape': { - 'crop': True, - 'method': 'contains' + "extract_shape": {"crop": True, "method": "contains"}, + "regrid": { + "lat_offset": True, + "lon_offset": True, + "scheme": "linear", + "target_grid": "0.1x0.1", }, - 'regrid': { - 'lat_offset': True, - 'lon_offset': True, - 'scheme': 'linear', - 'target_grid': '0.1x0.1' - } }, - 'general': { - 'custom_order': True, - 'extract_region': { - 'end_latitude': 52.2, - 'end_longitude': 11.9, - 'start_latitude': 46.3, - 'start_longitude': 4.1 + "general": { + "custom_order": True, + "extract_region": { + "end_latitude": 52.2, + "end_longitude": 11.9, + "start_latitude": 46.3, + "start_longitude": 4.1, }, - 'extract_shape': { - 'crop': True, - 'method': 'contains' + "extract_shape": {"crop": True, "method": "contains"}, + "regrid": { + "lat_offset": True, + "lon_offset": True, + "scheme": "linear", + "target_grid": "0.1x0.1", }, - 'regrid': { - 'lat_offset': True, - 'lon_offset': True, - 'scheme': 'linear', - 'target_grid': '0.1x0.1' - } - } - } + }, + }, } def test_result(self, forcing, tmp_path, sample_shape): - expected = LisfloodForcing(directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - shape=str(sample_shape), - PrefixPrecipitation='lisflood_pr.nc', - PrefixTavg='lisflood_tas.nc') + expected = LisfloodForcing( + directory=str(tmp_path), + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + shape=str(sample_shape), + PrefixPrecipitation="lisflood_pr.nc", + PrefixTavg="lisflood_tas.nc", + ) assert forcing == expected - def test_recipe_configured(self, forcing, mock_recipe_run, - reference_recipe, sample_shape): - actual = mock_recipe_run['data_during_run'] + def test_recipe_configured( + self, forcing, mock_recipe_run, reference_recipe, sample_shape + ): + actual = mock_recipe_run["data_during_run"] # Remove long description and absolute path so assert is easier - actual_desc = actual['documentation']['description'] - del actual['documentation']['description'] - actual_shapefile = actual['preprocessors']['general']['extract_shape'][ - 'shapefile'] + actual_desc = actual["documentation"]["description"] + del actual["documentation"]["description"] + actual_shapefile = actual["preprocessors"]["general"]["extract_shape"][ + "shapefile" + ] # Will also del other occurrences of shapefile due to extract shape object being shared between preprocessors - del actual['preprocessors']['general']['extract_shape']['shapefile'] + del actual["preprocessors"]["general"]["extract_shape"]["shapefile"] assert actual == reference_recipe assert actual_shapefile == sample_shape - assert 'LISFLOOD' in actual_desc + assert "LISFLOOD" in actual_desc def test_saved_yaml(self, forcing, tmp_path): saved_forcing = load(tmp_path) diff --git a/tests/forcing/test_marrmot.py b/tests/forcing/test_marrmot.py index fd332ee3..38734c93 100644 --- a/tests/forcing/test_marrmot.py +++ b/tests/forcing/test_marrmot.py @@ -10,10 +10,10 @@ def test_plot(): f = MarrmotForcing( - directory='.', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - forcing_file='marrmot.mat', + directory=".", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + forcing_file="marrmot.mat", ) with pytest.raises(NotImplementedError): f.plot() @@ -25,16 +25,14 @@ def mock_recipe_run(monkeypatch, tmp_path): recorder = {} class MockTaskOutput: - fake_forcing_path = str(tmp_path / 'marrmot.mat') - files = ( - OutputFile(fake_forcing_path), - ) + fake_forcing_path = str(tmp_path / "marrmot.mat") + files = (OutputFile(fake_forcing_path),) def mock_run(self): """Store recipe for inspection and return dummy output.""" nonlocal recorder - recorder['data_during_run'] = self.data - return {'diagnostic_daily/script': MockTaskOutput()} + recorder["data_during_run"] = self.data + return {"diagnostic_daily/script": MockTaskOutput()} monkeypatch.setattr(Recipe, "run", mock_run) return recorder @@ -44,83 +42,105 @@ class TestGenerate: @pytest.fixture def forcing(self, mock_recipe_run, sample_shape): return generate( - target_model='marrmot', - dataset='ERA5', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + target_model="marrmot", + dataset="ERA5", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, ) @pytest.fixture def reference_recipe(self): return { - 'diagnostics': { - 'diagnostic_daily': { - 'additional_datasets': [{'dataset': 'ERA5', - 'project': 'OBS6', - 'tier': 3, - 'type': 'reanaly', - 'version': 1}], - 'description': 'marrmot input ' - 'preprocessor for daily ' - 'data', - 'scripts': {'script': {'basin': 'Rhine', - 'script': 'hydrology/marrmot.py'}}, - 'variables': {'pr': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily', - 'start_year': 1989}, - 'psl': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily', - 'start_year': 1989}, - 'rsds': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily', - 'start_year': 1989}, - 'rsdt': {'end_year': 1999, - 'mip': 'CFday', - 'preprocessor': 'daily', - 'start_year': 1989}, - 'tas': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily', - 'start_year': 1989} - } - + "diagnostics": { + "diagnostic_daily": { + "additional_datasets": [ + { + "dataset": "ERA5", + "project": "OBS6", + "tier": 3, + "type": "reanaly", + "version": 1, + } + ], + "description": "marrmot input " "preprocessor for daily " "data", + "scripts": { + "script": {"basin": "Rhine", "script": "hydrology/marrmot.py"} + }, + "variables": { + "pr": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily", + "start_year": 1989, + }, + "psl": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily", + "start_year": 1989, + }, + "rsds": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily", + "start_year": 1989, + }, + "rsdt": { + "end_year": 1999, + "mip": "CFday", + "preprocessor": "daily", + "start_year": 1989, + }, + "tas": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily", + "start_year": 1989, + }, + }, + } + }, + "documentation": { + "authors": ["kalverla_peter", "camphuijsen_jaro", "alidoost_sarah"], + "projects": ["ewatercycle"], + "references": ["acknow_project"], + }, + "preprocessors": { + "daily": { + "extract_shape": { + "crop": True, + "method": "contains", + } } }, - 'documentation': {'authors': ['kalverla_peter', - 'camphuijsen_jaro', - 'alidoost_sarah'], - 'projects': ['ewatercycle'], - 'references': ['acknow_project']}, - 'preprocessors': {'daily': {'extract_shape': {'crop': True, - 'method': 'contains', - }}} } def test_result(self, forcing, tmp_path, sample_shape): expected = MarrmotForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - shape = str(sample_shape), - forcing_file='marrmot.mat' + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + shape=str(sample_shape), + forcing_file="marrmot.mat", ) assert forcing == expected - def test_recipe_configured(self, forcing, mock_recipe_run, reference_recipe, sample_shape): - actual = mock_recipe_run['data_during_run'] + def test_recipe_configured( + self, forcing, mock_recipe_run, reference_recipe, sample_shape + ): + actual = mock_recipe_run["data_during_run"] # Remove long description and absolute path so assert is easier - actual_desc = actual['documentation']['description'] - del actual['documentation']['description'] - actual_shapefile = actual['preprocessors']['daily']['extract_shape']['shapefile'] - del actual['preprocessors']['daily']['extract_shape']['shapefile'] + actual_desc = actual["documentation"]["description"] + del actual["documentation"]["description"] + actual_shapefile = actual["preprocessors"]["daily"]["extract_shape"][ + "shapefile" + ] + del actual["preprocessors"]["daily"]["extract_shape"]["shapefile"] assert actual == reference_recipe assert actual_shapefile == sample_shape - assert 'MARRMoT' in actual_desc + assert "MARRMoT" in actual_desc def test_saved_yaml(self, forcing, tmp_path): saved_forcing = load(tmp_path) @@ -133,40 +153,38 @@ def test_saved_yaml(self, forcing, tmp_path): def test_load_foreign(sample_shape, sample_marrmot_forcing_file): forcing_file = Path(sample_marrmot_forcing_file) actual = load_foreign( - target_model='marrmot', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + target_model="marrmot", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, directory=str(forcing_file.parent), - forcing_info={ - 'forcing_file': str(forcing_file.name) - } + forcing_info={"forcing_file": str(forcing_file.name)}, ) expected = MarrmotForcing( - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, directory=str(forcing_file.parent), - forcing_file=str(forcing_file.name) + forcing_file=str(forcing_file.name), ) assert actual == expected def test_load_foreign_without_forcing_info(sample_shape): actual = load_foreign( - target_model='marrmot', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + target_model="marrmot", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, - directory='/data' + directory="/data", ) expected = MarrmotForcing( - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, - directory='/data', - forcing_file='marrmot.mat' + directory="/data", + forcing_file="marrmot.mat", ) assert actual == expected diff --git a/tests/forcing/test_pcrglobwb.py b/tests/forcing/test_pcrglobwb.py index f3739ef4..e66a36e6 100644 --- a/tests/forcing/test_pcrglobwb.py +++ b/tests/forcing/test_pcrglobwb.py @@ -1,9 +1,9 @@ import numpy as np import pandas as pd import pytest +import xarray as xr from esmvalcore.experimental import Recipe from esmvalcore.experimental.recipe_output import DataFile -import xarray as xr from ewatercycle.forcing import generate from ewatercycle.forcing._pcrglobwb import PCRGlobWBForcing @@ -14,14 +14,12 @@ def create_netcdf(var_name, filename): lon = [[-99.83, -99.32], [-99.79, -99.23]] lat = [[42.25, 42.21], [42.63, 42.59]] ds = xr.Dataset( - { - var_name: (["longitude", "latitude", "time"], var) - }, + {var_name: (["longitude", "latitude", "time"], var)}, coords={ "lon": (["longitude", "latitude"], lon), "lat": (["longitude", "latitude"], lat), "time": pd.date_range("2014-09-06", periods=3), - } + }, ) ds.to_netcdf(filename) return DataFile(filename) @@ -34,15 +32,15 @@ def mock_recipe_run(monkeypatch, tmp_path): class MockTaskOutput: data_files = ( - create_netcdf('pr', tmp_path / 'pcrglobwb_pr.nc'), - create_netcdf('tas', tmp_path / 'pcrglobwb_tas.nc'), + create_netcdf("pr", tmp_path / "pcrglobwb_pr.nc"), + create_netcdf("tas", tmp_path / "pcrglobwb_tas.nc"), ) def mock_run(self): """Store recipe for inspection and return dummy output.""" nonlocal data - data['data_during_run'] = self.data - return {'diagnostic_daily/script': MockTaskOutput()} + data["data_during_run"] = self.data + return {"diagnostic_daily/script": MockTaskOutput()} monkeypatch.setattr(Recipe, "run", mock_run) return data @@ -52,33 +50,34 @@ class TestGenerateWithExtractRegion: @pytest.fixture def forcing(self, mock_recipe_run, sample_shape): return generate( - target_model='pcrglobwb', - dataset='ERA5', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + target_model="pcrglobwb", + dataset="ERA5", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, model_specific_options=dict( - start_time_climatology='1979-01-02T00:00:00Z', - end_time_climatology='1989-01-02T00:00:00Z', + start_time_climatology="1979-01-02T00:00:00Z", + end_time_climatology="1989-01-02T00:00:00Z", extract_region={ - 'start_longitude': 10, - 'end_longitude': 16.75, - 'start_latitude': 7.25, - 'end_latitude': 2.5, - } - ) + "start_longitude": 10, + "end_longitude": 16.75, + "start_latitude": 7.25, + "end_latitude": 2.5, + }, + ), ) def test_result(self, forcing, tmp_path, sample_shape): expected = PCRGlobWBForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=str(sample_shape), - precipitationNC='pcrglobwb_pr.nc', - temperatureNC='pcrglobwb_tas.nc' + precipitationNC="pcrglobwb_pr.nc", + temperatureNC="pcrglobwb_tas.nc", ) assert forcing == expected + # TODO test if recipe was generated correctlu # TODO test if yaml was written diff --git a/tests/forcing/test_wflow.py b/tests/forcing/test_wflow.py index f68a6fd4..bc18fc35 100644 --- a/tests/forcing/test_wflow.py +++ b/tests/forcing/test_wflow.py @@ -14,16 +14,14 @@ def mock_recipe_run(monkeypatch, tmp_path): data = {} class MockTaskOutput: - fake_forcing_path = str(tmp_path / 'wflow_forcing.nc') - data_files = ( - DataFile(fake_forcing_path), - ) + fake_forcing_path = str(tmp_path / "wflow_forcing.nc") + data_files = (DataFile(fake_forcing_path),) def mock_run(self): """Store recipe for inspection and return dummy output.""" nonlocal data - data['data_during_run'] = self.data - return {'wflow_daily/script': MockTaskOutput()} + data["data_during_run"] = self.data + return {"wflow_daily/script": MockTaskOutput()} monkeypatch.setattr(Recipe, "run", mock_run) return data @@ -33,88 +31,116 @@ class TestGenerateWithExtractRegion: @pytest.fixture def reference_recipe(self): return { - 'diagnostics': { - 'wflow_daily': { - 'additional_datasets': [{'dataset': 'ERA5', - 'project': 'OBS6', - 'tier': 3, - 'type': 'reanaly', - 'version': 1}], - 'description': 'WFlow input preprocessor for ' - 'daily data', - 'scripts': {'script': {'basin': 'Rhine', - 'dem_file': 'wflow_parameterset/meuse/staticmaps/wflow_dem.map', - 'regrid': 'area_weighted', - 'script': 'hydrology/wflow.py'}}, - 'variables': {'orog': {'mip': 'fx', - 'preprocessor': 'rough_cutout'}, - 'pr': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'rough_cutout', - 'start_year': 1989}, - 'psl': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'rough_cutout', - 'start_year': 1989}, - 'rsds': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'rough_cutout', - 'start_year': 1989}, - 'rsdt': {'end_year': 1999, - 'mip': 'CFday', - 'preprocessor': 'rough_cutout', - 'start_year': 1989}, - 'tas': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'rough_cutout', - 'start_year': 1989}}}}, - 'documentation': {'authors': ['kalverla_peter', - 'camphuijsen_jaro', - 'alidoost_sarah', - 'aerts_jerom', - 'andela_bouwe'], - 'description': 'Pre-processes climate data for the WFlow hydrological model.\n', - 'projects': ['ewatercycle'], - 'references': ['acknow_project']}, - 'preprocessors': {'rough_cutout': {'extract_region': {'end_latitude': 2.5, - 'end_longitude': 16.75, - 'start_latitude': 7.25, - 'start_longitude': 10} - } - } + "diagnostics": { + "wflow_daily": { + "additional_datasets": [ + { + "dataset": "ERA5", + "project": "OBS6", + "tier": 3, + "type": "reanaly", + "version": 1, + } + ], + "description": "WFlow input preprocessor for " "daily data", + "scripts": { + "script": { + "basin": "Rhine", + "dem_file": "wflow_parameterset/meuse/staticmaps/wflow_dem.map", + "regrid": "area_weighted", + "script": "hydrology/wflow.py", + } + }, + "variables": { + "orog": {"mip": "fx", "preprocessor": "rough_cutout"}, + "pr": { + "end_year": 1999, + "mip": "day", + "preprocessor": "rough_cutout", + "start_year": 1989, + }, + "psl": { + "end_year": 1999, + "mip": "day", + "preprocessor": "rough_cutout", + "start_year": 1989, + }, + "rsds": { + "end_year": 1999, + "mip": "day", + "preprocessor": "rough_cutout", + "start_year": 1989, + }, + "rsdt": { + "end_year": 1999, + "mip": "CFday", + "preprocessor": "rough_cutout", + "start_year": 1989, + }, + "tas": { + "end_year": 1999, + "mip": "day", + "preprocessor": "rough_cutout", + "start_year": 1989, + }, + }, + } + }, + "documentation": { + "authors": [ + "kalverla_peter", + "camphuijsen_jaro", + "alidoost_sarah", + "aerts_jerom", + "andela_bouwe", + ], + "description": "Pre-processes climate data for the WFlow hydrological model.\n", + "projects": ["ewatercycle"], + "references": ["acknow_project"], + }, + "preprocessors": { + "rough_cutout": { + "extract_region": { + "end_latitude": 2.5, + "end_longitude": 16.75, + "start_latitude": 7.25, + "start_longitude": 10, + } + } + }, } @pytest.fixture def forcing(self, mock_recipe_run, sample_shape): return generate( - target_model='wflow', - dataset='ERA5', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + target_model="wflow", + dataset="ERA5", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, model_specific_options=dict( - dem_file='wflow_parameterset/meuse/staticmaps/wflow_dem.map', + dem_file="wflow_parameterset/meuse/staticmaps/wflow_dem.map", extract_region={ - 'start_longitude': 10, - 'end_longitude': 16.75, - 'start_latitude': 7.25, - 'end_latitude': 2.5, - } - ) + "start_longitude": 10, + "end_longitude": 16.75, + "start_latitude": 7.25, + "end_latitude": 2.5, + }, + ), ) def test_result(self, forcing, tmp_path, sample_shape): expected = WflowForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - shape = str(sample_shape), - netcdfinput='wflow_forcing.nc' + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + shape=str(sample_shape), + netcdfinput="wflow_forcing.nc", ) assert forcing == expected def test_recipe_configured(self, forcing, mock_recipe_run, reference_recipe): - assert mock_recipe_run['data_during_run'] == reference_recipe + assert mock_recipe_run["data_during_run"] == reference_recipe def test_saved_yaml(self, forcing, tmp_path): saved_forcing = load(tmp_path) diff --git a/tests/models/test_abstract.py b/tests/models/test_abstract.py index 77a47c89..cfb929fc 100644 --- a/tests/models/test_abstract.py +++ b/tests/models/test_abstract.py @@ -1,8 +1,8 @@ import logging import weakref +from datetime import datetime, timezone from typing import Any, Iterable, Tuple from unittest.mock import patch -from datetime import datetime, timezone import numpy as np import pytest @@ -18,25 +18,25 @@ @pytest.fixture def setup_config(tmp_path): - CFG['parameterset_dir'] = tmp_path - CFG['ewatercycle_config'] = tmp_path / 'ewatercycle.yaml' + CFG["parameterset_dir"] = tmp_path + CFG["ewatercycle_config"] = tmp_path / "ewatercycle.yaml" yield CFG - CFG['ewatercycle_config'] = DEFAULT_CONFIG + CFG["ewatercycle_config"] = DEFAULT_CONFIG CFG.reload() class MockedModel(AbstractModel): - available_versions = ('0.4.2',) + available_versions = ("0.4.2",) - def __init__(self, version: str = '0.4.2', parameter_set: ParameterSet = None): + def __init__(self, version: str = "0.4.2", parameter_set: ParameterSet = None): super().__init__(version, parameter_set) def setup(self, *args, **kwargs) -> Tuple[str, str]: - if 'bmi' in kwargs: + if "bmi" in kwargs: # sub-class of AbstractModel should construct bmi # using grpc4bmi Docker or Singularity client - self.bmi = kwargs['bmi'] - return 'foobar.cfg', '.' + self.bmi = kwargs["bmi"] + return "foobar.cfg", "." def get_value_as_xarray(self, name: str) -> xr.DataArray: return xr.DataArray( @@ -44,28 +44,31 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: coords={ "latitude": [42.25, 42.21], "longitude": [-99.83, -99.32], - "time": '2014-09-06'}, + "time": "2014-09-06", + }, dims=["longitude", "latitude"], - name='Temperature', + name="Temperature", attrs=dict(units="degC"), ) - def _coords_to_indices(self, name: str, lat: Iterable[float], lon: Iterable[float]) -> Iterable[int]: + def _coords_to_indices( + self, name: str, lat: Iterable[float], lon: Iterable[float] + ) -> Iterable[int]: return np.array([0]) @property def parameters(self) -> Iterable[Tuple[str, Any]]: - return [('area', 42)] + return [("area", 42)] @pytest.fixture -@patch('basic_modeling_interface.Bmi') +@patch("basic_modeling_interface.Bmi") def bmi(MockedBmi): mocked_bmi = MockedBmi() mocked_bmi.get_start_time.return_value = 42.0 mocked_bmi.get_current_time.return_value = 43.0 mocked_bmi.get_end_time.return_value = 44.0 - mocked_bmi.get_time_units.return_value = 'days since 1970-01-01 00:00:00.0 00:00' + mocked_bmi.get_time_units.return_value = "days since 1970-01-01 00:00:00.0 00:00" return mocked_bmi @@ -78,29 +81,32 @@ def model(bmi: Bmi): def test_construct(): with pytest.raises(TypeError) as excinfo: - AbstractModel(version='0.4.2') + AbstractModel(version="0.4.2") msg = str(excinfo.value) assert "Can't instantiate abstract class" in msg - assert 'setup' in msg - assert 'parameters' in msg + assert "setup" in msg + assert "parameters" in msg def test_construct_with_unsupported_version(): with pytest.raises(ValueError) as excinfo: - MockedModel(version='1.2.3') + MockedModel(version="1.2.3") - assert "Supplied version 1.2.3 is not supported by this model. Available versions are ('0.4.2',)." in str(excinfo.value) + assert ( + "Supplied version 1.2.3 is not supported by this model. Available versions are ('0.4.2',)." + in str(excinfo.value) + ) def test_setup(model): result = model.setup() - expected = 'foobar.cfg', '.' + expected = "foobar.cfg", "." assert result == expected def test_initialize(model: MockedModel, bmi): - config_file = 'foobar.cfg' + config_file = "foobar.cfg" model.initialize(config_file) bmi.initialize.assert_called_once_with(config_file) @@ -131,7 +137,7 @@ def test_get_value(bmi, model: MockedModel): expected = np.array([[1.0, 2.0], [3.0, 4.0]]) bmi.get_value.return_value = expected - value = model.get_value('discharge') + value = model.get_value("discharge") assert_array_equal(value, expected) @@ -140,23 +146,23 @@ def test_get_value_at_coords(bmi, model: MockedModel): expected = np.array([1.0]) bmi.get_value_at_indices.return_value = expected - value = model.get_value_at_coords('discharge', [-99.83], [42.25]) + value = model.get_value_at_coords("discharge", [-99.83], [42.25]) assert_array_equal(value, expected) def test_set_value(model: MockedModel, bmi): value = np.array([1.0, 2.0]) - model.set_value('precipitation', value) + model.set_value("precipitation", value) - bmi.set_value.assert_called_once_with('precipitation', value) + bmi.set_value.assert_called_once_with("precipitation", value) def test_set_value_at_coords(model: MockedModel, bmi): value = np.array([1.0]) - model.set_value_at_coords('precipitation', [-99.83], [42.25], value) + model.set_value_at_coords("precipitation", [-99.83], [42.25], value) - bmi.set_value_at_indices.assert_called_once_with('precipitation', [0], value) + bmi.set_value_at_indices.assert_called_once_with("precipitation", [0], value) def test_start_time(model: MockedModel): @@ -180,7 +186,7 @@ def test_time(model: MockedModel): def test_time_units(model: MockedModel): units = model.time_units - assert units == 'days since 1970-01-01 00:00:00.0 00:00' + assert units == "days since 1970-01-01 00:00:00.0 00:00" def test_time_step(bmi, model: MockedModel): @@ -192,11 +198,11 @@ def test_time_step(bmi, model: MockedModel): def test_output_var_names(bmi, model: MockedModel): - bmi.get_output_var_names.return_value = ('discharge',) + bmi.get_output_var_names.return_value = ("discharge",) names = model.output_var_names - assert names == ('discharge',) + assert names == ("discharge",) def test_get_value_as_xarray(model: MockedModel): @@ -205,9 +211,10 @@ def test_get_value_as_xarray(model: MockedModel): coords={ "latitude": [42.25, 42.21], "longitude": [-99.83, -99.32], - "time": '2014-09-06'}, + "time": "2014-09-06", + }, dims=["longitude", "latitude"], - name='Temperature', + name="Temperature", attrs=dict(units="degC"), ) @@ -219,49 +226,49 @@ def test_get_value_as_xarray(model: MockedModel): def start_time_as_isostr(model: MockedModel): actual = model.start_time_as_isostr - expected = '1970-02-12T00:00:00Z' + expected = "1970-02-12T00:00:00Z" assert expected == actual def end_time_as_isostr(model: MockedModel): actual = model.end_time_as_isostr - expected = '1970-02-14T00:00:00Z' + expected = "1970-02-14T00:00:00Z" assert expected == actual def time_as_isostr(model: MockedModel): actual = model.time_as_isostr - expected = '1970-02-13T00:00:00Z' + expected = "1970-02-13T00:00:00Z" assert expected == actual def start_time_as_datetime(model: MockedModel): actual = model.start_time_as_isostr - expected = datetime(1970, 2, 12, tzinfo=timezone.utc) + expected = datetime(1970, 2, 12, tzinfo=timezone.utc) assert expected == actual def end_time_as_datetime(model: MockedModel): actual = model.end_time_as_isostr - expected = datetime(1970, 2, 14, tzinfo=timezone.utc) + expected = datetime(1970, 2, 14, tzinfo=timezone.utc) assert expected == actual def time_as_datetime(model: MockedModel): actual = model.time_as_isostr - expected = datetime(1970, 2, 13, tzinfo=timezone.utc) + expected = datetime(1970, 2, 13, tzinfo=timezone.utc) assert expected == actual def test_delete_model_resets_bmi(): - - class Object(): + class Object: """Target for weakref finalizer.""" + pass # Cannot use the `model` fixture, it doesn't play well with weakref @@ -277,53 +284,53 @@ class Object(): class TestCheckParameterSet: def test_correct_version(self, setup_config): ps = ParameterSet( - name='justatest', - directory='justatest', - config='justatest/config.ini', - target_model='mockedmodel', # == lowered class name - supported_model_versions={'0.4.2'} + name="justatest", + directory="justatest", + config="justatest/config.ini", + target_model="mockedmodel", # == lowered class name + supported_model_versions={"0.4.2"}, ) m = MockedModel(parameter_set=ps) assert m.parameter_set == ps def test_wrong_model(self, setup_config): ps = ParameterSet( - name='justatest', - directory='justatest', - config='justatest/config.ini', - target_model='wrongmodel', - supported_model_versions={'0.4.2'} + name="justatest", + directory="justatest", + config="justatest/config.ini", + target_model="wrongmodel", + supported_model_versions={"0.4.2"}, ) with pytest.raises(ValueError) as excinfo: MockedModel(parameter_set=ps) - expected = 'Parameter set has wrong target model' + expected = "Parameter set has wrong target model" assert expected in str(excinfo.value) def test_any_version(self, caplog, setup_config): ps = ParameterSet( - name='justatest', - directory='justatest', - config='justatest/config.ini', - target_model='mockedmodel', # == lowered class name - supported_model_versions=set() + name="justatest", + directory="justatest", + config="justatest/config.ini", + target_model="mockedmodel", # == lowered class name + supported_model_versions=set(), ) with caplog.at_level(logging.INFO): MockedModel(parameter_set=ps) - expected = 'is not explicitly listed in the supported model versions' + expected = "is not explicitly listed in the supported model versions" assert expected in caplog.text def test_unsupported_version(self, setup_config): ps = ParameterSet( - name='justatest', - directory='justatest', - config='justatest/config.ini', - target_model='mockedmodel', - supported_model_versions={'1.2.3'} + name="justatest", + directory="justatest", + config="justatest/config.ini", + target_model="mockedmodel", + supported_model_versions={"1.2.3"}, ) with pytest.raises(ValueError) as excinfo: MockedModel(parameter_set=ps) - expected = 'Parameter set is not compatible with version' + expected = "Parameter set is not compatible with version" assert expected in str(excinfo.value) diff --git a/tests/models/test_lisflood.py b/tests/models/test_lisflood.py index 77c50627..68391f18 100644 --- a/tests/models/test_lisflood.py +++ b/tests/models/test_lisflood.py @@ -66,9 +66,7 @@ def generate_forcing(self, tmp_path, parameterset: ParameterSet): @pytest.fixture def model(self, parameterset, generate_forcing): forcing = generate_forcing - m = Lisflood( - version="20.10", parameter_set=parameterset, forcing=forcing - ) + m = Lisflood(version="20.10", parameter_set=parameterset, forcing=forcing) yield m if m.bmi: # Clean up container @@ -119,20 +117,14 @@ def test_setup(self, model_with_setup, tmp_path): } assert find_values_in_xml(_cfg.config, "StepStart") == {"1"} assert find_values_in_xml(_cfg.config, "StepEnd") == {"11688"} - assert find_values_in_xml(_cfg.config, "PathMeteo") == { - f"{tmp_path}/forcing" - } + assert find_values_in_xml(_cfg.config, "PathMeteo") == {f"{tmp_path}/forcing"} assert find_values_in_xml(_cfg.config, "PathOut") == {str(config_dir)} - assert find_values_in_xml(_cfg.config, "IrrigationEfficiency") == { - "0.8" - } + assert find_values_in_xml(_cfg.config, "IrrigationEfficiency") == {"0.8"} assert find_values_in_xml(_cfg.config, "MaskMap") == { "$(PathMaps)/masksmall.map", "$(MaskMap)", } - assert find_values_in_xml(_cfg.config, "PrefixPrecipitation") == { - "mytp" - } + assert find_values_in_xml(_cfg.config, "PrefixPrecipitation") == {"mytp"} assert find_values_in_xml(_cfg.config, "PrefixTavg") == {"myta"} assert find_values_in_xml(_cfg.config, "PrefixE0") == {"mye0"} assert find_values_in_xml(_cfg.config, "PrefixES0") == {"es0"} @@ -181,9 +173,7 @@ def model(self, tmp_path, parameterset, generate_forcing): mask_file_in_ps = parameterset.directory / "maps/mask.map" shutil.copy(mask_file_in_ps, mask_dir / "mask.map") forcing = generate_forcing - m = Lisflood( - version="20.10", parameter_set=parameterset, forcing=forcing - ) + m = Lisflood(version="20.10", parameter_set=parameterset, forcing=forcing) yield m if m.bmi: # Clean up container @@ -193,12 +183,8 @@ def model(self, tmp_path, parameterset, generate_forcing): def model_with_setup(self, tmp_path, mocked_config, model: Lisflood): with patch.object( BmiClientSingularity, "__init__", return_value=None - ) as mocked_constructor, patch( - "datetime.datetime" - ) as mocked_datetime: - mocked_datetime.now.return_value = datetime( - 2021, 1, 2, 3, 4, 5 - ) + ) as mocked_constructor, patch("datetime.datetime") as mocked_datetime: + mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) config_file, config_dir = model.setup( MaskMap=str(tmp_path / "custommask/mask.map") ) @@ -210,9 +196,7 @@ def test_setup(self, model_with_setup, tmp_path): # Check setup returns expected_cfg_dir = CFG["output_dir"] / "lisflood_20210102_030405" assert config_dir == str(expected_cfg_dir) - assert config_file == str( - expected_cfg_dir / "lisflood_setting.xml" - ) + assert config_file == str(expected_cfg_dir / "lisflood_setting.xml") # Check container started mocked_constructor.assert_called_once_with( @@ -235,9 +219,7 @@ def test_setup(self, model_with_setup, tmp_path): assert find_values_in_xml(_cfg.config, "PathMeteo") == { f"{tmp_path}/forcing" } - assert find_values_in_xml(_cfg.config, "PathOut") == { - str(config_dir) - } + assert find_values_in_xml(_cfg.config, "PathOut") == {str(config_dir)} assert find_values_in_xml(_cfg.config, "IrrigationEfficiency") == { "0.75", "$(IrrigationEfficiency)", @@ -245,9 +227,7 @@ def test_setup(self, model_with_setup, tmp_path): assert find_values_in_xml(_cfg.config, "MaskMap") == { f"{tmp_path}/custommask/mask" } - assert find_values_in_xml(_cfg.config, "PrefixPrecipitation") == { - "mytp" - } + assert find_values_in_xml(_cfg.config, "PrefixPrecipitation") == {"mytp"} assert find_values_in_xml(_cfg.config, "PrefixTavg") == {"myta"} assert find_values_in_xml(_cfg.config, "PrefixE0") == {"mye0"} assert find_values_in_xml(_cfg.config, "PrefixES0") == {"es0"} diff --git a/tests/models/test_marrmotm01.py b/tests/models/test_marrmotm01.py index a40099ba..e9fa6109 100644 --- a/tests/models/test_marrmotm01.py +++ b/tests/models/test_marrmotm01.py @@ -10,13 +10,13 @@ from ewatercycle import CFG from ewatercycle.forcing import load_foreign -from ewatercycle.models.marrmot import Solver, MarrmotM01 +from ewatercycle.models.marrmot import MarrmotM01, Solver @pytest.fixture def mocked_config(tmp_path): - CFG['output_dir'] = tmp_path - CFG['container_engine'] = 'docker' + CFG["output_dir"] = tmp_path + CFG["container_engine"] = "docker" class TestWithDefaultsAndExampleData: @@ -26,13 +26,13 @@ def forcing_file(self, sample_marrmot_forcing_file): @pytest.fixture def generate_forcing(self, forcing_file): - forcing = load_foreign('marrmot', - directory=str(Path(forcing_file).parent), - start_time='1989-01-01T00:00:00Z', - end_time='1992-12-31T00:00:00Z', - forcing_info={ - 'forcing_file': str(Path(forcing_file).name) - }) + forcing = load_foreign( + "marrmot", + directory=str(Path(forcing_file).parent), + start_time="1989-01-01T00:00:00Z", + end_time="1992-12-31T00:00:00Z", + forcing_info={"forcing_file": str(Path(forcing_file).name)}, + ) return forcing @pytest.fixture @@ -45,16 +45,15 @@ def model(self, generate_forcing, mocked_config): @pytest.fixture def model_with_setup(self, model: MarrmotM01): - with patch('datetime.datetime') as mocked_datetime: + with patch("datetime.datetime") as mocked_datetime: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) cfg_file, cfg_dir = model.setup() return model, cfg_file, cfg_dir - def test_str(self, model, forcing_file): actual = str(model) - expected = "\n".join( + expected = "\n".join( [ "eWaterCycle MarrmotM01", "-------------------", @@ -69,32 +68,33 @@ def test_str(self, model, forcing_file): f" directory={str(Path(forcing_file).parent)}", " shape=None", " forcing_file=BMI_testcase_m01_BuffaloRiver_TN_USA.mat", - ]) + ] + ) assert actual == expected def test_parameters(self, model): expected = [ - ('maximum_soil_moisture_storage', 10.0), - ('initial_soil_moisture_storage', 5.0), - ('solver', Solver()), - ('start time', '1989-01-01T00:00:00Z'), - ('end time', '1992-12-31T00:00:00Z'), + ("maximum_soil_moisture_storage", 10.0), + ("initial_soil_moisture_storage", 5.0), + ("solver", Solver()), + ("start time", "1989-01-01T00:00:00Z"), + ("end time", "1992-12-31T00:00:00Z"), ] assert model.parameters == expected def test_setup(self, model_with_setup, forcing_file): model, cfg_file, cfg_dir = model_with_setup - expected_cfg_dir = CFG['output_dir'] / 'marrmot_20210102_030405' + expected_cfg_dir = CFG["output_dir"] / "marrmot_20210102_030405" assert cfg_dir == str(expected_cfg_dir) - assert cfg_file == str(expected_cfg_dir / 'marrmot-m01_config.mat') + assert cfg_file == str(expected_cfg_dir / "marrmot-m01_config.mat") assert model.bmi actual = loadmat(str(cfg_file)) expected_forcing = loadmat(forcing_file) - assert actual['model_name'] == "m_01_collie1_1p_1s" - assert_almost_equal(actual['time_start'], expected_forcing['time_start']) - assert_almost_equal(actual['time_end'], expected_forcing['time_end']) + assert actual["model_name"] == "m_01_collie1_1p_1s" + assert_almost_equal(actual["time_start"], expected_forcing["time_start"]) + assert_almost_equal(actual["time_end"], expected_forcing["time_end"]) # TODO compare forcings # assert_almost_equal(actual['forcing'], expected_forcing['forcing']) # TODO assert solver @@ -103,11 +103,11 @@ def test_setup(self, model_with_setup, forcing_file): def test_parameters_after_setup(self, model_with_setup): model = model_with_setup[0] expected = [ - ('maximum_soil_moisture_storage', 10.0), - ('initial_soil_moisture_storage', 5.0), - ('solver', Solver()), - ('start time', '1989-01-01T00:00:00Z'), - ('end time', '1992-12-31T00:00:00Z'), + ("maximum_soil_moisture_storage", 10.0), + ("initial_soil_moisture_storage", 5.0), + ("solver", Solver()), + ("start time", "1989-01-01T00:00:00Z"), + ("end time", "1992-12-31T00:00:00Z"), ] assert model.parameters == expected @@ -116,32 +116,28 @@ def test_get_value_as_xarray(self, model_with_setup): model.initialize(cfg_file) model.update() - actual = model.get_value_as_xarray('flux_out_Q') + actual = model.get_value_as_xarray("flux_out_Q") expected = xr.DataArray( data=[[11.91879913]], coords={ "longitude": [87.49], "latitude": [35.29], - "time": datetime(1989, 1, 2, tzinfo=timezone.utc) + "time": datetime(1989, 1, 2, tzinfo=timezone.utc), }, dims=["latitude", "longitude"], - name='flux_out_Q', - attrs={"units": 'mm day'}, + name="flux_out_Q", + attrs={"units": "mm day"}, ) assert_allclose(actual, expected) def test_setup_with_own_cfg_dir(self, tmp_path, mocked_config, model: MarrmotM01): - cfg_file, cfg_dir = model.setup( - cfg_dir=str(tmp_path) - ) + cfg_file, cfg_dir = model.setup(cfg_dir=str(tmp_path)) assert cfg_dir == str(tmp_path) def test_setup_create_cfg_dir(self, tmp_path, mocked_config, model: MarrmotM01): - work_dir = tmp_path / 'output' - cfg_file, cfg_dir = model.setup( - cfg_dir=str(work_dir) - ) + work_dir = tmp_path / "output" + cfg_file, cfg_dir = model.setup(cfg_dir=str(work_dir)) assert cfg_dir == str(work_dir) @@ -152,13 +148,13 @@ def forcing_file(self, sample_marrmot_forcing_file): @pytest.fixture def generate_forcing(self, forcing_file): - forcing = load_foreign('marrmot', - directory=str(Path(forcing_file).parent), - start_time='1989-01-01T00:00:00Z', - end_time='1992-12-31T00:00:00Z', - forcing_info={ - 'forcing_file': str(Path(forcing_file).name) - }) + forcing = load_foreign( + "marrmot", + directory=str(Path(forcing_file).parent), + start_time="1989-01-01T00:00:00Z", + end_time="1992-12-31T00:00:00Z", + forcing_info={"forcing_file": str(Path(forcing_file).name)}, + ) return forcing @pytest.fixture @@ -171,30 +167,30 @@ def model(self, generate_forcing, mocked_config): @pytest.fixture def model_with_setup(self, model: MarrmotM01): - with patch('datetime.datetime') as mocked_datetime: + with patch("datetime.datetime") as mocked_datetime: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) cfg_file, cfg_dir = model.setup( maximum_soil_moisture_storage=1234, initial_soil_moisture_storage=4321, - start_time='1990-01-01T00:00:00Z', - end_time='1991-12-31T00:00:00Z', + start_time="1990-01-01T00:00:00Z", + end_time="1991-12-31T00:00:00Z", ) return model, cfg_file, cfg_dir def test_setup(self, model_with_setup): model, cfg_file, cfg_dir = model_with_setup - expected_cfg_dir = CFG['output_dir'] / 'marrmot_20210102_030405' + expected_cfg_dir = CFG["output_dir"] / "marrmot_20210102_030405" assert cfg_dir == str(expected_cfg_dir) - assert cfg_file == str(expected_cfg_dir / 'marrmot-m01_config.mat') + assert cfg_file == str(expected_cfg_dir / "marrmot-m01_config.mat") assert model.bmi actual = loadmat(str(cfg_file)) - assert actual['model_name'] == "m_01_collie1_1p_1s" - assert actual['parameters'] == [[1234]] - assert actual['store_ini'] == [[4321]] - assert_almost_equal(actual['time_start'], [[1990, 1, 1, 0, 0, 0]]) - assert_almost_equal(actual['time_end'], [[1991, 12, 31, 0, 0, 0]]) + assert actual["model_name"] == "m_01_collie1_1p_1s" + assert actual["parameters"] == [[1234]] + assert actual["store_ini"] == [[4321]] + assert_almost_equal(actual["time_start"], [[1990, 1, 1, 0, 0, 0]]) + assert_almost_equal(actual["time_end"], [[1991, 12, 31, 0, 0, 0]]) class TestWithDatesOutsideRangeSetupAndExampleData: @@ -204,13 +200,13 @@ def forcing_file(self, sample_marrmot_forcing_file): @pytest.fixture def generate_forcing(self, forcing_file): - forcing = load_foreign('marrmot', - directory=str(Path(forcing_file).parent), - start_time='1989-01-01T00:00:00Z', - end_time='1992-12-31T00:00:00Z', - forcing_info={ - 'forcing_file': str(Path(forcing_file).name) - }) + forcing = load_foreign( + "marrmot", + directory=str(Path(forcing_file).parent), + start_time="1989-01-01T00:00:00Z", + end_time="1992-12-31T00:00:00Z", + forcing_info={"forcing_file": str(Path(forcing_file).name)}, + ) return forcing @pytest.fixture @@ -224,13 +220,13 @@ def model(self, generate_forcing, mocked_config): def test_setup_with_earlystart(self, model: MarrmotM01): with pytest.raises(ValueError) as excinfo: model.setup( - start_time='1980-01-01T00:00:00Z', + start_time="1980-01-01T00:00:00Z", ) - assert 'start_time outside forcing time range' in str(excinfo.value) + assert "start_time outside forcing time range" in str(excinfo.value) def test_setup_with_lateend(self, model: MarrmotM01): with pytest.raises(ValueError) as excinfo: model.setup( - end_time='2000-01-01T00:00:00Z', + end_time="2000-01-01T00:00:00Z", ) - assert 'end_time outside forcing time range' in str(excinfo.value) + assert "end_time outside forcing time range" in str(excinfo.value) diff --git a/tests/models/test_marrmotm14.py b/tests/models/test_marrmotm14.py index 9f75cdc4..82c33fe5 100644 --- a/tests/models/test_marrmotm14.py +++ b/tests/models/test_marrmotm14.py @@ -10,13 +10,13 @@ from ewatercycle import CFG from ewatercycle.forcing import load_foreign -from ewatercycle.models.marrmot import Solver, MarrmotM14 +from ewatercycle.models.marrmot import MarrmotM14, Solver @pytest.fixture def mocked_config(tmp_path): - CFG['output_dir'] = tmp_path - CFG['container_engine'] = 'docker' + CFG["output_dir"] = tmp_path + CFG["container_engine"] = "docker" class TestWithDefaultsAndExampleData: @@ -24,13 +24,13 @@ class TestWithDefaultsAndExampleData: def generate_forcing(self): # Downloaded from # https://github.com/wknoben/MARRMoT/blob/master/BMI/Config/BMI_testcase_m01_BuffaloRiver_TN_USA.mat - forcing = load_foreign('marrmot', - directory=f'{Path(__file__).parent}/data', - start_time='1989-01-01T00:00:00Z', - end_time='1992-12-31T00:00:00Z', - forcing_info={ - 'forcing_file': 'BMI_testcase_m01_BuffaloRiver_TN_USA.mat' - }) + forcing = load_foreign( + "marrmot", + directory=f"{Path(__file__).parent}/data", + start_time="1989-01-01T00:00:00Z", + end_time="1992-12-31T00:00:00Z", + forcing_info={"forcing_file": "BMI_testcase_m01_BuffaloRiver_TN_USA.mat"}, + ) return forcing @pytest.fixture @@ -44,7 +44,7 @@ def model(self, generate_forcing, mocked_config): @pytest.fixture def model_with_setup(self, model: MarrmotM14): - with patch('datetime.datetime') as mocked_datetime: + with patch("datetime.datetime") as mocked_datetime: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) cfg_file, cfg_dir = model.setup() @@ -52,18 +52,18 @@ def model_with_setup(self, model: MarrmotM14): def test_parameters(self, model): expected = [ - ('maximum_soil_moisture_storage', 1000.0), - ('threshold_flow_generation_evap_change', 0.5), - ('leakage_saturated_zone_flow_coefficient', 0.5), - ('zero_deficit_base_flow_speed', 100.0), - ('baseflow_coefficient', 0.5), - ('gamma_distribution_chi_parameter', 4.25), - ('gamma_distribution_phi_parameter', 2.5), - ('initial_upper_zone_storage', 900.0), - ('initial_saturated_zone_storage', 900.0), - ('solver', Solver()), - ('start time', '1989-01-01T00:00:00Z'), - ('end time', '1992-12-31T00:00:00Z'), + ("maximum_soil_moisture_storage", 1000.0), + ("threshold_flow_generation_evap_change", 0.5), + ("leakage_saturated_zone_flow_coefficient", 0.5), + ("zero_deficit_base_flow_speed", 100.0), + ("baseflow_coefficient", 0.5), + ("gamma_distribution_chi_parameter", 4.25), + ("gamma_distribution_phi_parameter", 2.5), + ("initial_upper_zone_storage", 900.0), + ("initial_saturated_zone_storage", 900.0), + ("solver", Solver()), + ("start time", "1989-01-01T00:00:00Z"), + ("end time", "1992-12-31T00:00:00Z"), ] assert model.parameters == expected @@ -71,16 +71,18 @@ def test_setup(self, model_with_setup): model, cfg_file, cfg_dir = model_with_setup actual = loadmat(str(cfg_file)) - forcing_file = f'{Path(__file__).parent}/data/BMI_testcase_m01_BuffaloRiver_TN_USA.mat' + forcing_file = ( + f"{Path(__file__).parent}/data/BMI_testcase_m01_BuffaloRiver_TN_USA.mat" + ) expected_forcing = loadmat(forcing_file) - expected_cfg_dir = CFG['output_dir'] / 'marrmot_20210102_030405' + expected_cfg_dir = CFG["output_dir"] / "marrmot_20210102_030405" assert cfg_dir == str(expected_cfg_dir) - assert cfg_file == str(expected_cfg_dir / 'marrmot-m14_config.mat') + assert cfg_file == str(expected_cfg_dir / "marrmot-m14_config.mat") assert model.bmi - assert actual['model_name'] == "m_14_topmodel_7p_2s" - assert_almost_equal(actual['time_start'], expected_forcing['time_start']) - assert_almost_equal(actual['time_end'], expected_forcing['time_end']) + assert actual["model_name"] == "m_14_topmodel_7p_2s" + assert_almost_equal(actual["time_start"], expected_forcing["time_start"]) + assert_almost_equal(actual["time_end"], expected_forcing["time_end"]) # TODO compare forcings # assert_almost_equal(actual['forcing'], expected_forcing['forcing']) # TODO assert solver @@ -89,18 +91,18 @@ def test_setup(self, model_with_setup): def test_parameters_after_setup(self, model_with_setup): model = model_with_setup[0] expected = [ - ('maximum_soil_moisture_storage', 1000.0), - ('threshold_flow_generation_evap_change', 0.5), - ('leakage_saturated_zone_flow_coefficient', 0.5), - ('zero_deficit_base_flow_speed', 100.0), - ('baseflow_coefficient', 0.5), - ('gamma_distribution_chi_parameter', 4.25), - ('gamma_distribution_phi_parameter', 2.5), - ('initial_upper_zone_storage', 900.0), - ('initial_saturated_zone_storage', 900.0), - ('solver', Solver()), - ('start time', '1989-01-01T00:00:00Z'), - ('end time', '1992-12-31T00:00:00Z'), + ("maximum_soil_moisture_storage", 1000.0), + ("threshold_flow_generation_evap_change", 0.5), + ("leakage_saturated_zone_flow_coefficient", 0.5), + ("zero_deficit_base_flow_speed", 100.0), + ("baseflow_coefficient", 0.5), + ("gamma_distribution_chi_parameter", 4.25), + ("gamma_distribution_phi_parameter", 2.5), + ("initial_upper_zone_storage", 900.0), + ("initial_saturated_zone_storage", 900.0), + ("solver", Solver()), + ("start time", "1989-01-01T00:00:00Z"), + ("end time", "1992-12-31T00:00:00Z"), ] assert model.parameters == expected @@ -109,32 +111,28 @@ def test_get_value_as_xarray(self, model_with_setup): model.initialize(cfg_file) model.update() - actual = model.get_value_as_xarray('flux_out_Q') + actual = model.get_value_as_xarray("flux_out_Q") expected = xr.DataArray( data=[[0.529399]], coords={ "longitude": [87.49], "latitude": [35.29], - "time": datetime(1989, 1, 2, tzinfo=timezone.utc) + "time": datetime(1989, 1, 2, tzinfo=timezone.utc), }, dims=["latitude", "longitude"], - name='flux_out_Q', - attrs={"units": 'mm day'}, + name="flux_out_Q", + attrs={"units": "mm day"}, ) assert_allclose(actual, expected) def test_setup_with_own_cfg_dir(self, tmp_path, mocked_config, model: MarrmotM14): - cfg_file, cfg_dir = model.setup( - cfg_dir=str(tmp_path) - ) + cfg_file, cfg_dir = model.setup(cfg_dir=str(tmp_path)) assert cfg_dir == str(tmp_path) def test_setup_create_cfg_dir(self, tmp_path, mocked_config, model: MarrmotM14): - work_dir = tmp_path / 'output' - cfg_file, cfg_dir = model.setup( - cfg_dir=str(work_dir) - ) + work_dir = tmp_path / "output" + cfg_file, cfg_dir = model.setup(cfg_dir=str(work_dir)) assert cfg_dir == str(work_dir) @@ -143,13 +141,13 @@ class TestWithCustomSetupAndExampleData: def generate_forcing(self): # Downloaded from # https://github.com/wknoben/MARRMoT/blob/master/BMI/Config/BMI_testcase_m01_BuffaloRiver_TN_USA.mat - forcing = load_foreign('marrmot', - directory=f'{Path(__file__).parent}/data', - start_time='1989-01-01T00:00:00Z', - end_time='1992-12-31T00:00:00Z', - forcing_info={ - 'forcing_file': 'BMI_testcase_m01_BuffaloRiver_TN_USA.mat' - }) + forcing = load_foreign( + "marrmot", + directory=f"{Path(__file__).parent}/data", + start_time="1989-01-01T00:00:00Z", + end_time="1992-12-31T00:00:00Z", + forcing_info={"forcing_file": "BMI_testcase_m01_BuffaloRiver_TN_USA.mat"}, + ) return forcing @pytest.fixture @@ -163,14 +161,14 @@ def model(self, generate_forcing, mocked_config): @pytest.fixture def model_with_setup(self, model: MarrmotM14): - with patch('datetime.datetime') as mocked_datetime: + with patch("datetime.datetime") as mocked_datetime: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) cfg_file, cfg_dir = model.setup( maximum_soil_moisture_storage=1234, initial_upper_zone_storage=4321, - start_time='1990-01-01T00:00:00Z', - end_time='1991-12-31T00:00:00Z', + start_time="1990-01-01T00:00:00Z", + end_time="1991-12-31T00:00:00Z", ) return model, cfg_file, cfg_dir @@ -179,15 +177,17 @@ def test_setup(self, model_with_setup): actual = loadmat(str(cfg_file)) - expected_cfg_dir = CFG['output_dir'] / 'marrmot_20210102_030405' + expected_cfg_dir = CFG["output_dir"] / "marrmot_20210102_030405" assert cfg_dir == str(expected_cfg_dir) - assert cfg_file == str(expected_cfg_dir / 'marrmot-m14_config.mat') + assert cfg_file == str(expected_cfg_dir / "marrmot-m14_config.mat") assert model.bmi - assert actual['model_name'] == "m_14_topmodel_7p_2s" - assert_array_equal(actual['parameters'], [[1234.0, 0.5, 0.5, 100.0, 0.5, 4.25, 2.5]]) - assert_array_equal(actual['store_ini'], [[4321, 900]]) - assert_almost_equal(actual['time_start'], [[1990, 1, 1, 0, 0, 0]]) - assert_almost_equal(actual['time_end'], [[1991, 12, 31, 0, 0, 0]]) + assert actual["model_name"] == "m_14_topmodel_7p_2s" + assert_array_equal( + actual["parameters"], [[1234.0, 0.5, 0.5, 100.0, 0.5, 4.25, 2.5]] + ) + assert_array_equal(actual["store_ini"], [[4321, 900]]) + assert_almost_equal(actual["time_start"], [[1990, 1, 1, 0, 0, 0]]) + assert_almost_equal(actual["time_end"], [[1991, 12, 31, 0, 0, 0]]) class TestWithDatesOutsideRangeSetupAndExampleData: @@ -195,13 +195,13 @@ class TestWithDatesOutsideRangeSetupAndExampleData: def generate_forcing(self): # Downloaded from # https://github.com/wknoben/MARRMoT/blob/master/BMI/Config/BMI_testcase_m01_BuffaloRiver_TN_USA.mat - forcing = load_foreign('marrmot', - directory=f'{Path(__file__).parent}/data', - start_time='1989-01-01T00:00:00Z', - end_time='1992-12-31T00:00:00Z', - forcing_info={ - 'forcing_file': 'BMI_testcase_m01_BuffaloRiver_TN_USA.mat' - }) + forcing = load_foreign( + "marrmot", + directory=f"{Path(__file__).parent}/data", + start_time="1989-01-01T00:00:00Z", + end_time="1992-12-31T00:00:00Z", + forcing_info={"forcing_file": "BMI_testcase_m01_BuffaloRiver_TN_USA.mat"}, + ) return forcing @pytest.fixture @@ -216,13 +216,13 @@ def model(self, generate_forcing, mocked_config): def test_setup_with_earlystart(self, model: MarrmotM14): with pytest.raises(ValueError) as excinfo: model.setup( - start_time='1980-01-01T00:00:00Z', + start_time="1980-01-01T00:00:00Z", ) - assert 'start_time outside forcing time range' in str(excinfo.value) + assert "start_time outside forcing time range" in str(excinfo.value) def test_setup_with_lateend(self, model: MarrmotM14): with pytest.raises(ValueError) as excinfo: model.setup( - end_time='2000-01-01T00:00:00Z', + end_time="2000-01-01T00:00:00Z", ) - assert 'end_time outside forcing time range' in str(excinfo.value) + assert "end_time outside forcing time range" in str(excinfo.value) diff --git a/tests/models/test_pcrglobwb.py b/tests/models/test_pcrglobwb.py index cf02c45c..09383dc1 100644 --- a/tests/models/test_pcrglobwb.py +++ b/tests/models/test_pcrglobwb.py @@ -52,9 +52,7 @@ def mocked_config(tmp_path): @pytest.fixture def parameter_set(mocked_config): - example_parameter_set = example_parameter_sets()[ - "pcrglobwb_rhinemeuse_30min" - ] + example_parameter_set = example_parameter_sets()["pcrglobwb_rhinemeuse_30min"] example_parameter_set.download() example_parameter_set.to_config() return example_parameter_set @@ -88,9 +86,9 @@ def initialized_model(model): def test_setup(model): - with patch.object( - BmiClientSingularity, "__init__", return_value=None - ), patch("datetime.datetime") as mocked_datetime: + with patch.object(BmiClientSingularity, "__init__", return_value=None), patch( + "datetime.datetime" + ) as mocked_datetime: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) cfg_file, cfg_dir = model.setup() @@ -101,23 +99,25 @@ def test_setup(model): def test_setup_withtimeoutexception(model, tmp_path): - with patch.object(BmiClientSingularity, "__init__", side_effect=FutureTimeoutError()), patch( - "datetime.datetime" - ) as mocked_datetime, pytest.raises(ValueError) as excinfo: + with patch.object( + BmiClientSingularity, "__init__", side_effect=FutureTimeoutError() + ), patch("datetime.datetime") as mocked_datetime, pytest.raises( + ValueError + ) as excinfo: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) model.setup() msg = str(excinfo.value) - assert 'docker pull ewatercycle/pcrg-grpc4bmi:setters' in msg - sif = tmp_path / 'ewatercycle-pcrg-grpc4bmi_setters.sif' + assert "docker pull ewatercycle/pcrg-grpc4bmi:setters" in msg + sif = tmp_path / "ewatercycle-pcrg-grpc4bmi_setters.sif" assert f"build {sif} docker://ewatercycle/pcrg-grpc4bmi:setters" in msg def test_setup_with_custom_cfg_dir(model, tmp_path): my_cfg_dir = str(tmp_path / "mycfgdir") - with patch.object( - BmiClientSingularity, "__init__", return_value=None - ), patch("datetime.datetime") as mocked_datetime: + with patch.object(BmiClientSingularity, "__init__", return_value=None), patch( + "datetime.datetime" + ) as mocked_datetime: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) cfg_file, cfg_dir = model.setup(cfg_dir=my_cfg_dir) @@ -133,8 +133,7 @@ def test_get_value_as_coords(initialized_model, caplog): result = model.get_value_at_coords("discharge", lon=[5.2], lat=[46.8]) msg = ( - "Requested point was lon: 5.2, lat: 46.8;" - " closest grid point is 5.00, 47.00." + "Requested point was lon: 5.2, lat: 46.8;" " closest grid point is 5.00, 47.00." ) assert msg in caplog.text diff --git a/tests/models/test_wflow.py b/tests/models/test_wflow.py index 694fd8f2..af4c35ee 100644 --- a/tests/models/test_wflow.py +++ b/tests/models/test_wflow.py @@ -109,10 +109,7 @@ def test_constructor_adds_api_riverrunoff(parameter_set, caplog): "Config file from parameter set is missing RiverRunoff option in API section" in caplog.text ) - assert ( - "added it with value '2, m/s option'" - in caplog.text - ) + assert "added it with value '2, m/s option'" in caplog.text def test_str(model, tmp_path): @@ -152,21 +149,21 @@ def test_setup(model): # Check content of config file cfg = CaseConfigParser() cfg.read(expected_cfg_file) - assert ( - cfg.get("API", "RiverRunoff") == "2, m/s" - ) + assert cfg.get("API", "RiverRunoff") == "2, m/s" def test_setup_withtimeoutexception(model, tmp_path): - with patch.object(BmiClientSingularity, "__init__", side_effect=FutureTimeoutError()), patch( - "datetime.datetime" - ) as mocked_datetime, pytest.raises(ValueError) as excinfo: + with patch.object( + BmiClientSingularity, "__init__", side_effect=FutureTimeoutError() + ), patch("datetime.datetime") as mocked_datetime, pytest.raises( + ValueError + ) as excinfo: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) model.setup() msg = str(excinfo.value) - assert 'docker pull ewatercycle/wflow-grpc4bmi:2020.1.1' in msg - sif = tmp_path / 'ewatercycle-wflow-grpc4bmi_2020.1.1.sif' + assert "docker pull ewatercycle/wflow-grpc4bmi:2020.1.1" in msg + sif = tmp_path / "ewatercycle-wflow-grpc4bmi_2020.1.1.sif" assert f"build {sif} docker://ewatercycle/wflow-grpc4bmi:2020.1.1" in msg @@ -189,9 +186,7 @@ def test_get_value_as_coords(initialized_model, caplog): with caplog.at_level(logging.DEBUG): result = model.get_value_at_coords("discharge", lon=[5.2], lat=[46.8]) - msg = ( - "Requested point was lon: 5.2, lat: 46.8; closest grid point is 5.00, 47.00." - ) + msg = "Requested point was lon: 5.2, lat: 46.8; closest grid point is 5.00, 47.00." assert msg in caplog.text assert result == np.array([1.0]) diff --git a/tests/observation/test_grdc.py b/tests/observation/test_grdc.py index b3db020a..ee0e5981 100644 --- a/tests/observation/test_grdc.py +++ b/tests/observation/test_grdc.py @@ -1,8 +1,8 @@ from datetime import datetime -import pandas as pd -import pytest import numpy as np +import pandas as pd +import pytest from pandas.testing import assert_frame_equal from ewatercycle import CFG @@ -11,9 +11,9 @@ @pytest.fixture def sample_grdc_file(tmp_path): - fn = tmp_path / '42424242_Q_Day.Cmd.txt' + fn = tmp_path / "42424242_Q_Day.Cmd.txt" # Sample with fictive data, but with same structure as real file - s = '''# Title: GRDC STATION DATA FILE + s = """# Title: GRDC STATION DATA FILE # -------------- # Format: DOS-ASCII # Field delimiter: ; @@ -51,8 +51,8 @@ def sample_grdc_file(tmp_path): YYYY-MM-DD;hh:mm; Value 2000-01-01;--:--; 123.000 2000-01-02;--:--; 456.000 -2000-01-03;--:--; -999.000''' - with open(fn, 'w', encoding='cp1252') as f: +2000-01-03;--:--; -999.000""" + with open(fn, "w", encoding="cp1252") as f: f.write(s) return fn @@ -61,84 +61,95 @@ def sample_grdc_file(tmp_path): def expected_results(tmp_path, sample_grdc_file): data = pd.DataFrame( - {'streamflow': [123., 456., np.NaN]}, - index = [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)], + {"streamflow": [123.0, 456.0, np.NaN]}, + index=[datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)], ) - data.index.rename('time', inplace=True) + data.index.rename("time", inplace=True) metadata = { - 'altitude_masl': 8.0, - 'country_code': 'NA', - 'dataSetContent': 'MEAN DAILY DISCHARGE (Q)', - 'file_generation_date': '2000-02-02', - 'grdc_catchment_area_in_km2': 4242.0, - 'grdc_file_name': str(tmp_path / sample_grdc_file), - 'grdc_latitude_in_arc_degree': 52.356154, - 'grdc_longitude_in_arc_degree': 4.955153, - 'id_from_grdc': 42424242, - 'last_update': '2000-02-01', - 'no_of_years': 1, - 'nrMeasurements': 'NA', - 'river_name': 'SOME RIVER', - 'station_name': 'SOME', - 'time_series': '2000-01 - 2000-01', - 'units': 'm³/s', - 'UserEndTime': '2000-02-01T00:00Z', - 'UserStartTime': '2000-01-01T00:00Z', - 'nrMissingData': 1, - } + "altitude_masl": 8.0, + "country_code": "NA", + "dataSetContent": "MEAN DAILY DISCHARGE (Q)", + "file_generation_date": "2000-02-02", + "grdc_catchment_area_in_km2": 4242.0, + "grdc_file_name": str(tmp_path / sample_grdc_file), + "grdc_latitude_in_arc_degree": 52.356154, + "grdc_longitude_in_arc_degree": 4.955153, + "id_from_grdc": 42424242, + "last_update": "2000-02-01", + "no_of_years": 1, + "nrMeasurements": "NA", + "river_name": "SOME RIVER", + "station_name": "SOME", + "time_series": "2000-01 - 2000-01", + "units": "m³/s", + "UserEndTime": "2000-02-01T00:00Z", + "UserStartTime": "2000-01-01T00:00Z", + "nrMissingData": 1, + } return data, metadata def test_get_grdc_data_with_datahome(tmp_path, expected_results): expected_data, expected_metadata = expected_results - result_data, result_metadata = get_grdc_data('42424242', '2000-01-01T00:00Z', '2000-02-01T00:00Z', data_home=str(tmp_path)) + result_data, result_metadata = get_grdc_data( + "42424242", "2000-01-01T00:00Z", "2000-02-01T00:00Z", data_home=str(tmp_path) + ) assert_frame_equal(result_data, expected_data) assert result_metadata == expected_metadata def test_get_grdc_data_with_CFG(expected_results, tmp_path): - CFG['grdc_location'] = str(tmp_path) + CFG["grdc_location"] = str(tmp_path) expected_data, expected_metadata = expected_results - result_data, result_metadata = get_grdc_data('42424242', '2000-01-01T00:00Z', '2000-02-01T00:00Z') + result_data, result_metadata = get_grdc_data( + "42424242", "2000-01-01T00:00Z", "2000-02-01T00:00Z" + ) assert_frame_equal(result_data, expected_data) assert result_metadata == expected_metadata def test_get_grdc_data_without_path(): - CFG['grdc_location'] = None + CFG["grdc_location"] = None with pytest.raises(ValueError) as excinfo: - get_grdc_data('42424242', '2000-01-01T00:00Z', '2000-02-01T00:00Z') + get_grdc_data("42424242", "2000-01-01T00:00Z", "2000-02-01T00:00Z") msg = str(excinfo.value) print(msg) - assert 'data_home' in msg - assert 'grdc_location' in msg + assert "data_home" in msg + assert "grdc_location" in msg def test_get_grdc_data_wrong_path(tmp_path): - CFG['grdc_location'] = f'{tmp_path}_data' + CFG["grdc_location"] = f"{tmp_path}_data" with pytest.raises(ValueError) as excinfo: - get_grdc_data('42424242', '2000-01-01T00:00Z', '2000-02-01T00:00Z') + get_grdc_data("42424242", "2000-01-01T00:00Z", "2000-02-01T00:00Z") msg = str(excinfo.value) print(msg) - assert 'directory' in msg + assert "directory" in msg def test_get_grdc_data_without_file(tmp_path): with pytest.raises(ValueError) as excinfo: - get_grdc_data('42424243', '2000-01-01T00:00Z', '2000-02-01T00:00Z', data_home=str(tmp_path)) + get_grdc_data( + "42424243", + "2000-01-01T00:00Z", + "2000-02-01T00:00Z", + data_home=str(tmp_path), + ) msg = str(excinfo.value) print(msg) - assert 'file' in msg + assert "file" in msg def test_get_grdc_dat_custom_column_name(expected_results, tmp_path): - CFG['grdc_location'] = str(tmp_path) - result_data, result_metadata = get_grdc_data('42424242', '2000-01-01T00:00Z', '2000-02-01T00:00Z', column='observation') + CFG["grdc_location"] = str(tmp_path) + result_data, result_metadata = get_grdc_data( + "42424242", "2000-01-01T00:00Z", "2000-02-01T00:00Z", column="observation" + ) expected_default_data, expected_metadata = expected_results - expected_data = expected_default_data.rename(columns={'streamflow': 'observation'}) + expected_data = expected_default_data.rename(columns={"streamflow": "observation"}) assert_frame_equal(result_data, expected_data) assert result_metadata == expected_metadata diff --git a/tests/parameter_sets/__init__.py b/tests/parameter_sets/__init__.py index 24421595..a3d5013c 100644 --- a/tests/parameter_sets/__init__.py +++ b/tests/parameter_sets/__init__.py @@ -7,7 +7,9 @@ def test_download_example_parameter_sets(tmp_path): download_example_parameter_sets() assert (tmp_path / "parameters" / "pcrglobwb_rhinemeuse_30min").exists() - assert (tmp_path / "parameters/pcrglobwb_rhinemeuse_30min/setup_natural_test.ini").exists() + assert ( + tmp_path / "parameters/pcrglobwb_rhinemeuse_30min/setup_natural_test.ini" + ).exists() # TODO test for the case where ewatercycle.yaml cfg is not writable diff --git a/tests/parameter_sets/test_default_parameterset.py b/tests/parameter_sets/test_default_parameterset.py index d850a1a7..19c6fe60 100644 --- a/tests/parameter_sets/test_default_parameterset.py +++ b/tests/parameter_sets/test_default_parameterset.py @@ -9,15 +9,15 @@ class TestDefaults: @pytest.fixture def mocked_config(self, tmp_path): - CFG['parameterset_dir'] = tmp_path - config = tmp_path / 'mymockedconfig.ini' - config.write_text('Something') + CFG["parameterset_dir"] = tmp_path + config = tmp_path / "mymockedconfig.ini" + config.write_text("Something") return config @pytest.fixture def parameter_set(self, tmp_path, mocked_config: Path): return ParameterSet( - name='justatest', + name="justatest", directory=str(tmp_path), config=mocked_config.name, ) @@ -45,8 +45,8 @@ def test_repr(self, parameter_set: ParameterSet, tmp_path): def test_str(self, parameter_set: ParameterSet, tmp_path): expected = ( - 'Parameter set\n' - '-------------\n' + "Parameter set\n" + "-------------\n" "name=justatest\n" f"directory={str(tmp_path)}\n" f"config={str(tmp_path)}/mymockedconfig.ini\n" @@ -60,18 +60,18 @@ def test_str(self, parameter_set: ParameterSet, tmp_path): class TestOutsideCFG: @pytest.fixture def mocked_config(self, tmp_path): - CFG['parameterset_dir'] = tmp_path / 'parameter-sets' - config = tmp_path / 'mymockedconfig.ini' - config.write_text('Something') + CFG["parameterset_dir"] = tmp_path / "parameter-sets" + config = tmp_path / "mymockedconfig.ini" + config.write_text("Something") return config @pytest.fixture def parameter_set(self, tmp_path, mocked_config: Path): return ParameterSet( - name='justatest', - directory=str(tmp_path / 'my-parameter-set'), + name="justatest", + directory=str(tmp_path / "my-parameter-set"), config=mocked_config.name, ) def test_directory(self, parameter_set: ParameterSet, tmp_path): - assert parameter_set.directory == tmp_path / 'my-parameter-set' + assert parameter_set.directory == tmp_path / "my-parameter-set" diff --git a/tests/parameter_sets/test_example.py b/tests/parameter_sets/test_example.py index 99087dc2..0e8fa4be 100644 --- a/tests/parameter_sets/test_example.py +++ b/tests/parameter_sets/test_example.py @@ -1,5 +1,5 @@ import logging -from unittest.mock import patch, Mock +from unittest.mock import Mock, patch import pytest @@ -9,7 +9,7 @@ @pytest.fixture def setup_config(tmp_path): - CFG['parameterset_dir'] = tmp_path + CFG["parameterset_dir"] = tmp_path yield CFG # Rollback changes made to CFG by tests CFG.reload() @@ -18,63 +18,70 @@ def setup_config(tmp_path): @pytest.fixture def example(setup_config): return ExampleParameterSet( - name='firstexample', - config_url='https://github.com/mymodelorg/mymodelrepo/raw/master/mymodelexample/config.ini', - datafiles_url='https://github.com/mymodelorg/mymodelrepo/trunk/mymodelexample', - directory='mymodelexample', - config='mymodelexample/config.ini', - supported_model_versions={'0.4.2'}, + name="firstexample", + config_url="https://github.com/mymodelorg/mymodelrepo/raw/master/mymodelexample/config.ini", + datafiles_url="https://github.com/mymodelorg/mymodelrepo/trunk/mymodelexample", + directory="mymodelexample", + config="mymodelexample/config.ini", + supported_model_versions={"0.4.2"}, ) def test_to_config(example): example.to_config() - assert 'firstexample' in CFG['parameter_sets'] + assert "firstexample" in CFG["parameter_sets"] expected = dict( - doi='N/A', - target_model='generic', - directory='mymodelexample', - config='mymodelexample/config.ini', - supported_model_versions={'0.4.2'}, + doi="N/A", + target_model="generic", + directory="mymodelexample", + config="mymodelexample/config.ini", + supported_model_versions={"0.4.2"}, ) - assert CFG['parameter_sets']['firstexample'] == expected + assert CFG["parameter_sets"]["firstexample"] == expected -@patch('urllib.request.urlopen') -@patch('subprocess.check_call') +@patch("urllib.request.urlopen") +@patch("subprocess.check_call") def test_download(mock_check_call, mock_urlopen, example, tmp_path): - ps_dir = tmp_path / 'mymodelexample' + ps_dir = tmp_path / "mymodelexample" r = Mock() - r.read.return_value = b'somecontent' + r.read.return_value = b"somecontent" mock_urlopen.return_value = r mock_check_call.side_effect = lambda _: ps_dir.mkdir() example.download() - mock_urlopen.assert_called_once_with('https://github.com/mymodelorg/mymodelrepo/raw/master/mymodelexample/config.ini') - mock_check_call.assert_called_once_with([ - 'svn', 'export', - 'https://github.com/mymodelorg/mymodelrepo/trunk/mymodelexample', - ps_dir - ]) - assert (ps_dir / 'config.ini').read_text() == 'somecontent' + mock_urlopen.assert_called_once_with( + "https://github.com/mymodelorg/mymodelrepo/raw/master/mymodelexample/config.ini" + ) + mock_check_call.assert_called_once_with( + [ + "svn", + "export", + "https://github.com/mymodelorg/mymodelrepo/trunk/mymodelexample", + ps_dir, + ] + ) + assert (ps_dir / "config.ini").read_text() == "somecontent" def test_download_already_exists(example, tmp_path): - ps_dir = tmp_path / 'mymodelexample' + ps_dir = tmp_path / "mymodelexample" ps_dir.mkdir() with pytest.raises(ValueError) as excinfo: example.download() - assert 'already exists, will not overwrite.' in str(excinfo.value) + assert "already exists, will not overwrite." in str(excinfo.value) -@patch('urllib.request.urlopen') -@patch('subprocess.check_call') -def test_download_already_exists_but_skipped(mock_check_call, mock_urlopen, example, tmp_path, caplog): - ps_dir = tmp_path / 'mymodelexample' +@patch("urllib.request.urlopen") +@patch("subprocess.check_call") +def test_download_already_exists_but_skipped( + mock_check_call, mock_urlopen, example, tmp_path, caplog +): + ps_dir = tmp_path / "mymodelexample" ps_dir.mkdir() with caplog.at_level(logging.INFO): @@ -83,4 +90,4 @@ def test_download_already_exists_but_skipped(mock_check_call, mock_urlopen, exam mock_urlopen.assert_not_called() mock_check_call.assert_not_called() - assert 'already exists, skipping download.' in caplog.text + assert "already exists, skipping download." in caplog.text diff --git a/tests/test_analysis.py b/tests/test_analysis.py index bd6c3117..a8143ab7 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -1,14 +1,15 @@ -from ewatercycle.analysis import hydrograph -from matplotlib.testing.decorators import image_comparison -import pandas as pd import numpy as np +import pandas as pd +from matplotlib.testing.decorators import image_comparison + +from ewatercycle.analysis import hydrograph @image_comparison( - baseline_images=['hydrograph'], + baseline_images=["hydrograph"], remove_text=True, - extensions=['png'], - savefig_kwarg={'bbox_inches':'tight'}, + extensions=["png"], + savefig_kwarg={"bbox_inches": "tight"}, ) def test_hydrograph(): ntime = 300 @@ -18,19 +19,19 @@ def test_hydrograph(): np.random.seed(20210416) discharge = { - 'discharge_a': pd.Series(np.linspace(0, 2, ntime), index=dti), - 'discharge_b': pd.Series(3*np.random.random(ntime)**2, index=dti), - 'discharge_c': pd.Series(2*np.random.random(ntime)**2, index=dti), - 'reference': pd.Series(np.random.random(ntime)**2, index=dti), + "discharge_a": pd.Series(np.linspace(0, 2, ntime), index=dti), + "discharge_b": pd.Series(3 * np.random.random(ntime) ** 2, index=dti), + "discharge_c": pd.Series(2 * np.random.random(ntime) ** 2, index=dti), + "reference": pd.Series(np.random.random(ntime) ** 2, index=dti), } df = pd.DataFrame(discharge) precipitation = { - 'precipitation_a': pd.Series(np.random.random(ntime)/20, index=dti), - 'precipitation_b': pd.Series(np.random.random(ntime)/30, index=dti), + "precipitation_a": pd.Series(np.random.random(ntime) / 20, index=dti), + "precipitation_b": pd.Series(np.random.random(ntime) / 30, index=dti), } df_pr = pd.DataFrame(precipitation) - hydrograph(df, reference='reference', precipitation=df_pr) + hydrograph(df, reference="reference", precipitation=df_pr) diff --git a/tests/test_config.py b/tests/test_config.py index 2ee7eab1..3fec590b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -3,13 +3,13 @@ """Tests for the ewatercycle_parametersetdb module. """ -from ewatercycle.parametersetdb.config import fetch, YamlConfig +from ewatercycle.parametersetdb.config import YamlConfig, fetch def test_fetch(yaml_config_url): result = fetch(yaml_config_url) - assert 'PEQ_Hupsel.dat' in result + assert "PEQ_Hupsel.dat" in result class TestYamlConfig: @@ -20,8 +20,8 @@ def test_construct_from_data_url(self, yaml_config_url, yaml_config): def test_save(self, tmpdir, yaml_config_url): conf = YamlConfig(yaml_config_url) - fn = tmpdir.join('myconfig.yml') + fn = tmpdir.join("myconfig.yml") conf.save(str(fn)) - assert 'PEQ_Hupsel.dat' in fn.read() + assert "PEQ_Hupsel.dat" in fn.read() diff --git a/tests/test_datafiles.py b/tests/test_datafiles.py index dd34726f..118e72ae 100644 --- a/tests/test_datafiles.py +++ b/tests/test_datafiles.py @@ -2,14 +2,14 @@ def test_SymlinkCopier(tmp_path): - source_dir = tmp_path / 'source' + source_dir = tmp_path / "source" source_dir.mkdir() - source_forcings = source_dir / 'forcings.csv' - source_forcings.write_text('dummy') + source_forcings = source_dir / "forcings.csv" + source_forcings.write_text("dummy") copier = SymlinkCopier(str(source_dir)) - target_dir = tmp_path / 'target' + target_dir = tmp_path / "target" copier.save(target_dir) - target_forcings = target_dir / 'forcings.csv' - assert target_forcings.read_text() == 'dummy' + target_forcings = target_dir / "forcings.csv" + assert target_forcings.read_text() == "dummy" diff --git a/tests/test_parameter_sets.py b/tests/test_parameter_sets.py index f0846895..d2952e49 100644 --- a/tests/test_parameter_sets.py +++ b/tests/test_parameter_sets.py @@ -4,97 +4,104 @@ from ewatercycle import CFG from ewatercycle.config import DEFAULT_CONFIG -from ewatercycle.parameter_sets import available_parameter_sets, get_parameter_set, example_parameter_sets, \ - download_example_parameter_sets, ExampleParameterSet +from ewatercycle.parameter_sets import ( + ExampleParameterSet, + available_parameter_sets, + download_example_parameter_sets, + example_parameter_sets, + get_parameter_set, +) @pytest.fixture def setup_config(tmp_path): - CFG['parameterset_dir'] = tmp_path - CFG['ewatercycle_config'] = tmp_path / 'ewatercycle.yaml' + CFG["parameterset_dir"] = tmp_path + CFG["ewatercycle_config"] = tmp_path / "ewatercycle.yaml" yield CFG - CFG['ewatercycle_config'] = DEFAULT_CONFIG + CFG["ewatercycle_config"] = DEFAULT_CONFIG CFG.reload() @pytest.fixture def mocked_parameterset_dir(setup_config, tmp_path): - ps1_dir = tmp_path / 'ps1' + ps1_dir = tmp_path / "ps1" ps1_dir.mkdir() - config1 = ps1_dir / 'mymockedconfig1.ini' - config1.write_text('Something') - ps2_dir = tmp_path / 'ps2' + config1 = ps1_dir / "mymockedconfig1.ini" + config1.write_text("Something") + ps2_dir = tmp_path / "ps2" ps2_dir.mkdir() - config2 = ps2_dir / 'mymockedconfig2.ini' - config2.write_text('Something else') - CFG['parameter_sets'] = { - 'ps1': { - 'directory': str(ps1_dir), - 'config': str(config1.relative_to(tmp_path)), - 'target_model': 'generic', - 'doi': 'somedoi1' + config2 = ps2_dir / "mymockedconfig2.ini" + config2.write_text("Something else") + CFG["parameter_sets"] = { + "ps1": { + "directory": str(ps1_dir), + "config": str(config1.relative_to(tmp_path)), + "target_model": "generic", + "doi": "somedoi1", }, - 'ps2': { - 'directory': str(ps2_dir), - 'config': str(config2.relative_to(tmp_path)), - 'target_model': 'generic', - 'doi': 'somedoi2' + "ps2": { + "directory": str(ps2_dir), + "config": str(config2.relative_to(tmp_path)), + "target_model": "generic", + "doi": "somedoi2", + }, + "ps3": { + "directory": str(tmp_path / "ps3"), + "config": "unavailable_config_file", + "target_model": "generic", + "doi": "somedoi3", }, - 'ps3': { - 'directory': str(tmp_path / 'ps3'), - 'config': 'unavailable_config_file', - 'target_model': 'generic', - 'doi': 'somedoi3' - } } class TestAvailableParameterSets: def test_filled(self, mocked_parameterset_dir): - names = available_parameter_sets('generic') - assert set(names) == {'ps1', 'ps2'} # ps3 is filtered due to not being available + names = available_parameter_sets("generic") + assert set(names) == { + "ps1", + "ps2", + } # ps3 is filtered due to not being available def test_no_config(self, tmp_path): # Load default config shipped with package - CFG['ewatercycle_config'] = DEFAULT_CONFIG + CFG["ewatercycle_config"] = DEFAULT_CONFIG CFG.reload() with pytest.raises(ValueError) as excinfo: available_parameter_sets() - assert 'No configuration file found' in str(excinfo.value) + assert "No configuration file found" in str(excinfo.value) def test_no_sets_in_config(self, setup_config): with pytest.raises(ValueError) as excinfo: available_parameter_sets() - assert 'No parameter sets defined in' in str(excinfo.value) + assert "No parameter sets defined in" in str(excinfo.value) def test_no_sets_for_model(self, mocked_parameterset_dir): with pytest.raises(ValueError) as excinfo: - available_parameter_sets('somemodel') + available_parameter_sets("somemodel") - assert 'No parameter sets defined for somemodel model in' in str(excinfo.value) + assert "No parameter sets defined for somemodel model in" in str(excinfo.value) class TestGetParameterSet: - def test_valid(self, mocked_parameterset_dir, tmp_path): - actual = get_parameter_set('ps1') + actual = get_parameter_set("ps1") - assert actual.name == 'ps1' - assert actual.directory == tmp_path / 'ps1' - assert actual.config == tmp_path / 'ps1' / 'mymockedconfig1.ini' - assert actual.doi == 'somedoi1' - assert actual.target_model == 'generic' + assert actual.name == "ps1" + assert actual.directory == tmp_path / "ps1" + assert actual.config == tmp_path / "ps1" / "mymockedconfig1.ini" + assert actual.doi == "somedoi1" + assert actual.target_model == "generic" def test_unknown(self, mocked_parameterset_dir): with pytest.raises(KeyError): - get_parameter_set('ps9999') + get_parameter_set("ps9999") def test_unavailable(self, mocked_parameterset_dir): with pytest.raises(ValueError): - get_parameter_set('ps3') + get_parameter_set("ps3") def test_example_parameter_sets(setup_config): @@ -104,10 +111,10 @@ def test_example_parameter_sets(setup_config): assert name == examples[name].name -@patch.object(ExampleParameterSet, 'download') +@patch.object(ExampleParameterSet, "download") def test_download_example_parameter_sets(mocked_download, setup_config, tmp_path): download_example_parameter_sets() assert mocked_download.call_count > 0 - assert CFG['ewatercycle_config'].read_text() == CFG.dump_to_yaml() - assert len(CFG['parameter_sets']) > 0 + assert CFG["ewatercycle_config"].read_text() == CFG.dump_to_yaml() + assert len(CFG["parameter_sets"]) > 0 diff --git a/tests/test_parameterset.py b/tests/test_parameterset.py index 7fb65ff9..1b26ab18 100644 --- a/tests/test_parameterset.py +++ b/tests/test_parameterset.py @@ -1,14 +1,16 @@ # -*- coding: utf-8 -*- from unittest import mock -from ewatercycle.parametersetdb import build_from_urls, ParameterSet +from ewatercycle.parametersetdb import ParameterSet, build_from_urls from ewatercycle.parametersetdb.datafiles import SubversionCopier def test_build_from_urls(yaml_config_url, yaml_config): pset = build_from_urls( - config_format='yaml', config_url=yaml_config_url, - datafiles_format='svn', datafiles_url='http://example.com', + config_format="yaml", + config_url=yaml_config_url, + datafiles_format="svn", + datafiles_url="http://example.com", ) assert isinstance(pset.df, SubversionCopier) @@ -17,17 +19,17 @@ def test_build_from_urls(yaml_config_url, yaml_config): class TestParameterSet: def test_save_config(self, sample_parameterset: ParameterSet, tmpdir): - fn = tmpdir.join('myconfig.yml') + fn = tmpdir.join("myconfig.yml") sample_parameterset.save_config(str(fn)) - assert 'PEQ_Hupsel.dat' in fn.read() + assert "PEQ_Hupsel.dat" in fn.read() - @mock.patch('subprocess.check_call') + @mock.patch("subprocess.check_call") def test_save_datafiles(self, mock_check_call, sample_parameterset: ParameterSet): - fn = '/somewhere/adirectory' + fn = "/somewhere/adirectory" sample_parameterset.save_datafiles(fn) - expected_args = ['svn', 'export', sample_parameterset.df.source, fn] + expected_args = ["svn", "export", sample_parameterset.df.source, fn] mock_check_call.assert_called_once_with(expected_args) From b3f70139b3952a9547ce341de3b0d4020b05843a Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 11 Aug 2021 09:39:32 +0200 Subject: [PATCH 06/33] Use nbqa-black to format notebooks instead of black As black replaces " with ', which makes invalid JSON --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6bbe54f2..3380d772 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,9 +5,6 @@ requires = [ ] build-backend = "setuptools.build_meta" -[tool.black] -include = '(\.pyi?|\.ipynb)$' - [tool.isort] profile = "black" multi_line_output = 3 From 565eb190f7cf393a74a29d904342317b7a80263e Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 11 Aug 2021 09:41:53 +0200 Subject: [PATCH 07/33] On release run pre-commit on all files --- .github/workflows/sonar.yml | 2 +- CONTRIBUTING.md | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sonar.yml b/.github/workflows/sonar.yml index 2cd235bb..9780a4df 100644 --- a/.github/workflows/sonar.yml +++ b/.github/workflows/sonar.yml @@ -25,7 +25,7 @@ jobs: shell: bash -l {0} run: | pip3 install -e .[dev] - - name: Run pre commit hooks like black formatter + - name: Run pre commit hooks like linters and black formatter uses: pre-commit/action@v2.0.3 - name: Tests with coverage run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e3da3a9d..0158444e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,24 +82,25 @@ This section is for maintainers of the package. 2. Determine what new version (major, minor or patch) to use. Package uses `semantic versioning `_. 3. Run ``bump2version `` to update version in package files. 4. Update CHANGELOG.md with changes between current and new version. -5. Commit & push changes to GitHub. -6. Wait for [GitHub +5. Make sure pre-commit hooks are green for all files by running ``pre-commit run --all-files``. +6. Commit & push changes to GitHub. +7. Wait for [GitHub actions](https://github.com/eWaterCycle/ewatercycle/actions?query=branch%3Amain+) to be completed and green. -7. Create a [GitHub release](https://github.com/eWaterCycle/ewatercycle/releases/new) +8. Create a [GitHub release](https://github.com/eWaterCycle/ewatercycle/releases/new) - Use version as title and tag version. - As description use intro text from README.md (to give context to Zenodo record) and changes from CHANGELOG.md -8. Create a PyPI release. +9. Create a PyPI release. 1. Create distribution archives with `python3 -m build`. 2. Upload archives to PyPI with `twine upload dist/*` (use your personal PyPI account). -9. Verify +10. Verify 1. Has [new Zenodo record](https://zenodo.org/search?page=1&size=20&q=ewatercycle) @@ -110,4 +111,4 @@ This section is for maintainers of the package. 3. Can new version be installed with pip using `pip3 install ewatercycle==`? -10. Celebrate +11. Celebrate From ca821b627118522562836e35ea4094a0a526632f Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 11 Aug 2021 09:47:26 +0200 Subject: [PATCH 08/33] Disable flake8 until its errors are fixed --- .pre-commit-config.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ee101a12..ca526909 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,10 +26,11 @@ repos: rev: '5.9.3' hooks: - id: isort - - repo: https://gitlab.com/pycqa/flake8 - rev: '3.9.2' - hooks: - - id: flake8 + # TODO renable when erros are fixed/ignored + # - repo: https://gitlab.com/pycqa/flake8 + # rev: '3.9.2' + # hooks: + # - id: flake8 - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.910 hooks: @@ -45,5 +46,6 @@ repos: additional_dependencies: [isort==5.9.3] - id: nbqa-mypy additional_dependencies: [mypy==0.910] - - id: nbqa-flake8 - additional_dependencies: [flake8==3.9.2] + # TODO renable when erros are fixed/ignored + # - id: nbqa-flake8 + # additional_dependencies: [flake8==3.9.2] From 22adbe7011f3ea8f5717ba4165a3b0eba977ead6 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 11 Aug 2021 13:11:26 +0200 Subject: [PATCH 09/33] Added flake8 plugins Selection from https://towardsdatascience.com/static-code-analysis-for-python-bdce10b8d287 --- .pre-commit-config.yaml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ca526909..9b788610 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,6 +31,17 @@ repos: # rev: '3.9.2' # hooks: # - id: flake8 + # additional_dependencies: [ + # flake8-annotations-complexity, flake8-cognitive-complexity, + # flake8-expression-complexity, + # flake8-docstrings, flake8-bugbear, flake8-eradicate, + # flake8-comprehensions, flake8-executable, flake8-raise, + # flake8-pathlib, flake8-pytest-style, flake8-implicit-str-concat, + # flake8-variables-names, pandas-vet, + # flake8-bandit, flake8-bugbear, + + # ] + # args: [--docstring-convention=google] - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.910 hooks: @@ -48,4 +59,18 @@ repos: additional_dependencies: [mypy==0.910] # TODO renable when erros are fixed/ignored # - id: nbqa-flake8 - # additional_dependencies: [flake8==3.9.2] + # additional_dependencies: [ + # flake8==3.9.2, + # flake8-annotations-complexity, flake8-cognitive-complexity, + # flake8-expression-complexity, + # flake8-docstrings, flake8-bugbear, flake8-eradicate, + # flake8-comprehensions, flake8-executable, flake8-raise, + # flake8-pathlib, flake8-pytest-style, flake8-implicit-str-concat, + # flake8-variables-names, pandas-vet, + # flake8-bandit, flake8-bugbear, + # ] + # args: [--docstring-convention=google] + - repo: https://github.com/regebro/pyroma + rev: "3.2" + hooks: + - id: pyroma From 107afa54a5d99be19a661d3d26cef036c381e01e Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 12 Aug 2021 11:44:29 +0200 Subject: [PATCH 10/33] Make isort, pylint and flake8 compatible with black See https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html --- pyproject.toml | 6 ++++++ setup.cfg | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 3380d772..cee6dc99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,3 +8,9 @@ build-backend = "setuptools.build_meta" [tool.isort] profile = "black" multi_line_output = 3 + +[tool.pylint.messages_control] +disable = "C0330, C0326" + +[tool.pylint.format] +max-line-length = "88" diff --git a/setup.cfg b/setup.cfg index 974df757..26c7b0e1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -110,3 +110,7 @@ builder = html [mypy] ignore_missing_imports = True files = src, tests + +[flake8] +max-line-length = 88 +extend-ignore = E203 From 9141a5d40982229cf803061473e3508947cfd5dc Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 12 Aug 2021 13:03:30 +0200 Subject: [PATCH 11/33] Use tabs in makefile --- .editorconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.editorconfig b/.editorconfig index 83e59afe..3f72fd3c 100644 --- a/.editorconfig +++ b/.editorconfig @@ -25,3 +25,6 @@ indent_size = 2 [*.{md,Rmd}] trim_trailing_whitespace = false + +[Makefile] +indent_style = tab From ba8cba807db971a04cb4087d4e0975c5e75b9d7b Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 12 Aug 2021 13:11:58 +0200 Subject: [PATCH 12/33] Replace prospector with pre-commit --- .prospector.yml | 29 ----------------------------- setup.cfg | 1 - 2 files changed, 30 deletions(-) delete mode 100644 .prospector.yml diff --git a/.prospector.yml b/.prospector.yml deleted file mode 100644 index 0ec2121e..00000000 --- a/.prospector.yml +++ /dev/null @@ -1,29 +0,0 @@ -# prospector configuration file - ---- - -output-format: grouped - -strictness: veryhigh -doc-warnings: true -test-warnings: true -member-warnings: false - -pyroma: - run: true - -pep8: - full: true - -mypy: - run: true - -pep257: - disable: [ - # Disable because not part of PEP257 official convention: - # see http://pep257.readthedocs.io/en/latest/error_codes.html - D203, # 1 blank line required before class docstring - D212, # Multi-line docstring summary should start at the first line - D213, # Multi-line docstring summary should start at the second line - D404, # First word of the docstring should not be This - ] diff --git a/setup.cfg b/setup.cfg index 26c7b0e1..f0b3f95a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -70,7 +70,6 @@ dev = isort nbsphinx pre-commit - prospector[with_pyroma,with_mypy] pycodestyle pytest pytest-cov From 7a4bab4ea57f680abdcad405fb42005a792f6b05 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 12 Aug 2021 13:12:29 +0200 Subject: [PATCH 13/33] Prefer `@pytest.fixture` over `@pytest.fixture()` --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index f0b3f95a..111da1bf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -113,3 +113,4 @@ files = src, tests [flake8] max-line-length = 88 extend-ignore = E203 +pytest-fixture-no-parentheses = False From 7f27ade4729e2acbb0ff09d02bd549fde9f8eca2 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 12 Aug 2021 13:17:38 +0200 Subject: [PATCH 14/33] Added pylint and bunch of flake8 plugins to pre commit config Using YAML anchor + alias for nb --- .pre-commit-config.yaml | 78 ++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9b788610..726e7da0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - repo: https://github.com/adrienverge/yamllint - rev: 'v1.26.0' + rev: "v1.26.0" hooks: - id: yamllint - repo: https://github.com/asottile/setup-cfg-fmt @@ -23,25 +23,51 @@ repos: hooks: - id: black - repo: https://github.com/PyCQA/isort - rev: '5.9.3' + rev: "5.9.3" hooks: - id: isort # TODO renable when erros are fixed/ignored - # - repo: https://gitlab.com/pycqa/flake8 - # rev: '3.9.2' - # hooks: - # - id: flake8 - # additional_dependencies: [ - # flake8-annotations-complexity, flake8-cognitive-complexity, - # flake8-expression-complexity, - # flake8-docstrings, flake8-bugbear, flake8-eradicate, - # flake8-comprehensions, flake8-executable, flake8-raise, - # flake8-pathlib, flake8-pytest-style, flake8-implicit-str-concat, - # flake8-variables-names, pandas-vet, - # flake8-bandit, flake8-bugbear, - - # ] - # args: [--docstring-convention=google] + - repo: https://github.com/pycqa/pylint + rev: "v2.9.6" + hooks: + - id: pylint + # TODO renable when erros are fixed/ignored + - repo: https://gitlab.com/pycqa/flake8 + rev: "3.9.2" + hooks: + - id: flake8 + additional_dependencies: + &fd [ + flake8-annotations-complexity, + flake8-bandit, + flake8-blind-except, + flake8-bugbear, + flake8-builtins, + flake8-cognitive-complexity, + flake8-comprehensions, + flake8-docstrings, + flake8-eradicate, + flake8-executable, + flake8-expression-complexity, + flake8-if-expr, + flake8-implicit-str-concat, + flake8-logging-format, + flake8-pathlib, + flake8-print, + flake8-pytest, + flake8-pytest-style, + # flake8-quotes, # conflicts with blacks double quote preference + flake8-raise, + flake8-return, + flake8-typing-imports, + flake8-variables-names, + flake8==3.9.2, + pandas-vet, + pep8-naming, + # wemake-python-styleguide, # conflicts with black + yesqa, + ] + args: &fa [--docstring-convention=google] - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.910 hooks: @@ -58,18 +84,12 @@ repos: - id: nbqa-mypy additional_dependencies: [mypy==0.910] # TODO renable when erros are fixed/ignored - # - id: nbqa-flake8 - # additional_dependencies: [ - # flake8==3.9.2, - # flake8-annotations-complexity, flake8-cognitive-complexity, - # flake8-expression-complexity, - # flake8-docstrings, flake8-bugbear, flake8-eradicate, - # flake8-comprehensions, flake8-executable, flake8-raise, - # flake8-pathlib, flake8-pytest-style, flake8-implicit-str-concat, - # flake8-variables-names, pandas-vet, - # flake8-bandit, flake8-bugbear, - # ] - # args: [--docstring-convention=google] + - id: nbqa-flake8 + additional_dependencies: *fd + args: *fa + # TODO renable when erros are fixed/ignored + - id: nbqa-pylint + additional_dependencies: [pylint==2.9.6] - repo: https://github.com/regebro/pyroma rev: "3.2" hooks: From 33cafd131339309ef36a810322a224d655165264 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 12 Aug 2021 13:26:49 +0200 Subject: [PATCH 15/33] Move config from arg to file + pytest-fixture config inverted --- .pre-commit-config.yaml | 2 -- setup.cfg | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 726e7da0..e07dd1d4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,7 +67,6 @@ repos: # wemake-python-styleguide, # conflicts with black yesqa, ] - args: &fa [--docstring-convention=google] - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.910 hooks: @@ -86,7 +85,6 @@ repos: # TODO renable when erros are fixed/ignored - id: nbqa-flake8 additional_dependencies: *fd - args: *fa # TODO renable when erros are fixed/ignored - id: nbqa-pylint additional_dependencies: [pylint==2.9.6] diff --git a/setup.cfg b/setup.cfg index 111da1bf..c1cf4e09 100644 --- a/setup.cfg +++ b/setup.cfg @@ -113,4 +113,5 @@ files = src, tests [flake8] max-line-length = 88 extend-ignore = E203 -pytest-fixture-no-parentheses = False +pytest-fixture-no-parentheses = True +docstring-convention = google From 13d2d42edd49810015dd55abf2121983099968d0 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 12 Aug 2021 14:02:40 +0200 Subject: [PATCH 16/33] Disable pylint and fix flake8 errors first --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e07dd1d4..bd3ce4f6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,10 +27,10 @@ repos: hooks: - id: isort # TODO renable when erros are fixed/ignored - - repo: https://github.com/pycqa/pylint - rev: "v2.9.6" - hooks: - - id: pylint + # - repo: https://github.com/pycqa/pylint + # rev: "v2.9.6" + # hooks: + # - id: pylint # TODO renable when erros are fixed/ignored - repo: https://gitlab.com/pycqa/flake8 rev: "3.9.2" @@ -86,8 +86,8 @@ repos: - id: nbqa-flake8 additional_dependencies: *fd # TODO renable when erros are fixed/ignored - - id: nbqa-pylint - additional_dependencies: [pylint==2.9.6] + # - id: nbqa-pylint + # additional_dependencies: [pylint==2.9.6] - repo: https://github.com/regebro/pyroma rev: "3.2" hooks: From c74b2f54510ad38fe060971f33e98a1d3b77d490 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 12 Aug 2021 14:07:32 +0200 Subject: [PATCH 17/33] Ignore docstring checks in tests/ + Move google doc style to where pydocstyle expects it --- pyproject.toml | 3 +++ setup.cfg | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cee6dc99..9a76df37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,3 +14,6 @@ disable = "C0330, C0326" [tool.pylint.format] max-line-length = "88" + +[tool.pydocstyle] +convention = "google" diff --git a/setup.cfg b/setup.cfg index c1cf4e09..e23162f9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -114,4 +114,5 @@ files = src, tests max-line-length = 88 extend-ignore = E203 pytest-fixture-no-parentheses = True -docstring-convention = google +per-file-ignores = + tests/**: S101,D100,D101,D102,D103,D104 From 35afd9909a21516f7638da6c34f97284b2166493 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 12 Aug 2021 14:08:22 +0200 Subject: [PATCH 18/33] Fix flake8 errors --- docs/conf.py | 56 ++++++++++++++-------------------------------------- 1 file changed, 15 insertions(+), 41 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index a1bdf3e8..ab9573af 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,3 +1,4 @@ +# noqa: D100 # -*- coding: utf-8 -*- # # ewatercycle documentation build configuration file, created by @@ -16,19 +17,15 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -import os import sys +from pathlib import Path -here = os.path.dirname(__file__) -sys.path.insert(0, os.path.abspath(os.path.join(here, "..", "src"))) +src = Path(__file__) / ".." / "src" +sys.path.insert(0, str(src.absolute())) # -- General configuration ------------------------------------------------ -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. @@ -46,15 +43,14 @@ # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # -# source_suffix = ['.rst', '.md'] -source_suffix = ".rst" +source_suffix = [".rst"] # The master toctree document. master_doc = "index" # General information about the project. project = "ewatercycle" -copyright = "2018, Netherlands eScience Center & Delft University of Technology" +copyright = "2018, Netherlands eScience Center & Delft University of Technology" # noqa A001,VNE003 author = "Stefan Verhoeven" # The version info for the project you're documenting, acts as replacement for @@ -87,10 +83,10 @@ # -- Run apidoc plug-in manually, as readthedocs doesn't support it ------- # See https://github.com/rtfd/readthedocs.org/issues/1139 -def run_apidoc(_): - here = os.path.dirname(__file__) - out = os.path.abspath(os.path.join(here, "apidocs")) - src = os.path.abspath(os.path.join(here, "..", "src", "ewatercycle")) +def run_apidoc(_): # noqa: D103 + here = Path(__file__) + out = (here / "apidocs").absolute() + sourcedir = src / "ewatercycle" ignore_paths = [] @@ -102,7 +98,7 @@ def run_apidoc(_): "--implicit-namespaces", "-o", out, - src, + sourcedir, ] + ignore_paths try: @@ -118,7 +114,7 @@ def run_apidoc(_): apidoc.main(argv) -def setup(app): +def setup(app): # noqa: D103 app.connect("builder-inited", run_apidoc) @@ -127,16 +123,9 @@ def setup(app): # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -# html_theme = 'alabaster' html_theme = "sphinx_rtd_theme" html_logo = "examples/logo.png" -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} - # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". @@ -163,21 +152,6 @@ def setup(app): # -- Options for LaTeX output --------------------------------------------- -# latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -# -# 'papersize': 'letterpaper', -# The font size ('10pt', '11pt' or '12pt'). -# -# 'pointsize': '10pt', -# Additional stuff for the LaTeX preamble. -# -# 'preamble': '', -# Latex figure (float) alignment -# -# 'figure_align': 'htbp', -# } - # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). @@ -259,9 +233,9 @@ def setup(app): intersphinx_mapping = { "cf_units": ("https://scitools.org.uk/cf-units/docs/latest/", None), - "esmvalcore": (f"https://docs.esmvaltool.org/projects/esmvalcore/en/latest/", None), - "esmvaltool": (f"https://docs.esmvaltool.org/en/latest/", None), - "grpc4bmi": (f"https://grpc4bmi.readthedocs.io/en/latest/", None), + "esmvalcore": ("https://docs.esmvaltool.org/projects/esmvalcore/en/latest/", None), + "esmvaltool": ("https://docs.esmvaltool.org/en/latest/", None), + "grpc4bmi": ("https://grpc4bmi.readthedocs.io/en/latest/", None), "iris": ("https://scitools-iris.readthedocs.io/en/latest/", None), "lime": ("https://lime-ml.readthedocs.io/en/latest/", None), "basic_modeling_interface": ("https://bmi.readthedocs.io/en/latest/", None), From 096ea900dad215f35dcf03f4753400130e01e62f Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 12 Aug 2021 14:35:29 +0200 Subject: [PATCH 19/33] Reduce flake8 warnings --- src/ewatercycle/observation/grdc.py | 145 ++++++++++++++-------------- 1 file changed, 74 insertions(+), 71 deletions(-) diff --git a/src/ewatercycle/observation/grdc.py b/src/ewatercycle/observation/grdc.py index 6facb2db..29c2ef5f 100644 --- a/src/ewatercycle/observation/grdc.py +++ b/src/ewatercycle/observation/grdc.py @@ -1,3 +1,4 @@ +"""Global Runoff Data Centre module.""" import logging import os from typing import Dict, Tuple, Union @@ -9,6 +10,8 @@ logger = logging.getLogger(__name__) +MetaDataType = Dict[str, Union[str, int, float]] + def get_grdc_data( station_id: str, @@ -17,7 +20,7 @@ def get_grdc_data( parameter: str = "Q", data_home: str = None, column: str = "streamflow", -) -> Tuple[pd.core.frame.DataFrame, Dict[str, Union[str, int, float]]]: +) -> Tuple[pd.core.frame.DataFrame, MetaDataType]: """Get river discharge data from Global Runoff Data Centre (GRDC). Requires the GRDC daily data files in a local directory. The GRDC daily data @@ -47,7 +50,9 @@ def get_grdc_data( from ewatercycle.observation.grdc import get_grdc_data - df, meta = get_grdc_data('6335020', '2000-01-01T00:00Z', '2001-01-01T00:00Z', data_home='.') + df, meta = get_grdc_data('6335020', + '2000-01-01T00:00Z', + '2001-01-01T00:00Z') df.describe() streamflow count 4382.000000 @@ -86,8 +91,8 @@ def get_grdc_data( data_path = to_absolute_path(CFG["grdc_location"]) else: raise ValueError( - f"Provide the grdc path using `data_home` argument " - f"or using `grdc_location` in ewatercycle configuration file." + "Provide the grdc path using `data_home` argument" + "or using `grdc_location` in ewatercycle configuration file." ) if not data_path.exists(): @@ -120,14 +125,14 @@ def get_grdc_data( def _grdc_read(grdc_station_path, start, end, column): - with open(grdc_station_path, "r", encoding="cp1252", errors="ignore") as file: + with grdc_station_path.open("r", encoding="cp1252", errors="ignore") as file: data = file.read() metadata = _grdc_metadata_reader(grdc_station_path, data) - allLines = data.split("\n") + all_lines = data.split("\n") header = 0 - for i, line in enumerate(allLines): + for i, line in enumerate(all_lines): if line.startswith("# DATA"): header = i + 1 break @@ -142,8 +147,8 @@ def _grdc_read(grdc_station_path, start, end, column): na_values="-999", ) grdc_station_df = pd.DataFrame( - {column: grdc_data[" Value"].values}, - index=grdc_data["YYYY-MM-DD"].values, + {column: grdc_data[" Value"].array}, + index=grdc_data["YYYY-MM-DD"].array, ) grdc_station_df.index.rename("time", inplace=True) @@ -153,8 +158,7 @@ def _grdc_read(grdc_station_path, start, end, column): return metadata, grdc_station_select -def _grdc_metadata_reader(grdc_station_path, allLines): - """ +def _grdc_metadata_reader(grdc_station_path, all_lines): # Initiating a dictionary that will contain all GRDC attributes. # This function is based on earlier work by Rolf Hut. # https://github.com/RolfHut/GRDC2NetCDF/blob/master/GRDC2NetCDF.py @@ -163,14 +167,13 @@ def _grdc_metadata_reader(grdc_station_path, allLines): # from Utrecht University. # https://github.com/edwinkost/discharge_analysis_IWMI # Modified by Susan Branchett - """ # initiating a dictionary that will contain all GRDC attributes: - attributeGRDC = {} + attribute_grdc = {} # split the content of the file into several lines - allLines = allLines.replace("\r", "") - allLines = allLines.split("\n") + all_lines = all_lines.replace("\r", "") + all_lines = all_lines.split("\n") # get grdc ids (from files) and check their consistency with their # file names @@ -178,8 +181,8 @@ def _grdc_metadata_reader(grdc_station_path, allLines): os.path.basename(grdc_station_path).split(".")[0].split("_")[0] ) id_from_grdc = None - if id_from_file_name == int(allLines[8].split(":")[1].strip()): - id_from_grdc = int(allLines[8].split(":")[1].strip()) + if id_from_file_name == int(all_lines[8].split(":")[1].strip()): + id_from_grdc = int(all_lines[8].split(":")[1].strip()) else: print( "GRDC station " @@ -191,92 +194,92 @@ def _grdc_metadata_reader(grdc_station_path, allLines): if id_from_grdc is not None: - attributeGRDC["grdc_file_name"] = str(grdc_station_path) - attributeGRDC["id_from_grdc"] = id_from_grdc + attribute_grdc["grdc_file_name"] = str(grdc_station_path) + attribute_grdc["id_from_grdc"] = id_from_grdc try: - attributeGRDC["file_generation_date"] = str( - allLines[6].split(":")[1].strip() + attribute_grdc["file_generation_date"] = str( + all_lines[6].split(":")[1].strip() ) - except: - attributeGRDC["file_generation_date"] = "NA" + except IndexError: + attribute_grdc["file_generation_date"] = "NA" try: - attributeGRDC["river_name"] = str(allLines[9].split(":")[1].strip()) - except: - attributeGRDC["river_name"] = "NA" + attribute_grdc["river_name"] = str(all_lines[9].split(":")[1].strip()) + except IndexError: + attribute_grdc["river_name"] = "NA" try: - attributeGRDC["station_name"] = str(allLines[10].split(":")[1].strip()) - except: - attributeGRDC["station_name"] = "NA" + attribute_grdc["station_name"] = str(all_lines[10].split(":")[1].strip()) + except IndexError: + attribute_grdc["station_name"] = "NA" try: - attributeGRDC["country_code"] = str(allLines[11].split(":")[1].strip()) - except: - attributeGRDC["country_code"] = "NA" + attribute_grdc["country_code"] = str(all_lines[11].split(":")[1].strip()) + except IndexError: + attribute_grdc["country_code"] = "NA" try: - attributeGRDC["grdc_latitude_in_arc_degree"] = float( - allLines[12].split(":")[1].strip() + attribute_grdc["grdc_latitude_in_arc_degree"] = float( + all_lines[12].split(":")[1].strip() ) - except: - attributeGRDC["grdc_latitude_in_arc_degree"] = "NA" + except IndexError: + attribute_grdc["grdc_latitude_in_arc_degree"] = "NA" try: - attributeGRDC["grdc_longitude_in_arc_degree"] = float( - allLines[13].split(":")[1].strip() + attribute_grdc["grdc_longitude_in_arc_degree"] = float( + all_lines[13].split(":")[1].strip() ) - except: - attributeGRDC["grdc_longitude_in_arc_degree"] = "NA" + except IndexError: + attribute_grdc["grdc_longitude_in_arc_degree"] = "NA" try: - attributeGRDC["grdc_catchment_area_in_km2"] = float( - allLines[14].split(":")[1].strip() + attribute_grdc["grdc_catchment_area_in_km2"] = float( + all_lines[14].split(":")[1].strip() ) - if attributeGRDC["grdc_catchment_area_in_km2"] <= 0.0: - attributeGRDC["grdc_catchment_area_in_km2"] = "NA" - except: - attributeGRDC["grdc_catchment_area_in_km2"] = "NA" + if attribute_grdc["grdc_catchment_area_in_km2"] <= 0.0: + attribute_grdc["grdc_catchment_area_in_km2"] = "NA" + except IndexError: + attribute_grdc["grdc_catchment_area_in_km2"] = "NA" try: - attributeGRDC["altitude_masl"] = float(allLines[15].split(":")[1].strip()) - except: - attributeGRDC["altitude_masl"] = "NA" + attribute_grdc["altitude_masl"] = float(all_lines[15].split(":")[1].strip()) + except IndexError: + attribute_grdc["altitude_masl"] = "NA" try: - attributeGRDC["dataSetContent"] = str(allLines[20].split(":")[1].strip()) - except: - attributeGRDC["dataSetContent"] = "NA" + attribute_grdc["dataSetContent"] = str(all_lines[20].split(":")[1].strip()) + except IndexError: + attribute_grdc["dataSetContent"] = "NA" try: - attributeGRDC["units"] = str(allLines[22].split(":")[1].strip()) - except: - attributeGRDC["units"] = "NA" + attribute_grdc["units"] = str(all_lines[22].split(":")[1].strip()) + except IndexError: + attribute_grdc["units"] = "NA" try: - attributeGRDC["time_series"] = str(allLines[23].split(":")[1].strip()) - except: - attributeGRDC["time_series"] = "NA" + attribute_grdc["time_series"] = str(all_lines[23].split(":")[1].strip()) + except IndexError: + attribute_grdc["time_series"] = "NA" try: - attributeGRDC["no_of_years"] = int(allLines[24].split(":")[1].strip()) - except: - attributeGRDC["no_of_years"] = "NA" + attribute_grdc["no_of_years"] = int(all_lines[24].split(":")[1].strip()) + except IndexError: + attribute_grdc["no_of_years"] = "NA" try: - attributeGRDC["last_update"] = str(allLines[25].split(":")[1].strip()) - except: - attributeGRDC["last_update"] = "NA" + attribute_grdc["last_update"] = str(all_lines[25].split(":")[1].strip()) + except IndexError: + attribute_grdc["last_update"] = "NA" try: - attributeGRDC["nrMeasurements"] = int( - str(allLines[38].split(":")[1].strip()) + attribute_grdc["nrMeasurements"] = int( + str(all_lines[38].split(":")[1].strip()) ) - except: - attributeGRDC["nrMeasurements"] = "NA" + except IndexError: + attribute_grdc["nrMeasurements"] = "NA" - return attributeGRDC + return attribute_grdc def _count_missing_data(df, column): @@ -295,8 +298,8 @@ def _log_metadata(metadata): f"The river name is: {metadata['river_name']}." f"The coordinates are: {coords}." f"The catchment area in km2 is: {metadata['grdc_catchment_area_in_km2']}. " - f"There are {metadata['nrMissingData']} missing values " - f"during {metadata['UserStartTime']}_{metadata['UserEndTime']} at this station. " + f"There are {metadata['nrMissingData']} missing values during " + f"{metadata['UserStartTime']}_{metadata['UserEndTime']} at this station. " f"See the metadata for more information." ) logger.info("%s", message) From 59abbe3c57070ad5310f6bf2d4311c812fbfb5aa Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 12 Aug 2021 14:59:51 +0200 Subject: [PATCH 20/33] Fix doc generator + Fix bunch of lint warnings --- docs/conf.py | 10 ++--- setup.cfg | 4 +- src/ewatercycle/models/lisflood.py | 64 ++++++++++++++++++++---------- src/ewatercycle/models/marrmot.py | 14 ++++--- 4 files changed, 58 insertions(+), 34 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index ab9573af..df6cdbca 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ import sys from pathlib import Path -src = Path(__file__) / ".." / "src" +src = Path(__file__).parent / ".." / "src" sys.path.insert(0, str(src.absolute())) @@ -84,9 +84,9 @@ # -- Run apidoc plug-in manually, as readthedocs doesn't support it ------- # See https://github.com/rtfd/readthedocs.org/issues/1139 def run_apidoc(_): # noqa: D103 - here = Path(__file__) + here = Path(__file__).parent out = (here / "apidocs").absolute() - sourcedir = src / "ewatercycle" + source_dir = (here / ".." / "src" / "ewatercycle").absolute() ignore_paths = [] @@ -97,8 +97,8 @@ def run_apidoc(_): # noqa: D103 "-M", "--implicit-namespaces", "-o", - out, - sourcedir, + str(out), + str(source_dir), ] + ignore_paths try: diff --git a/setup.cfg b/setup.cfg index e23162f9..a89c4a9b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -112,7 +112,7 @@ files = src, tests [flake8] max-line-length = 88 -extend-ignore = E203 +extend-ignore = E203,S101 pytest-fixture-no-parentheses = True per-file-ignores = - tests/**: S101,D100,D101,D102,D103,D104 + tests/**: D100,D101,D102,D103,D104 diff --git a/src/ewatercycle/models/lisflood.py b/src/ewatercycle/models/lisflood.py index bf8e36bd..904dc0a0 100644 --- a/src/ewatercycle/models/lisflood.py +++ b/src/ewatercycle/models/lisflood.py @@ -1,3 +1,4 @@ +"""Module with Lisflood model.""" import datetime import logging import xml.etree.ElementTree as ET @@ -24,12 +25,13 @@ class Lisflood(AbstractModel[LisfloodForcing]): """eWaterCycle implementation of Lisflood hydrological model. Args: - version: pick a version for which an ewatercycle grpc4bmi docker image is available. + version: pick a version for which an grpc4bmi docker image is available. parameter_set: LISFLOOD input files. Any included forcing data will be ignored. forcing: a LisfloodForcing object. Example: - See examples/lisflood.ipynb in `ewatercycle repository `_ + See examples/lisflood.ipynb in + `ewatercycle repository `_ """ available_versions = ("20.10",) @@ -64,29 +66,35 @@ def _get_textvar_value(self, name: str): # unable to subclass with more specialized arguments so ignore type def setup( # type: ignore self, - IrrigationEfficiency: str = None, + IrrigationEfficiency: str = None, # noqa: N803 start_time: str = None, end_time: str = None, MaskMap: str = None, cfg_dir: str = None, ) -> Tuple[str, str]: - """Configure model run + """Configure model run. - 1. Creates config file and config directory based on the forcing variables and time range + 1. Creates config file and config directory + based on the forcing variables and time range. 2. Start bmi container and store as :py:attr:`bmi` Args: - IrrigationEfficiency: Field application irrigation efficiency max 1, ~0.90 drip irrigation, ~0.75 sprinkling - start_time: Start time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is used. - end_time: End time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. + IrrigationEfficiency: Field application irrigation efficiency. + max 1, ~0.90 drip irrigation, ~0.75 sprinkling + start_time: Start time of model in UTC and ISO format string + e.g. 'YYYY-MM-DDTHH:MM:SSZ'. + If not given then forcing start time is used. + end_time: End time of model in UTC and ISO format string + e.g. 'YYYY-MM-DDTHH:MM:SSZ'. + If not given then forcing end time is used. MaskMap: Mask map to use instead of one supplied in parameter set. - Path to a NetCDF or pcraster file with same dimensions as parameter set map files and a boolean variable. + Path to a NetCDF or pcraster file with + same dimensions as parameter set map files and a boolean variable. cfg_dir: a run directory given by user or created for user. Returns: Path to config file and path to config directory """ - # TODO forcing can be a part of parameter_set cfg_dir_as_path = to_absolute_path(cfg_dir) if cfg_dir else None cfg_dir_as_path = _generate_workdir(cfg_dir_as_path) @@ -130,7 +138,7 @@ def setup( # type: ignore return str(config_file), str(cfg_dir_as_path) def _check_forcing(self, forcing): - """ "Check forcing argument and get path, start and end time of forcing data.""" + """Checks forcing argument and get path, start/end time of forcing data.""" # TODO check if mask has same grid as forcing files, # if not warn users to run reindex_forcings if isinstance(forcing, LisfloodForcing): @@ -141,7 +149,8 @@ def _check_forcing(self, forcing): self._end = get_time(forcing.end_time) else: raise TypeError( - f"Unknown forcing type: {forcing}. Please supply a LisfloodForcing object." + f"Unknown forcing type: {forcing}. " + "Please supply a LisfloodForcing object." ) def _create_lisflood_config( @@ -149,10 +158,10 @@ def _create_lisflood_config( cfg_dir: Path, start_time_iso: str = None, end_time_iso: str = None, - IrrigationEfficiency: str = None, + IrrigationEfficiency: str = None, # noqa: N803 MaskMap: str = None, ) -> Path: - """Create lisflood config file""" + """Create lisflood config file.""" assert self.parameter_set is not None assert self.forcing is not None # overwrite dates if given @@ -233,7 +242,7 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: shape = self.bmi.get_grid_shape(grid) # Extract the data and store it in an xarray DataArray - da = xr.DataArray( + return xr.DataArray( data=np.reshape(self.bmi.get_value(name), shape), coords={ "longitude": self.bmi.get_grid_x(grid), @@ -245,12 +254,10 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: attrs={"units": self.bmi.get_var_units(name)}, ) - return da - def _coords_to_indices( self, name: str, lat: Iterable[float], lon: Iterable[float] ) -> Iterable[int]: - """Converts lat/lon values to index. + """Convert lat/lon values to index. Args: lat: Latitudinal value @@ -284,7 +291,7 @@ def parameters(self) -> Iterable[Tuple[str, Any]]: assert self.parameter_set is not None assert self.forcing is not None # TODO fix issue #60 - parameters = [ + return [ ( "IrrigationEfficiency", self._get_textvar_value("IrrigationEfficiency"), @@ -293,7 +300,6 @@ def parameters(self) -> Iterable[Tuple[str, Any]]: ("start_time", self._start.strftime("%Y-%m-%dT%H:%M:%SZ")), ("end_time", self._end.strftime("%Y-%m-%dT%H:%M:%SZ")), ] - return parameters # TODO it needs fix regarding forcing @@ -325,11 +331,14 @@ def parameters(self) -> Iterable[Tuple[str, Any]]: def _generate_workdir(cfg_dir: Path = None) -> Path: - """ + """Creates or makes sure workdir exists. Args: cfg_dir: If cfg dir is None then create sub-directory in CFG['output_dir'] + Returns: + absolute path of workdir + """ if cfg_dir is None: scratch_dir = CFG["output_dir"] @@ -343,13 +352,24 @@ def _generate_workdir(cfg_dir: Path = None) -> Path: class XmlConfig(AbstractConfig): - """Config container where config is read/saved in xml format""" + """Config container where config is read/saved in xml format.""" def __init__(self, source): + """Config container where config is read/saved in xml format. + + Args: + source: file to read from + """ super().__init__(source) self.tree = ET.parse(source) self.config: ET.Element = self.tree.getroot() """XML element used to make changes to the config""" def save(self, target): + """Save xml to file. + + Args: + target: file to save to + + """ self.tree.write(target) diff --git a/src/ewatercycle/models/marrmot.py b/src/ewatercycle/models/marrmot.py index 79ec15e6..521c0087 100644 --- a/src/ewatercycle/models/marrmot.py +++ b/src/ewatercycle/models/marrmot.py @@ -430,8 +430,12 @@ def _create_marrmot_config( Args: cfg_dir: a run directory given by user or created for user. - start_time_iso: Start time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is used. - end_time_iso: End time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. + start_time_iso: Start time of model in UTC and ISO format string + e.g. 'YYYY-MM-DDTHH:MM:SSZ'. + If not given then forcing start time is used. + end_time_iso: End time of model in UTC and ISO format string + e.g. 'YYYY-MM-DDTHH:MM:SSZ'. + If not given then forcing end time is used. Returns: Path for Marrmot config file @@ -510,12 +514,12 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: @property def parameters(self) -> Iterable[Tuple[str, Any]]: """List the parameters for this model.""" - p: List[Tuple[str, Any]] = list(zip(M14_PARAMS, self._parameters)) - p += [ + pars: List[Tuple[str, Any]] = list(zip(M14_PARAMS, self._parameters)) + pars += [ ("initial_upper_zone_storage", self.store_ini[0]), ("initial_saturated_zone_storage", self.store_ini[1]), ("solver", self.solver), ("start time", self.forcing_start_time.strftime("%Y-%m-%dT%H:%M:%SZ")), ("end time", self.forcing_end_time.strftime("%Y-%m-%dT%H:%M:%SZ")), ] - return p + return pars From 05938c8b763c81ae2b856ed9ded88b485dbc358e Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 12 Aug 2021 15:05:30 +0200 Subject: [PATCH 21/33] More of the same --- src/ewatercycle/models/marrmot.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/ewatercycle/models/marrmot.py b/src/ewatercycle/models/marrmot.py index 521c0087..215e9f52 100644 --- a/src/ewatercycle/models/marrmot.py +++ b/src/ewatercycle/models/marrmot.py @@ -389,7 +389,7 @@ def setup( # type: ignore return str(config_file), str(cfg_dir_as_path) def _check_forcing(self, forcing): - """ "Check forcing argument and get path, start and end time of forcing data.""" + """Check forcing argument and get path, start and end time of forcing data.""" if isinstance(forcing, MarrmotForcing): forcing_dir = to_absolute_path(forcing.directory) self.forcing_file = str(forcing_dir / forcing.forcing_file) @@ -398,7 +398,8 @@ def _check_forcing(self, forcing): self.forcing_end_time = get_time(forcing.end_time) else: raise TypeError( - f"Unknown forcing type: {forcing}. Please supply a MarrmotForcing object." + f"Unknown forcing type: {forcing}. " + "Please supply a MarrmotForcing object." ) # parse start/end time forcing_data = sio.loadmat(self.forcing_file, mat_dtype=True) @@ -406,13 +407,21 @@ def _check_forcing(self, forcing): if len(forcing_data["parameters"]) == len(self._parameters): self._parameters = forcing_data["parameters"] else: - message = f"The length of parameters in forcing {self.forcing_file} does not match the length of M14 parameters that is seven." + message = ( + "The length of parameters in forcing " + f"{self.forcing_file} does not match " + "the length of M14 parameters that is seven." + ) logger.warning("%s", message) if "store_ini" in forcing_data: if len(forcing_data["store_ini"]) == len(self.store_ini): self.store_ini = forcing_data["store_ini"] else: - message = f"The length of initial stores in forcing {self.forcing_file} does not match the length of M14 iniatial stores that is two." + message = ( + "The length of initial stores in forcing " + f"{self.forcing_file} does not match " + "the length of M14 iniatial stores that is two." + ) logger.warning("%s", message) if "solver" in forcing_data: forcing_solver = forcing_data["solver"] From a69a5c9322f77896486ba90e818874c532450389 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 12 Aug 2021 15:50:58 +0200 Subject: [PATCH 22/33] Fixing lint errors in grdc Fixes #241 --- src/ewatercycle/observation/grdc.py | 32 ++++++++++++++--------------- tests/observation/test_grdc.py | 25 ++++++++-------------- 2 files changed, 25 insertions(+), 32 deletions(-) diff --git a/src/ewatercycle/observation/grdc.py b/src/ewatercycle/observation/grdc.py index 29c2ef5f..1152c9b2 100644 --- a/src/ewatercycle/observation/grdc.py +++ b/src/ewatercycle/observation/grdc.py @@ -84,7 +84,7 @@ def get_grdc_data( 'UserStartTime': '2000-01-01T00:00Z', 'UserEndTime': '2001-01-01T00:00Z', 'nrMissingData': 0} - """ + """ # noqa: E501 if data_home: data_path = to_absolute_path(data_home) elif CFG["grdc_location"]: @@ -201,36 +201,36 @@ def _grdc_metadata_reader(grdc_station_path, all_lines): attribute_grdc["file_generation_date"] = str( all_lines[6].split(":")[1].strip() ) - except IndexError: + except (IndexError, ValueError): attribute_grdc["file_generation_date"] = "NA" try: attribute_grdc["river_name"] = str(all_lines[9].split(":")[1].strip()) - except IndexError: + except (IndexError, ValueError): attribute_grdc["river_name"] = "NA" try: attribute_grdc["station_name"] = str(all_lines[10].split(":")[1].strip()) - except IndexError: + except (IndexError, ValueError): attribute_grdc["station_name"] = "NA" try: attribute_grdc["country_code"] = str(all_lines[11].split(":")[1].strip()) - except IndexError: + except (IndexError, ValueError): attribute_grdc["country_code"] = "NA" try: attribute_grdc["grdc_latitude_in_arc_degree"] = float( all_lines[12].split(":")[1].strip() ) - except IndexError: + except (IndexError, ValueError): attribute_grdc["grdc_latitude_in_arc_degree"] = "NA" try: attribute_grdc["grdc_longitude_in_arc_degree"] = float( all_lines[13].split(":")[1].strip() ) - except IndexError: + except (IndexError, ValueError): attribute_grdc["grdc_longitude_in_arc_degree"] = "NA" try: @@ -239,44 +239,44 @@ def _grdc_metadata_reader(grdc_station_path, all_lines): ) if attribute_grdc["grdc_catchment_area_in_km2"] <= 0.0: attribute_grdc["grdc_catchment_area_in_km2"] = "NA" - except IndexError: + except (IndexError, ValueError): attribute_grdc["grdc_catchment_area_in_km2"] = "NA" try: attribute_grdc["altitude_masl"] = float(all_lines[15].split(":")[1].strip()) - except IndexError: + except (IndexError, ValueError): attribute_grdc["altitude_masl"] = "NA" try: attribute_grdc["dataSetContent"] = str(all_lines[20].split(":")[1].strip()) - except IndexError: + except (IndexError, ValueError): attribute_grdc["dataSetContent"] = "NA" try: attribute_grdc["units"] = str(all_lines[22].split(":")[1].strip()) - except IndexError: + except (IndexError, ValueError): attribute_grdc["units"] = "NA" try: attribute_grdc["time_series"] = str(all_lines[23].split(":")[1].strip()) - except IndexError: + except (IndexError, ValueError): attribute_grdc["time_series"] = "NA" try: attribute_grdc["no_of_years"] = int(all_lines[24].split(":")[1].strip()) - except IndexError: + except (IndexError, ValueError): attribute_grdc["no_of_years"] = "NA" try: attribute_grdc["last_update"] = str(all_lines[25].split(":")[1].strip()) - except IndexError: + except (IndexError, ValueError): attribute_grdc["last_update"] = "NA" try: attribute_grdc["nrMeasurements"] = int( - str(all_lines[38].split(":")[1].strip()) + str(all_lines[33].split(":")[1].strip()) ) - except IndexError: + except (IndexError, ValueError): attribute_grdc["nrMeasurements"] = "NA" return attribute_grdc diff --git a/tests/observation/test_grdc.py b/tests/observation/test_grdc.py index ee0e5981..21682bea 100644 --- a/tests/observation/test_grdc.py +++ b/tests/observation/test_grdc.py @@ -13,7 +13,7 @@ def sample_grdc_file(tmp_path): fn = tmp_path / "42424242_Q_Day.Cmd.txt" # Sample with fictive data, but with same structure as real file - s = """# Title: GRDC STATION DATA FILE + body = """# Title: GRDC STATION DATA FILE # -------------- # Format: DOS-ASCII # Field delimiter: ; @@ -51,9 +51,9 @@ def sample_grdc_file(tmp_path): YYYY-MM-DD;hh:mm; Value 2000-01-01;--:--; 123.000 2000-01-02;--:--; 456.000 -2000-01-03;--:--; -999.000""" - with open(fn, "w", encoding="cp1252") as f: - f.write(s) +2000-01-03;--:--; -999.000""" # noqa: E800 + with fn.open("w", encoding="cp1252") as f: + f.write(body) return fn @@ -77,7 +77,7 @@ def expected_results(tmp_path, sample_grdc_file): "id_from_grdc": 42424242, "last_update": "2000-02-01", "no_of_years": 1, - "nrMeasurements": "NA", + "nrMeasurements": 3, "river_name": "SOME RIVER", "station_name": "SOME", "time_series": "2000-01 - 2000-01", @@ -99,7 +99,7 @@ def test_get_grdc_data_with_datahome(tmp_path, expected_results): assert result_metadata == expected_metadata -def test_get_grdc_data_with_CFG(expected_results, tmp_path): +def test_get_grdc_data_with_cfg(expected_results, tmp_path): CFG["grdc_location"] = str(tmp_path) expected_data, expected_metadata = expected_results result_data, result_metadata = get_grdc_data( @@ -112,10 +112,9 @@ def test_get_grdc_data_with_CFG(expected_results, tmp_path): def test_get_grdc_data_without_path(): CFG["grdc_location"] = None - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match=r"Provide the grdc path") as excinfo: get_grdc_data("42424242", "2000-01-01T00:00Z", "2000-02-01T00:00Z") msg = str(excinfo.value) - print(msg) assert "data_home" in msg assert "grdc_location" in msg @@ -123,24 +122,18 @@ def test_get_grdc_data_without_path(): def test_get_grdc_data_wrong_path(tmp_path): CFG["grdc_location"] = f"{tmp_path}_data" - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match=r"The grdc directory .* does not exist!"): get_grdc_data("42424242", "2000-01-01T00:00Z", "2000-02-01T00:00Z") - msg = str(excinfo.value) - print(msg) - assert "directory" in msg def test_get_grdc_data_without_file(tmp_path): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match="The grdc file .* does not exist!"): get_grdc_data( "42424243", "2000-01-01T00:00Z", "2000-02-01T00:00Z", data_home=str(tmp_path), ) - msg = str(excinfo.value) - print(msg) - assert "file" in msg def test_get_grdc_dat_custom_column_name(expected_results, tmp_path): From 6695108e361057b17227ff7b5968f31147de0cde Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 12 Aug 2021 15:57:10 +0200 Subject: [PATCH 23/33] Make flake8 just print errors instead of dying --- .pre-commit-config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bd3ce4f6..00dd59ca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,6 +67,8 @@ repos: # wemake-python-styleguide, # conflicts with black yesqa, ] + verbose: true + args: &fa [--statistics, --exit-zero] - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.910 hooks: @@ -85,6 +87,7 @@ repos: # TODO renable when erros are fixed/ignored - id: nbqa-flake8 additional_dependencies: *fd + args: *fa # TODO renable when erros are fixed/ignored # - id: nbqa-pylint # additional_dependencies: [pylint==2.9.6] From ddbd76e92a88aac288dc45190f8db9d72840e573 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 16 Aug 2021 09:30:07 +0200 Subject: [PATCH 24/33] Fix line lengths in ewatercycle.models --- src/ewatercycle/models/abstract.py | 13 +++-- src/ewatercycle/models/lisflood.py | 26 ++++++--- src/ewatercycle/models/marrmot.py | 84 +++++++++++++++++++---------- src/ewatercycle/models/pcrglobwb.py | 8 +-- src/ewatercycle/models/wflow.py | 15 +++--- 5 files changed, 96 insertions(+), 50 deletions(-) diff --git a/src/ewatercycle/models/abstract.py b/src/ewatercycle/models/abstract.py index 0f11e6a2..e1f4ef5f 100644 --- a/src/ewatercycle/models/abstract.py +++ b/src/ewatercycle/models/abstract.py @@ -151,7 +151,8 @@ def _coords_to_indices( """ raise NotImplementedError( - "Method to convert from coordinates to model indices not implemented for this model." + "Method to convert from coordinates to model indices " + "not implemented for this model." ) @abstractmethod @@ -263,13 +264,15 @@ def _check_parameter_set(self): ) if self.parameter_set.supported_model_versions == set(): logger.info( - f"Model version {self.version} is not explicitly listed in the supported model versions " - f"of this parameter set. This can lead to compatibility issues." + f"Model version {self.version} is not explicitly listed in the " + "supported model versions of this parameter set. " + "This can lead to compatibility issues." ) elif self.version not in self.parameter_set.supported_model_versions: raise ValueError( - f"Parameter set is not compatible with version {self.version} of model, " - f"parameter set only supports {self.parameter_set.supported_model_versions}" + "Parameter set is not compatible with version {self.version} of " + "model, parameter set only supports " + f"{self.parameter_set.supported_model_versions}." ) def _check_version(self): diff --git a/src/ewatercycle/models/lisflood.py b/src/ewatercycle/models/lisflood.py index 904dc0a0..1820621e 100644 --- a/src/ewatercycle/models/lisflood.py +++ b/src/ewatercycle/models/lisflood.py @@ -303,13 +303,17 @@ def parameters(self) -> Iterable[Tuple[str, Any]]: # TODO it needs fix regarding forcing -# def reindex_forcings(mask_map: Path, forcing: LisfloodForcing, output_dir: Path = None) -> Path: +# def reindex_forcings( +# mask_map: Path, forcing: LisfloodForcing, output_dir: Path = None +# ) -> Path: # """Reindex forcing files to match mask map grid # Args: -# mask_map: Path to NetCDF file used a boolean map that defines model boundaries. +# mask_map: Path to NetCDF file used a boolean map that defines model +# boundaries. # forcing: Forcing data from ESMValTool -# output_dir: Directory where to write the re-indexed files, given by user or created for user +# output_dir: Directory where to write the re-indexed files, given by user +# or created for user # Returns: # Output dir with re-indexed files. @@ -321,12 +325,18 @@ def parameters(self) -> Iterable[Tuple[str, Any]]: # dataset = data_file.load_xarray() # out_fn = output_dir / data_file.filename.name # var_name = list(dataset.data_vars.keys())[0] -# encoding = {var_name: {"zlib": True, "complevel": 4, "chunksizes": (1,) + dataset[var_name].shape[1:]}} +# encoding = { +# var_name: { +# "zlib": True, +# "complevel": 4, +# "chunksizes": (1,) + dataset[var_name].shape[1:], +# } +# } # dataset.reindex( -# {"lat": mask["lat"], "lon": mask["lon"]}, -# method="nearest", -# tolerance=1e-2, -# ).to_netcdf(out_fn, encoding=encoding) +# {"lat": mask["lat"], "lon": mask["lon"]}, +# method="nearest", +# tolerance=1e-2, +# ).to_netcdf(out_fn, encoding=encoding) # return output_dir diff --git a/src/ewatercycle/models/marrmot.py b/src/ewatercycle/models/marrmot.py index 215e9f52..1fcb8076 100644 --- a/src/ewatercycle/models/marrmot.py +++ b/src/ewatercycle/models/marrmot.py @@ -21,8 +21,8 @@ @dataclass class Solver: - """Solver, for current implementations see - `here `_. + """Solver, for current implementations see `here + `_. """ name: str = "createOdeApprox_IE" @@ -33,7 +33,8 @@ class Solver: def _generate_cfg_dir(cfg_dir: Path = None) -> Path: """ Args: - cfg_dir: If cfg dir is None or does not exist then create sub-directory in CFG['output_dir'] + cfg_dir: If cfg dir is None or does not exist then create sub-directory + in CFG['output_dir'] """ if cfg_dir is None: scratch_dir = CFG["output_dir"] @@ -47,17 +48,21 @@ def _generate_cfg_dir(cfg_dir: Path = None) -> Path: class MarrmotM01(AbstractModel[MarrmotForcing]): - """eWaterCycle implementation of Marrmot Collie River 1 (traditional bucket) hydrological model. + """eWaterCycle implementation of Marrmot Collie River 1 (traditional bucket) model. - It sets MarrmotM01 parameter with an initial value that is the mean value of the range specfied in `model parameter range file `_. + It sets MarrmotM01 parameter with an initial value that is the mean value of + the range specfied in `model parameter range file + `_. Args: - version: pick a version for which an ewatercycle grpc4bmi docker image is available. - forcing: a MarrmotForcing object. - If forcing file contains parameter and other settings, those are used and can be changed in :py:meth:`setup`. + version: pick a version for which an ewatercycle grpc4bmi docker image + is available. forcing: a MarrmotForcing object. If forcing file contains + parameter and other settings, those are used and can be changed in + :py:meth:`setup`. Example: - See examples/marrmotM01.ipynb in `ewatercycle repository `_ + See examples/marrmotM01.ipynb in `ewatercycle repository + `_ """ model_name = "m_01_collie1_1p_1s" @@ -97,16 +102,23 @@ def setup( # type: ignore ) -> Tuple[str, str]: """Configure model run. - 1. Creates config file and config directory based on the forcing variables and time range + 1. Creates config file and config directory based on the forcing + variables and time range 2. Start bmi container and store as :py:attr:`bmi` Args: - maximum_soil_moisture_storage: in mm. Range is specfied in `model parameter range file `_. + maximum_soil_moisture_storage: in mm. Range is specfied in `model + parameter range file + `_. initial_soil_moisture_storage: in mm. - start_time: Start time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is used. - end_time: End time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. + start_time: Start time of model in UTC and ISO format string e.g. + 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is + used. + end_time: End time of model in UTC and ISO format string e.g. + 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. solver: Solver settings cfg_dir: a run directory given by user or created for user. + Returns: Path to config file and path to config directory """ @@ -150,7 +162,8 @@ def _check_forcing(self, forcing): self.forcing_end_time = get_time(forcing.end_time) else: raise TypeError( - f"Unknown forcing type: {forcing}. Please supply a MarrmotForcing object." + f"Unknown forcing type: {forcing}. Please supply a " + " MarrmotForcing object." ) # parse start/end time forcing_data = sio.loadmat(self.forcing_file, mat_dtype=True) @@ -169,13 +182,16 @@ def _create_marrmot_config( ) -> Path: """Write model configuration file. - Adds the model parameters to forcing file for the given period - and writes this information to a model configuration file. + Adds the model parameters to forcing file for the given period and + writes this information to a model configuration file. Args: cfg_dir: a run directory given by user or created for user. - start_time_iso: Start time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is used. - end_time_iso: End time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. + start_time_iso: Start time of model in UTC and ISO format string + e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is + used. + end_time_iso: End time of model in UTC and ISO format string e.g. + 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. Returns: Path for Marrmot config file @@ -278,15 +294,20 @@ def parameters(self) -> Iterable[Tuple[str, Any]]: class MarrmotM14(AbstractModel[MarrmotForcing]): """eWaterCycle implementation of Marrmot Top Model hydrological model. - It sets MarrmotM14 parameter with an initial value that is the mean value of the range specfied in `model parameter range file `_. + It sets MarrmotM14 parameter with an initial value that is the mean value of + the range specfied in `model parameter range file + `_. Args: - version: pick a version for which an ewatercycle grpc4bmi docker image is available. + version: pick a version for which an ewatercycle grpc4bmi docker image + is available. forcing: a MarrmotForcing object. - If forcing file contains parameter and other settings, those are used and can be changed in :py:meth:`setup`. + If forcing file contains parameter and other settings, those are used + and can be changed in :py:meth:`setup`. Example: - See examples/marrmotM14.ipynb in `ewatercycle repository `_ + See examples/marrmotM14.ipynb in `ewatercycle repository + `_ """ model_name = "m_14_topmodel_7p_2s" @@ -333,12 +354,15 @@ def setup( # type: ignore ) -> Tuple[str, str]: """Configure model run. - 1. Creates config file and config directory based on the forcing variables and time range + 1. Creates config file and config directory based on the forcing + variables and time range 2. Start bmi container and store as :py:attr:`bmi` Args: - maximum_soil_moisture_storage: in mm. Range is specfied in `model parameter range file `_. - threshold_flow_generation_evap_change. + maximum_soil_moisture_storage: in mm. Range is specfied in `model + parameter range file + `_. + threshold_flow_generation_evap_change. leakage_saturated_zone_flow_coefficient: in mm/d. zero_deficit_base_flow_speed: in mm/d. baseflow_coefficient: in mm-1. @@ -346,10 +370,14 @@ def setup( # type: ignore gamma_distribution_phi_parameter. initial_upper_zone_storage: in mm. initial_saturated_zone_storage: in mm. - start_time: Start time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is used. - end_time: End time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. - solver: Solver settings + start_time: Start time of model in UTC and ISO format string e.g. + 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is + used. + end_time: End time of model in UTC and ISO format string e.g. + 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. + solver: Solver settings cfg_dir: a run directory given by user or created for user. + Returns: Path to config file and path to config directory """ diff --git a/src/ewatercycle/models/pcrglobwb.py b/src/ewatercycle/models/pcrglobwb.py index 3d75a464..cf1be017 100644 --- a/src/ewatercycle/models/pcrglobwb.py +++ b/src/ewatercycle/models/pcrglobwb.py @@ -26,7 +26,8 @@ class PCRGlobWB(AbstractModel[PCRGlobWBForcing]): Args: version: pick a version from :py:attr:`~available_versions` - parameter_set: instance of :py:class:`~ewatercycle.parameter_sets.default.ParameterSet`. + parameter_set: instance of + :py:class:`~ewatercycle.parameter_sets.default.ParameterSet`. forcing: ewatercycle forcing container; see :py:mod:`ewatercycle.forcing`. @@ -135,8 +136,9 @@ def setup(self, cfg_dir: str = None, **kwargs) -> Tuple[str, str]: # type: igno "Couldn't spawn container within allocated time limit " "(15 seconds). You may try pulling the docker image with" f" `docker pull {self.docker_image}` or call `singularity " - f"build {self._singularity_image(CFG['singularity_dir'])} docker://{self.docker_image}`" - "if you're using singularity, and then try again." + f"build {self._singularity_image(CFG['singularity_dir'])} " + f"docker://{self.docker_image}` if you're using singularity," + " and then try again." ) return str(cfg_file), str(work_dir) diff --git a/src/ewatercycle/models/wflow.py b/src/ewatercycle/models/wflow.py index 68596738..02aeee82 100644 --- a/src/ewatercycle/models/wflow.py +++ b/src/ewatercycle/models/wflow.py @@ -26,7 +26,8 @@ class Wflow(AbstractModel[WflowForcing]): Args: version: pick a version from :py:attr:`~available_versions` - parameter_set: instance of :py:class:`~ewatercycle.parameter_sets.default.ParameterSet`. + parameter_set: instance of + :py:class:`~ewatercycle.parameter_sets.default.ParameterSet`. forcing: instance of :py:class:`~WflowForcing` or None. If None, it is assumed that forcing is included with the parameter_set. """ @@ -80,13 +81,14 @@ def _setup_default_config(self): if self.version == "2020.1.1": if not cfg.has_section("API"): logger.warning( - "Config file from parameter set is missing API section, adding section" + "Config file from parameter set is missing API section, " + "adding section" ) cfg.add_section("API") if not cfg.has_option("API", "RiverRunoff"): logger.warning( - "Config file from parameter set is missing RiverRunoff option in API section, " - "added it with value '2, m/s option'" + "Config file from parameter set is missing RiverRunoff " + "option in API section, added it with value '2, m/s option'" ) cfg.set("API", "RiverRunoff", "2, m/s") @@ -126,8 +128,9 @@ def setup(self, cfg_dir: str = None, **kwargs) -> Tuple[str, str]: # type: igno "Couldn't spawn container within allocated time limit " "(15 seconds). You may try pulling the docker image with" f" `docker pull {self.docker_image}` or call `singularity " - f"build {self._singularity_image(CFG['singularity_dir'])} docker://{self.docker_image}`" - "if you're using singularity, and then try again." + f"build {self._singularity_image(CFG['singularity_dir'])} " + f"docker://{self.docker_image}` if you're using singularity," + " and then try again." ) return ( From f5b3fc6aa2ac16bd5c088f682840cfbc256c6e41 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 16 Aug 2021 09:57:17 +0200 Subject: [PATCH 25/33] Fix line lengths in config --- src/ewatercycle/config/__init__.py | 25 ++++++++++++++---------- src/ewatercycle/config/_config_object.py | 5 +++-- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/ewatercycle/config/__init__.py b/src/ewatercycle/config/__init__.py index 53b0b9ff..2e81d90a 100644 --- a/src/ewatercycle/config/__init__.py +++ b/src/ewatercycle/config/__init__.py @@ -1,8 +1,9 @@ """Config ****** -Configuration of eWaterCycle is done via the :py:class:`~eWaterCycle.config.Config` object. -The global configuration can be imported from the :py:mod:`eWaterCycle` module as :py:data:`~ewatercycle.CFG`: +Configuration of eWaterCycle is done via the +:py:class:`~eWaterCycle.config.Config` object. The global configuration can be +imported from the :py:mod:`eWaterCycle` module as :py:data:`~ewatercycle.CFG`: .. code-block:: python @@ -17,8 +18,9 @@ By default all values are initialized as ``None``. -:py:data:`~ewatercycle.CFG` is essentially a python dictionary with a few extra functions, similar to :py:mod:`matplotlib.rcParams`. -This means that values can be updated like this: +:py:data:`~ewatercycle.CFG` is essentially a python dictionary with a few extra +functions, similar to :py:mod:`matplotlib.rcParams`. This means that values can +be updated like this: .. code-block:: python @@ -26,8 +28,10 @@ >>> CFG['output_dir'] PosixPath('/home/user/output') -Notice that :py:data:`~ewatercycle.CFG` automatically converts the path to an instance of ``pathlib.Path`` and expands the home directory. -All values entered into the config are validated to prevent mistakes, for example, it will warn you if you make a typo in the key: +Notice that :py:data:`~ewatercycle.CFG` automatically converts the path to an +instance of ``pathlib.Path`` and expands the home directory. All values entered +into the config are validated to prevent mistakes, for example, it will warn you +if you make a typo in the key: .. code-block:: python @@ -41,9 +45,9 @@ >>> CFG['output_dir'] = 123 InvalidConfigParameter: Key `output_dir`: Expected a path, but got 123 -By default, the config is loaded from the default location (i.e. ``~/.config/ewatercycle/ewatercycle.yaml``). -If it does not exist, it falls back to the default values. -to load a different file: +By default, the config is loaded from the default location (i.e. +``~/.config/ewatercycle/ewatercycle.yaml``). If it does not exist, it falls back +to the default values. to load a different file: .. code-block:: python @@ -74,7 +78,8 @@ container_engine: singularity singularity_dir: /data/singularity-images output_dir: /scratch - # Created with cd /data/singularity-images && singularity pull docker://ewatercycle/wflow-grpc4bmi:2020.1.1 + # Created with cd /data/singularity-images && + # singularity pull docker://ewatercycle/wflow-grpc4bmi:2020.1.1 wflow.singularity_images: wflow-grpc4bmi_2020.1.1.sif wflow.docker_images: ewatercycle/wflow-grpc4bmi:2020.1.1 """ diff --git a/src/ewatercycle/config/_config_object.py b/src/ewatercycle/config/_config_object.py index 81b64faf..d5e99310 100644 --- a/src/ewatercycle/config/_config_object.py +++ b/src/ewatercycle/config/_config_object.py @@ -94,8 +94,9 @@ def save_to_file(self, config_file: Optional[Union[os.PathLike, str]] = None): Args: config_file: File to write configuration object to. - If not given then will try to use `CFG['ewatercycle_config']` location - and if `CFG['ewatercycle_config']` is not set then will use the location in users home directory. + If not given then will try to use `CFG['ewatercycle_config']` + location and if `CFG['ewatercycle_config']` is not set then will use + the location in users home directory. """ # Exclude own path from dump old_config_file = self.get("ewatercycle_config", None) From 644e14da2077f8fa6d5a42870f94654f331d18f0 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 16 Aug 2021 16:22:18 +0200 Subject: [PATCH 26/33] Fix line lengths in ewatercycle.parameter_sets --- src/ewatercycle/parameter_sets/__init__.py | 26 ++++++++++++-------- src/ewatercycle/parameter_sets/_example.py | 7 +++--- src/ewatercycle/parameter_sets/_lisflood.py | 4 +-- src/ewatercycle/parameter_sets/_pcrglobwb.py | 4 +-- src/ewatercycle/parameter_sets/_wflow.py | 4 +-- src/ewatercycle/parameter_sets/default.py | 13 ++++++---- 6 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/ewatercycle/parameter_sets/__init__.py b/src/ewatercycle/parameter_sets/__init__.py index 14d1aad3..9c043d8b 100644 --- a/src/ewatercycle/parameter_sets/__init__.py +++ b/src/ewatercycle/parameter_sets/__init__.py @@ -38,9 +38,10 @@ def available_parameter_sets(target_model: str = None) -> Tuple[str, ...]: if CFG["ewatercycle_config"] == DEFAULT_CONFIG: raise ValueError(f"No configuration file found.") raise ValueError( - f'No parameter sets defined in {CFG["ewatercycle_config"]}. ' - f"Use `ewatercycle.parareter_sets.download_example_parameter_sets` to download examples " - f"or define your own or ask whoever setup the ewatercycle system to do it." + f'No parameter sets defined in {CFG["ewatercycle_config"]}. Use ' + "`ewatercycle.parareter_sets.download_example_parameter_sets` to download" + " examples or define your own or ask whoever setup the ewatercycle " + "system to do it." ) # TODO explain somewhere how to add new parameter sets filtered = tuple( @@ -50,9 +51,10 @@ def available_parameter_sets(target_model: str = None) -> Tuple[str, ...]: ) if not filtered: raise ValueError( - f'No parameter sets defined for {target_model} model in {CFG["ewatercycle_config"]}. ' - f"Use `ewatercycle.parareter_sets.download_example_parameter_sets` to download examples " - f"or define your own or ask whoever setup the ewatercycle system to do it." + f'No parameter sets defined in {CFG["ewatercycle_config"]}. Use ' + "`ewatercycle.parareter_sets.download_example_parameter_sets` to download" + " examples or define your own or ask whoever setup the ewatercycle " + "system to do it." ) return filtered @@ -87,7 +89,9 @@ def download_parameter_sets(zenodo_doi: str, target_model: str, config: str): def example_parameter_sets() -> Dict[str, ExampleParameterSet]: - """Lists example parameter sets that can be downloaded with :py:func:`~download_example_parameter_sets`.""" + """Lists the available example parameter sets. + + They can be downloaded with :py:func:`~download_example_parameter_sets`.""" # TODO how to add a new model docs should be updated with this part examples = chain( _wflow.example_parameter_sets(), @@ -100,11 +104,13 @@ def example_parameter_sets() -> Dict[str, ExampleParameterSet]: def download_example_parameter_sets(skip_existing=True): """Downloads all of the example parameter sets and adds them to the config_file. - Downloads to `parameterset_dir` directory defined in :py:data:`ewatercycle.config.CFG`. + Downloads to `parameterset_dir` directory defined in + :py:data:`ewatercycle.config.CFG`. Args: - skip_existing: When true will not download any parameter set which already has a local directory. - When false will raise ValueError exception when parameter set already exists. + skip_existing: When true will not download any parameter set which + already has a local directory. When false will raise ValueError + exception when parameter set already exists. """ examples = example_parameter_sets() diff --git a/src/ewatercycle/parameter_sets/_example.py b/src/ewatercycle/parameter_sets/_example.py index cbcefe52..78e12208 100644 --- a/src/ewatercycle/parameter_sets/_example.py +++ b/src/ewatercycle/parameter_sets/_example.py @@ -36,8 +36,8 @@ def download(self, skip_existing=False): if not skip_existing: raise ValueError( f"Directory {self.directory} for parameter set {self.name}" - f" already exists, will not overwrite. " - f"Try again with skip_existing=True or remove {self.directory} directory." + f" already exists, will not overwrite. Try again with " + f"skip_existing=True or remove {self.directory} directory." ) logger.info( @@ -50,7 +50,8 @@ def download(self, skip_existing=False): ) subprocess.check_call(["svn", "export", self.datafiles_url, self.directory]) - # TODO replace subversion with alternative see https://stackoverflow.com/questions/33066582/how-to-download-a-folder-from-github/48948711 + # TODO replace subversion with alternative, see + # https://stackoverflow.com/questions/33066582/how-to-download-a-folder-from-github/48948711 response = request.urlopen(self.config_url) self.config.write_text(response.read().decode()) diff --git a/src/ewatercycle/parameter_sets/_lisflood.py b/src/ewatercycle/parameter_sets/_lisflood.py index c8f54028..34fba55e 100644 --- a/src/ewatercycle/parameter_sets/_lisflood.py +++ b/src/ewatercycle/parameter_sets/_lisflood.py @@ -11,9 +11,9 @@ def example_parameter_sets() -> Iterable[ExampleParameterSet]: name="lisflood_fraser", # Relative to CFG['parameterset_dir'] config="lisflood_fraser/settings_lat_lon-Run.xml", - datafiles_url="https://github.com/ec-jrc/lisflood-usecases/trunk/LF_lat_lon_UseCase", + datafiles_url="https://github.com/ec-jrc/lisflood-usecases/trunk/LF_lat_lon_UseCase", # pylint: disable=C0301 # Raw url to config file - config_url="https://github.com/ec-jrc/lisflood-usecases/raw/master/LF_lat_lon_UseCase/settings_lat_lon-Run.xml", + config_url="https://github.com/ec-jrc/lisflood-usecases/raw/master/LF_lat_lon_UseCase/settings_lat_lon-Run.xml", # pylint: disable=C0301 doi="N/A", target_model="lisflood", supported_model_versions={"20.10"}, diff --git a/src/ewatercycle/parameter_sets/_pcrglobwb.py b/src/ewatercycle/parameter_sets/_pcrglobwb.py index 035352a7..acc36bcb 100644 --- a/src/ewatercycle/parameter_sets/_pcrglobwb.py +++ b/src/ewatercycle/parameter_sets/_pcrglobwb.py @@ -11,9 +11,9 @@ def example_parameter_sets() -> Iterable[ExampleParameterSet]: name="pcrglobwb_rhinemeuse_30min", # Relative to CFG['parameterset_dir'] config="pcrglobwb_rhinemeuse_30min/setup_natural_test.ini", - datafiles_url="https://github.com/UU-Hydro/PCR-GLOBWB_input_example/trunk/RhineMeuse30min", + datafiles_url="https://github.com/UU-Hydro/PCR-GLOBWB_input_example/trunk/RhineMeuse30min", # pylint: disable=C0301 # Raw url to config file - config_url="https://raw.githubusercontent.com/UU-Hydro/PCR-GLOBWB_input_example/master/ini_and_batch_files_for_pcrglobwb_course/rhine_meuse_30min_using_input_example/setup_natural_test.ini", + config_url="https://raw.githubusercontent.com/UU-Hydro/PCR-GLOBWB_input_example/master/ini_and_batch_files_for_pcrglobwb_course/rhine_meuse_30min_using_input_example/setup_natural_test.ini", # pylint: disable=C0301 doi="https://doi.org/10.5281/zenodo.1045339", target_model="pcrglobwb", supported_model_versions={"setters"}, diff --git a/src/ewatercycle/parameter_sets/_wflow.py b/src/ewatercycle/parameter_sets/_wflow.py index 74f7758e..de9fbb15 100644 --- a/src/ewatercycle/parameter_sets/_wflow.py +++ b/src/ewatercycle/parameter_sets/_wflow.py @@ -11,9 +11,9 @@ def example_parameter_sets() -> Iterable[ExampleParameterSet]: name="wflow_rhine_sbm_nc", # Relative to CFG['parameterset_dir'] config="wflow_rhine_sbm_nc/wflow_sbm_NC.ini", - datafiles_url="https://github.com/openstreams/wflow/trunk/examples/wflow_rhine_sbm_nc", + datafiles_url="https://github.com/openstreams/wflow/trunk/examples/wflow_rhine_sbm_nc", # pylint: disable=C0301 # Raw url to config file - config_url="https://github.com/openstreams/wflow/raw/master/examples/wflow_rhine_sbm_nc/wflow_sbm_NC.ini", + config_url="https://github.com/openstreams/wflow/raw/master/examples/wflow_rhine_sbm_nc/wflow_sbm_NC.ini", # pylint: disable=C0301 doi="N/A", target_model="wflow", supported_model_versions={"2020.1.1"}, diff --git a/src/ewatercycle/parameter_sets/default.py b/src/ewatercycle/parameter_sets/default.py index f84da59a..e62b0a1f 100644 --- a/src/ewatercycle/parameter_sets/default.py +++ b/src/ewatercycle/parameter_sets/default.py @@ -12,12 +12,15 @@ class ParameterSet: name (str): Name of parameter set directory (Path): Location on disk where files of parameter set are stored. If Path is relative then relative to CFG['parameterset_dir']. - config (Path): Model configuration file which uses files from :py:attr:`~directory`. - If Path is relative then relative to CFG['parameterset_dir']. - doi (str): Persistent identifier of parameter set. For a example a DOI for a Zenodo record. + config (Path): Model configuration file which uses files from + :py:attr:`~directory`. If Path is relative then relative to + CFG['parameterset_dir']. + doi (str): Persistent identifier of parameter set. For a example a DOI + for a Zenodo record. target_model (str): Name of model that parameter set can work with - supported_model_versions (Set[str]): Set of model versions that are supported by this parameter set. - If not set then parameter set will be supported by all versions of model + supported_model_versions (Set[str]): Set of model versions that are + supported by this parameter set. If not set then parameter set will be + supported by all versions of model """ def __init__( From 6786f7a3b7efaa888f55ee0c81cfc03e4e406bef Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 16 Aug 2021 16:26:35 +0200 Subject: [PATCH 27/33] fix line lengths in ewatercycle.forcing --- src/ewatercycle/forcing/_default.py | 4 ++-- src/ewatercycle/forcing/_lisflood.py | 12 +++++++----- src/ewatercycle/forcing/_pcrglobwb.py | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/ewatercycle/forcing/_default.py b/src/ewatercycle/forcing/_default.py index 64476dda..19b8fff6 100644 --- a/src/ewatercycle/forcing/_default.py +++ b/src/ewatercycle/forcing/_default.py @@ -1,7 +1,6 @@ """Forcing related functionality for default models""" import logging from copy import copy -from pathlib import Path from typing import Optional from ruamel.yaml import YAML @@ -75,7 +74,8 @@ def save(self): except ValueError: clone.shape = None logger.info( - f"Shapefile {self.shape} is not in forcing directory {self.directory}. So, it won't be saved in {target}." + f"Shapefile {self.shape} is not in forcing directory " + f"{self.directory}. So, it won't be saved in {target}." ) with open(target, "w") as f: diff --git a/src/ewatercycle/forcing/_lisflood.py b/src/ewatercycle/forcing/_lisflood.py index 2185fff4..28f7e6cc 100644 --- a/src/ewatercycle/forcing/_lisflood.py +++ b/src/ewatercycle/forcing/_lisflood.py @@ -63,9 +63,10 @@ def generate( # type: ignore run_lisvap: bool = False, ) -> "LisfloodForcing": """ - extract_region (dict): Region specification, dictionary must contain `start_longitude`, - `end_longitude`, `start_latitude`, `end_latitude` - run_lisvap (bool): if lisvap should be run. Default is False. Running lisvap is not supported yet. + extract_region (dict): Region specification, dictionary must contain + `start_longitude`, `end_longitude`, `start_latitude`, `end_latitude` + run_lisvap (bool): if lisvap should be run. Default is False. + Running lisvap is not supported yet. TODO add regrid options so forcing can be generated for parameter set TODO that is not on a 0.1x0.1 grid """ @@ -123,8 +124,9 @@ def generate( # type: ignore raise NotImplementedError("Dont know how to run LISVAP.") else: message = ( - f"Parameter `run_lisvap` is set to False. No forcing data will be generator for 'e0', 'es0' and 'et0'. " - f"However, the recipe creates LISVAP input data that can be found in {directory}." + "Parameter `run_lisvap` is set to False. No forcing data will be " + "generated for 'e0', 'es0' and 'et0'. However, the recipe creates " + f"LISVAP input data that can be found in {directory}." ) logger.warning("%s", message) return LisfloodForcing( diff --git a/src/ewatercycle/forcing/_pcrglobwb.py b/src/ewatercycle/forcing/_pcrglobwb.py index b29e87c9..7577be8c 100644 --- a/src/ewatercycle/forcing/_pcrglobwb.py +++ b/src/ewatercycle/forcing/_pcrglobwb.py @@ -43,7 +43,7 @@ def generate( # type: ignore end_time: str, shape: str, start_time_climatology: str, # TODO make optional, default to start_time - end_time_climatology: str, # TODO make optional, defaults to start_time + 1 year + end_time_climatology: str, # TODO make optional, defaults to start_time + 1 y extract_region: dict = None, ) -> "PCRGlobWBForcing": """ @@ -106,7 +106,7 @@ def generate( # type: ignore # generate forcing data and retrieve useful information recipe_output = recipe.run() - # TODO dont open recipe output files, but use standard name from ESMValTool diagnostic + # TODO dont open recipe output, but use standard name from ESMValTool directory, forcing_files = data_files_from_recipe_output(recipe_output) # instantiate forcing object based on generated data From e6b91cfe4febae3debeaee3e50520e2b3eb18a17 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 16 Aug 2021 16:28:31 +0200 Subject: [PATCH 28/33] Fix remaining line lengths in ewatercycle package --- src/ewatercycle/observation/usgs.py | 9 +++++---- src/ewatercycle/parametersetdb/__init__.py | 3 ++- src/ewatercycle/util.py | 6 ++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/ewatercycle/observation/usgs.py b/src/ewatercycle/observation/usgs.py index d89db225..7904c594 100644 --- a/src/ewatercycle/observation/usgs.py +++ b/src/ewatercycle/observation/usgs.py @@ -8,9 +8,10 @@ def get_usgs_data(station_id, start_date, end_date, parameter="00060", cache_dir=None): - """ - Get river discharge data from the - `U.S. Geological Survey Water Services `_ (USGS) rest web service. + """Get river discharge data from the USGS REST web service. + + See `U.S. Geological Survey Water Services + `_ (USGS) Parameters ---------- @@ -42,7 +43,7 @@ def get_usgs_data(station_id, start_date, end_date, parameter="00060", cache_dir station: Little Beaver Creek near East Liverpool OH stationid: 03109500 location: (40.6758974, -80.5406244) - """ + """ # noqa: E501 if cache_dir is None: cache_dir = os.environ["USGS_DATA_HOME"] diff --git a/src/ewatercycle/parametersetdb/__init__.py b/src/ewatercycle/parametersetdb/__init__.py index ee48235c..75f709c4 100644 --- a/src/ewatercycle/parametersetdb/__init__.py +++ b/src/ewatercycle/parametersetdb/__init__.py @@ -38,7 +38,8 @@ def save_config(self, target): def config(self) -> Any: """Configuration as dictionary. - To make changes to configuration before saving set the config keys and/or values. + To make changes to configuration before saving set the config keys + and/or values. Can be a nested dict. """ diff --git a/src/ewatercycle/util.py b/src/ewatercycle/util.py index a1649a93..e1e67996 100644 --- a/src/ewatercycle/util.py +++ b/src/ewatercycle/util.py @@ -63,7 +63,8 @@ def get_time(time_iso: str) -> datetime: time = parse(time_iso) if not time.tzname() == "UTC": raise ValueError( - f"The time is not in UTC. The ISO format for a UTC time is 'YYYY-MM-DDTHH:MM:SSZ'" + "The time is not in UTC. The ISO format for a UTC time " + "is 'YYYY-MM-DDTHH:MM:SSZ'" ) return time @@ -130,7 +131,8 @@ def to_absolute_path( input_path: Input string path that can be a relative or absolute path. parent: Optional parent path of the input path must_exist: Optional argument to check if the input path exists. - must_be_in_parent: Optional argument to check if the input path is subpath of parent path + must_be_in_parent: Optional argument to check if the input path is + subpath of parent path Returns: The input path that is an absolute path and a :py:class:`pathlib.Path` object. From 0f5bf7810b4a2598908f5e01d6ef8e4ece1333fe Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 16 Aug 2021 16:46:10 +0200 Subject: [PATCH 29/33] noqa instead of pylint-specific --- src/ewatercycle/parameter_sets/_lisflood.py | 4 ++-- src/ewatercycle/parameter_sets/_pcrglobwb.py | 4 ++-- src/ewatercycle/parameter_sets/_wflow.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ewatercycle/parameter_sets/_lisflood.py b/src/ewatercycle/parameter_sets/_lisflood.py index 34fba55e..f808c655 100644 --- a/src/ewatercycle/parameter_sets/_lisflood.py +++ b/src/ewatercycle/parameter_sets/_lisflood.py @@ -11,9 +11,9 @@ def example_parameter_sets() -> Iterable[ExampleParameterSet]: name="lisflood_fraser", # Relative to CFG['parameterset_dir'] config="lisflood_fraser/settings_lat_lon-Run.xml", - datafiles_url="https://github.com/ec-jrc/lisflood-usecases/trunk/LF_lat_lon_UseCase", # pylint: disable=C0301 + datafiles_url="https://github.com/ec-jrc/lisflood-usecases/trunk/LF_lat_lon_UseCase", # noqa: E501 # Raw url to config file - config_url="https://github.com/ec-jrc/lisflood-usecases/raw/master/LF_lat_lon_UseCase/settings_lat_lon-Run.xml", # pylint: disable=C0301 + config_url="https://github.com/ec-jrc/lisflood-usecases/raw/master/LF_lat_lon_UseCase/settings_lat_lon-Run.xml", # noqa: E501 doi="N/A", target_model="lisflood", supported_model_versions={"20.10"}, diff --git a/src/ewatercycle/parameter_sets/_pcrglobwb.py b/src/ewatercycle/parameter_sets/_pcrglobwb.py index acc36bcb..105f9158 100644 --- a/src/ewatercycle/parameter_sets/_pcrglobwb.py +++ b/src/ewatercycle/parameter_sets/_pcrglobwb.py @@ -11,9 +11,9 @@ def example_parameter_sets() -> Iterable[ExampleParameterSet]: name="pcrglobwb_rhinemeuse_30min", # Relative to CFG['parameterset_dir'] config="pcrglobwb_rhinemeuse_30min/setup_natural_test.ini", - datafiles_url="https://github.com/UU-Hydro/PCR-GLOBWB_input_example/trunk/RhineMeuse30min", # pylint: disable=C0301 + datafiles_url="https://github.com/UU-Hydro/PCR-GLOBWB_input_example/trunk/RhineMeuse30min", # noqa: E501 # Raw url to config file - config_url="https://raw.githubusercontent.com/UU-Hydro/PCR-GLOBWB_input_example/master/ini_and_batch_files_for_pcrglobwb_course/rhine_meuse_30min_using_input_example/setup_natural_test.ini", # pylint: disable=C0301 + config_url="https://raw.githubusercontent.com/UU-Hydro/PCR-GLOBWB_input_example/master/ini_and_batch_files_for_pcrglobwb_course/rhine_meuse_30min_using_input_example/setup_natural_test.ini", # noqa: E501 doi="https://doi.org/10.5281/zenodo.1045339", target_model="pcrglobwb", supported_model_versions={"setters"}, diff --git a/src/ewatercycle/parameter_sets/_wflow.py b/src/ewatercycle/parameter_sets/_wflow.py index de9fbb15..9fbad5f5 100644 --- a/src/ewatercycle/parameter_sets/_wflow.py +++ b/src/ewatercycle/parameter_sets/_wflow.py @@ -11,9 +11,9 @@ def example_parameter_sets() -> Iterable[ExampleParameterSet]: name="wflow_rhine_sbm_nc", # Relative to CFG['parameterset_dir'] config="wflow_rhine_sbm_nc/wflow_sbm_NC.ini", - datafiles_url="https://github.com/openstreams/wflow/trunk/examples/wflow_rhine_sbm_nc", # pylint: disable=C0301 + datafiles_url="https://github.com/openstreams/wflow/trunk/examples/wflow_rhine_sbm_nc", # noqa: E501 # Raw url to config file - config_url="https://github.com/openstreams/wflow/raw/master/examples/wflow_rhine_sbm_nc/wflow_sbm_NC.ini", # pylint: disable=C0301 + config_url="https://github.com/openstreams/wflow/raw/master/examples/wflow_rhine_sbm_nc/wflow_sbm_NC.ini", # noqa: E501 doi="N/A", target_model="wflow", supported_model_versions={"2020.1.1"}, From 8e47e24db26e62c5d7379bbe830356719761d542 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 16 Aug 2021 16:49:55 +0200 Subject: [PATCH 30/33] Fix line lengths in tests --- tests/conftest.py | 2 +- tests/forcing/test_lisflood.py | 3 ++- tests/forcing/test_wflow.py | 4 ++-- tests/models/test_abstract.py | 4 ++-- tests/parameter_sets/test_example.py | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 03c8330d..52d07ff1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ @pytest.fixture def yaml_config_url(): - return "data:text/plain,data: data/PEQ_Hupsel.dat\nparameters:\n cW: 200\n cV: 4\n cG: 5.0e+6\n cQ: 10\n cS: 4\n dG0: 1250\n cD: 1500\n aS: 0.01\n st: loamy_sand\nstart: 367416 # 2011120000\nend: 368904 # 2012020000\nstep: 1\n" + return "data:text/plain,data: data/PEQ_Hupsel.dat\nparameters:\n cW: 200\n cV: 4\n cG: 5.0e+6\n cQ: 10\n cS: 4\n dG0: 1250\n cD: 1500\n aS: 0.01\n st: loamy_sand\nstart: 367416 # 2011120000\nend: 368904 # 2012020000\nstep: 1\n" # noqa: E501 @pytest.fixture diff --git a/tests/forcing/test_lisflood.py b/tests/forcing/test_lisflood.py index 5292470f..23407775 100644 --- a/tests/forcing/test_lisflood.py +++ b/tests/forcing/test_lisflood.py @@ -256,7 +256,8 @@ def test_recipe_configured( actual_shapefile = actual["preprocessors"]["general"]["extract_shape"][ "shapefile" ] - # Will also del other occurrences of shapefile due to extract shape object being shared between preprocessors + # Will also del other occurrences of shapefile due to extract shape object + # being shared between preprocessors del actual["preprocessors"]["general"]["extract_shape"]["shapefile"] assert actual == reference_recipe diff --git a/tests/forcing/test_wflow.py b/tests/forcing/test_wflow.py index bc18fc35..734ab27c 100644 --- a/tests/forcing/test_wflow.py +++ b/tests/forcing/test_wflow.py @@ -46,7 +46,7 @@ def reference_recipe(self): "scripts": { "script": { "basin": "Rhine", - "dem_file": "wflow_parameterset/meuse/staticmaps/wflow_dem.map", + "dem_file": "wflow_parameterset/meuse/staticmaps/wflow_dem.map", # noqa: E501 "regrid": "area_weighted", "script": "hydrology/wflow.py", } @@ -94,7 +94,7 @@ def reference_recipe(self): "aerts_jerom", "andela_bouwe", ], - "description": "Pre-processes climate data for the WFlow hydrological model.\n", + "description": "Pre-processes climate data for the WFlow hydrological model.\n", # noqa: E501 "projects": ["ewatercycle"], "references": ["acknow_project"], }, diff --git a/tests/models/test_abstract.py b/tests/models/test_abstract.py index cfb929fc..878ba724 100644 --- a/tests/models/test_abstract.py +++ b/tests/models/test_abstract.py @@ -93,8 +93,8 @@ def test_construct_with_unsupported_version(): MockedModel(version="1.2.3") assert ( - "Supplied version 1.2.3 is not supported by this model. Available versions are ('0.4.2',)." - in str(excinfo.value) + "Supplied version 1.2.3 is not supported by this model. " + "Available versions are ('0.4.2',)." in str(excinfo.value) ) diff --git a/tests/parameter_sets/test_example.py b/tests/parameter_sets/test_example.py index 0e8fa4be..a658e71d 100644 --- a/tests/parameter_sets/test_example.py +++ b/tests/parameter_sets/test_example.py @@ -19,7 +19,7 @@ def setup_config(tmp_path): def example(setup_config): return ExampleParameterSet( name="firstexample", - config_url="https://github.com/mymodelorg/mymodelrepo/raw/master/mymodelexample/config.ini", + config_url="https://github.com/mymodelorg/mymodelrepo/raw/master/mymodelexample/config.ini", # noqa: E501 datafiles_url="https://github.com/mymodelorg/mymodelrepo/trunk/mymodelexample", directory="mymodelexample", config="mymodelexample/config.ini", From b0f5f8d1989fea49d5769ff25ff4f40ce841f3c9 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Mon, 16 Aug 2021 17:30:51 +0200 Subject: [PATCH 31/33] fix error message --- src/ewatercycle/parameter_sets/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ewatercycle/parameter_sets/__init__.py b/src/ewatercycle/parameter_sets/__init__.py index 9c043d8b..13f5b6d7 100644 --- a/src/ewatercycle/parameter_sets/__init__.py +++ b/src/ewatercycle/parameter_sets/__init__.py @@ -51,9 +51,10 @@ def available_parameter_sets(target_model: str = None) -> Tuple[str, ...]: ) if not filtered: raise ValueError( - f'No parameter sets defined in {CFG["ewatercycle_config"]}. Use ' - "`ewatercycle.parareter_sets.download_example_parameter_sets` to download" - " examples or define your own or ask whoever setup the ewatercycle " + f"No parameter sets defined for {target_model} model in " + f"{CFG['ewatercycle_config']}. Use " + "`ewatercycle.parareter_sets.download_example_parameter_sets` to download " + "examples or define your own or ask whoever setup the ewatercycle " "system to do it." ) return filtered From a6f0436184cbecdc4218c94171f2055acf4dc9a2 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 17 Aug 2021 10:09:09 +0200 Subject: [PATCH 32/33] Force showing of warnings of nbqa-flake8 --- .pre-commit-config.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 00dd59ca..bc2671ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,12 +26,12 @@ repos: rev: "5.9.3" hooks: - id: isort - # TODO renable when erros are fixed/ignored + # TODO renable when errors are fixed/ignored # - repo: https://github.com/pycqa/pylint # rev: "v2.9.6" # hooks: # - id: pylint - # TODO renable when erros are fixed/ignored + # TODO renable when errors are fixed/ignored - repo: https://gitlab.com/pycqa/flake8 rev: "3.9.2" hooks: @@ -84,11 +84,12 @@ repos: additional_dependencies: [isort==5.9.3] - id: nbqa-mypy additional_dependencies: [mypy==0.910] - # TODO renable when erros are fixed/ignored + # TODO renable when errors are fixed/ignored - id: nbqa-flake8 additional_dependencies: *fd args: *fa - # TODO renable when erros are fixed/ignored + verbose: true + # TODO renable when errors are fixed/ignored # - id: nbqa-pylint # additional_dependencies: [pylint==2.9.6] - repo: https://github.com/regebro/pyroma From 04723c56d4eaa782b662aca8d857fe5505c231a3 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 17 Aug 2021 10:26:38 +0200 Subject: [PATCH 33/33] Update .pre-commit-config.yaml Co-authored-by: Peter Kalverla --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 00dd59ca..3e9c62c6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -78,7 +78,7 @@ repos: rev: 1.1.0 hooks: - id: nbqa-black - # Match version of black used for .py and .pynb + # Match version of black used for .py and .ipynb additional_dependencies: [black==21.7b0] - id: nbqa-isort additional_dependencies: [isort==5.9.3]