diff --git a/.editorconfig b/.editorconfig index 83e59afe..3f72fd3c 100644 --- a/.editorconfig +++ b/.editorconfig @@ -25,3 +25,6 @@ indent_size = 2 [*.{md,Rmd}] trim_trailing_whitespace = false + +[Makefile] +indent_style = tab diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb70b2ac..6cfa0861 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,10 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - name: Python package on: push: branches: [main] pull_request: - types: [opened, synchronize, reopened] + types: [opened, synchronize, reopened] jobs: build: @@ -19,18 +16,18 @@ jobs: fail-fast: false name: Run tests in conda environment ${{ matrix.python-version }} steps: - - uses: actions/checkout@v2 - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: ewatercycle - environment-file: environment.yml - python-version: ${{ matrix.python-version }} - miniconda-version: "latest" - channels: conda-forge - - name: Install dependencies - shell: bash -l {0} - run: | - pip3 install -e .[dev] - - name: Test with pytest - run: pytest - shell: bash -l {0} + - uses: actions/checkout@v2 + - uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: ewatercycle + environment-file: environment.yml + python-version: ${{ matrix.python-version }} + miniconda-version: "latest" + channels: conda-forge + - name: Install dependencies + shell: bash -l {0} + run: | + pip3 install -e .[dev] + - name: Test with pytest + run: pytest + shell: bash -l {0} diff --git a/.github/workflows/sonar.yml b/.github/workflows/sonar.yml index 48f49546..9780a4df 100644 --- a/.github/workflows/sonar.yml +++ b/.github/workflows/sonar.yml @@ -4,34 +4,38 @@ on: push: branches: [main] pull_request: - types: [opened, synchronize, reopened] + types: [opened, synchronize, reopened] jobs: sonarcloud: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up Python - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: ewatercycle - environment-file: environment.yml - python-version: 3.8 - miniconda-version: "latest" - channels: conda-forge - - name: Install dependencies - shell: bash -l {0} - run: | - pip3 install -e .[dev] - - name: Tests with coverage - run: pytest --cov --cov-report term --cov-report xml --junitxml=xunit-result.xml - shell: bash -l {0} - - name: Correct coverage paths - run: sed -i "s+$PWD/++g" coverage.xml - - name: SonarCloud Scan - uses: sonarsource/sonarcloud-github-action@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python + uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: ewatercycle + environment-file: environment.yml + python-version: 3.8 + miniconda-version: "latest" + channels: conda-forge + - name: Install dependencies + shell: bash -l {0} + run: | + pip3 install -e .[dev] + - name: Run pre commit hooks like linters and black formatter + uses: pre-commit/action@v2.0.3 + - name: Tests with coverage + run: | + pytest --cov --cov-report term --cov-report xml \ + --junitxml=xunit-result.xml + shell: bash -l {0} + - name: Correct coverage paths + run: sed -i "s+$PWD/++g" coverage.xml + - name: SonarCloud Scan + uses: sonarsource/sonarcloud-github-action@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..27bfe5a5 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,98 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.4.0 + hooks: + - id: check-added-large-files + - id: check-ast + - id: check-case-conflict + - id: check-merge-conflict + - id: debug-statements + - id: end-of-file-fixer + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - repo: https://github.com/adrienverge/yamllint + rev: "v1.26.0" + hooks: + - id: yamllint + - repo: https://github.com/asottile/setup-cfg-fmt + rev: v1.17.0 + hooks: + - id: setup-cfg-fmt + - repo: https://github.com/psf/black + rev: 21.7b0 + hooks: + - id: black + - repo: https://github.com/PyCQA/isort + rev: "5.9.3" + hooks: + - id: isort + # TODO renable when errors are fixed/ignored + # - repo: https://github.com/pycqa/pylint + # rev: "v2.9.6" + # hooks: + # - id: pylint + # TODO renable when errors are fixed/ignored + - repo: https://gitlab.com/pycqa/flake8 + rev: "3.9.2" + hooks: + - id: flake8 + additional_dependencies: + &fd [ + flake8-annotations-complexity, + flake8-bandit, + flake8-blind-except, + flake8-bugbear, + flake8-builtins, + flake8-cognitive-complexity, + flake8-comprehensions, + flake8-docstrings, + flake8-eradicate, + flake8-executable, + flake8-expression-complexity, + flake8-if-expr, + flake8-implicit-str-concat, + flake8-logging-format, + flake8-pathlib, + flake8-print, + flake8-pytest, + flake8-pytest-style, + # flake8-quotes, # conflicts with blacks double quote preference + flake8-raise, + flake8-return, + flake8-typing-imports, + flake8-variables-names, + flake8==3.9.2, + pandas-vet, + pep8-naming, + # wemake-python-styleguide, # conflicts with black + yesqa, + ] + verbose: true + args: &fa [--statistics, --exit-zero] + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.910 + hooks: + - id: mypy + additional_dependencies: [types-python-dateutil] + - repo: https://github.com/nbQA-dev/nbQA + rev: 1.1.0 + hooks: + - id: nbqa-black + # Match version of black used for .py and .ipynb + additional_dependencies: [black==21.7b0] + - id: nbqa-isort + additional_dependencies: [isort==5.9.3] + - id: nbqa-mypy + additional_dependencies: [mypy==0.910] + # TODO renable when errors are fixed/ignored + - id: nbqa-flake8 + additional_dependencies: *fd + args: *fa + verbose: true + # TODO renable when errors are fixed/ignored + # - id: nbqa-pylint + # additional_dependencies: [pylint==2.9.6] + - repo: https://github.com/regebro/pyroma + rev: "3.2" + hooks: + - id: pyroma diff --git a/.prospector.yml b/.prospector.yml deleted file mode 100644 index 94063ef9..00000000 --- a/.prospector.yml +++ /dev/null @@ -1,29 +0,0 @@ -# prospector configuration file - ---- - -output-format: grouped - -strictness: veryhigh -doc-warnings: true -test-warnings: true -member-warnings: false - -pyroma: - run: true - -pep8: - full: true - -mypy: - run: true - -pep257: - disable: [ - # Disable because not part of PEP257 official convention: - # see http://pep257.readthedocs.io/en/latest/error_codes.html - D203, # 1 blank line required before class docstring - D212, # Multi-line docstring summary should start at the first line - D213, # Multi-line docstring summary should start at the second line - D404, # First word of the docstring should not be This - ] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 187a6c4e..0158444e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,6 +55,7 @@ The sections below outline the steps in each case. and [here](https://help.github.com/articles/syncing-a-fork/)); 4. install the package in editable mode and its dependencies with `pip3 install -e .[dev]`; +4. make sure pre commit hook is installed by running `pre-commit install`, causes linting and formatting to be applied during commit; 5. make sure the existing tests still work by running `pytest`; 6. make sure the existing documentation can still by generated without warnings by running `cd docs && make html`. [Pandoc](https://pandoc.org/) is required to generate docs, it can be installed with ``conda install -c conda-forge pandoc`` ; @@ -81,24 +82,25 @@ This section is for maintainers of the package. 2. Determine what new version (major, minor or patch) to use. Package uses `semantic versioning `_. 3. Run ``bump2version `` to update version in package files. 4. Update CHANGELOG.md with changes between current and new version. -5. Commit & push changes to GitHub. -6. Wait for [GitHub +5. Make sure pre-commit hooks are green for all files by running ``pre-commit run --all-files``. +6. Commit & push changes to GitHub. +7. Wait for [GitHub actions](https://github.com/eWaterCycle/ewatercycle/actions?query=branch%3Amain+) to be completed and green. -7. Create a [GitHub release](https://github.com/eWaterCycle/ewatercycle/releases/new) +8. Create a [GitHub release](https://github.com/eWaterCycle/ewatercycle/releases/new) - Use version as title and tag version. - As description use intro text from README.md (to give context to Zenodo record) and changes from CHANGELOG.md -8. Create a PyPI release. +9. Create a PyPI release. 1. Create distribution archives with `python3 -m build`. 2. Upload archives to PyPI with `twine upload dist/*` (use your personal PyPI account). -9. Verify +10. Verify 1. Has [new Zenodo record](https://zenodo.org/search?page=1&size=20&q=ewatercycle) @@ -109,4 +111,4 @@ This section is for maintainers of the package. 3. Can new version be installed with pip using `pip3 install ewatercycle==`? -10. Celebrate +11. Celebrate diff --git a/LICENSE b/LICENSE index baecc0b5..e5599134 100644 --- a/LICENSE +++ b/LICENSE @@ -14,4 +14,3 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/docs/conf.py b/docs/conf.py index c89a35c2..df6cdbca 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,3 +1,4 @@ +# noqa: D100 # -*- coding: utf-8 -*- # # ewatercycle documentation build configuration file, created by @@ -16,46 +17,41 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -import os import sys +from pathlib import Path -here = os.path.dirname(__file__) -sys.path.insert(0, os.path.abspath(os.path.join(here, '..', 'src'))) +src = Path(__file__).parent / ".." / "src" +sys.path.insert(0, str(src.absolute())) # -- General configuration ------------------------------------------------ -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.napoleon', - 'nbsphinx', - 'sphinx.ext.intersphinx', - 'sphinx_copybutton', + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "nbsphinx", + "sphinx.ext.intersphinx", + "sphinx_copybutton", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = [".rst"] # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'ewatercycle' -copyright = u'2018, Netherlands eScience Center & Delft University of Technology' -author = u'Stefan Verhoeven' +project = "ewatercycle" +copyright = "2018, Netherlands eScience Center & Delft University of Technology" # noqa A001,VNE003 +author = "Stefan Verhoeven" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -63,7 +59,7 @@ # # The short X.Y version. # The full version, including alpha/beta/rc tags. -version = '1.1.1' +version = "1.1.1" release = version # The language for content autogenerated by Sphinx. Refer to documentation @@ -76,10 +72,10 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -87,10 +83,10 @@ # -- Run apidoc plug-in manually, as readthedocs doesn't support it ------- # See https://github.com/rtfd/readthedocs.org/issues/1139 -def run_apidoc(_): - here = os.path.dirname(__file__) - out = os.path.abspath(os.path.join(here, 'apidocs')) - src = os.path.abspath(os.path.join(here, '..', 'src', 'ewatercycle')) +def run_apidoc(_): # noqa: D103 + here = Path(__file__).parent + out = (here / "apidocs").absolute() + source_dir = (here / ".." / "src" / "ewatercycle").absolute() ignore_paths = [] @@ -100,23 +96,26 @@ def run_apidoc(_): "-e", "-M", "--implicit-namespaces", - "-o", out, - src + "-o", + str(out), + str(source_dir), ] + ignore_paths try: # Sphinx 1.7+ from sphinx.ext import apidoc + apidoc.main(argv) except ImportError: # Sphinx 1.6 (and earlier) from sphinx import apidoc + argv.insert(0, apidoc.__file__) apidoc.main(argv) -def setup(app): - app.connect('builder-inited', run_apidoc) +def setup(app): # noqa: D103 + app.connect("builder-inited", run_apidoc) # -- Options for HTML output ---------------------------------------------- @@ -124,20 +123,13 @@ def setup(app): # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -#html_theme = 'alabaster' -html_theme = 'sphinx_rtd_theme' -html_logo = 'examples/logo.png' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} +html_theme = "sphinx_rtd_theme" +html_logo = "examples/logo.png" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -145,9 +137,9 @@ def setup(app): # This is required for the alabaster theme # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars html_sidebars = { - '**': [ - 'relations.html', # needs 'show_related': True theme option to display - 'searchbox.html', + "**": [ + "relations.html", # needs 'show_related': True theme option to display + "searchbox.html", ] } @@ -155,35 +147,22 @@ def setup(app): # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. -htmlhelp_basename = 'ewatercycle_doc' +htmlhelp_basename = "ewatercycle_doc" # -- Options for LaTeX output --------------------------------------------- -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'ewatercycle.tex', u'ewatercycle Documentation', - u'Stefan Verhoeven', 'manual'), + ( + master_doc, + "ewatercycle.tex", + "ewatercycle Documentation", + "Stefan Verhoeven", + "manual", + ), ] @@ -191,10 +170,7 @@ def setup(app): # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'ewatercycle', u'ewatercycle Documentation', - [author], 1) -] +man_pages = [(master_doc, "ewatercycle", "ewatercycle Documentation", [author], 1)] # -- Options for Texinfo output ------------------------------------------- @@ -203,67 +179,71 @@ def setup(app): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'ewatercycle', u'ewatercycle Documentation', - author, 'ewatercycle', 'Python utilities to gather input files for running a hydrology model', - 'Miscellaneous'), + ( + master_doc, + "ewatercycle", + "ewatercycle Documentation", + author, + "ewatercycle", + "Python utilities to gather input files for running a hydrology model", + "Miscellaneous", + ), ] autodoc_mock_imports = [ - 'basic_modeling_interface', - 'cftime', - 'dask', - 'esmvalcore', - 'fiona', - 'dateutil', - 'shapely', - 'hydrostats', - 'matplotlib', - 'numpy', - 'pandas', - 'pyoos', - 'grpc4bmi', - 'grpc', - 'ruamel.yaml', - 'scipy', - 'xarray', + "basic_modeling_interface", + "cftime", + "dask", + "esmvalcore", + "fiona", + "dateutil", + "shapely", + "hydrostats", + "matplotlib", + "numpy", + "pandas", + "pyoos", + "grpc4bmi", + "grpc", + "ruamel.yaml", + "scipy", + "xarray", ] # Prevent alphabetic sorting of (@data)class attributes/methods -autodoc_member_order = 'bysource' +autodoc_member_order = "bysource" # nbsphinx configuration values cf. https://nbsphinx.readthedocs.io/en/0.8.6/usage.html -nbsphinx_execute = 'never' +nbsphinx_execute = "never" # Hacky way to 'remove' the cell count from the prompt. # Inspired by https://github.com/spatialaudio/nbsphinx/issues/126 -nbsphinx_prompt_width = '0' -nbsphinx_input_prompt = '%s In:' -nbsphinx_output_prompt = '%s Out:' +nbsphinx_prompt_width = "0" +nbsphinx_input_prompt = "%s In:" +nbsphinx_output_prompt = "%s Out:" # Nice formatting of model-specific input parameters napoleon_custom_sections = [ - ('hype', 'params_style'), - ('lisflood', 'params_style'), - ('marrmot', 'params_style'), - ('pcrglobwb', 'params_style'), - ('wflow', 'params_style'), + ("hype", "params_style"), + ("lisflood", "params_style"), + ("marrmot", "params_style"), + ("pcrglobwb", "params_style"), + ("wflow", "params_style"), ] intersphinx_mapping = { - 'cf_units': ('https://scitools.org.uk/cf-units/docs/latest/', None), - 'esmvalcore': - (f'https://docs.esmvaltool.org/projects/esmvalcore/en/latest/', - None), - 'esmvaltool': (f'https://docs.esmvaltool.org/en/latest/', None), - 'grpc4bmi': (f'https://grpc4bmi.readthedocs.io/en/latest/', None), - 'iris': ('https://scitools-iris.readthedocs.io/en/latest/', None), - 'lime': ('https://lime-ml.readthedocs.io/en/latest/', None), - 'basic_modeling_interface': ('https://bmi.readthedocs.io/en/latest/', None), - 'matplotlib': ('https://matplotlib.org/', None), - 'numpy': ('https://numpy.org/doc/stable/', None), - 'pandas': ('https://pandas.pydata.org/pandas-docs/dev', None), - 'python': ('https://docs.python.org/3/', None), - 'scipy': ('https://docs.scipy.org/doc/scipy/reference/', None), - 'seaborn': ('https://seaborn.pydata.org/', None), - 'sklearn': ('https://scikit-learn.org/stable', None), + "cf_units": ("https://scitools.org.uk/cf-units/docs/latest/", None), + "esmvalcore": ("https://docs.esmvaltool.org/projects/esmvalcore/en/latest/", None), + "esmvaltool": ("https://docs.esmvaltool.org/en/latest/", None), + "grpc4bmi": ("https://grpc4bmi.readthedocs.io/en/latest/", None), + "iris": ("https://scitools-iris.readthedocs.io/en/latest/", None), + "lime": ("https://lime-ml.readthedocs.io/en/latest/", None), + "basic_modeling_interface": ("https://bmi.readthedocs.io/en/latest/", None), + "matplotlib": ("https://matplotlib.org/", None), + "numpy": ("https://numpy.org/doc/stable/", None), + "pandas": ("https://pandas.pydata.org/pandas-docs/dev", None), + "python": ("https://docs.python.org/3/", None), + "scipy": ("https://docs.scipy.org/doc/scipy/reference/", None), + "seaborn": ("https://seaborn.pydata.org/", None), + "sklearn": ("https://scikit-learn.org/stable", None), } diff --git a/docs/examples/data/Rhine/Rhine.prj b/docs/examples/data/Rhine/Rhine.prj index a30c00a5..8f73f480 100644 --- a/docs/examples/data/Rhine/Rhine.prj +++ b/docs/examples/data/Rhine/Rhine.prj @@ -1 +1 @@ -GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]] \ No newline at end of file +GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]] diff --git a/docs/examples/pcrglobwb_merrimack.ini b/docs/examples/pcrglobwb_merrimack.ini index d1177570..a6158e07 100644 --- a/docs/examples/pcrglobwb_merrimack.ini +++ b/docs/examples/pcrglobwb_merrimack.ini @@ -2,7 +2,7 @@ # Set the pcrglobwb output directory in an absolute path. outputDir = /data/output -# Set the input directory map in an absolute path. +# Set the input directory map in an absolute path. # - The input forcing and parameter directories and files will be relative to this. inputDir = /data/input @@ -10,14 +10,14 @@ inputDir = /data/input # - Spatial resolution and coverage are based on this map: cloneMap = global_05min/cloneMaps/merrimack_05min.map # The area/landmask of interest: -# If None, area/landmask is limited for cells with ldd value. +# If None, area/landmask is limited for cells with ldd value. landmask = global_05min/cloneMaps/merrimack_05min.map # netcdf attributes for output files: institution = Department of Physical Geography, Utrecht University title = PCR-GLOBWB 2 output (not coupled to MODFLOW) -description = by Edwin H. Sutanudjaja (contact: e.h.sutanudjaja@uu.nl) +description = by Edwin H. Sutanudjaja (contact: e.h.sutanudjaja@uu.nl) startTime = 2002-01-01 endTime = 2002-12-31 @@ -44,12 +44,12 @@ temperatureNC = global_05min/meteo/pcrglobwb_OBS6_ERA5_reanaly_1_day_tas_2002- referenceETPotMethod = Hamon refETPotFileNC = 'test' -# variable names in the forcing files (optional) +# variable names in the forcing files (optional) precipitationVariableName = pr temperatureVariableName = tas -referenceEPotVariableName = potentialEvaporation +referenceEPotVariableName = potentialEvaporation -# conversion constants and factors to correct forcing values (optional) so that the units are in m.day-1 and degree Celcius +# conversion constants and factors to correct forcing values (optional) so that the units are in m.day-1 and degree Celcius precipitationConstant = 0.0 precipitationFactor = 1.0 temperatureConstant = -273.15 @@ -59,26 +59,26 @@ referenceEPotFactor = 1.0 [meteoDownscalingOptions] -# This section is for a 5 arcmin run, for downscaling meteorological forcing at 30 arcmin to 5 arcmin. - -downscalePrecipitation = True -downscaleTemperature = True -downscaleReferenceETPot = True - -# downscaling (based on the digital elevation model): -# The downscaling will be performed by providing the "cellIds" (meteoDownscaleIds) of lower resolution cells. -meteoDownscaleIds = global_05min/meteo/downscaling_from_30min/uniqueIds_30min.map -highResolutionDEM = global_05min/meteo/downscaling_from_30min/gtopo05min.map - -# lapse rates: -temperLapseRateNC = global_05min/meteo/downscaling_from_30min/temperature_slope.nc -precipLapseRateNC = global_05min/meteo/downscaling_from_30min/precipitation_slope.nc - -# downscaling criteria (TODO: remove these): -temperatCorrelNC = global_05min/meteo/downscaling_from_30min/temperature_correl.nc -precipitCorrelNC = global_05min/meteo/downscaling_from_30min/precipitation_correl.nc - -# windows length (unit: arc-degree) for smoothing/averaging forcing data (not recommended): +# This section is for a 5 arcmin run, for downscaling meteorological forcing at 30 arcmin to 5 arcmin. + +downscalePrecipitation = True +downscaleTemperature = True +downscaleReferenceETPot = True + +# downscaling (based on the digital elevation model): +# The downscaling will be performed by providing the "cellIds" (meteoDownscaleIds) of lower resolution cells. +meteoDownscaleIds = global_05min/meteo/downscaling_from_30min/uniqueIds_30min.map +highResolutionDEM = global_05min/meteo/downscaling_from_30min/gtopo05min.map + +# lapse rates: +temperLapseRateNC = global_05min/meteo/downscaling_from_30min/temperature_slope.nc +precipLapseRateNC = global_05min/meteo/downscaling_from_30min/precipitation_slope.nc + +# downscaling criteria (TODO: remove these): +temperatCorrelNC = global_05min/meteo/downscaling_from_30min/temperature_correl.nc +precipitCorrelNC = global_05min/meteo/downscaling_from_30min/precipitation_correl.nc + +# windows length (unit: arc-degree) for smoothing/averaging forcing data (not recommended): smoothingWindowsLength = 0 @@ -88,16 +88,16 @@ debugWaterBalance = True numberOfUpperSoilLayers = 2 # soil and parameters -# - they are used for all land cover types, unless they are are defined in certain land cover type options -# (e.g. different/various soil types for agriculture areas) +# - they are used for all land cover types, unless they are are defined in certain land cover type options +# (e.g. different/various soil types for agriculture areas) topographyNC = global_05min/landSurface/topography/topography_parameters_5_arcmin_october_2015.nc soilPropertiesNC = global_05min/landSurface/soil/soilProperties5ArcMin.nc includeIrrigation = True -# netcdf time series for historical expansion of irrigation areas (unit: hectares). -# Note: The resolution of this map must be consisten with the resolution of cellArea. +# netcdf time series for historical expansion of irrigation areas (unit: hectares). +# Note: The resolution of this map must be consisten with the resolution of cellArea. historicalIrrigationArea = global_05min/waterUse/irrigation/irrigated_areas/irrigationArea05ArcMin.nc # a pcraster map/value defining irrigation efficiency (dimensionless) - optional @@ -118,10 +118,10 @@ livestockWaterDemandFile = global_05min/waterUse/waterDemand/livestock/livestock desalinationWater = global_05min/waterUse/desalination/desalination_water_version_april_2015.nc -# zone IDs (scale) at which allocations of groundwater and surface water (as well as desalinated water) are performed +# zone IDs (scale) at which allocations of groundwater and surface water (as well as desalinated water) are performed allocationSegmentsForGroundSurfaceWater = global_05min/waterUse/abstraction_zones/abstraction_zones_60min_05min.map -# pcraster maps defining the partitioning of groundwater - surface water source +# pcraster maps defining the partitioning of groundwater - surface water source # # - predefined surface water - groundwater partitioning for irrigation demand (e.g. based on Siebert, Global Map of Irrigation Areas version 5) irrigationSurfaceWaterAbstractionFractionData = global_05min/waterUse/source_partitioning/surface_water_fraction_for_irrigation/AEI_SWFRAC.map @@ -164,9 +164,9 @@ fracVegCover = global_05min/landSurface/landCover/naturalTall/vegf_tall.map minSoilDepthFrac = global_30min/landSurface/landCover/naturalTall/minf_tall_permafrost.map maxSoilDepthFrac = global_30min/landSurface/landCover/naturalTall/maxf_tall.map rootFraction1 = global_05min/landSurface/landCover/naturalTall/rfrac1_tall.map -rootFraction2 = global_05min/landSurface/landCover/naturalTall/rfrac2_tall.map +rootFraction2 = global_05min/landSurface/landCover/naturalTall/rfrac2_tall.map maxRootDepth = 1.0 -# Note: The maxRootDepth is not used for non irrigated land cover type. +# Note: The maxRootDepth is not used for non irrigated land cover type. # # Parameters for the Arno's scheme: arnoBeta = None @@ -209,9 +209,9 @@ fracVegCover = global_05min/landSurface/landCover/naturalShort/vegf_short.m minSoilDepthFrac = global_30min/landSurface/landCover/naturalShort/minf_short_permafrost.map maxSoilDepthFrac = global_30min/landSurface/landCover/naturalShort/maxf_short.map rootFraction1 = global_05min/landSurface/landCover/naturalShort/rfrac1_short.map -rootFraction2 = global_05min/landSurface/landCover/naturalShort/rfrac2_short.map +rootFraction2 = global_05min/landSurface/landCover/naturalShort/rfrac2_short.map maxRootDepth = 0.5 -# Note: The maxRootDepth is not used for non irrigated land cover type. +# Note: The maxRootDepth is not used for non irrigated land cover type. # # Parameters for the Arno's scheme: arnoBeta = None @@ -325,14 +325,14 @@ debugWaterBalance = True groundwaterPropertiesNC = global_05min/groundwater/properties/groundwaterProperties5ArcMin.nc # The file will containspecificYield (m3.m-3), kSatAquifer, recessionCoeff (day-1) # -# - minimum value for groundwater recession coefficient (day-1) +# - minimum value for groundwater recession coefficient (day-1) minRecessionCoeff = 1.0e-4 # some options for constraining groundwater abstraction limitFossilGroundWaterAbstraction = True estimateOfRenewableGroundwaterCapacity = 0.0 estimateOfTotalGroundwaterThickness = global_05min/groundwater/aquifer_thickness_estimate/thickness_05min.map -# minimum and maximum total groundwater thickness +# minimum and maximum total groundwater thickness minimumTotalGroundwaterThickness = 100. maximumTotalGroundwaterThickness = None @@ -350,11 +350,11 @@ avgTotalGroundwaterAbstractionIni = global_05min/initialConditions/avgTo avgTotalGroundwaterAllocationLongIni = global_05min/initialConditions/avgTotalGroundwaterAllocationLong_1999-12-31.map avgTotalGroundwaterAllocationShortIni = global_05min/initialConditions/avgTotalGroundwaterAllocationShort_1999-12-31.map # -# additional initial conditions (needed only for MODFLOW run) +# additional initial conditions (needed only for MODFLOW run) relativeGroundwaterHeadIni = global_05min/initialConditions/relativeGroundwaterHead_1999-12-31.map baseflowIni = global_05min/initialConditions/baseflow_1999-12-31.map -# zonal IDs (scale) at which zonal allocation of groundwater is performed +# zonal IDs (scale) at which zonal allocation of groundwater is performed allocationSegmentsForGroundwater = global_05min/waterUse/abstraction_zones/abstraction_zones_30min_05min.map # assumption for the minimum transmissivity value (unit: m2/day) that can be extracted (via capillary rise and/or groundwater abstraction) - optional @@ -389,7 +389,7 @@ floodplainManningsN = 0.07 # channel gradient gradient = global_05min/routing/channel_properties/channel_gradient.map -# constant channel depth +# constant channel depth constantChannelDepth = global_05min/routing/channel_properties/bankfull_depth.map # constant channel width (optional) @@ -400,15 +400,15 @@ minimumChannelWidth = global_05min/routing/channel_properties/bankfull_width.ma # channel properties for flooding bankfullCapacity = None -# - If None, it will be estimated from (bankfull) channel depth (m) and width (m) +# - If None, it will be estimated from (bankfull) channel depth (m) and width (m) -# files for relative elevation (above minimum dem) +# files for relative elevation (above minimum dem) relativeElevationFiles = global_05min/routing/channel_properties/dzRel%04d.map relativeElevationLevels = 0.0, 0.01, 0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 1.00 -# composite crop factors for WaterBodies: +# composite crop factors for WaterBodies: cropCoefficientWaterNC = global_30min/routing/kc_surface_water/cropCoefficientForOpenWater.nc minCropWaterKC = 1.00 @@ -436,7 +436,7 @@ avgLakeReservoirOutflowLongIni = global_05min/initialConditions/avgLakeReservoir # # number of days (timesteps) that have been performed for spinning up initial conditions in the routing module (i.e. channelStorageIni, avgDischargeLongIni, avgDischargeShortIni, etc.) timestepsToAvgDischargeIni = global_05min/initialConditions/timestepsToAvgDischarge_1999-12-31.map -# Note that: +# Note that: # - maximum number of days (timesteps) to calculate long term average flow values (default: 5 years = 5 * 365 days = 1825) # - maximum number of days (timesteps) to calculate short term average values (default: 1 month = 1 * 30 days = 30) @@ -464,6 +464,3 @@ outAnnuaMaxNC = None # netcdf format and zlib setup formatNetCDF = NETCDF4 zlib = True - - - diff --git a/docs/observations.rst b/docs/observations.rst index 960518eb..cb943a1d 100644 --- a/docs/observations.rst +++ b/docs/observations.rst @@ -7,7 +7,7 @@ USGS ---- The `U.S. Geological Survey Water Services `_ provides public discharge data for a large number of US based stations. In eWaterCycle we make use of the `USGS web service `_ to automatically retrieve this data. -The Discharge timestamp is corrected to the UTC timezone. Units are converted from cubic feet per second to cubic meter per second. +The Discharge timestamp is corrected to the UTC timezone. Units are converted from cubic feet per second to cubic meter per second. GRDC ---- diff --git a/docs/user_guide.ipynb b/docs/user_guide.ipynb index 5fa31a6f..9de9101e 100644 --- a/docs/user_guide.ipynb +++ b/docs/user_guide.ipynb @@ -809,7 +809,7 @@ " station_id=grdc_station_id,\n", " start_time=\"1990-01-01T00:00:00Z\", # or: model_instance.start_time_as_isostr\n", " end_time=\"1990-12-15T00:00:00Z\",\n", - " column=\"GRDC\"\n", + " column=\"GRDC\",\n", ")\n", "\n", "observations.head()" @@ -1012,4 +1012,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/environment.yml b/environment.yml index ccfd8f99..1da148ff 100644 --- a/environment.yml +++ b/environment.yml @@ -9,5 +9,6 @@ dependencies: - esmvaltool-python>=2.3.0 - subversion # Pin esmpy so we dont get forced to run all parallel tasks on single cpu - # Can be removed once https://github.com/ESMValGroup/ESMValCore/issues/1208 is resolved. + # Can be removed once + # https://github.com/ESMValGroup/ESMValCore/issues/1208 is resolved. - esmpy!=8.1.0 diff --git a/pyproject.toml b/pyproject.toml index de11f324..9a76df37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,3 +5,15 @@ requires = [ ] build-backend = "setuptools.build_meta" +[tool.isort] +profile = "black" +multi_line_output = 3 + +[tool.pylint.messages_control] +disable = "C0330, C0326" + +[tool.pylint.format] +max-line-length = "88" + +[tool.pydocstyle] +convention = "google" diff --git a/setup.cfg b/setup.cfg index e653b2eb..a89c4a9b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,23 +2,20 @@ name = ewatercycle version = 1.1.1 description = A Python package for running and validating a hydrology model +long_description = file: README.md +long_description_content_type = text/markdown +url = https://www.ewatercycle.org/ author = Stefan Verhoeven author_email = s.verhoeven@esciencecenter.nl -url = https://www.ewatercycle.org/ license = Apache Software License 2.0 -long_description = file: README.md -long_description_content_type = text/markdown -keywords = - ewatercycle - FAIR - BMI - Geoscience +license_file = LICENSE classifiers = Development Status :: 2 - Pre-Alpha Intended Audience :: Developers License :: OSI Approved :: Apache Software License Natural Language :: English Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 @@ -26,22 +23,26 @@ classifiers = Topic :: Scientific/Engineering :: GIS Topic :: Scientific/Engineering :: Hydrology Typing :: Typed +keywords = + ewatercycle + FAIR + BMI + Geoscience project_urls = Bug Tracker = https://github.com/eWaterCycle/ewatercycle/issues Documentation = https://ewatercycle.readthedocs.io/ Source Code = https://github.com/eWaterCycle/ewatercycle [options] -zip_safe = False -python_requires = >=3.7 -package_dir = - = src packages = find: install_requires = + Fiona + Shapely basic_modeling_interface cftime esmvaltool>=2.3.0 grpc4bmi>=0.2.12,<0.3 + grpcio hydrostats matplotlib numpy @@ -51,42 +52,38 @@ install_requires = ruamel.yaml scipy xarray - Fiona - Shapely - grpcio +python_requires = >=3.7 +package_dir = + = src +zip_safe = False + +[options.packages.find] +where = src [options.extras_require] dev = - # Test + black + build + bump2version deepdiff + ipython + isort + nbsphinx + pre-commit + pycodestyle pytest pytest-cov pytest-mypy pytest-runner - types-python-dateutil - # Linters - isort - prospector[with_pyroma,with_mypy] - pycodestyle - yapf - # Dependencies for documentation generation - nbsphinx recommonmark sphinx sphinx_rtd_theme - # ipython syntax highlighting is required in doc notebooks - ipython - # release - bump2version - build twine + types-python-dateutil [options.package_data] * = *.yaml -[options.packages.find] -where = src - [coverage:run] branch = True source = src @@ -103,7 +100,6 @@ addopts = --cov-report html --junit-xml=xunit-result.xml -# Define `python setup.py build_sphinx` [build_sphinx] source-dir = docs build-dir = docs/_build @@ -113,3 +109,10 @@ builder = html [mypy] ignore_missing_imports = True files = src, tests + +[flake8] +max-line-length = 88 +extend-ignore = E203,S101 +pytest-fixture-no-parentheses = True +per-file-ignores = + tests/**: D100,D101,D102,D103,D104 diff --git a/src/ewatercycle/analysis/__init__.py b/src/ewatercycle/analysis/__init__.py index 3c661230..5c62dd9f 100644 --- a/src/ewatercycle/analysis/__init__.py +++ b/src/ewatercycle/analysis/__init__.py @@ -1,8 +1,9 @@ -import pandas as pd import os -from typing import Union, Tuple -from hydrostats import metrics +from typing import Tuple, Union + import matplotlib.pyplot as plt +import pandas as pd +from hydrostats import metrics from matplotlib.dates import DateFormatter @@ -12,9 +13,9 @@ def hydrograph( reference: str, precipitation: pd.DataFrame = None, dpi: int = None, - title: str = 'Hydrograph', - discharge_units: str = 'm$^3$ s$^{-1}$', - precipitation_units: str = 'mm day$^{-1}$', + title: str = "Hydrograph", + discharge_units: str = "m$^3$ s$^{-1}$", + precipitation_units: str = "mm day$^{-1}$", figsize: Tuple[float, float] = (10, 10), filename: Union[os.PathLike, str] = None, **kwargs, @@ -68,11 +69,11 @@ def hydrograph( ncols=1, dpi=dpi, figsize=figsize, - gridspec_kw={'height_ratios': [3, 1]}, + gridspec_kw={"height_ratios": [3, 1]}, ) ax.set_title(title) - ax.set_ylabel(f'Discharge ({discharge_units})') + ax.set_ylabel(f"Discharge ({discharge_units})") y_obs.plot(ax=ax, **kwargs) y_sim.plot(ax=ax, **kwargs) @@ -83,11 +84,11 @@ def hydrograph( if precipitation is not None: ax_pr = ax.twinx() ax_pr.invert_yaxis() - ax_pr.set_ylabel(f'Precipitation ({precipitation_units})') + ax_pr.set_ylabel(f"Precipitation ({precipitation_units})") prop_cycler = ax._get_lines.prop_cycler for pr_label, pr_timeseries in precipitation.iteritems(): - color = next(prop_cycler)['color'] + color = next(prop_cycler)["color"] ax_pr.bar( pr_timeseries.index.values, pr_timeseries.values, @@ -106,33 +107,34 @@ def hydrograph( labels = labels_pr + labels # Put the legend outside the plot - ax.legend(handles, labels, bbox_to_anchor=(1.10, 1), loc='upper left') + ax.legend(handles, labels, bbox_to_anchor=(1.10, 1), loc="upper left") # set formatting for xticks date_fmt = DateFormatter("%Y-%m") ax.xaxis.set_major_formatter(date_fmt) - ax.tick_params(axis='x', rotation=30) + ax.tick_params(axis="x", rotation=30) # calculate metrics for data table underneath plot def calc_metric(metric) -> float: return y_sim.apply(metric, observed_array=y_obs) - metrs = pd.DataFrame({ - 'nse': calc_metric(metrics.nse), - 'kge_2009': calc_metric(metrics.kge_2009), - 'sa': calc_metric(metrics.sa), - 'me': calc_metric(metrics.me), - }) + metrs = pd.DataFrame( + { + "nse": calc_metric(metrics.nse), + "kge_2009": calc_metric(metrics.kge_2009), + "sa": calc_metric(metrics.sa), + "me": calc_metric(metrics.me), + } + ) # convert data in dataframe to strings - cell_text = [[f'{item:.2f}' for item in row[1]] - for row in metrs.iterrows()] + cell_text = [[f"{item:.2f}" for item in row[1]] for row in metrs.iterrows()] table = ax_tbl.table( cellText=cell_text, rowLabels=metrs.index, colLabels=metrs.columns, - loc='center', + loc="center", ) ax_tbl.set_axis_off() @@ -140,6 +142,6 @@ def calc_metric(metric) -> float: table.scale(1, 1.5) if filename is not None: - fig.savefig(filename, bbox_inches='tight', dpi=dpi) + fig.savefig(filename, bbox_inches="tight", dpi=dpi) return fig, (ax, ax_tbl) diff --git a/src/ewatercycle/config/__init__.py b/src/ewatercycle/config/__init__.py index ed5d1a3a..2e81d90a 100644 --- a/src/ewatercycle/config/__init__.py +++ b/src/ewatercycle/config/__init__.py @@ -1,8 +1,9 @@ """Config ****** -Configuration of eWaterCycle is done via the :py:class:`~eWaterCycle.config.Config` object. -The global configuration can be imported from the :py:mod:`eWaterCycle` module as :py:data:`~ewatercycle.CFG`: +Configuration of eWaterCycle is done via the +:py:class:`~eWaterCycle.config.Config` object. The global configuration can be +imported from the :py:mod:`eWaterCycle` module as :py:data:`~ewatercycle.CFG`: .. code-block:: python @@ -17,8 +18,9 @@ By default all values are initialized as ``None``. -:py:data:`~ewatercycle.CFG` is essentially a python dictionary with a few extra functions, similar to :py:mod:`matplotlib.rcParams`. -This means that values can be updated like this: +:py:data:`~ewatercycle.CFG` is essentially a python dictionary with a few extra +functions, similar to :py:mod:`matplotlib.rcParams`. This means that values can +be updated like this: .. code-block:: python @@ -26,8 +28,10 @@ >>> CFG['output_dir'] PosixPath('/home/user/output') -Notice that :py:data:`~ewatercycle.CFG` automatically converts the path to an instance of ``pathlib.Path`` and expands the home directory. -All values entered into the config are validated to prevent mistakes, for example, it will warn you if you make a typo in the key: +Notice that :py:data:`~ewatercycle.CFG` automatically converts the path to an +instance of ``pathlib.Path`` and expands the home directory. All values entered +into the config are validated to prevent mistakes, for example, it will warn you +if you make a typo in the key: .. code-block:: python @@ -41,9 +45,9 @@ >>> CFG['output_dir'] = 123 InvalidConfigParameter: Key `output_dir`: Expected a path, but got 123 -By default, the config is loaded from the default location (i.e. ``~/.config/ewatercycle/ewatercycle.yaml``). -If it does not exist, it falls back to the default values. -to load a different file: +By default, the config is loaded from the default location (i.e. +``~/.config/ewatercycle/ewatercycle.yaml``). If it does not exist, it falls back +to the default values. to load a different file: .. code-block:: python @@ -74,17 +78,12 @@ container_engine: singularity singularity_dir: /data/singularity-images output_dir: /scratch - # Created with cd /data/singularity-images && singularity pull docker://ewatercycle/wflow-grpc4bmi:2020.1.1 + # Created with cd /data/singularity-images && + # singularity pull docker://ewatercycle/wflow-grpc4bmi:2020.1.1 wflow.singularity_images: wflow-grpc4bmi_2020.1.1.sif wflow.docker_images: ewatercycle/wflow-grpc4bmi:2020.1.1 """ -from ._config_object import CFG, Config, SYSTEM_CONFIG, USER_HOME_CONFIG, DEFAULT_CONFIG +from ._config_object import CFG, DEFAULT_CONFIG, SYSTEM_CONFIG, USER_HOME_CONFIG, Config -__all__ = [ - 'CFG', - 'Config', - 'DEFAULT_CONFIG', - 'SYSTEM_CONFIG', - 'USER_HOME_CONFIG' -] +__all__ = ["CFG", "Config", "DEFAULT_CONFIG", "SYSTEM_CONFIG", "USER_HOME_CONFIG"] diff --git a/src/ewatercycle/config/_config_object.py b/src/ewatercycle/config/_config_object.py index c5020e84..d5e99310 100644 --- a/src/ewatercycle/config/_config_object.py +++ b/src/ewatercycle/config/_config_object.py @@ -4,15 +4,15 @@ from io import StringIO from logging import getLogger from pathlib import Path -from typing import Union, Optional, TextIO +from typing import Optional, TextIO, Union from ruamel.yaml import YAML -from ._validators import _validators -from ._validated_config import ValidatedConfig - from ewatercycle.util import to_absolute_path +from ._validated_config import ValidatedConfig +from ._validators import _validators + logger = getLogger(__name__) @@ -26,7 +26,7 @@ class Config(ValidatedConfig): _validate = _validators @classmethod - def _load_user_config(cls, filename: Union[os.PathLike, str]) -> 'Config': + def _load_user_config(cls, filename: Union[os.PathLike, str]) -> "Config": """Load user configuration from the given file. The config is cleared and updated in-place. @@ -38,7 +38,7 @@ def _load_user_config(cls, filename: Union[os.PathLike, str]) -> 'Config': """ new = cls() mapping = read_config_file(filename) - mapping['ewatercycle_config'] = filename + mapping["ewatercycle_config"] = filename new.update(CFG_DEFAULT) new.update(mapping) @@ -46,7 +46,7 @@ def _load_user_config(cls, filename: Union[os.PathLike, str]) -> 'Config': return new @classmethod - def _load_default_config(cls, filename: Union[os.PathLike, str]) -> 'Config': + def _load_default_config(cls, filename: Union[os.PathLike, str]) -> "Config": """Load the default configuration.""" new = cls() mapping = read_config_file(filename) @@ -58,7 +58,7 @@ def load_from_file(self, filename: Union[os.PathLike, str]) -> None: """Load user configuration from the given file.""" path = to_absolute_path(str(filename)) if not path.exists(): - raise FileNotFoundError(f'Cannot find: `{filename}') + raise FileNotFoundError(f"Cannot find: `{filename}") self.clear() self.update(CFG_DEFAULT) @@ -66,12 +66,11 @@ def load_from_file(self, filename: Union[os.PathLike, str]) -> None: def reload(self) -> None: """Reload the config file.""" - filename = self.get('ewatercycle_config', DEFAULT_CONFIG) + filename = self.get("ewatercycle_config", DEFAULT_CONFIG) self.load_from_file(filename) def dump_to_yaml(self) -> str: - """Dumps YAML formatted string of Config object - """ + """Dumps YAML formatted string of Config object""" stream = StringIO() self._save_to_stream(stream) return stream.getvalue() @@ -87,7 +86,7 @@ def _save_to_stream(self, stream: TextIO): cp["output_dir"] = str(cp["output_dir"]) cp["parameterset_dir"] = str(cp["parameterset_dir"]) - yaml = YAML(typ='safe') + yaml = YAML(typ="safe") yaml.dump(cp, stream) def save_to_file(self, config_file: Optional[Union[os.PathLike, str]] = None): @@ -95,17 +94,20 @@ def save_to_file(self, config_file: Optional[Union[os.PathLike, str]] = None): Args: config_file: File to write configuration object to. - If not given then will try to use `CFG['ewatercycle_config']` location - and if `CFG['ewatercycle_config']` is not set then will use the location in users home directory. + If not given then will try to use `CFG['ewatercycle_config']` + location and if `CFG['ewatercycle_config']` is not set then will use + the location in users home directory. """ # Exclude own path from dump old_config_file = self.get("ewatercycle_config", None) if config_file is None: - config_file = USER_HOME_CONFIG if old_config_file is None else old_config_file + config_file = ( + USER_HOME_CONFIG if old_config_file is None else old_config_file + ) if config_file == DEFAULT_CONFIG: - raise PermissionError(f'Not allowed to write to {config_file}', config_file) + raise PermissionError(f"Not allowed to write to {config_file}", config_file) with open(config_file, "w") as f: self._save_to_stream(f) @@ -119,10 +121,10 @@ def read_config_file(config_file: Union[os.PathLike, str]) -> dict: """Read config user file and store settings in a dictionary.""" config_file = to_absolute_path(str(config_file)) if not config_file.exists(): - raise IOError(f'Config file `{config_file}` does not exist.') + raise IOError(f"Config file `{config_file}` does not exist.") - with open(config_file, 'r') as file: - yaml = YAML(typ='safe') + with open(config_file, "r") as file: + yaml = YAML(typ="safe") cfg = yaml.load(file) return cfg @@ -137,15 +139,17 @@ def find_user_config(sources: tuple) -> Optional[os.PathLike]: return None -FILENAME = 'ewatercycle.yaml' +FILENAME = "ewatercycle.yaml" -USER_HOME_CONFIG = Path.home() / os.environ.get('XDG_CONFIG_HOME', '.config') / 'ewatercycle' / FILENAME -SYSTEM_CONFIG = Path('/etc') / FILENAME - -SOURCES = ( - USER_HOME_CONFIG, - SYSTEM_CONFIG +USER_HOME_CONFIG = ( + Path.home() + / os.environ.get("XDG_CONFIG_HOME", ".config") + / "ewatercycle" + / FILENAME ) +SYSTEM_CONFIG = Path("/etc") / FILENAME + +SOURCES = (USER_HOME_CONFIG, SYSTEM_CONFIG) USER_CONFIG = find_user_config(SOURCES) DEFAULT_CONFIG = Path(__file__).parent / FILENAME diff --git a/src/ewatercycle/config/_validated_config.py b/src/ewatercycle/config/_validated_config.py index b527f410..ae02089f 100644 --- a/src/ewatercycle/config/_validated_config.py +++ b/src/ewatercycle/config/_validated_config.py @@ -38,7 +38,8 @@ def __setitem__(self, key, val): raise InvalidConfigParameter(f"Key `{key}`: {verr}") from None except KeyError: raise InvalidConfigParameter( - f"`{key}` is not a valid config parameter.") from None + f"`{key}` is not a valid config parameter." + ) from None self._mapping[key] = cval @@ -50,15 +51,15 @@ def __repr__(self): """Return canonical string representation.""" class_name = self.__class__.__name__ indent = len(class_name) + 1 - repr_split = pprint.pformat(self._mapping, indent=1, - width=80 - indent).split('\n') - repr_indented = ('\n' + ' ' * indent).join(repr_split) - return '{}({})'.format(class_name, repr_indented) + repr_split = pprint.pformat(self._mapping, indent=1, width=80 - indent).split( + "\n" + ) + repr_indented = ("\n" + " " * indent).join(repr_split) + return "{}({})".format(class_name, repr_indented) def __str__(self): """Return string representation.""" - return '\n'.join( - map('{0[0]}: {0[1]}'.format, sorted(self._mapping.items()))) + return "\n".join(map("{0[0]}: {0[1]}".format, sorted(self._mapping.items()))) def __iter__(self): """Yield sorted list of keys.""" diff --git a/src/ewatercycle/config/_validators.py b/src/ewatercycle/config/_validators.py index c176e4c4..5b45f3ea 100644 --- a/src/ewatercycle/config/_validators.py +++ b/src/ewatercycle/config/_validators.py @@ -65,26 +65,18 @@ def func(inp): if allow_stringlist: # Sometimes, a list of colors might be a single string # of single-letter colornames. So give that a shot. - inp = [ - scalar_validator(val.strip()) - for val in inp - if val.strip() - ] + inp = [scalar_validator(val.strip()) for val in inp if val.strip()] else: raise # Allow any ordered sequence type -- generators, np.ndarray, pd.Series # -- but not sets, whose iteration order is non-deterministic. - elif isinstance(inp, Iterable) and not isinstance( - inp, (set, frozenset) - ): + elif isinstance(inp, Iterable) and not isinstance(inp, (set, frozenset)): # The condition on this list comprehension will preserve the # behavior of filtering out any empty strings (behavior was # from the original validate_stringlist()), while allowing # any non-string/text scalar values such as numbers and arrays. inp = [ - scalar_validator(val) - for val in inp - if not isinstance(val, str) or val + scalar_validator(val) for val in inp if not isinstance(val, str) or val ] else: raise ValidationError( @@ -101,9 +93,7 @@ def func(inp): func.__name__ = "{}list".format(scalar_validator.__name__) except AttributeError: # class instance. func.__name__ = "{}List".format(type(scalar_validator).__name__) - func.__qualname__ = ( - func.__qualname__.rsplit(".", 1)[0] + "." + func.__name__ - ) + func.__qualname__ = func.__qualname__.rsplit(".", 1)[0] + "." + func.__name__ if docstring is not None: docstring = scalar_validator.__doc__ func.__doc__ = docstring diff --git a/src/ewatercycle/forcing/__init__.py b/src/ewatercycle/forcing/__init__.py index 84d51ea9..ab8495c8 100644 --- a/src/ewatercycle/forcing/__init__.py +++ b/src/ewatercycle/forcing/__init__.py @@ -1,13 +1,13 @@ from pathlib import Path -from typing import Optional, Dict, Type +from typing import Dict, Optional, Type from ruamel.yaml import YAML -from ._default import DefaultForcing, FORCING_YAML -from . import _hype, _lisflood, _marrmot, _pcrglobwb, _wflow - from ewatercycle.util import to_absolute_path +from . import _hype, _lisflood, _marrmot, _pcrglobwb, _wflow +from ._default import FORCING_YAML, DefaultForcing + FORCING_CLASSES: Dict[str, Type[DefaultForcing]] = { "hype": _hype.HypeForcing, "lisflood": _lisflood.LisfloodForcing, @@ -37,18 +37,20 @@ def load(directory: str) -> DefaultForcing: forcing_info = yaml.load(source / FORCING_YAML) forcing_info.directory = source if forcing_info.shape: - forcing_info.shape = to_absolute_path(forcing_info.shape, parent = source) + forcing_info.shape = to_absolute_path(forcing_info.shape, parent=source) return forcing_info # Or load_custom , load_external, load_???., from_external, import_forcing, -def load_foreign(target_model, - start_time: str, - end_time: str, - directory: str = '.', - shape: str = None, - forcing_info: Optional[Dict] = None) -> DefaultForcing: +def load_foreign( + target_model, + start_time: str, + end_time: str, + directory: str = ".", + shape: str = None, + forcing_info: Optional[Dict] = None, +) -> DefaultForcing: """Load existing forcing data generated from an external source. Args: @@ -105,8 +107,9 @@ def load_foreign(target_model, constructor = FORCING_CLASSES.get(target_model, None) if constructor is None: raise NotImplementedError( - f'Target model `{target_model}` is not supported by the ' - 'eWatercycle forcing generator.') + f"Target model `{target_model}` is not supported by the " + "eWatercycle forcing generator." + ) if forcing_info is None: forcing_info = {} return constructor( # type: ignore # each subclass can have different forcing_info @@ -118,12 +121,14 @@ def load_foreign(target_model, ) -def generate(target_model: str, - dataset: str, - start_time: str, - end_time: str, - shape: str, - model_specific_options: Optional[Dict] = None) -> DefaultForcing: +def generate( + target_model: str, + dataset: str, + start_time: str, + end_time: str, + shape: str, + model_specific_options: Optional[Dict] = None, +) -> DefaultForcing: """Generate forcing data with ESMValTool. Args: @@ -146,19 +151,23 @@ def generate(target_model: str, constructor = FORCING_CLASSES.get(target_model, None) if constructor is None: raise NotImplementedError( - f'Target model `{target_model}` is not supported by the ' - 'eWatercycle forcing generator') + f"Target model `{target_model}` is not supported by the " + "eWatercycle forcing generator" + ) if model_specific_options is None: model_specific_options = {} - forcing_info = constructor.generate(dataset, start_time, end_time, shape, - **model_specific_options) + forcing_info = constructor.generate( + dataset, start_time, end_time, shape, **model_specific_options + ) forcing_info.save() return forcing_info # Append docstrings of with model-specific options to existing docstring load_foreign.__doc__ += "".join( # type:ignore - [f"\n {k}: {v.__init__.__doc__}" for k, v in FORCING_CLASSES.items()]) + [f"\n {k}: {v.__init__.__doc__}" for k, v in FORCING_CLASSES.items()] +) generate.__doc__ += "".join( # type:ignore - [f"\n {k}: {v.generate.__doc__}" for k, v in FORCING_CLASSES.items()]) + [f"\n {k}: {v.generate.__doc__}" for k, v in FORCING_CLASSES.items()] +) diff --git a/src/ewatercycle/forcing/_default.py b/src/ewatercycle/forcing/_default.py index e06b2199..19b8fff6 100644 --- a/src/ewatercycle/forcing/_default.py +++ b/src/ewatercycle/forcing/_default.py @@ -1,8 +1,7 @@ """Forcing related functionality for default models""" +import logging from copy import copy -from pathlib import Path from typing import Optional -import logging from ruamel.yaml import YAML @@ -10,7 +9,7 @@ logger = logging.getLogger(__name__) -FORCING_YAML = 'ewatercycle_forcing.yaml' +FORCING_YAML = "ewatercycle_forcing.yaml" class DefaultForcing: @@ -24,11 +23,14 @@ class DefaultForcing: 'YYYY-MM-DDTHH:MM:SSZ'. shape: Path to a shape file. Used for spatial selection. """ - def __init__(self, - start_time: str, - end_time: str, - directory: str, - shape: Optional[str] = None): + + def __init__( + self, + start_time: str, + end_time: str, + directory: str, + shape: Optional[str] = None, + ): self.start_time = start_time self.end_time = end_time self.directory = to_absolute_path(directory) @@ -52,7 +54,7 @@ def generate( end_time: str, shape: str, **model_specific_options, - ) -> 'DefaultForcing': + ) -> "DefaultForcing": """Generate forcing data with ESMValTool.""" raise NotImplementedError("No default forcing generator available.") @@ -71,9 +73,12 @@ def save(self): clone.shape = str(clone.shape.relative_to(self.directory)) except ValueError: clone.shape = None - logger.info(f"Shapefile {self.shape} is not in forcing directory {self.directory}. So, it won't be saved in {target}.") + logger.info( + f"Shapefile {self.shape} is not in forcing directory " + f"{self.directory}. So, it won't be saved in {target}." + ) - with open(target, 'w') as f: + with open(target, "w") as f: yaml.dump(clone, f) return target diff --git a/src/ewatercycle/forcing/_hype.py b/src/ewatercycle/forcing/_hype.py index 5e2d91b0..cf80894e 100644 --- a/src/ewatercycle/forcing/_hype.py +++ b/src/ewatercycle/forcing/_hype.py @@ -12,6 +12,7 @@ class HypeForcing(DefaultForcing): """Container for hype forcing data.""" + def __init__( self, start_time: str, @@ -20,55 +21,58 @@ def __init__( shape: Optional[str] = None, ): """ - None: Hype does not have model-specific load options. + None: Hype does not have model-specific load options. """ super().__init__(start_time, end_time, directory, shape) @classmethod def generate( # type: ignore - cls, dataset: str, start_time: str, end_time: str, - shape: str) -> 'HypeForcing': + cls, dataset: str, start_time: str, end_time: str, shape: str + ) -> "HypeForcing": """ - None: Hype does not have model-specific generate options. + None: Hype does not have model-specific generate options. """ # load the ESMValTool recipe recipe_name = "hydrology/recipe_hype.yml" recipe = get_recipe(recipe_name) # model-specific updates to the recipe - preproc_names = ('preprocessor', 'temperature', 'water') + preproc_names = ("preprocessor", "temperature", "water") for preproc_name in preproc_names: - recipe.data['preprocessors'][preproc_name]['extract_shape'][ - 'shapefile'] = to_absolute_path(shape) + recipe.data["preprocessors"][preproc_name]["extract_shape"][ + "shapefile" + ] = to_absolute_path(shape) - recipe.data['datasets'] = [DATASETS[dataset]] + recipe.data["datasets"] = [DATASETS[dataset]] - variables = recipe.data['diagnostics']['hype']['variables'] - var_names = 'tas', 'tasmin', 'tasmax', 'pr' + variables = recipe.data["diagnostics"]["hype"]["variables"] + var_names = "tas", "tasmin", "tasmax", "pr" startyear = get_time(start_time).year for var_name in var_names: - variables[var_name]['start_year'] = startyear + variables[var_name]["start_year"] = startyear endyear = get_time(end_time).year for var_name in var_names: - variables[var_name]['end_year'] = endyear + variables[var_name]["end_year"] = endyear # generate forcing data and retreive useful information recipe_output = recipe.run() # TODO return files created by ESMValTOOL which are needed by Hype Model # forcing_path = list(recipe_output['...........']).data_files[0] - forcing_path = '/foobar.txt' + forcing_path = "/foobar.txt" forcing_file = Path(forcing_path).name directory = str(Path(forcing_file).parent) # instantiate forcing object based on generated data - return HypeForcing(directory=directory, - start_time=str(startyear), - end_time=str(endyear), - shape=shape) + return HypeForcing( + directory=directory, + start_time=str(startyear), + end_time=str(endyear), + shape=shape, + ) def plot(self): - raise NotImplementedError('Dont know how to plot') + raise NotImplementedError("Dont know how to plot") diff --git a/src/ewatercycle/forcing/_lisflood.py b/src/ewatercycle/forcing/_lisflood.py index 9ecc1b19..28f7e6cc 100644 --- a/src/ewatercycle/forcing/_lisflood.py +++ b/src/ewatercycle/forcing/_lisflood.py @@ -1,16 +1,22 @@ """Forcing related functionality for lisflood""" -from typing import Optional import logging +from typing import Optional from esmvalcore.experimental import get_recipe -from ..util import data_files_from_recipe_output, get_extents, get_time, to_absolute_path +from ..util import ( + data_files_from_recipe_output, + get_extents, + get_time, + to_absolute_path, +) from ._default import DefaultForcing from .datasets import DATASETS logger = logging.getLogger(__name__) + class LisfloodForcing(DefaultForcing): """Container for lisflood forcing data.""" @@ -21,23 +27,23 @@ def __init__( end_time: str, directory: str, shape: Optional[str] = None, - PrefixPrecipitation: str = 'pr.nc', - PrefixTavg: str = 'tas.nc', - PrefixE0: str = 'e0.nc', - PrefixES0: str = 'es0.nc', - PrefixET0: str = 'et0.nc', + PrefixPrecipitation: str = "pr.nc", + PrefixTavg: str = "tas.nc", + PrefixE0: str = "e0.nc", + PrefixES0: str = "es0.nc", + PrefixET0: str = "et0.nc", ): """ - PrefixPrecipitation: Path to a NetCDF or pcraster file with - precipitation data - PrefixTavg: Path to a NetCDF or pcraster file with average - temperature data - PrefixE0: Path to a NetCDF or pcraster file with potential - evaporation rate from open water surface data - PrefixES0: Path to a NetCDF or pcraster file with potential - evaporation rate from bare soil surface data - PrefixET0: Path to a NetCDF or pcraster file with potential - (reference) evapotranspiration rate data + PrefixPrecipitation: Path to a NetCDF or pcraster file with + precipitation data + PrefixTavg: Path to a NetCDF or pcraster file with average + temperature data + PrefixE0: Path to a NetCDF or pcraster file with potential + evaporation rate from open water surface data + PrefixES0: Path to a NetCDF or pcraster file with potential + evaporation rate from bare soil surface data + PrefixET0: Path to a NetCDF or pcraster file with potential + (reference) evapotranspiration rate data """ super().__init__(start_time, end_time, directory, shape) self.PrefixPrecipitation = PrefixPrecipitation @@ -55,47 +61,56 @@ def generate( # type: ignore shape: str, extract_region: dict = None, run_lisvap: bool = False, - ) -> 'LisfloodForcing': + ) -> "LisfloodForcing": """ - extract_region (dict): Region specification, dictionary must contain `start_longitude`, - `end_longitude`, `start_latitude`, `end_latitude` - run_lisvap (bool): if lisvap should be run. Default is False. Running lisvap is not supported yet. - TODO add regrid options so forcing can be generated for parameter set - TODO that is not on a 0.1x0.1 grid + extract_region (dict): Region specification, dictionary must contain + `start_longitude`, `end_longitude`, `start_latitude`, `end_latitude` + run_lisvap (bool): if lisvap should be run. Default is False. + Running lisvap is not supported yet. + TODO add regrid options so forcing can be generated for parameter set + TODO that is not on a 0.1x0.1 grid """ # load the ESMValTool recipe recipe_name = "hydrology/recipe_lisflood.yml" recipe = get_recipe(recipe_name) # model-specific updates to the recipe - preproc_names = ('general', 'daily_water', 'daily_temperature', - 'daily_radiation', 'daily_windspeed') + preproc_names = ( + "general", + "daily_water", + "daily_temperature", + "daily_radiation", + "daily_windspeed", + ) basin = to_absolute_path(shape).stem for preproc_name in preproc_names: - recipe.data['preprocessors'][preproc_name]['extract_shape'][ - 'shapefile'] = shape - recipe.data['diagnostics']['diagnostic_daily']['scripts']['script'][ - 'catchment'] = basin + recipe.data["preprocessors"][preproc_name]["extract_shape"][ + "shapefile" + ] = shape + recipe.data["diagnostics"]["diagnostic_daily"]["scripts"]["script"][ + "catchment" + ] = basin if extract_region is None: extract_region = get_extents(shape) for preproc_name in preproc_names: - recipe.data['preprocessors'][preproc_name][ - 'extract_region'] = extract_region + recipe.data["preprocessors"][preproc_name][ + "extract_region" + ] = extract_region - recipe.data['datasets'] = [DATASETS[dataset]] + recipe.data["datasets"] = [DATASETS[dataset]] - variables = recipe.data['diagnostics']['diagnostic_daily']['variables'] - var_names = 'pr', 'tas', 'tasmax', 'tasmin', 'tdps', 'uas', 'vas', 'rsds' + variables = recipe.data["diagnostics"]["diagnostic_daily"]["variables"] + var_names = "pr", "tas", "tasmax", "tasmin", "tdps", "uas", "vas", "rsds" startyear = get_time(start_time).year for var_name in var_names: - variables[var_name]['start_year'] = startyear + variables[var_name]["start_year"] = startyear endyear = get_time(end_time).year for var_name in var_names: - variables[var_name]['end_year'] = endyear + variables[var_name]["end_year"] = endyear # generate forcing data and retrieve useful information recipe_output = recipe.run() @@ -106,11 +121,13 @@ def generate( # type: ignore # instantiate forcing object based on generated data if run_lisvap: - raise NotImplementedError('Dont know how to run LISVAP.') + raise NotImplementedError("Dont know how to run LISVAP.") else: message = ( - f"Parameter `run_lisvap` is set to False. No forcing data will be generator for 'e0', 'es0' and 'et0'. " - f"However, the recipe creates LISVAP input data that can be found in {directory}.") + "Parameter `run_lisvap` is set to False. No forcing data will be " + "generated for 'e0', 'es0' and 'et0'. However, the recipe creates " + f"LISVAP input data that can be found in {directory}." + ) logger.warning("%s", message) return LisfloodForcing( directory=directory, @@ -121,6 +138,5 @@ def generate( # type: ignore PrefixTavg=forcing_files["tas"], ) - def plot(self): - raise NotImplementedError('Dont know how to plot') + raise NotImplementedError("Dont know how to plot") diff --git a/src/ewatercycle/forcing/_marrmot.py b/src/ewatercycle/forcing/_marrmot.py index 32f80f9c..2fa21cb2 100644 --- a/src/ewatercycle/forcing/_marrmot.py +++ b/src/ewatercycle/forcing/_marrmot.py @@ -12,18 +12,19 @@ class MarrmotForcing(DefaultForcing): """Container for marrmot forcing data.""" + def __init__( self, start_time: str, end_time: str, directory: str, shape: Optional[str] = None, - forcing_file: Optional[str] = 'marrmot.mat', + forcing_file: Optional[str] = "marrmot.mat", ): """ - forcing_file: Matlab file that contains forcings for Marrmot - models. See format forcing file in `model implementation - `_. + forcing_file: Matlab file that contains forcings for Marrmot + models. See format forcing file in `model implementation + `_. """ super().__init__(start_time, end_time, directory, shape) self.forcing_file = forcing_file @@ -35,9 +36,9 @@ def generate( # type: ignore start_time: str, end_time: str, shape: str, - ) -> 'MarrmotForcing': + ) -> "MarrmotForcing": """ - None: Marrmot does not have model-specific generate options. + None: Marrmot does not have model-specific generate options. """ # load the ESMValTool recipe @@ -46,24 +47,25 @@ def generate( # type: ignore # model-specific updates to the recipe basin = to_absolute_path(shape).stem - recipe.data['preprocessors']['daily']['extract_shape'][ - 'shapefile'] = shape - recipe.data['diagnostics']['diagnostic_daily']['scripts']['script'][ - 'basin'] = basin + recipe.data["preprocessors"]["daily"]["extract_shape"]["shapefile"] = shape + recipe.data["diagnostics"]["diagnostic_daily"]["scripts"]["script"][ + "basin" + ] = basin - recipe.data['diagnostics']['diagnostic_daily'][ - 'additional_datasets'] = [DATASETS[dataset]] + recipe.data["diagnostics"]["diagnostic_daily"]["additional_datasets"] = [ + DATASETS[dataset] + ] - variables = recipe.data['diagnostics']['diagnostic_daily']['variables'] - var_names = 'tas', 'pr', 'psl', 'rsds', 'rsdt' + variables = recipe.data["diagnostics"]["diagnostic_daily"]["variables"] + var_names = "tas", "pr", "psl", "rsds", "rsdt" startyear = get_time(start_time).year for var_name in var_names: - variables[var_name]['start_year'] = startyear + variables[var_name]["start_year"] = startyear endyear = get_time(end_time).year for var_name in var_names: - variables[var_name]['end_year'] = endyear + variables[var_name]["end_year"] = endyear # generate forcing data and retrieve useful information recipe_output = recipe.run() @@ -72,11 +74,13 @@ def generate( # type: ignore directory = str(Path(forcing_file).parent) # instantiate forcing object based on generated data - return MarrmotForcing(directory=directory, - start_time=start_time, - end_time=end_time, - shape=shape, - forcing_file=forcing_file.name) + return MarrmotForcing( + directory=directory, + start_time=start_time, + end_time=end_time, + shape=shape, + forcing_file=forcing_file.name, + ) def plot(self): - raise NotImplementedError('Dont know how to plot') + raise NotImplementedError("Dont know how to plot") diff --git a/src/ewatercycle/forcing/_pcrglobwb.py b/src/ewatercycle/forcing/_pcrglobwb.py index c3657f83..7577be8c 100644 --- a/src/ewatercycle/forcing/_pcrglobwb.py +++ b/src/ewatercycle/forcing/_pcrglobwb.py @@ -5,7 +5,12 @@ from esmvalcore.experimental import get_recipe -from ..util import data_files_from_recipe_output, get_extents, get_time, to_absolute_path +from ..util import ( + data_files_from_recipe_output, + get_extents, + get_time, + to_absolute_path, +) from ._default import DefaultForcing from .datasets import DATASETS @@ -38,7 +43,7 @@ def generate( # type: ignore end_time: str, shape: str, start_time_climatology: str, # TODO make optional, default to start_time - end_time_climatology: str, # TODO make optional, defaults to start_time + 1 year + end_time_climatology: str, # TODO make optional, defaults to start_time + 1 y extract_region: dict = None, ) -> "PCRGlobWBForcing": """ @@ -62,9 +67,9 @@ def generate( # type: ignore ) if dataset is not None: - recipe.data["diagnostics"]["diagnostic_daily"][ - "additional_datasets" - ] = [DATASETS[dataset]] + recipe.data["diagnostics"]["diagnostic_daily"]["additional_datasets"] = [ + DATASETS[dataset] + ] basin = to_absolute_path(shape).stem recipe.data["diagnostics"]["diagnostic_daily"]["scripts"]["script"][ @@ -101,7 +106,7 @@ def generate( # type: ignore # generate forcing data and retrieve useful information recipe_output = recipe.run() - # TODO dont open recipe output files, but use standard name from ESMValTool diagnostic + # TODO dont open recipe output, but use standard name from ESMValTool directory, forcing_files = data_files_from_recipe_output(recipe_output) # instantiate forcing object based on generated data diff --git a/src/ewatercycle/forcing/_wflow.py b/src/ewatercycle/forcing/_wflow.py index de23453c..75049536 100644 --- a/src/ewatercycle/forcing/_wflow.py +++ b/src/ewatercycle/forcing/_wflow.py @@ -11,6 +11,7 @@ class WflowForcing(DefaultForcing): """Container for wflow forcing data.""" + def __init__( self, start_time: str, @@ -24,14 +25,14 @@ def __init__( Inflow: Optional[str] = None, ): """ - netcdfinput (str) = "inmaps.nc": Path to forcing file." - Precipitation (str) = "/pr": Variable name of precipitation data in - input file. - EvapoTranspiration (str) = "/pet": Variable name of - evapotranspiration data in input file. - Temperature (str) = "/tas": Variable name of temperature data in - input file. - Inflow (str) = None: Variable name of inflow data in input file. + netcdfinput (str) = "inmaps.nc": Path to forcing file." + Precipitation (str) = "/pr": Variable name of precipitation data in + input file. + EvapoTranspiration (str) = "/pet": Variable name of + evapotranspiration data in input file. + Temperature (str) = "/tas": Variable name of temperature data in + input file. + Inflow (str) = None: Variable name of inflow data in input file. """ super().__init__(start_time, end_time, directory, shape) self.netcdfinput = netcdfinput @@ -49,76 +50,78 @@ def generate( # type: ignore shape: str, dem_file: str, extract_region: Dict[str, float] = None, - ) -> 'WflowForcing': + ) -> "WflowForcing": """ - dem_file (str): Name of the dem_file to use. Also defines the basin - param. - extract_region (dict): Region specification, dictionary must - contain `start_longitude`, `end_longitude`, `start_latitude`, - `end_latitude` + dem_file (str): Name of the dem_file to use. Also defines the basin + param. + extract_region (dict): Region specification, dictionary must + contain `start_longitude`, `end_longitude`, `start_latitude`, + `end_latitude` """ # load the ESMValTool recipe recipe_name = "hydrology/recipe_wflow.yml" recipe = get_recipe(recipe_name) basin = to_absolute_path(shape).stem - recipe.data['diagnostics']['wflow_daily']['scripts']['script'][ - 'basin'] = basin + recipe.data["diagnostics"]["wflow_daily"]["scripts"]["script"]["basin"] = basin # model-specific updates - script = recipe.data['diagnostics']['wflow_daily']['scripts']['script'] - script['dem_file'] = dem_file + script = recipe.data["diagnostics"]["wflow_daily"]["scripts"]["script"] + script["dem_file"] = dem_file if extract_region is None: extract_region = get_extents(shape) - recipe.data['preprocessors']['rough_cutout'][ - 'extract_region'] = extract_region + recipe.data["preprocessors"]["rough_cutout"]["extract_region"] = extract_region - recipe.data['diagnostics']['wflow_daily']['additional_datasets'] = [ + recipe.data["diagnostics"]["wflow_daily"]["additional_datasets"] = [ DATASETS[dataset] ] - variables = recipe.data['diagnostics']['wflow_daily']['variables'] - var_names = 'tas', 'pr', 'psl', 'rsds', 'rsdt' + variables = recipe.data["diagnostics"]["wflow_daily"]["variables"] + var_names = "tas", "pr", "psl", "rsds", "rsdt" startyear = get_time(start_time).year for var_name in var_names: - variables[var_name]['start_year'] = startyear + variables[var_name]["start_year"] = startyear endyear = get_time(end_time).year for var_name in var_names: - variables[var_name]['end_year'] = endyear + variables[var_name]["end_year"] = endyear # generate forcing data and retreive useful information recipe_output = recipe.run() - forcing_data = recipe_output['wflow_daily/script'].data_files[0] + forcing_data = recipe_output["wflow_daily/script"].data_files[0] forcing_file = forcing_data.path directory = str(forcing_file.parent) # instantiate forcing object based on generated data - return WflowForcing(directory=directory, - start_time=start_time, - end_time=end_time, - shape=shape, - netcdfinput=forcing_file.name) + return WflowForcing( + directory=directory, + start_time=start_time, + end_time=end_time, + shape=shape, + netcdfinput=forcing_file.name, + ) def __str__(self): """Nice formatting of the forcing data object.""" - return "\n".join([ - "Forcing data for Wflow", - "----------------------", - f"Directory: {self.directory}", - f"Start time: {self.start_time}", - f"End time: {self.end_time}", - f"Shapefile: {self.shape}", - f"Additional information for model config:", - f" - netcdfinput: {self.netcdfinput}", - f" - Precipitation: {self.Precipitation}", - f" - Temperature: {self.Temperature}", - f" - EvapoTranspiration: {self.EvapoTranspiration}", - f" - Inflow: {self.Inflow}", - ]) + return "\n".join( + [ + "Forcing data for Wflow", + "----------------------", + f"Directory: {self.directory}", + f"Start time: {self.start_time}", + f"End time: {self.end_time}", + f"Shapefile: {self.shape}", + f"Additional information for model config:", + f" - netcdfinput: {self.netcdfinput}", + f" - Precipitation: {self.Precipitation}", + f" - Temperature: {self.Temperature}", + f" - EvapoTranspiration: {self.EvapoTranspiration}", + f" - Inflow: {self.Inflow}", + ] + ) def plot(self): - raise NotImplementedError('Dont know how to plot') + raise NotImplementedError("Dont know how to plot") diff --git a/src/ewatercycle/forcing/datasets.py b/src/ewatercycle/forcing/datasets.py index 70d80858..1d87d1d6 100644 --- a/src/ewatercycle/forcing/datasets.py +++ b/src/ewatercycle/forcing/datasets.py @@ -4,18 +4,18 @@ """ DATASETS = { - 'ERA5': { - 'dataset': 'ERA5', - 'project': 'OBS6', - 'tier': 3, - 'type': 'reanaly', - 'version': 1 + "ERA5": { + "dataset": "ERA5", + "project": "OBS6", + "tier": 3, + "type": "reanaly", + "version": 1, }, - 'ERA-Interim': { - 'dataset': 'ERA-Interim', - 'project': 'OBS6', - 'tier': 3, - 'type': 'reanaly', - 'version': 1 + "ERA-Interim": { + "dataset": "ERA-Interim", + "project": "OBS6", + "tier": 3, + "type": "reanaly", + "version": 1, }, } diff --git a/src/ewatercycle/models/__init__.py b/src/ewatercycle/models/__init__.py index 0c72ea4b..9dd9f9de 100644 --- a/src/ewatercycle/models/__init__.py +++ b/src/ewatercycle/models/__init__.py @@ -1,5 +1,4 @@ from .lisflood import Lisflood -from .wflow import Wflow -from .marrmot import MarrmotM01 -from .marrmot import MarrmotM14 +from .marrmot import MarrmotM01, MarrmotM14 from .pcrglobwb import PCRGlobWB +from .wflow import Wflow diff --git a/src/ewatercycle/models/abstract.py b/src/ewatercycle/models/abstract.py index 78137ba1..e1f4ef5f 100644 --- a/src/ewatercycle/models/abstract.py +++ b/src/ewatercycle/models/abstract.py @@ -2,33 +2,33 @@ import textwrap from abc import ABCMeta, abstractmethod from datetime import datetime -from typing import Tuple, Iterable, Any, TypeVar, Generic, Optional, ClassVar, Set +from typing import Any, ClassVar, Generic, Iterable, Optional, Set, Tuple, TypeVar import numpy as np import xarray as xr -from cftime import num2date from basic_modeling_interface import Bmi +from cftime import num2date from ewatercycle.forcing import DefaultForcing from ewatercycle.parameter_sets import ParameterSet logger = logging.getLogger(__name__) -ForcingT = TypeVar('ForcingT', bound=DefaultForcing) +ForcingT = TypeVar("ForcingT", bound=DefaultForcing) class AbstractModel(Generic[ForcingT], metaclass=ABCMeta): - """Abstract class of a eWaterCycle model. + """Abstract class of a eWaterCycle model.""" - """ available_versions: ClassVar[Tuple[str, ...]] = tuple() """Versions of model that are available in this class""" - def __init__(self, - version: str, - parameter_set: ParameterSet = None, - forcing: Optional[ForcingT] = None, - ): + def __init__( + self, + version: str, + parameter_set: ParameterSet = None, + forcing: Optional[ForcingT] = None, + ): self.version = version self.parameter_set = parameter_set self.forcing: Optional[ForcingT] = forcing @@ -51,9 +51,9 @@ def __str__(self): "-------------------", f"Version = {self.version}", "Parameter set = ", - textwrap.indent(str(self.parameter_set), ' '), + textwrap.indent(str(self.parameter_set), " "), "Forcing = ", - textwrap.indent(str(self.forcing), ' '), + textwrap.indent(str(self.forcing), " "), ] ) @@ -99,7 +99,9 @@ def get_value(self, name: str) -> np.ndarray: """ return self.bmi.get_value(name) - def get_value_at_coords(self, name, lat: Iterable[float], lon: Iterable[float]) -> np.ndarray: + def get_value_at_coords( + self, name, lat: Iterable[float], lon: Iterable[float] + ) -> np.ndarray: """Get a copy of values of the given variable at lat/lon coordinates. Args: @@ -122,7 +124,9 @@ def set_value(self, name: str, value: np.ndarray) -> None: """ self.bmi.set_value(name, value) - def set_value_at_coords(self, name: str, lat: Iterable[float], lon: Iterable[float], values: np.ndarray) -> None: + def set_value_at_coords( + self, name: str, lat: Iterable[float], lon: Iterable[float], values: np.ndarray + ) -> None: """Specify a new value for a model variable at at lat/lon coordinates. Args: @@ -136,7 +140,9 @@ def set_value_at_coords(self, name: str, lat: Iterable[float], lon: Iterable[flo indices = np.array(indices) self.bmi.set_value_at_indices(name, indices, values) - def _coords_to_indices(self, name: str, lat: Iterable[float], lon: Iterable[float]) -> Iterable[int]: + def _coords_to_indices( + self, name: str, lat: Iterable[float], lon: Iterable[float] + ) -> Iterable[int]: """Converts lat/lon values to index. Args: @@ -144,7 +150,10 @@ def _coords_to_indices(self, name: str, lat: Iterable[float], lon: Iterable[floa lon: Longitudinal value """ - raise NotImplementedError("Method to convert from coordinates to model indices not implemented for this model.") + raise NotImplementedError( + "Method to convert from coordinates to model indices " + "not implemented for this model." + ) @abstractmethod def get_value_as_xarray(self, name: str) -> xr.DataArray: @@ -218,27 +227,30 @@ def time_as_isostr(self) -> str: @property def start_time_as_datetime(self) -> datetime: - """Start time of the model as a datetime object. - """ - return num2date(self.bmi.get_start_time(), - self.bmi.get_time_units(), - only_use_cftime_datetimes=False) + """Start time of the model as a datetime object.""" + return num2date( + self.bmi.get_start_time(), + self.bmi.get_time_units(), + only_use_cftime_datetimes=False, + ) @property def end_time_as_datetime(self) -> datetime: - """End time of the model as a datetime object'. - """ - return num2date(self.bmi.get_end_time(), - self.bmi.get_time_units(), - only_use_cftime_datetimes=False) + """End time of the model as a datetime object'.""" + return num2date( + self.bmi.get_end_time(), + self.bmi.get_time_units(), + only_use_cftime_datetimes=False, + ) @property def time_as_datetime(self) -> datetime: - """Current time of the model as a datetime object'. - """ - return num2date(self.bmi.get_current_time(), - self.bmi.get_time_units(), - only_use_cftime_datetimes=False) + """Current time of the model as a datetime object'.""" + return num2date( + self.bmi.get_current_time(), + self.bmi.get_time_units(), + only_use_cftime_datetimes=False, + ) def _check_parameter_set(self): if not self.parameter_set: @@ -246,17 +258,26 @@ def _check_parameter_set(self): return model_name = self.__class__.__name__.lower() if model_name != self.parameter_set.target_model: - raise ValueError(f'Parameter set has wrong target model, ' - f'expected {model_name} got {self.parameter_set.target_model}') + raise ValueError( + f"Parameter set has wrong target model, " + f"expected {model_name} got {self.parameter_set.target_model}" + ) if self.parameter_set.supported_model_versions == set(): - logger.info(f'Model version {self.version} is not explicitly listed in the supported model versions ' - f'of this parameter set. This can lead to compatibility issues.') + logger.info( + f"Model version {self.version} is not explicitly listed in the " + "supported model versions of this parameter set. " + "This can lead to compatibility issues." + ) elif self.version not in self.parameter_set.supported_model_versions: raise ValueError( - f'Parameter set is not compatible with version {self.version} of model, ' - f'parameter set only supports {self.parameter_set.supported_model_versions}') + "Parameter set is not compatible with version {self.version} of " + "model, parameter set only supports " + f"{self.parameter_set.supported_model_versions}." + ) def _check_version(self): if self.version not in self.available_versions: - raise ValueError(f'Supplied version {self.version} is not supported by this model. ' - f'Available versions are {self.available_versions}.') + raise ValueError( + f"Supplied version {self.version} is not supported by this model. " + f"Available versions are {self.available_versions}." + ) diff --git a/src/ewatercycle/models/lisflood.py b/src/ewatercycle/models/lisflood.py index ff9f041d..1820621e 100644 --- a/src/ewatercycle/models/lisflood.py +++ b/src/ewatercycle/models/lisflood.py @@ -1,3 +1,4 @@ +"""Module with Lisflood model.""" import datetime import logging import xml.etree.ElementTree as ET @@ -24,12 +25,13 @@ class Lisflood(AbstractModel[LisfloodForcing]): """eWaterCycle implementation of Lisflood hydrological model. Args: - version: pick a version for which an ewatercycle grpc4bmi docker image is available. + version: pick a version for which an grpc4bmi docker image is available. parameter_set: LISFLOOD input files. Any included forcing data will be ignored. forcing: a LisfloodForcing object. Example: - See examples/lisflood.ipynb in `ewatercycle repository `_ + See examples/lisflood.ipynb in + `ewatercycle repository `_ """ available_versions = ("20.10",) @@ -64,29 +66,35 @@ def _get_textvar_value(self, name: str): # unable to subclass with more specialized arguments so ignore type def setup( # type: ignore self, - IrrigationEfficiency: str = None, + IrrigationEfficiency: str = None, # noqa: N803 start_time: str = None, end_time: str = None, MaskMap: str = None, cfg_dir: str = None, ) -> Tuple[str, str]: - """Configure model run + """Configure model run. - 1. Creates config file and config directory based on the forcing variables and time range + 1. Creates config file and config directory + based on the forcing variables and time range. 2. Start bmi container and store as :py:attr:`bmi` Args: - IrrigationEfficiency: Field application irrigation efficiency max 1, ~0.90 drip irrigation, ~0.75 sprinkling - start_time: Start time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is used. - end_time: End time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. + IrrigationEfficiency: Field application irrigation efficiency. + max 1, ~0.90 drip irrigation, ~0.75 sprinkling + start_time: Start time of model in UTC and ISO format string + e.g. 'YYYY-MM-DDTHH:MM:SSZ'. + If not given then forcing start time is used. + end_time: End time of model in UTC and ISO format string + e.g. 'YYYY-MM-DDTHH:MM:SSZ'. + If not given then forcing end time is used. MaskMap: Mask map to use instead of one supplied in parameter set. - Path to a NetCDF or pcraster file with same dimensions as parameter set map files and a boolean variable. + Path to a NetCDF or pcraster file with + same dimensions as parameter set map files and a boolean variable. cfg_dir: a run directory given by user or created for user. Returns: Path to config file and path to config directory """ - # TODO forcing can be a part of parameter_set cfg_dir_as_path = to_absolute_path(cfg_dir) if cfg_dir else None cfg_dir_as_path = _generate_workdir(cfg_dir_as_path) @@ -130,7 +138,7 @@ def setup( # type: ignore return str(config_file), str(cfg_dir_as_path) def _check_forcing(self, forcing): - """ "Check forcing argument and get path, start and end time of forcing data.""" + """Checks forcing argument and get path, start/end time of forcing data.""" # TODO check if mask has same grid as forcing files, # if not warn users to run reindex_forcings if isinstance(forcing, LisfloodForcing): @@ -141,7 +149,8 @@ def _check_forcing(self, forcing): self._end = get_time(forcing.end_time) else: raise TypeError( - f"Unknown forcing type: {forcing}. Please supply a LisfloodForcing object." + f"Unknown forcing type: {forcing}. " + "Please supply a LisfloodForcing object." ) def _create_lisflood_config( @@ -149,10 +158,10 @@ def _create_lisflood_config( cfg_dir: Path, start_time_iso: str = None, end_time_iso: str = None, - IrrigationEfficiency: str = None, + IrrigationEfficiency: str = None, # noqa: N803 MaskMap: str = None, ) -> Path: - """Create lisflood config file""" + """Create lisflood config file.""" assert self.parameter_set is not None assert self.forcing is not None # overwrite dates if given @@ -194,9 +203,7 @@ def _create_lisflood_config( # input for lisflood if "PrefixPrecipitation" in textvar_name: - textvar.set( - "value", Path(self.forcing.PrefixPrecipitation).stem - ) + textvar.set("value", Path(self.forcing.PrefixPrecipitation).stem) if "PrefixTavg" in textvar_name: textvar.set("value", Path(self.forcing.PrefixTavg).stem) @@ -235,7 +242,7 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: shape = self.bmi.get_grid_shape(grid) # Extract the data and store it in an xarray DataArray - da = xr.DataArray( + return xr.DataArray( data=np.reshape(self.bmi.get_value(name), shape), coords={ "longitude": self.bmi.get_grid_x(grid), @@ -247,12 +254,10 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: attrs={"units": self.bmi.get_var_units(name)}, ) - return da - def _coords_to_indices( self, name: str, lat: Iterable[float], lon: Iterable[float] ) -> Iterable[int]: - """Converts lat/lon values to index. + """Convert lat/lon values to index. Args: lat: Latitudinal value @@ -286,7 +291,7 @@ def parameters(self) -> Iterable[Tuple[str, Any]]: assert self.parameter_set is not None assert self.forcing is not None # TODO fix issue #60 - parameters = [ + return [ ( "IrrigationEfficiency", self._get_textvar_value("IrrigationEfficiency"), @@ -295,17 +300,20 @@ def parameters(self) -> Iterable[Tuple[str, Any]]: ("start_time", self._start.strftime("%Y-%m-%dT%H:%M:%SZ")), ("end_time", self._end.strftime("%Y-%m-%dT%H:%M:%SZ")), ] - return parameters # TODO it needs fix regarding forcing -# def reindex_forcings(mask_map: Path, forcing: LisfloodForcing, output_dir: Path = None) -> Path: +# def reindex_forcings( +# mask_map: Path, forcing: LisfloodForcing, output_dir: Path = None +# ) -> Path: # """Reindex forcing files to match mask map grid # Args: -# mask_map: Path to NetCDF file used a boolean map that defines model boundaries. +# mask_map: Path to NetCDF file used a boolean map that defines model +# boundaries. # forcing: Forcing data from ESMValTool -# output_dir: Directory where to write the re-indexed files, given by user or created for user +# output_dir: Directory where to write the re-indexed files, given by user +# or created for user # Returns: # Output dir with re-indexed files. @@ -317,21 +325,30 @@ def parameters(self) -> Iterable[Tuple[str, Any]]: # dataset = data_file.load_xarray() # out_fn = output_dir / data_file.filename.name # var_name = list(dataset.data_vars.keys())[0] -# encoding = {var_name: {"zlib": True, "complevel": 4, "chunksizes": (1,) + dataset[var_name].shape[1:]}} +# encoding = { +# var_name: { +# "zlib": True, +# "complevel": 4, +# "chunksizes": (1,) + dataset[var_name].shape[1:], +# } +# } # dataset.reindex( -# {"lat": mask["lat"], "lon": mask["lon"]}, -# method="nearest", -# tolerance=1e-2, -# ).to_netcdf(out_fn, encoding=encoding) +# {"lat": mask["lat"], "lon": mask["lon"]}, +# method="nearest", +# tolerance=1e-2, +# ).to_netcdf(out_fn, encoding=encoding) # return output_dir def _generate_workdir(cfg_dir: Path = None) -> Path: - """ + """Creates or makes sure workdir exists. Args: cfg_dir: If cfg dir is None then create sub-directory in CFG['output_dir'] + Returns: + absolute path of workdir + """ if cfg_dir is None: scratch_dir = CFG["output_dir"] @@ -339,21 +356,30 @@ def _generate_workdir(cfg_dir: Path = None) -> Path: timestamp = datetime.datetime.now(datetime.timezone.utc).strftime( "%Y%m%d_%H%M%S" ) - cfg_dir = to_absolute_path( - f"lisflood_{timestamp}", parent=Path(scratch_dir) - ) + cfg_dir = to_absolute_path(f"lisflood_{timestamp}", parent=Path(scratch_dir)) cfg_dir.mkdir(parents=True, exist_ok=True) return cfg_dir class XmlConfig(AbstractConfig): - """Config container where config is read/saved in xml format""" + """Config container where config is read/saved in xml format.""" def __init__(self, source): + """Config container where config is read/saved in xml format. + + Args: + source: file to read from + """ super().__init__(source) self.tree = ET.parse(source) self.config: ET.Element = self.tree.getroot() """XML element used to make changes to the config""" def save(self, target): + """Save xml to file. + + Args: + target: file to save to + + """ self.tree.write(target) diff --git a/src/ewatercycle/models/marrmot.py b/src/ewatercycle/models/marrmot.py index dcce15c8..1fcb8076 100644 --- a/src/ewatercycle/models/marrmot.py +++ b/src/ewatercycle/models/marrmot.py @@ -1,8 +1,8 @@ -from dataclasses import asdict, dataclass import datetime +import logging +from dataclasses import asdict, dataclass from pathlib import Path from typing import Any, Iterable, List, Tuple -import logging import numpy as np import scipy.io as sio @@ -21,10 +21,11 @@ @dataclass class Solver: - """Solver, for current implementations see - `here `_. + """Solver, for current implementations see `here + `_. """ - name: str = 'createOdeApprox_IE' + + name: str = "createOdeApprox_IE" resnorm_tolerance: float = 0.1 resnorm_maxiter: float = 6.0 @@ -32,37 +33,45 @@ class Solver: def _generate_cfg_dir(cfg_dir: Path = None) -> Path: """ Args: - cfg_dir: If cfg dir is None or does not exist then create sub-directory in CFG['output_dir'] + cfg_dir: If cfg dir is None or does not exist then create sub-directory + in CFG['output_dir'] """ if cfg_dir is None: - scratch_dir = CFG['output_dir'] + scratch_dir = CFG["output_dir"] # TODO this timestamp isnot safe for parallel processing - timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S") - cfg_dir = to_absolute_path(f'marrmot_{timestamp}', parent=Path(scratch_dir)) + timestamp = datetime.datetime.now(datetime.timezone.utc).strftime( + "%Y%m%d_%H%M%S" + ) + cfg_dir = to_absolute_path(f"marrmot_{timestamp}", parent=Path(scratch_dir)) cfg_dir.mkdir(parents=True, exist_ok=True) return cfg_dir class MarrmotM01(AbstractModel[MarrmotForcing]): - """eWaterCycle implementation of Marrmot Collie River 1 (traditional bucket) hydrological model. + """eWaterCycle implementation of Marrmot Collie River 1 (traditional bucket) model. - It sets MarrmotM01 parameter with an initial value that is the mean value of the range specfied in `model parameter range file `_. + It sets MarrmotM01 parameter with an initial value that is the mean value of + the range specfied in `model parameter range file + `_. Args: - version: pick a version for which an ewatercycle grpc4bmi docker image is available. - forcing: a MarrmotForcing object. - If forcing file contains parameter and other settings, those are used and can be changed in :py:meth:`setup`. + version: pick a version for which an ewatercycle grpc4bmi docker image + is available. forcing: a MarrmotForcing object. If forcing file contains + parameter and other settings, those are used and can be changed in + :py:meth:`setup`. Example: - See examples/marrmotM01.ipynb in `ewatercycle repository `_ + See examples/marrmotM01.ipynb in `ewatercycle repository + `_ """ + model_name = "m_01_collie1_1p_1s" """Name of model in Matlab code.""" - available_versions = ("2020.11", ) + available_versions = ("2020.11",) """Versions for which ewatercycle grpc4bmi docker images are available.""" def __init__(self, version: str, forcing: MarrmotForcing): - """Construct MarrmotM01 with initial values. """ + """Construct MarrmotM01 with initial values.""" super().__init__(version, forcing=forcing) self._parameters = [1000.0] self.store_ini = [900.0] @@ -73,38 +82,43 @@ def __init__(self, version: str, forcing: MarrmotForcing): self._set_docker_image() def _set_docker_image(self): - images = { - '2020.11': 'ewatercycle/marrmot-grpc4bmi:2020.11' - } + images = {"2020.11": "ewatercycle/marrmot-grpc4bmi:2020.11"} self.docker_image = images[self.version] def _set_singularity_image(self): - images = { - '2020.11': 'ewatercycle-marrmot-grpc4bmi_2020.11.sif' - } - if CFG.get('singularity_dir'): - self.singularity_image = CFG['singularity_dir'] / images[self.version] + images = {"2020.11": "ewatercycle-marrmot-grpc4bmi_2020.11.sif"} + if CFG.get("singularity_dir"): + self.singularity_image = CFG["singularity_dir"] / images[self.version] # unable to subclass with more specialized arguments so ignore type - def setup(self, # type: ignore - maximum_soil_moisture_storage: float = None, - initial_soil_moisture_storage: float = None, - start_time: str = None, - end_time: str = None, - solver: Solver = None, - cfg_dir: str = None) -> Tuple[str, str]: + def setup( # type: ignore + self, + maximum_soil_moisture_storage: float = None, + initial_soil_moisture_storage: float = None, + start_time: str = None, + end_time: str = None, + solver: Solver = None, + cfg_dir: str = None, + ) -> Tuple[str, str]: """Configure model run. - 1. Creates config file and config directory based on the forcing variables and time range + 1. Creates config file and config directory based on the forcing + variables and time range 2. Start bmi container and store as :py:attr:`bmi` Args: - maximum_soil_moisture_storage: in mm. Range is specfied in `model parameter range file `_. + maximum_soil_moisture_storage: in mm. Range is specfied in `model + parameter range file + `_. initial_soil_moisture_storage: in mm. - start_time: Start time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is used. - end_time: End time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. + start_time: Start time of model in UTC and ISO format string e.g. + 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is + used. + end_time: End time of model in UTC and ISO format string e.g. + 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. solver: Solver settings cfg_dir: a run directory given by user or created for user. + Returns: Path to config file and path to config directory """ @@ -119,14 +133,14 @@ def setup(self, # type: ignore cfg_dir_as_path = _generate_cfg_dir(cfg_dir_as_path) config_file = self._create_marrmot_config(cfg_dir_as_path, start_time, end_time) - if CFG['container_engine'].lower() == 'singularity': + if CFG["container_engine"].lower() == "singularity": message = f"The singularity image {self.singularity_image} does not exist." assert self.singularity_image.exists(), message self.bmi = BmiClientSingularity( image=str(self.singularity_image), work_dir=str(cfg_dir_as_path), ) - elif CFG['container_engine'].lower() == 'docker': + elif CFG["container_engine"].lower() == "docker": self.bmi = BmiClientDocker( image=self.docker_image, image_port=55555, @@ -139,7 +153,7 @@ def setup(self, # type: ignore return str(config_file), str(cfg_dir_as_path) def _check_forcing(self, forcing): - """"Check forcing argument and get path, start and end time of forcing data.""" + """ "Check forcing argument and get path, start and end time of forcing data.""" if isinstance(forcing, MarrmotForcing): forcing_dir = to_absolute_path(forcing.directory) self.forcing_file = str(forcing_dir / forcing.forcing_file) @@ -148,30 +162,36 @@ def _check_forcing(self, forcing): self.forcing_end_time = get_time(forcing.end_time) else: raise TypeError( - f"Unknown forcing type: {forcing}. Please supply a MarrmotForcing object." + f"Unknown forcing type: {forcing}. Please supply a " + " MarrmotForcing object." ) # parse start/end time forcing_data = sio.loadmat(self.forcing_file, mat_dtype=True) - if 'parameters' in forcing_data: - self._parameters = forcing_data['parameters'][0] - if 'store_ini' in forcing_data: - self.store_ini = forcing_data['store_ini'][0] - if 'solver' in forcing_data: - forcing_solver = forcing_data['solver'] - self.solver.name = forcing_solver['name'][0][0][0] - self.solver.resnorm_tolerance = forcing_solver['resnorm_tolerance'][0][0][0] - self.solver.resnorm_maxiter = forcing_solver['resnorm_maxiter'][0][0][0] - - def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_time_iso: str = None) -> Path: + if "parameters" in forcing_data: + self._parameters = forcing_data["parameters"][0] + if "store_ini" in forcing_data: + self.store_ini = forcing_data["store_ini"][0] + if "solver" in forcing_data: + forcing_solver = forcing_data["solver"] + self.solver.name = forcing_solver["name"][0][0][0] + self.solver.resnorm_tolerance = forcing_solver["resnorm_tolerance"][0][0][0] + self.solver.resnorm_maxiter = forcing_solver["resnorm_maxiter"][0][0][0] + + def _create_marrmot_config( + self, cfg_dir: Path, start_time_iso: str = None, end_time_iso: str = None + ) -> Path: """Write model configuration file. - Adds the model parameters to forcing file for the given period - and writes this information to a model configuration file. + Adds the model parameters to forcing file for the given period and + writes this information to a model configuration file. Args: cfg_dir: a run directory given by user or created for user. - start_time_iso: Start time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is used. - end_time_iso: End time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. + start_time_iso: Start time of model in UTC and ISO format string + e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is + used. + end_time_iso: End time of model in UTC and ISO format string e.g. + 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. Returns: Path for Marrmot config file @@ -182,7 +202,7 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ if start_time_iso is not None: start_time = get_time(start_time_iso) if self.forcing_start_time <= start_time <= self.forcing_end_time: - forcing_data['time_start'][0][0:6] = [ + forcing_data["time_start"][0][0:6] = [ start_time.year, start_time.month, start_time.day, @@ -192,11 +212,11 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ ] self.forcing_start_time = start_time else: - raise ValueError('start_time outside forcing time range') + raise ValueError("start_time outside forcing time range") if end_time_iso is not None: end_time = get_time(end_time_iso) if self.forcing_start_time <= end_time <= self.forcing_end_time: - forcing_data['time_end'][0][0:6] = [ + forcing_data["time_end"][0][0:6] = [ end_time.year, end_time.month, end_time.day, @@ -206,7 +226,7 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ ] self.forcing_end_time = end_time else: - raise ValueError('end_time outside forcing time range') + raise ValueError("end_time outside forcing time range") # combine forcing and model parameters forcing_data.update( @@ -216,17 +236,18 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ store_ini=self.store_ini, ) - config_file = cfg_dir / 'marrmot-m01_config.mat' + config_file = cfg_dir / "marrmot-m01_config.mat" sio.savemat(config_file, forcing_data) return config_file def get_value_as_xarray(self, name: str) -> xr.DataArray: """Return the value as xarray object.""" - marrmot_vars = {'S(t)', 'flux_out_Q', 'flux_out_Ea', 'wb'} + marrmot_vars = {"S(t)", "flux_out_Q", "flux_out_Ea", "wb"} if name not in marrmot_vars: raise NotImplementedError( "Variable '{}' is not implemented. " - "Please choose one of {}.".format(name, marrmot_vars)) + "Please choose one of {}.".format(name, marrmot_vars) + ) # Get time information time_units = self.bmi.get_time_units() @@ -239,7 +260,7 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: coords={ "longitude": self.bmi.get_grid_x(grid), "latitude": self.bmi.get_grid_y(grid), - "time": num2date(self.bmi.get_current_time(), time_units) + "time": num2date(self.bmi.get_current_time(), time_units), }, dims=["latitude", "longitude"], name=name, @@ -250,44 +271,52 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: def parameters(self) -> Iterable[Tuple[str, Any]]: """List the parameters for this model.""" p = [ - ('maximum_soil_moisture_storage', self._parameters[0]), - ('initial_soil_moisture_storage', self.store_ini[0]), - ('solver', self.solver), - ('start time', self.forcing_start_time.strftime("%Y-%m-%dT%H:%M:%SZ")), - ('end time', self.forcing_end_time.strftime("%Y-%m-%dT%H:%M:%SZ")), + ("maximum_soil_moisture_storage", self._parameters[0]), + ("initial_soil_moisture_storage", self.store_ini[0]), + ("solver", self.solver), + ("start time", self.forcing_start_time.strftime("%Y-%m-%dT%H:%M:%SZ")), + ("end time", self.forcing_end_time.strftime("%Y-%m-%dT%H:%M:%SZ")), ] return p -M14_PARAMS = ('maximum_soil_moisture_storage', - 'threshold_flow_generation_evap_change', - 'leakage_saturated_zone_flow_coefficient', - 'zero_deficit_base_flow_speed', - 'baseflow_coefficient', - 'gamma_distribution_chi_parameter', - 'gamma_distribution_phi_parameter') +M14_PARAMS = ( + "maximum_soil_moisture_storage", + "threshold_flow_generation_evap_change", + "leakage_saturated_zone_flow_coefficient", + "zero_deficit_base_flow_speed", + "baseflow_coefficient", + "gamma_distribution_chi_parameter", + "gamma_distribution_phi_parameter", +) class MarrmotM14(AbstractModel[MarrmotForcing]): """eWaterCycle implementation of Marrmot Top Model hydrological model. - It sets MarrmotM14 parameter with an initial value that is the mean value of the range specfied in `model parameter range file `_. + It sets MarrmotM14 parameter with an initial value that is the mean value of + the range specfied in `model parameter range file + `_. Args: - version: pick a version for which an ewatercycle grpc4bmi docker image is available. + version: pick a version for which an ewatercycle grpc4bmi docker image + is available. forcing: a MarrmotForcing object. - If forcing file contains parameter and other settings, those are used and can be changed in :py:meth:`setup`. + If forcing file contains parameter and other settings, those are used + and can be changed in :py:meth:`setup`. Example: - See examples/marrmotM14.ipynb in `ewatercycle repository `_ + See examples/marrmotM14.ipynb in `ewatercycle repository + `_ """ + model_name = "m_14_topmodel_7p_2s" """Name of model in Matlab code.""" - available_versions = ("2020.11", ) + available_versions = ("2020.11",) """Versions for which ewatercycle grpc4bmi docker images are available.""" def __init__(self, version: str, forcing: MarrmotForcing): - """Construct MarrmotM14 with initial values. """ + """Construct MarrmotM14 with initial values.""" super().__init__(version, forcing=forcing) self._parameters = [1000.0, 0.5, 0.5, 100.0, 0.5, 4.25, 2.5] self.store_ini = [900.0, 900.0] @@ -298,41 +327,42 @@ def __init__(self, version: str, forcing: MarrmotForcing): self._set_docker_image() def _set_docker_image(self): - images = { - '2020.11': 'ewatercycle/marrmot-grpc4bmi:2020.11' - } + images = {"2020.11": "ewatercycle/marrmot-grpc4bmi:2020.11"} self.docker_image = images[self.version] def _set_singularity_image(self): - images = { - '2020.11': 'ewatercycle-marrmot-grpc4bmi_2020.11.sif' - } - if CFG.get('singularity_dir'): - self.singularity_image = CFG['singularity_dir'] / images[self.version] + images = {"2020.11": "ewatercycle-marrmot-grpc4bmi_2020.11.sif"} + if CFG.get("singularity_dir"): + self.singularity_image = CFG["singularity_dir"] / images[self.version] # unable to subclass with more specialized arguments so ignore type - def setup(self, # type: ignore - maximum_soil_moisture_storage: float = None, - threshold_flow_generation_evap_change: float = None, - leakage_saturated_zone_flow_coefficient: float = None, - zero_deficit_base_flow_speed: float = None, - baseflow_coefficient: float = None, - gamma_distribution_chi_parameter: float = None, - gamma_distribution_phi_parameter: float = None, - initial_upper_zone_storage: float = None, - initial_saturated_zone_storage: float = None, - start_time: str = None, - end_time: str = None, - solver: Solver = None, - cfg_dir: str = None) -> Tuple[str, str]: + def setup( # type: ignore + self, + maximum_soil_moisture_storage: float = None, + threshold_flow_generation_evap_change: float = None, + leakage_saturated_zone_flow_coefficient: float = None, + zero_deficit_base_flow_speed: float = None, + baseflow_coefficient: float = None, + gamma_distribution_chi_parameter: float = None, + gamma_distribution_phi_parameter: float = None, + initial_upper_zone_storage: float = None, + initial_saturated_zone_storage: float = None, + start_time: str = None, + end_time: str = None, + solver: Solver = None, + cfg_dir: str = None, + ) -> Tuple[str, str]: """Configure model run. - 1. Creates config file and config directory based on the forcing variables and time range + 1. Creates config file and config directory based on the forcing + variables and time range 2. Start bmi container and store as :py:attr:`bmi` Args: - maximum_soil_moisture_storage: in mm. Range is specfied in `model parameter range file `_. - threshold_flow_generation_evap_change. + maximum_soil_moisture_storage: in mm. Range is specfied in `model + parameter range file + `_. + threshold_flow_generation_evap_change. leakage_saturated_zone_flow_coefficient: in mm/d. zero_deficit_base_flow_speed: in mm/d. baseflow_coefficient: in mm-1. @@ -340,10 +370,14 @@ def setup(self, # type: ignore gamma_distribution_phi_parameter. initial_upper_zone_storage: in mm. initial_saturated_zone_storage: in mm. - start_time: Start time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is used. - end_time: End time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. - solver: Solver settings + start_time: Start time of model in UTC and ISO format string e.g. + 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is + used. + end_time: End time of model in UTC and ISO format string e.g. + 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. + solver: Solver settings cfg_dir: a run directory given by user or created for user. + Returns: Path to config file and path to config directory """ @@ -363,14 +397,14 @@ def setup(self, # type: ignore cfg_dir_as_path = _generate_cfg_dir(cfg_dir_as_path) config_file = self._create_marrmot_config(cfg_dir_as_path, start_time, end_time) - if CFG['container_engine'].lower() == 'singularity': + if CFG["container_engine"].lower() == "singularity": message = f"The singularity image {self.singularity_image} does not exist." assert self.singularity_image.exists(), message self.bmi = BmiClientSingularity( image=str(self.singularity_image), work_dir=str(cfg_dir_as_path), ) - elif CFG['container_engine'].lower() == 'docker': + elif CFG["container_engine"].lower() == "docker": self.bmi = BmiClientDocker( image=self.docker_image, image_port=55555, @@ -383,7 +417,7 @@ def setup(self, # type: ignore return str(config_file), str(cfg_dir_as_path) def _check_forcing(self, forcing): - """"Check forcing argument and get path, start and end time of forcing data.""" + """Check forcing argument and get path, start and end time of forcing data.""" if isinstance(forcing, MarrmotForcing): forcing_dir = to_absolute_path(forcing.directory) self.forcing_file = str(forcing_dir / forcing.forcing_file) @@ -392,29 +426,40 @@ def _check_forcing(self, forcing): self.forcing_end_time = get_time(forcing.end_time) else: raise TypeError( - f"Unknown forcing type: {forcing}. Please supply a MarrmotForcing object." + f"Unknown forcing type: {forcing}. " + "Please supply a MarrmotForcing object." ) # parse start/end time forcing_data = sio.loadmat(self.forcing_file, mat_dtype=True) - if 'parameters' in forcing_data: - if len(forcing_data['parameters']) == len(self._parameters): - self._parameters = forcing_data['parameters'] + if "parameters" in forcing_data: + if len(forcing_data["parameters"]) == len(self._parameters): + self._parameters = forcing_data["parameters"] else: - message = f"The length of parameters in forcing {self.forcing_file} does not match the length of M14 parameters that is seven." + message = ( + "The length of parameters in forcing " + f"{self.forcing_file} does not match " + "the length of M14 parameters that is seven." + ) logger.warning("%s", message) - if 'store_ini' in forcing_data: - if len(forcing_data['store_ini']) == len(self.store_ini): - self.store_ini = forcing_data['store_ini'] + if "store_ini" in forcing_data: + if len(forcing_data["store_ini"]) == len(self.store_ini): + self.store_ini = forcing_data["store_ini"] else: - message = f"The length of initial stores in forcing {self.forcing_file} does not match the length of M14 iniatial stores that is two." + message = ( + "The length of initial stores in forcing " + f"{self.forcing_file} does not match " + "the length of M14 iniatial stores that is two." + ) logger.warning("%s", message) - if 'solver' in forcing_data: - forcing_solver = forcing_data['solver'] - self.solver.name = forcing_solver['name'][0][0][0] - self.solver.resnorm_tolerance = forcing_solver['resnorm_tolerance'][0][0][0] - self.solver.resnorm_maxiter = forcing_solver['resnorm_maxiter'][0][0][0] - - def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_time_iso: str = None) -> Path: + if "solver" in forcing_data: + forcing_solver = forcing_data["solver"] + self.solver.name = forcing_solver["name"][0][0][0] + self.solver.resnorm_tolerance = forcing_solver["resnorm_tolerance"][0][0][0] + self.solver.resnorm_maxiter = forcing_solver["resnorm_maxiter"][0][0][0] + + def _create_marrmot_config( + self, cfg_dir: Path, start_time_iso: str = None, end_time_iso: str = None + ) -> Path: """Write model configuration file. Adds the model parameters to forcing file for the given period @@ -422,8 +467,12 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ Args: cfg_dir: a run directory given by user or created for user. - start_time_iso: Start time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is used. - end_time_iso: End time of model in UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used. + start_time_iso: Start time of model in UTC and ISO format string + e.g. 'YYYY-MM-DDTHH:MM:SSZ'. + If not given then forcing start time is used. + end_time_iso: End time of model in UTC and ISO format string + e.g. 'YYYY-MM-DDTHH:MM:SSZ'. + If not given then forcing end time is used. Returns: Path for Marrmot config file @@ -434,7 +483,7 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ if start_time_iso is not None: start_time = get_time(start_time_iso) if self.forcing_start_time <= start_time <= self.forcing_end_time: - forcing_data['time_start'][0][0:6] = [ + forcing_data["time_start"][0][0:6] = [ start_time.year, start_time.month, start_time.day, @@ -444,11 +493,11 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ ] self.forcing_start_time = start_time else: - raise ValueError('start_time outside forcing time range') + raise ValueError("start_time outside forcing time range") if end_time_iso is not None: end_time = get_time(end_time_iso) if self.forcing_start_time <= end_time <= self.forcing_end_time: - forcing_data['time_end'][0][0:6] = [ + forcing_data["time_end"][0][0:6] = [ end_time.year, end_time.month, end_time.day, @@ -458,7 +507,7 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ ] self.forcing_end_time = end_time else: - raise ValueError('end_time outside forcing time range') + raise ValueError("end_time outside forcing time range") # combine forcing and model parameters forcing_data.update( @@ -468,17 +517,18 @@ def _create_marrmot_config(self, cfg_dir: Path, start_time_iso: str = None, end_ store_ini=self.store_ini, ) - config_file = cfg_dir / 'marrmot-m14_config.mat' + config_file = cfg_dir / "marrmot-m14_config.mat" sio.savemat(config_file, forcing_data) return config_file def get_value_as_xarray(self, name: str) -> xr.DataArray: """Return the value as xarray object.""" - marrmot_vars = {'S(t)', 'flux_out_Q', 'flux_out_Ea', 'wb'} + marrmot_vars = {"S(t)", "flux_out_Q", "flux_out_Ea", "wb"} if name not in marrmot_vars: raise NotImplementedError( "Variable '{}' is not implemented. " - "Please choose one of {}.".format(name, marrmot_vars)) + "Please choose one of {}.".format(name, marrmot_vars) + ) # Get time information time_units = self.bmi.get_time_units() @@ -491,7 +541,7 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: coords={ "longitude": self.bmi.get_grid_x(grid), "latitude": self.bmi.get_grid_y(grid), - "time": num2date(self.bmi.get_current_time(), time_units) + "time": num2date(self.bmi.get_current_time(), time_units), }, dims=["latitude", "longitude"], name=name, @@ -501,12 +551,12 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: @property def parameters(self) -> Iterable[Tuple[str, Any]]: """List the parameters for this model.""" - p: List[Tuple[str, Any]] = list(zip(M14_PARAMS, self._parameters)) - p += [ - ('initial_upper_zone_storage', self.store_ini[0]), - ('initial_saturated_zone_storage', self.store_ini[1]), - ('solver', self.solver), - ('start time', self.forcing_start_time.strftime("%Y-%m-%dT%H:%M:%SZ")), - ('end time', self.forcing_end_time.strftime("%Y-%m-%dT%H:%M:%SZ")), + pars: List[Tuple[str, Any]] = list(zip(M14_PARAMS, self._parameters)) + pars += [ + ("initial_upper_zone_storage", self.store_ini[0]), + ("initial_saturated_zone_storage", self.store_ini[1]), + ("solver", self.solver), + ("start time", self.forcing_start_time.strftime("%Y-%m-%dT%H:%M:%SZ")), + ("end time", self.forcing_end_time.strftime("%Y-%m-%dT%H:%M:%SZ")), ] - return p + return pars diff --git a/src/ewatercycle/models/pcrglobwb.py b/src/ewatercycle/models/pcrglobwb.py index fe3ffcaf..cf1be017 100644 --- a/src/ewatercycle/models/pcrglobwb.py +++ b/src/ewatercycle/models/pcrglobwb.py @@ -26,7 +26,8 @@ class PCRGlobWB(AbstractModel[PCRGlobWBForcing]): Args: version: pick a version from :py:attr:`~available_versions` - parameter_set: instance of :py:class:`~ewatercycle.parameter_sets.default.ParameterSet`. + parameter_set: instance of + :py:class:`~ewatercycle.parameter_sets.default.ParameterSet`. forcing: ewatercycle forcing container; see :py:mod:`ewatercycle.forcing`. @@ -135,8 +136,9 @@ def setup(self, cfg_dir: str = None, **kwargs) -> Tuple[str, str]: # type: igno "Couldn't spawn container within allocated time limit " "(15 seconds). You may try pulling the docker image with" f" `docker pull {self.docker_image}` or call `singularity " - f"build {self._singularity_image(CFG['singularity_dir'])} docker://{self.docker_image}`" - "if you're using singularity, and then try again." + f"build {self._singularity_image(CFG['singularity_dir'])} " + f"docker://{self.docker_image}` if you're using singularity," + " and then try again." ) return str(cfg_file), str(work_dir) @@ -159,9 +161,7 @@ def _update_config(self, **kwargs): ) if "routing_method" in kwargs: - cfg.set( - "routingOptions", "routingMethod", kwargs["routing_method"] - ) + cfg.set("routingOptions", "routingMethod", kwargs["routing_method"]) if "dynamic_flood_plain" in kwargs: cfg.set( @@ -203,7 +203,7 @@ def _start_container(self): ) elif CFG["container_engine"] == "singularity": self.bmi = BmiClientSingularity( - image=self._singularity_image(CFG['singularity_dir']), + image=self._singularity_image(CFG["singularity_dir"]), work_dir=str(self.work_dir), input_dirs=additional_input_dirs, timeout=15, diff --git a/src/ewatercycle/models/wflow.py b/src/ewatercycle/models/wflow.py index 37933207..02aeee82 100644 --- a/src/ewatercycle/models/wflow.py +++ b/src/ewatercycle/models/wflow.py @@ -26,7 +26,8 @@ class Wflow(AbstractModel[WflowForcing]): Args: version: pick a version from :py:attr:`~available_versions` - parameter_set: instance of :py:class:`~ewatercycle.parameter_sets.default.ParameterSet`. + parameter_set: instance of + :py:class:`~ewatercycle.parameter_sets.default.ParameterSet`. forcing: instance of :py:class:`~WflowForcing` or None. If None, it is assumed that forcing is included with the parameter_set. """ @@ -80,13 +81,14 @@ def _setup_default_config(self): if self.version == "2020.1.1": if not cfg.has_section("API"): logger.warning( - "Config file from parameter set is missing API section, adding section" + "Config file from parameter set is missing API section, " + "adding section" ) cfg.add_section("API") if not cfg.has_option("API", "RiverRunoff"): logger.warning( - "Config file from parameter set is missing RiverRunoff option in API section, " - "added it with value '2, m/s option'" + "Config file from parameter set is missing RiverRunoff " + "option in API section, added it with value '2, m/s option'" ) cfg.set("API", "RiverRunoff", "2, m/s") @@ -126,8 +128,9 @@ def setup(self, cfg_dir: str = None, **kwargs) -> Tuple[str, str]: # type: igno "Couldn't spawn container within allocated time limit " "(15 seconds). You may try pulling the docker image with" f" `docker pull {self.docker_image}` or call `singularity " - f"build {self._singularity_image(CFG['singularity_dir'])} docker://{self.docker_image}`" - "if you're using singularity, and then try again." + f"build {self._singularity_image(CFG['singularity_dir'])} " + f"docker://{self.docker_image}` if you're using singularity," + " and then try again." ) return ( @@ -166,7 +169,7 @@ def _start_container(self): ) elif CFG["container_engine"] == "singularity": self.bmi = BmiClientSingularity( - image=self._singularity_image(CFG['singularity_dir']), + image=self._singularity_image(CFG["singularity_dir"]), work_dir=str(self.work_dir), timeout=15, ) diff --git a/src/ewatercycle/observation/grdc.py b/src/ewatercycle/observation/grdc.py index f05344b7..1152c9b2 100644 --- a/src/ewatercycle/observation/grdc.py +++ b/src/ewatercycle/observation/grdc.py @@ -1,20 +1,26 @@ +"""Global Runoff Data Centre module.""" +import logging import os from typing import Dict, Tuple, Union -import logging import pandas as pd + from ewatercycle import CFG from ewatercycle.util import get_time, to_absolute_path logger = logging.getLogger(__name__) +MetaDataType = Dict[str, Union[str, int, float]] + -def get_grdc_data(station_id: str, - start_time: str, - end_time: str, - parameter: str = 'Q', - data_home: str = None, - column: str = 'streamflow') -> Tuple[pd.core.frame.DataFrame, Dict[str, Union[str, int, float]]]: +def get_grdc_data( + station_id: str, + start_time: str, + end_time: str, + parameter: str = "Q", + data_home: str = None, + column: str = "streamflow", +) -> Tuple[pd.core.frame.DataFrame, MetaDataType]: """Get river discharge data from Global Runoff Data Centre (GRDC). Requires the GRDC daily data files in a local directory. The GRDC daily data @@ -44,7 +50,9 @@ def get_grdc_data(station_id: str, from ewatercycle.observation.grdc import get_grdc_data - df, meta = get_grdc_data('6335020', '2000-01-01T00:00Z', '2001-01-01T00:00Z', data_home='.') + df, meta = get_grdc_data('6335020', + '2000-01-01T00:00Z', + '2001-01-01T00:00Z') df.describe() streamflow count 4382.000000 @@ -76,31 +84,32 @@ def get_grdc_data(station_id: str, 'UserStartTime': '2000-01-01T00:00Z', 'UserEndTime': '2001-01-01T00:00Z', 'nrMissingData': 0} - """ + """ # noqa: E501 if data_home: data_path = to_absolute_path(data_home) elif CFG["grdc_location"]: data_path = to_absolute_path(CFG["grdc_location"]) else: raise ValueError( - f'Provide the grdc path using `data_home` argument ' - f'or using `grdc_location` in ewatercycle configuration file.' - ) + "Provide the grdc path using `data_home` argument" + "or using `grdc_location` in ewatercycle configuration file." + ) if not data_path.exists(): - raise ValueError(f'The grdc directory {data_path} does not exist!') + raise ValueError(f"The grdc directory {data_path} does not exist!") # Read the raw data raw_file = data_path / f"{station_id}_{parameter}_Day.Cmd.txt" if not raw_file.exists(): - raise ValueError(f'The grdc file {raw_file} does not exist!') + raise ValueError(f"The grdc file {raw_file} does not exist!") # Convert the raw data to an xarray metadata, df = _grdc_read( raw_file, start=get_time(start_time).date(), end=get_time(end_time).date(), - column=column) + column=column, + ) # Add start/end_time to metadata metadata["UserStartTime"] = start_time @@ -116,33 +125,32 @@ def get_grdc_data(station_id: str, def _grdc_read(grdc_station_path, start, end, column): - with open( - grdc_station_path, 'r', encoding='cp1252', - errors='ignore') as file: + with grdc_station_path.open("r", encoding="cp1252", errors="ignore") as file: data = file.read() metadata = _grdc_metadata_reader(grdc_station_path, data) - allLines = data.split('\n') + all_lines = data.split("\n") header = 0 - for i, line in enumerate(allLines): - if line.startswith('# DATA'): + for i, line in enumerate(all_lines): + if line.startswith("# DATA"): header = i + 1 break # Import GRDC data into dataframe and modify dataframe format grdc_data = pd.read_csv( grdc_station_path, - encoding='cp1252', + encoding="cp1252", skiprows=header, - delimiter=';', - parse_dates=['YYYY-MM-DD'], - na_values='-999') + delimiter=";", + parse_dates=["YYYY-MM-DD"], + na_values="-999", + ) grdc_station_df = pd.DataFrame( - {column: grdc_data[' Value'].values}, - index = grdc_data['YYYY-MM-DD'].values, - ) - grdc_station_df.index.rename('time', inplace=True) + {column: grdc_data[" Value"].array}, + index=grdc_data["YYYY-MM-DD"].array, + ) + grdc_station_df.index.rename("time", inplace=True) # Select GRDC station data that matches the forecast results Date grdc_station_select = grdc_station_df.loc[start:end] @@ -150,8 +158,7 @@ def _grdc_read(grdc_station_path, start, end, column): return metadata, grdc_station_select -def _grdc_metadata_reader(grdc_station_path, allLines): - """ +def _grdc_metadata_reader(grdc_station_path, all_lines): # Initiating a dictionary that will contain all GRDC attributes. # This function is based on earlier work by Rolf Hut. # https://github.com/RolfHut/GRDC2NetCDF/blob/master/GRDC2NetCDF.py @@ -160,117 +167,119 @@ def _grdc_metadata_reader(grdc_station_path, allLines): # from Utrecht University. # https://github.com/edwinkost/discharge_analysis_IWMI # Modified by Susan Branchett - """ # initiating a dictionary that will contain all GRDC attributes: - attributeGRDC = {} + attribute_grdc = {} # split the content of the file into several lines - allLines = allLines.replace("\r", "") - allLines = allLines.split("\n") + all_lines = all_lines.replace("\r", "") + all_lines = all_lines.split("\n") # get grdc ids (from files) and check their consistency with their # file names id_from_file_name = int( - os.path.basename(grdc_station_path).split(".")[0].split("_")[0]) + os.path.basename(grdc_station_path).split(".")[0].split("_")[0] + ) id_from_grdc = None - if id_from_file_name == int(allLines[8].split(":")[1].strip()): - id_from_grdc = int(allLines[8].split(":")[1].strip()) + if id_from_file_name == int(all_lines[8].split(":")[1].strip()): + id_from_grdc = int(all_lines[8].split(":")[1].strip()) else: - print("GRDC station " + str(id_from_file_name) + " (" + - str(grdc_station_path) + ") is NOT used.") + print( + "GRDC station " + + str(id_from_file_name) + + " (" + + str(grdc_station_path) + + ") is NOT used." + ) if id_from_grdc is not None: - attributeGRDC["grdc_file_name"] = str(grdc_station_path) - attributeGRDC["id_from_grdc"] = id_from_grdc + attribute_grdc["grdc_file_name"] = str(grdc_station_path) + attribute_grdc["id_from_grdc"] = id_from_grdc try: - attributeGRDC["file_generation_date"] = \ - str(allLines[6].split(":")[1].strip()) - except: - attributeGRDC["file_generation_date"] = "NA" + attribute_grdc["file_generation_date"] = str( + all_lines[6].split(":")[1].strip() + ) + except (IndexError, ValueError): + attribute_grdc["file_generation_date"] = "NA" try: - attributeGRDC["river_name"] = \ - str(allLines[9].split(":")[1].strip()) - except: - attributeGRDC["river_name"] = "NA" + attribute_grdc["river_name"] = str(all_lines[9].split(":")[1].strip()) + except (IndexError, ValueError): + attribute_grdc["river_name"] = "NA" try: - attributeGRDC["station_name"] = \ - str(allLines[10].split(":")[1].strip()) - except: - attributeGRDC["station_name"] = "NA" + attribute_grdc["station_name"] = str(all_lines[10].split(":")[1].strip()) + except (IndexError, ValueError): + attribute_grdc["station_name"] = "NA" try: - attributeGRDC["country_code"] = \ - str(allLines[11].split(":")[1].strip()) - except: - attributeGRDC["country_code"] = "NA" + attribute_grdc["country_code"] = str(all_lines[11].split(":")[1].strip()) + except (IndexError, ValueError): + attribute_grdc["country_code"] = "NA" try: - attributeGRDC["grdc_latitude_in_arc_degree"] = \ - float(allLines[12].split(":")[1].strip()) - except: - attributeGRDC["grdc_latitude_in_arc_degree"] = "NA" + attribute_grdc["grdc_latitude_in_arc_degree"] = float( + all_lines[12].split(":")[1].strip() + ) + except (IndexError, ValueError): + attribute_grdc["grdc_latitude_in_arc_degree"] = "NA" try: - attributeGRDC["grdc_longitude_in_arc_degree"] = \ - float(allLines[13].split(":")[1].strip()) - except: - attributeGRDC["grdc_longitude_in_arc_degree"] = "NA" + attribute_grdc["grdc_longitude_in_arc_degree"] = float( + all_lines[13].split(":")[1].strip() + ) + except (IndexError, ValueError): + attribute_grdc["grdc_longitude_in_arc_degree"] = "NA" try: - attributeGRDC["grdc_catchment_area_in_km2"] = \ - float(allLines[14].split(":")[1].strip()) - if attributeGRDC["grdc_catchment_area_in_km2"] <= 0.0: - attributeGRDC["grdc_catchment_area_in_km2"] = "NA" - except: - attributeGRDC["grdc_catchment_area_in_km2"] = "NA" + attribute_grdc["grdc_catchment_area_in_km2"] = float( + all_lines[14].split(":")[1].strip() + ) + if attribute_grdc["grdc_catchment_area_in_km2"] <= 0.0: + attribute_grdc["grdc_catchment_area_in_km2"] = "NA" + except (IndexError, ValueError): + attribute_grdc["grdc_catchment_area_in_km2"] = "NA" try: - attributeGRDC["altitude_masl"] = \ - float(allLines[15].split(":")[1].strip()) - except: - attributeGRDC["altitude_masl"] = "NA" + attribute_grdc["altitude_masl"] = float(all_lines[15].split(":")[1].strip()) + except (IndexError, ValueError): + attribute_grdc["altitude_masl"] = "NA" try: - attributeGRDC["dataSetContent"] = \ - str(allLines[20].split(":")[1].strip()) - except: - attributeGRDC["dataSetContent"] = "NA" + attribute_grdc["dataSetContent"] = str(all_lines[20].split(":")[1].strip()) + except (IndexError, ValueError): + attribute_grdc["dataSetContent"] = "NA" try: - attributeGRDC["units"] = str(allLines[22].split(":")[1].strip()) - except: - attributeGRDC["units"] = "NA" + attribute_grdc["units"] = str(all_lines[22].split(":")[1].strip()) + except (IndexError, ValueError): + attribute_grdc["units"] = "NA" try: - attributeGRDC["time_series"] = \ - str(allLines[23].split(":")[1].strip()) - except: - attributeGRDC["time_series"] = "NA" + attribute_grdc["time_series"] = str(all_lines[23].split(":")[1].strip()) + except (IndexError, ValueError): + attribute_grdc["time_series"] = "NA" try: - attributeGRDC["no_of_years"] = \ - int(allLines[24].split(":")[1].strip()) - except: - attributeGRDC["no_of_years"] = "NA" + attribute_grdc["no_of_years"] = int(all_lines[24].split(":")[1].strip()) + except (IndexError, ValueError): + attribute_grdc["no_of_years"] = "NA" try: - attributeGRDC["last_update"] = \ - str(allLines[25].split(":")[1].strip()) - except: - attributeGRDC["last_update"] = "NA" + attribute_grdc["last_update"] = str(all_lines[25].split(":")[1].strip()) + except (IndexError, ValueError): + attribute_grdc["last_update"] = "NA" try: - attributeGRDC["nrMeasurements"] = \ - int(str(allLines[38].split(":")[1].strip())) - except: - attributeGRDC["nrMeasurements"] = "NA" + attribute_grdc["nrMeasurements"] = int( + str(all_lines[33].split(":")[1].strip()) + ) + except (IndexError, ValueError): + attribute_grdc["nrMeasurements"] = "NA" - return attributeGRDC + return attribute_grdc def _count_missing_data(df, column): @@ -281,15 +290,16 @@ def _count_missing_data(df, column): def _log_metadata(metadata): """Print some information about data.""" coords = ( - metadata['grdc_latitude_in_arc_degree'], - metadata['grdc_longitude_in_arc_degree'] - ) + metadata["grdc_latitude_in_arc_degree"], + metadata["grdc_longitude_in_arc_degree"], + ) message = ( f"GRDC station {metadata['id_from_grdc']} is selected. " f"The river name is: {metadata['river_name']}." f"The coordinates are: {coords}." f"The catchment area in km2 is: {metadata['grdc_catchment_area_in_km2']}. " - f"There are {metadata['nrMissingData']} missing values " - f"during {metadata['UserStartTime']}_{metadata['UserEndTime']} at this station. " - f"See the metadata for more information.") + f"There are {metadata['nrMissingData']} missing values during " + f"{metadata['UserStartTime']}_{metadata['UserEndTime']} at this station. " + f"See the metadata for more information." + ) logger.info("%s", message) diff --git a/src/ewatercycle/observation/usgs.py b/src/ewatercycle/observation/usgs.py index 6a541963..7904c594 100644 --- a/src/ewatercycle/observation/usgs.py +++ b/src/ewatercycle/observation/usgs.py @@ -1,20 +1,17 @@ import os from datetime import datetime -import xarray as xr import numpy as np +import xarray as xr from pyoos.collectors.usgs.usgs_rest import UsgsRest from pyoos.parsers.waterml import WaterML11ToPaegan -def get_usgs_data(station_id, - start_date, - end_date, - parameter='00060', - cache_dir=None): - """ - Get river discharge data from the - `U.S. Geological Survey Water Services `_ (USGS) rest web service. +def get_usgs_data(station_id, start_date, end_date, parameter="00060", cache_dir=None): + """Get river discharge data from the USGS REST web service. + + See `U.S. Geological Survey Water Services + `_ (USGS) Parameters ---------- @@ -46,34 +43,53 @@ def get_usgs_data(station_id, station: Little Beaver Creek near East Liverpool OH stationid: 03109500 location: (40.6758974, -80.5406244) - """ + """ # noqa: E501 if cache_dir is None: - cache_dir = os.environ['USGS_DATA_HOME'] + cache_dir = os.environ["USGS_DATA_HOME"] # Check if we have the netcdf data netcdf = os.path.join( - cache_dir, "USGS_" + station_id + "_" + parameter + "_" + start_date + - "_" + end_date + ".nc") + cache_dir, + "USGS_" + + station_id + + "_" + + parameter + + "_" + + start_date + + "_" + + end_date + + ".nc", + ) if os.path.exists(netcdf): return xr.open_dataset(netcdf) # Download the data if needed out = os.path.join( - cache_dir, "USGS_" + station_id + "_" + parameter + "_" + start_date + - "_" + end_date + ".wml") + cache_dir, + "USGS_" + + station_id + + "_" + + parameter + + "_" + + start_date + + "_" + + end_date + + ".wml", + ) if not os.path.exists(out): collector = UsgsRest() collector.filter( start=datetime.strptime(start_date, "%Y-%m-%d"), end=datetime.strptime(end_date, "%Y-%m-%d"), variables=[parameter], - features=[station_id]) + features=[station_id], + ) data = collector.raw() - with open(out, 'w') as file: + with open(out, "w") as file: file.write(data) collector.clear() else: - with open(out, 'r') as file: + with open(out, "r") as file: data = file.read() # Convert the raw data to an xarray @@ -86,26 +102,26 @@ def get_usgs_data(station_id, station = data.elements[0] # Unit conversion from cubic feet to cubic meter per second - values = np.array([ - float(point.members[0]['value']) / 35.315 - for point in station.elements - ], - dtype=np.float32) + values = np.array( + [float(point.members[0]["value"]) / 35.315 for point in station.elements], + dtype=np.float32, + ) times = [point.time for point in station.elements] attrs = { - 'units': 'cubic meters per second', + "units": "cubic meters per second", } # Create the xarray dataset - ds = xr.Dataset({'streamflow': (['time'], values, attrs)}, - coords={'time': times}) + ds = xr.Dataset( + {"streamflow": (["time"], values, attrs)}, coords={"time": times} + ) # Set some nice attributes - ds.attrs['title'] = 'USGS Data from streamflow data' - ds.attrs['station'] = station.name - ds.attrs['stationid'] = station.get_uid() - ds.attrs['location'] = (station.location.y, station.location.x) + ds.attrs["title"] = "USGS Data from streamflow data" + ds.attrs["station"] = station.name + ds.attrs["stationid"] = station.get_uid() + ds.attrs["location"] = (station.location.y, station.location.x) ds.to_netcdf(netcdf) diff --git a/src/ewatercycle/parameter_sets/__init__.py b/src/ewatercycle/parameter_sets/__init__.py index d9d178ff..13f5b6d7 100644 --- a/src/ewatercycle/parameter_sets/__init__.py +++ b/src/ewatercycle/parameter_sets/__init__.py @@ -4,10 +4,11 @@ from typing import Dict, Tuple from ewatercycle import CFG -from . import _pcrglobwb, _lisflood, _wflow + +from ..config import DEFAULT_CONFIG, SYSTEM_CONFIG, USER_HOME_CONFIG +from . import _lisflood, _pcrglobwb, _wflow from ._example import ExampleParameterSet from .default import ParameterSet -from ..config import DEFAULT_CONFIG, SYSTEM_CONFIG, USER_HOME_CONFIG logger = getLogger(__name__) @@ -34,11 +35,14 @@ def available_parameter_sets(target_model: str = None) -> Tuple[str, ...]: """ all_parameter_sets = _parse_parametersets() if not all_parameter_sets: - if CFG['ewatercycle_config'] == DEFAULT_CONFIG: - raise ValueError(f'No configuration file found.') - raise ValueError(f'No parameter sets defined in {CFG["ewatercycle_config"]}. ' - f'Use `ewatercycle.parareter_sets.download_example_parameter_sets` to download examples ' - f'or define your own or ask whoever setup the ewatercycle system to do it.') + if CFG["ewatercycle_config"] == DEFAULT_CONFIG: + raise ValueError(f"No configuration file found.") + raise ValueError( + f'No parameter sets defined in {CFG["ewatercycle_config"]}. Use ' + "`ewatercycle.parareter_sets.download_example_parameter_sets` to download" + " examples or define your own or ask whoever setup the ewatercycle " + "system to do it." + ) # TODO explain somewhere how to add new parameter sets filtered = tuple( name @@ -46,9 +50,13 @@ def available_parameter_sets(target_model: str = None) -> Tuple[str, ...]: if ps.is_available and (target_model is None or ps.target_model == target_model) ) if not filtered: - raise ValueError(f'No parameter sets defined for {target_model} model in {CFG["ewatercycle_config"]}. ' - f'Use `ewatercycle.parareter_sets.download_example_parameter_sets` to download examples ' - f'or define your own or ask whoever setup the ewatercycle system to do it.') + raise ValueError( + f"No parameter sets defined for {target_model} model in " + f"{CFG['ewatercycle_config']}. Use " + "`ewatercycle.parareter_sets.download_example_parameter_sets` to download " + "examples or define your own or ask whoever setup the ewatercycle " + "system to do it." + ) return filtered @@ -82,8 +90,9 @@ def download_parameter_sets(zenodo_doi: str, target_model: str, config: str): def example_parameter_sets() -> Dict[str, ExampleParameterSet]: - """Lists example parameter sets that can be downloaded with :py:func:`~download_example_parameter_sets`. - """ + """Lists the available example parameter sets. + + They can be downloaded with :py:func:`~download_example_parameter_sets`.""" # TODO how to add a new model docs should be updated with this part examples = chain( _wflow.example_parameter_sets(), @@ -96,11 +105,13 @@ def example_parameter_sets() -> Dict[str, ExampleParameterSet]: def download_example_parameter_sets(skip_existing=True): """Downloads all of the example parameter sets and adds them to the config_file. - Downloads to `parameterset_dir` directory defined in :py:data:`ewatercycle.config.CFG`. + Downloads to `parameterset_dir` directory defined in + :py:data:`ewatercycle.config.CFG`. Args: - skip_existing: When true will not download any parameter set which already has a local directory. - When false will raise ValueError exception when parameter set already exists. + skip_existing: When true will not download any parameter set which + already has a local directory. When false will raise ValueError + exception when parameter set already exists. """ examples = example_parameter_sets() diff --git a/src/ewatercycle/parameter_sets/_example.py b/src/ewatercycle/parameter_sets/_example.py index c1df98e2..78e12208 100644 --- a/src/ewatercycle/parameter_sets/_example.py +++ b/src/ewatercycle/parameter_sets/_example.py @@ -5,6 +5,7 @@ from urllib import request from ewatercycle import CFG + from .default import ParameterSet logger = getLogger(__name__) @@ -22,7 +23,9 @@ def __init__( doi="N/A", target_model="generic", ): - super().__init__(name, directory, config, doi, target_model, supported_model_versions) + super().__init__( + name, directory, config, doi, target_model, supported_model_versions + ) self.config_url = config_url """URL where model configuration file can be downloaded""" self.datafiles_url = datafiles_url @@ -31,19 +34,24 @@ def __init__( def download(self, skip_existing=False): if self.directory.exists(): if not skip_existing: - raise ValueError(f"Directory {self.directory} for parameter set {self.name}" - f" already exists, will not overwrite. " - f"Try again with skip_existing=True or remove {self.directory} directory.") + raise ValueError( + f"Directory {self.directory} for parameter set {self.name}" + f" already exists, will not overwrite. Try again with " + f"skip_existing=True or remove {self.directory} directory." + ) - logger.info(f'Directory {self.directory} for parameter set {self.name}' - f' already exists, skipping download.') + logger.info( + f"Directory {self.directory} for parameter set {self.name}" + f" already exists, skipping download." + ) return logger.info( f"Downloading example parameter set {self.name} to {self.directory}..." ) subprocess.check_call(["svn", "export", self.datafiles_url, self.directory]) - # TODO replace subversion with alternative see https://stackoverflow.com/questions/33066582/how-to-download-a-folder-from-github/48948711 + # TODO replace subversion with alternative, see + # https://stackoverflow.com/questions/33066582/how-to-download-a-folder-from-github/48948711 response = request.urlopen(self.config_url) self.config.write_text(response.read().decode()) diff --git a/src/ewatercycle/parameter_sets/_lisflood.py b/src/ewatercycle/parameter_sets/_lisflood.py index bfb37255..f808c655 100644 --- a/src/ewatercycle/parameter_sets/_lisflood.py +++ b/src/ewatercycle/parameter_sets/_lisflood.py @@ -11,11 +11,11 @@ def example_parameter_sets() -> Iterable[ExampleParameterSet]: name="lisflood_fraser", # Relative to CFG['parameterset_dir'] config="lisflood_fraser/settings_lat_lon-Run.xml", - datafiles_url="https://github.com/ec-jrc/lisflood-usecases/trunk/LF_lat_lon_UseCase", + datafiles_url="https://github.com/ec-jrc/lisflood-usecases/trunk/LF_lat_lon_UseCase", # noqa: E501 # Raw url to config file - config_url="https://github.com/ec-jrc/lisflood-usecases/raw/master/LF_lat_lon_UseCase/settings_lat_lon-Run.xml", + config_url="https://github.com/ec-jrc/lisflood-usecases/raw/master/LF_lat_lon_UseCase/settings_lat_lon-Run.xml", # noqa: E501 doi="N/A", target_model="lisflood", - supported_model_versions={"20.10"} + supported_model_versions={"20.10"}, ) ] diff --git a/src/ewatercycle/parameter_sets/_pcrglobwb.py b/src/ewatercycle/parameter_sets/_pcrglobwb.py index c70a7aeb..105f9158 100644 --- a/src/ewatercycle/parameter_sets/_pcrglobwb.py +++ b/src/ewatercycle/parameter_sets/_pcrglobwb.py @@ -11,11 +11,11 @@ def example_parameter_sets() -> Iterable[ExampleParameterSet]: name="pcrglobwb_rhinemeuse_30min", # Relative to CFG['parameterset_dir'] config="pcrglobwb_rhinemeuse_30min/setup_natural_test.ini", - datafiles_url="https://github.com/UU-Hydro/PCR-GLOBWB_input_example/trunk/RhineMeuse30min", + datafiles_url="https://github.com/UU-Hydro/PCR-GLOBWB_input_example/trunk/RhineMeuse30min", # noqa: E501 # Raw url to config file - config_url="https://raw.githubusercontent.com/UU-Hydro/PCR-GLOBWB_input_example/master/ini_and_batch_files_for_pcrglobwb_course/rhine_meuse_30min_using_input_example/setup_natural_test.ini", + config_url="https://raw.githubusercontent.com/UU-Hydro/PCR-GLOBWB_input_example/master/ini_and_batch_files_for_pcrglobwb_course/rhine_meuse_30min_using_input_example/setup_natural_test.ini", # noqa: E501 doi="https://doi.org/10.5281/zenodo.1045339", target_model="pcrglobwb", - supported_model_versions={"setters"} + supported_model_versions={"setters"}, ) ] diff --git a/src/ewatercycle/parameter_sets/_wflow.py b/src/ewatercycle/parameter_sets/_wflow.py index 8634b524..9fbad5f5 100644 --- a/src/ewatercycle/parameter_sets/_wflow.py +++ b/src/ewatercycle/parameter_sets/_wflow.py @@ -11,11 +11,11 @@ def example_parameter_sets() -> Iterable[ExampleParameterSet]: name="wflow_rhine_sbm_nc", # Relative to CFG['parameterset_dir'] config="wflow_rhine_sbm_nc/wflow_sbm_NC.ini", - datafiles_url="https://github.com/openstreams/wflow/trunk/examples/wflow_rhine_sbm_nc", + datafiles_url="https://github.com/openstreams/wflow/trunk/examples/wflow_rhine_sbm_nc", # noqa: E501 # Raw url to config file - config_url="https://github.com/openstreams/wflow/raw/master/examples/wflow_rhine_sbm_nc/wflow_sbm_NC.ini", + config_url="https://github.com/openstreams/wflow/raw/master/examples/wflow_rhine_sbm_nc/wflow_sbm_NC.ini", # noqa: E501 doi="N/A", target_model="wflow", - supported_model_versions={"2020.1.1"} + supported_model_versions={"2020.1.1"}, ) ] diff --git a/src/ewatercycle/parameter_sets/default.py b/src/ewatercycle/parameter_sets/default.py index cccce94a..e62b0a1f 100644 --- a/src/ewatercycle/parameter_sets/default.py +++ b/src/ewatercycle/parameter_sets/default.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Set, Optional +from typing import Optional, Set from ewatercycle import CFG from ewatercycle.util import to_absolute_path @@ -12,12 +12,15 @@ class ParameterSet: name (str): Name of parameter set directory (Path): Location on disk where files of parameter set are stored. If Path is relative then relative to CFG['parameterset_dir']. - config (Path): Model configuration file which uses files from :py:attr:`~directory`. - If Path is relative then relative to CFG['parameterset_dir']. - doi (str): Persistent identifier of parameter set. For a example a DOI for a Zenodo record. + config (Path): Model configuration file which uses files from + :py:attr:`~directory`. If Path is relative then relative to + CFG['parameterset_dir']. + doi (str): Persistent identifier of parameter set. For a example a DOI + for a Zenodo record. target_model (str): Name of model that parameter set can work with - supported_model_versions (Set[str]): Set of model versions that are supported by this parameter set. - If not set then parameter set will be supported by all versions of model + supported_model_versions (Set[str]): Set of model versions that are + supported by this parameter set. If not set then parameter set will be + supported by all versions of model """ def __init__( @@ -30,11 +33,17 @@ def __init__( supported_model_versions: Optional[Set[str]] = None, ): self.name = name - self.directory = to_absolute_path(directory, parent = CFG.get("parameterset_dir"), must_be_in_parent=False) - self.config = to_absolute_path(config, parent = CFG.get("parameterset_dir"), must_be_in_parent=False) + self.directory = to_absolute_path( + directory, parent=CFG.get("parameterset_dir"), must_be_in_parent=False + ) + self.config = to_absolute_path( + config, parent=CFG.get("parameterset_dir"), must_be_in_parent=False + ) self.doi = doi self.target_model = target_model - self.supported_model_versions = set() if supported_model_versions is None else supported_model_versions + self.supported_model_versions = ( + set() if supported_model_versions is None else supported_model_versions + ) def __repr__(self): options = ", ".join(f"{k}={v!s}" for k, v in self.__dict__.items()) @@ -54,4 +63,3 @@ def __str__(self): def is_available(self) -> bool: """Tests if directory and config file is available on this machine""" return self.directory.exists() and self.config.exists() - diff --git a/src/ewatercycle/parametersetdb/__init__.py b/src/ewatercycle/parametersetdb/__init__.py index c63a9bb2..75f709c4 100644 --- a/src/ewatercycle/parametersetdb/__init__.py +++ b/src/ewatercycle/parametersetdb/__init__.py @@ -2,8 +2,8 @@ """Documentation about ewatercycle_parametersetdb""" from typing import Any -from ewatercycle.parametersetdb.config import AbstractConfig, CONFIG_FORMATS -from ewatercycle.parametersetdb.datafiles import AbstractCopier, DATAFILES_FORMATS +from ewatercycle.parametersetdb.config import CONFIG_FORMATS, AbstractConfig +from ewatercycle.parametersetdb.datafiles import DATAFILES_FORMATS, AbstractCopier class ParameterSet: @@ -38,14 +38,17 @@ def save_config(self, target): def config(self) -> Any: """Configuration as dictionary. - To make changes to configuration before saving set the config keys and/or values. + To make changes to configuration before saving set the config keys + and/or values. Can be a nested dict. """ return self._cfg.config -def build_from_urls(config_format, config_url, datafiles_format, datafiles_url) -> ParameterSet: +def build_from_urls( + config_format, config_url, datafiles_format, datafiles_url +) -> ParameterSet: """Construct ParameterSet based on urls Args: diff --git a/src/ewatercycle/parametersetdb/config.py b/src/ewatercycle/parametersetdb/config.py index 105d0313..284160f8 100644 --- a/src/ewatercycle/parametersetdb/config.py +++ b/src/ewatercycle/parametersetdb/config.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -from configparser import ConfigParser from abc import ABC, abstractmethod +from configparser import ConfigParser +from typing import Any, Dict, Type from urllib.request import urlopen -from typing import Any, Type, Dict from ruamel.yaml import YAML @@ -11,6 +11,7 @@ class CaseConfigParser(ConfigParser): """Case sensitive config parser See https://stackoverflow.com/questions/1611799/preserve-case-in-configparser """ + def optionxform(self, optionstr): return optionstr @@ -47,8 +48,8 @@ def save(self, target: str): class IniConfig(AbstractConfig): - """Config container where config is read/saved in ini format. - """ + """Config container where config is read/saved in ini format.""" + def __init__(self, source): super().__init__(source) body = fetch(source) @@ -56,12 +57,13 @@ def __init__(self, source): self.config.read_string(body) def save(self, target): - with open(target, 'w') as f: + with open(target, "w") as f: self.config.write(f) class YamlConfig(AbstractConfig): """Config container where config is read/saved in yaml format""" + yaml = YAML() def __init__(self, source): @@ -70,11 +72,11 @@ def __init__(self, source): self.config = self.yaml.load(body) def save(self, target): - with open(target, 'w') as f: + with open(target, "w") as f: self.yaml.dump(self.config, f) CONFIG_FORMATS: Dict[str, Type[AbstractConfig]] = { - 'ini': IniConfig, - 'yaml': YamlConfig, + "ini": IniConfig, + "yaml": YamlConfig, } diff --git a/src/ewatercycle/parametersetdb/datafiles.py b/src/ewatercycle/parametersetdb/datafiles.py index 193c568f..cf705b8a 100644 --- a/src/ewatercycle/parametersetdb/datafiles.py +++ b/src/ewatercycle/parametersetdb/datafiles.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- import os import subprocess -from abc import abstractmethod, ABC -from typing import Type, Dict +from abc import ABC, abstractmethod +from typing import Dict, Type class AbstractCopier(ABC): @@ -28,22 +28,22 @@ def save(self, target: str): class SubversionCopier(AbstractCopier): - """Uses subversion export to copy files from source to target - """ + """Uses subversion export to copy files from source to target""" + def save(self, target): if os.path.exists(target): - raise Exception('Target directory already exists, will not overwrite') - subprocess.check_call(['svn', 'export', self.source, target]) + raise Exception("Target directory already exists, will not overwrite") + subprocess.check_call(["svn", "export", self.source, target]) class SymlinkCopier(AbstractCopier): - """Creates symlink from source to target - """ + """Creates symlink from source to target""" + def save(self, target): os.symlink(self.source, target) DATAFILES_FORMATS: Dict[str, Type[AbstractCopier]] = { - 'svn': SubversionCopier, - 'symlink': SymlinkCopier, + "svn": SubversionCopier, + "symlink": SymlinkCopier, } diff --git a/src/ewatercycle/util.py b/src/ewatercycle/util.py index 0eddcfc4..e1e67996 100644 --- a/src/ewatercycle/util.py +++ b/src/ewatercycle/util.py @@ -1,11 +1,10 @@ +from datetime import datetime from os import PathLike -from typing import Any, Iterable, Tuple, Dict from pathlib import Path +from typing import Any, Dict, Iterable, Tuple import fiona import numpy as np -from datetime import datetime - from dateutil.parser import parse from esmvalcore.experimental.recipe_output import RecipeOutput from shapely import geometry @@ -49,9 +48,7 @@ def find_closest_point( dx = np.diff(grid_longitudes).max() * 111 # (1 degree ~ 111km) dy = np.diff(grid_latitudes).max() * 111 # (1 degree ~ 111km) if distance > max(dx, dy) * 2: - raise ValueError( - f"Point {point_longitude, point_latitude} outside model grid." - ) + raise ValueError(f"Point {point_longitude, point_latitude} outside model grid.") return idx_lon, idx_lat @@ -66,7 +63,8 @@ def get_time(time_iso: str) -> datetime: time = parse(time_iso) if not time.tzname() == "UTC": raise ValueError( - f"The time is not in UTC. The ISO format for a UTC time is 'YYYY-MM-DDTHH:MM:SSZ'" + "The time is not in UTC. The ISO format for a UTC time " + "is 'YYYY-MM-DDTHH:MM:SSZ'" ) return time @@ -121,14 +119,20 @@ def data_files_from_recipe_output( return directory, forcing_files -def to_absolute_path(input_path: str, parent: Path = None, must_exist: bool = False, must_be_in_parent=True) -> Path: +def to_absolute_path( + input_path: str, + parent: Path = None, + must_exist: bool = False, + must_be_in_parent=True, +) -> Path: """Parse input string as :py:class:`pathlib.Path` object. Args: input_path: Input string path that can be a relative or absolute path. parent: Optional parent path of the input path must_exist: Optional argument to check if the input path exists. - must_be_in_parent: Optional argument to check if the input path is subpath of parent path + must_be_in_parent: Optional argument to check if the input path is + subpath of parent path Returns: The input path that is an absolute path and a :py:class:`pathlib.Path` object. @@ -140,6 +144,8 @@ def to_absolute_path(input_path: str, parent: Path = None, must_exist: bool = Fa try: pathlike.relative_to(parent) except ValueError: - raise ValueError(f"Input path {input_path} is not a subpath of parent {parent}") + raise ValueError( + f"Input path {input_path} is not a subpath of parent {parent}" + ) return pathlike.expanduser().resolve(strict=must_exist) diff --git a/src/ewatercycle/version.py b/src/ewatercycle/version.py index 2a283e93..2d53356f 100644 --- a/src/ewatercycle/version.py +++ b/src/ewatercycle/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -__version__ = '1.1.1' +__version__ = "1.1.1" diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 2da62aae..bef1d412 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -5,8 +5,8 @@ import pytest from ewatercycle import CFG - from ewatercycle.config._config_object import Config +from ewatercycle.config._validated_config import InvalidConfigParameter from ewatercycle.config._validators import ( _listify_validator, validate_float, @@ -18,9 +18,6 @@ validate_string_or_none, ) -from ewatercycle.config._validated_config import ( - InvalidConfigParameter, ) - def generate_validator_testcases(valid): # The code for this function was taken from matplotlib (v3.3) and modified @@ -30,45 +27,54 @@ def generate_validator_testcases(valid): validation_tests = ( { - 'validator': - _listify_validator(validate_float, n_items=2), - 'success': - ((_, [1.5, 2.5]) - for _ in ('1.5, 2.5', [1.5, 2.5], [1.5, 2.5], (1.5, 2.5), - np.array((1.5, 2.5)))), - 'fail': ((_, ValueError) for _ in ('fail', ('a', 1), (1, 2, 3))) + "validator": _listify_validator(validate_float, n_items=2), + "success": ( + (_, [1.5, 2.5]) + for _ in ( + "1.5, 2.5", + [1.5, 2.5], + [1.5, 2.5], + (1.5, 2.5), + np.array((1.5, 2.5)), + ) + ), + "fail": ((_, ValueError) for _ in ("fail", ("a", 1), (1, 2, 3))), }, { - 'validator': - _listify_validator(validate_float, n_items=2), - 'success': - ((_, [1.5, 2.5]) - for _ in ('1.5, 2.5', [1.5, 2.5], [1.5, 2.5], (1.5, 2.5), - np.array((1.5, 2.5)))), - 'fail': ((_, ValueError) for _ in ('fail', ('a', 1), (1, 2, 3))) + "validator": _listify_validator(validate_float, n_items=2), + "success": ( + (_, [1.5, 2.5]) + for _ in ( + "1.5, 2.5", + [1.5, 2.5], + [1.5, 2.5], + (1.5, 2.5), + np.array((1.5, 2.5)), + ) + ), + "fail": ((_, ValueError) for _ in ("fail", ("a", 1), (1, 2, 3))), }, { - 'validator': - _listify_validator(validate_int, n_items=2), - 'success': - ((_, [1, 2]) - for _ in ('1, 2', [1.5, 2.5], [1, 2], (1, 2), np.array((1, 2)))), - 'fail': ((_, ValueError) for _ in ('fail', ('a', 1), (1, 2, 3))) + "validator": _listify_validator(validate_int, n_items=2), + "success": ( + (_, [1, 2]) + for _ in ("1, 2", [1.5, 2.5], [1, 2], (1, 2), np.array((1, 2))) + ), + "fail": ((_, ValueError) for _ in ("fail", ("a", 1), (1, 2, 3))), }, { - 'validator': validate_int_or_none, - 'success': ((None, None), ), - 'fail': (), + "validator": validate_int_or_none, + "success": ((None, None),), + "fail": (), }, { - 'validator': - validate_path, - 'success': ( - ('a/b/c', Path.cwd() / 'a' / 'b' / 'c'), - ('/a/b/c/', Path('/', 'a', 'b', 'c')), - ('~/', Path.home()), + "validator": validate_path, + "success": ( + ("a/b/c", Path.cwd() / "a" / "b" / "c"), + ("/a/b/c/", Path("/", "a", "b", "c")), + ("~/", Path.home()), ), - 'fail': ( + "fail": ( (None, ValueError), (123, ValueError), (False, ValueError), @@ -76,58 +82,57 @@ def generate_validator_testcases(valid): ), }, { - 'validator': validate_path_or_none, - 'success': ((None, None), ), - 'fail': (), + "validator": validate_path_or_none, + "success": ((None, None),), + "fail": (), }, { - 'validator': - _listify_validator(validate_string), - 'success': ( - ('', []), - ('a,b', ['a', 'b']), - ('abc', ['abc']), - ('abc, ', ['abc']), - ('abc, ,', ['abc']), - (['a', 'b'], ['a', 'b']), - (('a', 'b'), ['a', 'b']), - (iter(['a', 'b']), ['a', 'b']), - (np.array(['a', 'b']), ['a', 'b']), - ((1, 2), ['1', '2']), - (np.array([1, 2]), ['1', '2']), + "validator": _listify_validator(validate_string), + "success": ( + ("", []), + ("a,b", ["a", "b"]), + ("abc", ["abc"]), + ("abc, ", ["abc"]), + ("abc, ,", ["abc"]), + (["a", "b"], ["a", "b"]), + (("a", "b"), ["a", "b"]), + (iter(["a", "b"]), ["a", "b"]), + (np.array(["a", "b"]), ["a", "b"]), + ((1, 2), ["1", "2"]), + (np.array([1, 2]), ["1", "2"]), ), - 'fail': ( + "fail": ( (set(), ValueError), (1, ValueError), - ) + ), }, { - 'validator': validate_string_or_none, - 'success': ((None, None), ), - 'fail': (), + "validator": validate_string_or_none, + "success": ((None, None),), + "fail": (), }, ) for validator_dict in validation_tests: - validator = validator_dict['validator'] + validator = validator_dict["validator"] if valid: - for arg, target in validator_dict['success']: + for arg, target in validator_dict["success"]: yield validator, arg, target else: - for arg, error_type in validator_dict['fail']: + for arg, error_type in validator_dict["fail"]: yield validator, arg, error_type -@pytest.mark.parametrize('validator, arg, target', - generate_validator_testcases(True)) +@pytest.mark.parametrize("validator, arg, target", generate_validator_testcases(True)) def test_validator_valid(validator, arg, target): """Test valid cases for the validators.""" res = validator(arg) assert res == target -@pytest.mark.parametrize('validator, arg, exception_type', - generate_validator_testcases(False)) +@pytest.mark.parametrize( + "validator, arg, exception_type", generate_validator_testcases(False) +) def test_validator_invalid(validator, arg, exception_type): """Test invalid cases for the validators.""" with pytest.raises(exception_type): @@ -138,17 +143,17 @@ def test_config_object(): """Test that the config is of the right type.""" assert isinstance(CFG, MutableMapping) - del CFG['output_dir'] - assert 'output_dir' not in CFG + del CFG["output_dir"] + assert "output_dir" not in CFG CFG.reload() - assert 'output_dir' in CFG + assert "output_dir" in CFG def test_config_update(): """Test whether `config.update` raises the correct exception.""" - config = Config({'output_dir': 'directory'}) - fail_dict = {'output_dir': 123} + config = Config({"output_dir": "directory"}) + fail_dict = {"output_dir": 123} with pytest.raises(InvalidConfigParameter): config.update(fail_dict) @@ -157,19 +162,19 @@ def test_config_update(): def test_config_class(): """Test that the validators turn strings into paths.""" config = { - 'container_engine': 'docker', - 'grdc_location': 'path/to/grdc_location', - 'output_dir': 'path/to/output_dir', - 'singularity_dir': 'path/to/singularity_dir', - 'parameterset_dir': 'path/to/parameter_sets', - 'parameter_sets': {} + "container_engine": "docker", + "grdc_location": "path/to/grdc_location", + "output_dir": "path/to/output_dir", + "singularity_dir": "path/to/singularity_dir", + "parameterset_dir": "path/to/parameter_sets", + "parameter_sets": {}, } cfg = Config(config) - assert isinstance(cfg['container_engine'], str) - assert isinstance(cfg['grdc_location'], Path) - assert isinstance(cfg['output_dir'], Path) - assert isinstance(cfg['singularity_dir'], Path) - assert isinstance(cfg['parameterset_dir'], Path) - assert isinstance(cfg['parameter_sets'], dict) + assert isinstance(cfg["container_engine"], str) + assert isinstance(cfg["grdc_location"], Path) + assert isinstance(cfg["output_dir"], Path) + assert isinstance(cfg["singularity_dir"], Path) + assert isinstance(cfg["parameterset_dir"], Path) + assert isinstance(cfg["parameter_sets"], dict) diff --git a/tests/conftest.py b/tests/conftest.py index 1cf25937..52d07ff1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,44 +8,61 @@ @pytest.fixture def yaml_config_url(): - return "data:text/plain,data: data/PEQ_Hupsel.dat\nparameters:\n cW: 200\n cV: 4\n cG: 5.0e+6\n cQ: 10\n cS: 4\n dG0: 1250\n cD: 1500\n aS: 0.01\n st: loamy_sand\nstart: 367416 # 2011120000\nend: 368904 # 2012020000\nstep: 1\n" + return "data:text/plain,data: data/PEQ_Hupsel.dat\nparameters:\n cW: 200\n cV: 4\n cG: 5.0e+6\n cQ: 10\n cS: 4\n dG0: 1250\n cD: 1500\n aS: 0.01\n st: loamy_sand\nstart: 367416 # 2011120000\nend: 368904 # 2012020000\nstep: 1\n" # noqa: E501 @pytest.fixture def yaml_config(): - return OrderedDict([ - ('data', 'data/PEQ_Hupsel.dat'), - ('parameters', OrderedDict([ - ('cW', 200), - ('cV', 4), - ('cG', 5000000.0), - ('cQ', 10), - ('cS', 4), - ('dG0', 1250), - ('cD', 1500), - ('aS', 0.01), - ('st', 'loamy_sand') - ])), - ('start', 367416), - ('end', 368904), - ('step', 1) - ]) + return OrderedDict( + [ + ("data", "data/PEQ_Hupsel.dat"), + ( + "parameters", + OrderedDict( + [ + ("cW", 200), + ("cV", 4), + ("cG", 5000000.0), + ("cQ", 10), + ("cS", 4), + ("dG0", 1250), + ("cD", 1500), + ("aS", 0.01), + ("st", "loamy_sand"), + ] + ), + ), + ("start", 367416), + ("end", 368904), + ("step", 1), + ] + ) @pytest.fixture def sample_parameterset(yaml_config_url): return build_from_urls( - config_format='yaml', config_url=yaml_config_url, - datafiles_format='svn', datafiles_url='http://example.com', + config_format="yaml", + config_url=yaml_config_url, + datafiles_format="svn", + datafiles_url="http://example.com", ) @pytest.fixture def sample_shape(): - return str(Path(__file__).parents[1] / 'docs' / 'examples' / 'data' / 'Rhine' / 'Rhine.shp') + return str( + Path(__file__).parents[1] / "docs" / "examples" / "data" / "Rhine" / "Rhine.shp" + ) + @pytest.fixture def sample_marrmot_forcing_file(): # Downloaded from # https://github.com/wknoben/MARRMoT/blob/master/BMI/Config/BMI_testcase_m01_BuffaloRiver_TN_USA.mat - return str(Path(__file__).parent / 'models' / 'data' / 'BMI_testcase_m01_BuffaloRiver_TN_USA.mat') + return str( + Path(__file__).parent + / "models" + / "data" + / "BMI_testcase_m01_BuffaloRiver_TN_USA.mat" + ) diff --git a/tests/forcing/test_default.py b/tests/forcing/test_default.py index a635974a..c7eb9b90 100644 --- a/tests/forcing/test_default.py +++ b/tests/forcing/test_default.py @@ -1,55 +1,73 @@ import logging + import pytest -from ewatercycle.forcing import generate, load_foreign, DefaultForcing, load, FORCING_YAML +from ewatercycle.forcing import ( + FORCING_YAML, + DefaultForcing, + generate, + load, + load_foreign, +) def test_generate_unknown_model(sample_shape): with pytest.raises(NotImplementedError): generate( target_model="unknown", - dataset='ERA5', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - shape=sample_shape + dataset="ERA5", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + shape=sample_shape, ) def test_load_foreign_unknown(): with pytest.raises(NotImplementedError) as excinfo: load_foreign( - target_model='unknown', - directory='/data/unknown-forcings-case1', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z' + target_model="unknown", + directory="/data/unknown-forcings-case1", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", ) - assert 'Target model `unknown` is not supported by the eWatercycle forcing generator' in str(excinfo.value) + assert ( + "Target model `unknown` is not supported by the eWatercycle forcing generator" + in str(excinfo.value) + ) @pytest.fixture def sample_forcing_yaml_content(): - return ''.join([ - '!DefaultForcing\n', - "start_time: '1989-01-02T00:00:00Z'\n", - "end_time: '1999-01-02T00:00:00Z'\n", - "shape:\n" - ]) + return "".join( + [ + "!DefaultForcing\n", + "start_time: '1989-01-02T00:00:00Z'\n", + "end_time: '1999-01-02T00:00:00Z'\n", + "shape:\n", + ] + ) + @pytest.fixture def sample_forcing_yaml_content_with_shape(): - return ''.join([ - '!DefaultForcing\n', - "start_time: '1989-01-02T00:00:00Z'\n", - "end_time: '1999-01-02T00:00:00Z'\n", - "shape: myshape.shp\n" - ]) - -def test_save_with_shapefile_outside_forcing_dir(sample_shape, tmp_path, sample_forcing_yaml_content, caplog): + return "".join( + [ + "!DefaultForcing\n", + "start_time: '1989-01-02T00:00:00Z'\n", + "end_time: '1999-01-02T00:00:00Z'\n", + "shape: myshape.shp\n", + ] + ) + + +def test_save_with_shapefile_outside_forcing_dir( + sample_shape, tmp_path, sample_forcing_yaml_content, caplog +): forcing = DefaultForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - shape=sample_shape + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + shape=sample_shape, ) with caplog.at_level(logging.INFO): forcing.save() @@ -58,15 +76,18 @@ def test_save_with_shapefile_outside_forcing_dir(sample_shape, tmp_path, sample_ written = file.read_text() expected = sample_forcing_yaml_content assert written == expected - assert 'is not in forcing directory' in caplog.text + assert "is not in forcing directory" in caplog.text -def test_save_with_shapefile_inside_forcing_dir(tmp_path, sample_forcing_yaml_content_with_shape, caplog): + +def test_save_with_shapefile_inside_forcing_dir( + tmp_path, sample_forcing_yaml_content_with_shape, caplog +): forcing = DefaultForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - shape=str(tmp_path / "myshape.shp") + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + shape=str(tmp_path / "myshape.shp"), ) with caplog.at_level(logging.INFO): forcing.save() @@ -75,15 +96,15 @@ def test_save_with_shapefile_inside_forcing_dir(tmp_path, sample_forcing_yaml_co written = file.read_text() expected = sample_forcing_yaml_content_with_shape assert written == expected - assert 'is not in forcing directory' not in caplog.text + assert "is not in forcing directory" not in caplog.text def test_save_without_shapefile(tmp_path, sample_forcing_yaml_content): forcing = DefaultForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", ) forcing.save() @@ -99,8 +120,8 @@ def test_load(tmp_path, sample_forcing_yaml_content): result = load(tmp_path) expected = DefaultForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", ) assert result == expected @@ -111,10 +132,9 @@ def test_load_with_shape(tmp_path, sample_forcing_yaml_content_with_shape): result = load(tmp_path) expected = DefaultForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - shape=tmp_path / "myshape.shp" - + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + shape=tmp_path / "myshape.shp", ) assert result == expected diff --git a/tests/forcing/test_lisflood.py b/tests/forcing/test_lisflood.py index 6d1736dd..23407775 100644 --- a/tests/forcing/test_lisflood.py +++ b/tests/forcing/test_lisflood.py @@ -1,9 +1,9 @@ import numpy as np import pandas as pd import pytest +import xarray as xr from esmvalcore.experimental import Recipe from esmvalcore.experimental.recipe_output import DataFile -import xarray as xr from ewatercycle.forcing import generate, load from ewatercycle.forcing._lisflood import LisfloodForcing @@ -11,9 +11,9 @@ def test_plot(): f = LisfloodForcing( - directory='.', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + directory=".", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", ) with pytest.raises(NotImplementedError): f.plot() @@ -23,12 +23,14 @@ def create_netcdf(var_name, filename): var = 15 + 8 * np.random.randn(2, 2, 3) lon = [[-99.83, -99.32], [-99.79, -99.23]] lat = [[42.25, 42.21], [42.63, 42.59]] - ds = xr.Dataset({var_name: (["longitude", "latitude", "time"], var)}, - coords={ - "lon": (["longitude", "latitude"], lon), - "lat": (["longitude", "latitude"], lat), - "time": pd.date_range("2014-09-06", periods=3), - }) + ds = xr.Dataset( + {var_name: (["longitude", "latitude", "time"], var)}, + coords={ + "lon": (["longitude", "latitude"], lon), + "lat": (["longitude", "latitude"], lat), + "time": pd.date_range("2014-09-06", periods=3), + }, + ) ds.to_netcdf(filename) return DataFile(filename) @@ -40,15 +42,15 @@ def mock_recipe_run(monkeypatch, tmp_path): # TODO add lisvap input files once implemented, see issue #96 class MockTaskOutput: data_files = ( - create_netcdf('pr', tmp_path / 'lisflood_pr.nc'), - create_netcdf('tas', tmp_path / 'lisflood_tas.nc'), + create_netcdf("pr", tmp_path / "lisflood_pr.nc"), + create_netcdf("tas", tmp_path / "lisflood_tas.nc"), ) def mock_run(self): """Store recipe for inspection and return dummy output.""" nonlocal data - data['data_during_run'] = self.data - return {'diagnostic_daily/script': MockTaskOutput()} + data["data_during_run"] = self.data + return {"diagnostic_daily/script": MockTaskOutput()} monkeypatch.setattr(Recipe, "run", mock_run) return data @@ -58,226 +60,209 @@ class TestGenerateRegionFromShapeFile: @pytest.fixture def forcing(self, mock_recipe_run, sample_shape): return generate( - target_model='lisflood', - dataset='ERA5', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + target_model="lisflood", + dataset="ERA5", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, ) @pytest.fixture def reference_recipe(self): return { - 'datasets': [{ - 'dataset': 'ERA5', - 'project': 'OBS6', - 'tier': 3, - 'type': 'reanaly', - 'version': 1 - }], - 'diagnostics': { - 'diagnostic_daily': { - 'description': - 'LISFLOOD input ' - 'preprocessor for ' - 'ERA-Interim and ERA5 ' - 'data', - 'scripts': { - 'script': { - 'catchment': 'Rhine', - 'script': 'hydrology/lisflood.py' + "datasets": [ + { + "dataset": "ERA5", + "project": "OBS6", + "tier": 3, + "type": "reanaly", + "version": 1, + } + ], + "diagnostics": { + "diagnostic_daily": { + "description": "LISFLOOD input " + "preprocessor for " + "ERA-Interim and ERA5 " + "data", + "scripts": { + "script": { + "catchment": "Rhine", + "script": "hydrology/lisflood.py", } }, - 'variables': { - 'pr': { - 'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily_water', - 'start_year': 1989 + "variables": { + "pr": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily_water", + "start_year": 1989, }, - 'rsds': { - 'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily_radiation', - 'start_year': 1989 + "rsds": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily_radiation", + "start_year": 1989, }, - 'tas': { - 'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily_temperature', - 'start_year': 1989 + "tas": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily_temperature", + "start_year": 1989, }, - 'tasmax': { - 'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily_temperature', - 'start_year': 1989 + "tasmax": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily_temperature", + "start_year": 1989, }, - 'tasmin': { - 'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily_temperature', - 'start_year': 1989 + "tasmin": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily_temperature", + "start_year": 1989, }, - 'tdps': { - 'end_year': 1999, - 'mip': 'Eday', - 'preprocessor': 'daily_temperature', - 'start_year': 1989 + "tdps": { + "end_year": 1999, + "mip": "Eday", + "preprocessor": "daily_temperature", + "start_year": 1989, }, - 'uas': { - 'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily_windspeed', - 'start_year': 1989 + "uas": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily_windspeed", + "start_year": 1989, }, - 'vas': { - 'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily_windspeed', - 'start_year': 1989 - } - } + "vas": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily_windspeed", + "start_year": 1989, + }, + }, } }, - 'documentation': { - 'authors': - ['verhoeven_stefan', 'kalverla_peter', 'andela_bouwe'], - 'projects': ['ewatercycle'], - 'references': ['acknow_project'] + "documentation": { + "authors": ["verhoeven_stefan", "kalverla_peter", "andela_bouwe"], + "projects": ["ewatercycle"], + "references": ["acknow_project"], }, - 'preprocessors': { - 'daily_radiation': { - 'convert_units': { - 'units': 'J m-2 ' - 'day-1' + "preprocessors": { + "daily_radiation": { + "convert_units": {"units": "J m-2 " "day-1"}, + "custom_order": True, + "extract_region": { + "end_latitude": 52.2, + "end_longitude": 11.9, + "start_latitude": 46.3, + "start_longitude": 4.1, }, - 'custom_order': True, - 'extract_region': { - 'end_latitude': 52.2, - 'end_longitude': 11.9, - 'start_latitude': 46.3, - 'start_longitude': 4.1 + "extract_shape": {"crop": True, "method": "contains"}, + "regrid": { + "lat_offset": True, + "lon_offset": True, + "scheme": "linear", + "target_grid": "0.1x0.1", }, - 'extract_shape': { - 'crop': True, - 'method': 'contains' - }, - 'regrid': { - 'lat_offset': True, - 'lon_offset': True, - 'scheme': 'linear', - 'target_grid': '0.1x0.1' - } }, - 'daily_temperature': { - 'convert_units': { - 'units': 'degC' - }, - 'custom_order': True, - 'extract_region': { - 'end_latitude': 52.2, - 'end_longitude': 11.9, - 'start_latitude': 46.3, - 'start_longitude': 4.1 + "daily_temperature": { + "convert_units": {"units": "degC"}, + "custom_order": True, + "extract_region": { + "end_latitude": 52.2, + "end_longitude": 11.9, + "start_latitude": 46.3, + "start_longitude": 4.1, }, - 'extract_shape': { - 'crop': True, - 'method': 'contains' + "extract_shape": {"crop": True, "method": "contains"}, + "regrid": { + "lat_offset": True, + "lon_offset": True, + "scheme": "linear", + "target_grid": "0.1x0.1", }, - 'regrid': { - 'lat_offset': True, - 'lon_offset': True, - 'scheme': 'linear', - 'target_grid': '0.1x0.1' - } }, - 'daily_water': { - 'convert_units': { - 'units': 'kg m-2 d-1' + "daily_water": { + "convert_units": {"units": "kg m-2 d-1"}, + "custom_order": True, + "extract_region": { + "end_latitude": 52.2, + "end_longitude": 11.9, + "start_latitude": 46.3, + "start_longitude": 4.1, }, - 'custom_order': True, - 'extract_region': { - 'end_latitude': 52.2, - 'end_longitude': 11.9, - 'start_latitude': 46.3, - 'start_longitude': 4.1 + "extract_shape": {"crop": True, "method": "contains"}, + "regrid": { + "lat_offset": True, + "lon_offset": True, + "scheme": "linear", + "target_grid": "0.1x0.1", }, - 'extract_shape': { - 'crop': True, - 'method': 'contains' - }, - 'regrid': { - 'lat_offset': True, - 'lon_offset': True, - 'scheme': 'linear', - 'target_grid': '0.1x0.1' - } }, - 'daily_windspeed': { - 'custom_order': True, - 'extract_region': { - 'end_latitude': 52.2, - 'end_longitude': 11.9, - 'start_latitude': 46.3, - 'start_longitude': 4.1 + "daily_windspeed": { + "custom_order": True, + "extract_region": { + "end_latitude": 52.2, + "end_longitude": 11.9, + "start_latitude": 46.3, + "start_longitude": 4.1, }, - 'extract_shape': { - 'crop': True, - 'method': 'contains' + "extract_shape": {"crop": True, "method": "contains"}, + "regrid": { + "lat_offset": True, + "lon_offset": True, + "scheme": "linear", + "target_grid": "0.1x0.1", }, - 'regrid': { - 'lat_offset': True, - 'lon_offset': True, - 'scheme': 'linear', - 'target_grid': '0.1x0.1' - } }, - 'general': { - 'custom_order': True, - 'extract_region': { - 'end_latitude': 52.2, - 'end_longitude': 11.9, - 'start_latitude': 46.3, - 'start_longitude': 4.1 + "general": { + "custom_order": True, + "extract_region": { + "end_latitude": 52.2, + "end_longitude": 11.9, + "start_latitude": 46.3, + "start_longitude": 4.1, }, - 'extract_shape': { - 'crop': True, - 'method': 'contains' + "extract_shape": {"crop": True, "method": "contains"}, + "regrid": { + "lat_offset": True, + "lon_offset": True, + "scheme": "linear", + "target_grid": "0.1x0.1", }, - 'regrid': { - 'lat_offset': True, - 'lon_offset': True, - 'scheme': 'linear', - 'target_grid': '0.1x0.1' - } - } - } + }, + }, } def test_result(self, forcing, tmp_path, sample_shape): - expected = LisfloodForcing(directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - shape=str(sample_shape), - PrefixPrecipitation='lisflood_pr.nc', - PrefixTavg='lisflood_tas.nc') + expected = LisfloodForcing( + directory=str(tmp_path), + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + shape=str(sample_shape), + PrefixPrecipitation="lisflood_pr.nc", + PrefixTavg="lisflood_tas.nc", + ) assert forcing == expected - def test_recipe_configured(self, forcing, mock_recipe_run, - reference_recipe, sample_shape): - actual = mock_recipe_run['data_during_run'] + def test_recipe_configured( + self, forcing, mock_recipe_run, reference_recipe, sample_shape + ): + actual = mock_recipe_run["data_during_run"] # Remove long description and absolute path so assert is easier - actual_desc = actual['documentation']['description'] - del actual['documentation']['description'] - actual_shapefile = actual['preprocessors']['general']['extract_shape'][ - 'shapefile'] - # Will also del other occurrences of shapefile due to extract shape object being shared between preprocessors - del actual['preprocessors']['general']['extract_shape']['shapefile'] + actual_desc = actual["documentation"]["description"] + del actual["documentation"]["description"] + actual_shapefile = actual["preprocessors"]["general"]["extract_shape"][ + "shapefile" + ] + # Will also del other occurrences of shapefile due to extract shape object + # being shared between preprocessors + del actual["preprocessors"]["general"]["extract_shape"]["shapefile"] assert actual == reference_recipe assert actual_shapefile == sample_shape - assert 'LISFLOOD' in actual_desc + assert "LISFLOOD" in actual_desc def test_saved_yaml(self, forcing, tmp_path): saved_forcing = load(tmp_path) diff --git a/tests/forcing/test_marrmot.py b/tests/forcing/test_marrmot.py index fd332ee3..38734c93 100644 --- a/tests/forcing/test_marrmot.py +++ b/tests/forcing/test_marrmot.py @@ -10,10 +10,10 @@ def test_plot(): f = MarrmotForcing( - directory='.', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - forcing_file='marrmot.mat', + directory=".", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + forcing_file="marrmot.mat", ) with pytest.raises(NotImplementedError): f.plot() @@ -25,16 +25,14 @@ def mock_recipe_run(monkeypatch, tmp_path): recorder = {} class MockTaskOutput: - fake_forcing_path = str(tmp_path / 'marrmot.mat') - files = ( - OutputFile(fake_forcing_path), - ) + fake_forcing_path = str(tmp_path / "marrmot.mat") + files = (OutputFile(fake_forcing_path),) def mock_run(self): """Store recipe for inspection and return dummy output.""" nonlocal recorder - recorder['data_during_run'] = self.data - return {'diagnostic_daily/script': MockTaskOutput()} + recorder["data_during_run"] = self.data + return {"diagnostic_daily/script": MockTaskOutput()} monkeypatch.setattr(Recipe, "run", mock_run) return recorder @@ -44,83 +42,105 @@ class TestGenerate: @pytest.fixture def forcing(self, mock_recipe_run, sample_shape): return generate( - target_model='marrmot', - dataset='ERA5', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + target_model="marrmot", + dataset="ERA5", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, ) @pytest.fixture def reference_recipe(self): return { - 'diagnostics': { - 'diagnostic_daily': { - 'additional_datasets': [{'dataset': 'ERA5', - 'project': 'OBS6', - 'tier': 3, - 'type': 'reanaly', - 'version': 1}], - 'description': 'marrmot input ' - 'preprocessor for daily ' - 'data', - 'scripts': {'script': {'basin': 'Rhine', - 'script': 'hydrology/marrmot.py'}}, - 'variables': {'pr': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily', - 'start_year': 1989}, - 'psl': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily', - 'start_year': 1989}, - 'rsds': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily', - 'start_year': 1989}, - 'rsdt': {'end_year': 1999, - 'mip': 'CFday', - 'preprocessor': 'daily', - 'start_year': 1989}, - 'tas': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'daily', - 'start_year': 1989} - } - + "diagnostics": { + "diagnostic_daily": { + "additional_datasets": [ + { + "dataset": "ERA5", + "project": "OBS6", + "tier": 3, + "type": "reanaly", + "version": 1, + } + ], + "description": "marrmot input " "preprocessor for daily " "data", + "scripts": { + "script": {"basin": "Rhine", "script": "hydrology/marrmot.py"} + }, + "variables": { + "pr": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily", + "start_year": 1989, + }, + "psl": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily", + "start_year": 1989, + }, + "rsds": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily", + "start_year": 1989, + }, + "rsdt": { + "end_year": 1999, + "mip": "CFday", + "preprocessor": "daily", + "start_year": 1989, + }, + "tas": { + "end_year": 1999, + "mip": "day", + "preprocessor": "daily", + "start_year": 1989, + }, + }, + } + }, + "documentation": { + "authors": ["kalverla_peter", "camphuijsen_jaro", "alidoost_sarah"], + "projects": ["ewatercycle"], + "references": ["acknow_project"], + }, + "preprocessors": { + "daily": { + "extract_shape": { + "crop": True, + "method": "contains", + } } }, - 'documentation': {'authors': ['kalverla_peter', - 'camphuijsen_jaro', - 'alidoost_sarah'], - 'projects': ['ewatercycle'], - 'references': ['acknow_project']}, - 'preprocessors': {'daily': {'extract_shape': {'crop': True, - 'method': 'contains', - }}} } def test_result(self, forcing, tmp_path, sample_shape): expected = MarrmotForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - shape = str(sample_shape), - forcing_file='marrmot.mat' + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + shape=str(sample_shape), + forcing_file="marrmot.mat", ) assert forcing == expected - def test_recipe_configured(self, forcing, mock_recipe_run, reference_recipe, sample_shape): - actual = mock_recipe_run['data_during_run'] + def test_recipe_configured( + self, forcing, mock_recipe_run, reference_recipe, sample_shape + ): + actual = mock_recipe_run["data_during_run"] # Remove long description and absolute path so assert is easier - actual_desc = actual['documentation']['description'] - del actual['documentation']['description'] - actual_shapefile = actual['preprocessors']['daily']['extract_shape']['shapefile'] - del actual['preprocessors']['daily']['extract_shape']['shapefile'] + actual_desc = actual["documentation"]["description"] + del actual["documentation"]["description"] + actual_shapefile = actual["preprocessors"]["daily"]["extract_shape"][ + "shapefile" + ] + del actual["preprocessors"]["daily"]["extract_shape"]["shapefile"] assert actual == reference_recipe assert actual_shapefile == sample_shape - assert 'MARRMoT' in actual_desc + assert "MARRMoT" in actual_desc def test_saved_yaml(self, forcing, tmp_path): saved_forcing = load(tmp_path) @@ -133,40 +153,38 @@ def test_saved_yaml(self, forcing, tmp_path): def test_load_foreign(sample_shape, sample_marrmot_forcing_file): forcing_file = Path(sample_marrmot_forcing_file) actual = load_foreign( - target_model='marrmot', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + target_model="marrmot", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, directory=str(forcing_file.parent), - forcing_info={ - 'forcing_file': str(forcing_file.name) - } + forcing_info={"forcing_file": str(forcing_file.name)}, ) expected = MarrmotForcing( - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, directory=str(forcing_file.parent), - forcing_file=str(forcing_file.name) + forcing_file=str(forcing_file.name), ) assert actual == expected def test_load_foreign_without_forcing_info(sample_shape): actual = load_foreign( - target_model='marrmot', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + target_model="marrmot", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, - directory='/data' + directory="/data", ) expected = MarrmotForcing( - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, - directory='/data', - forcing_file='marrmot.mat' + directory="/data", + forcing_file="marrmot.mat", ) assert actual == expected diff --git a/tests/forcing/test_pcrglobwb.py b/tests/forcing/test_pcrglobwb.py index f3739ef4..e66a36e6 100644 --- a/tests/forcing/test_pcrglobwb.py +++ b/tests/forcing/test_pcrglobwb.py @@ -1,9 +1,9 @@ import numpy as np import pandas as pd import pytest +import xarray as xr from esmvalcore.experimental import Recipe from esmvalcore.experimental.recipe_output import DataFile -import xarray as xr from ewatercycle.forcing import generate from ewatercycle.forcing._pcrglobwb import PCRGlobWBForcing @@ -14,14 +14,12 @@ def create_netcdf(var_name, filename): lon = [[-99.83, -99.32], [-99.79, -99.23]] lat = [[42.25, 42.21], [42.63, 42.59]] ds = xr.Dataset( - { - var_name: (["longitude", "latitude", "time"], var) - }, + {var_name: (["longitude", "latitude", "time"], var)}, coords={ "lon": (["longitude", "latitude"], lon), "lat": (["longitude", "latitude"], lat), "time": pd.date_range("2014-09-06", periods=3), - } + }, ) ds.to_netcdf(filename) return DataFile(filename) @@ -34,15 +32,15 @@ def mock_recipe_run(monkeypatch, tmp_path): class MockTaskOutput: data_files = ( - create_netcdf('pr', tmp_path / 'pcrglobwb_pr.nc'), - create_netcdf('tas', tmp_path / 'pcrglobwb_tas.nc'), + create_netcdf("pr", tmp_path / "pcrglobwb_pr.nc"), + create_netcdf("tas", tmp_path / "pcrglobwb_tas.nc"), ) def mock_run(self): """Store recipe for inspection and return dummy output.""" nonlocal data - data['data_during_run'] = self.data - return {'diagnostic_daily/script': MockTaskOutput()} + data["data_during_run"] = self.data + return {"diagnostic_daily/script": MockTaskOutput()} monkeypatch.setattr(Recipe, "run", mock_run) return data @@ -52,33 +50,34 @@ class TestGenerateWithExtractRegion: @pytest.fixture def forcing(self, mock_recipe_run, sample_shape): return generate( - target_model='pcrglobwb', - dataset='ERA5', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + target_model="pcrglobwb", + dataset="ERA5", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, model_specific_options=dict( - start_time_climatology='1979-01-02T00:00:00Z', - end_time_climatology='1989-01-02T00:00:00Z', + start_time_climatology="1979-01-02T00:00:00Z", + end_time_climatology="1989-01-02T00:00:00Z", extract_region={ - 'start_longitude': 10, - 'end_longitude': 16.75, - 'start_latitude': 7.25, - 'end_latitude': 2.5, - } - ) + "start_longitude": 10, + "end_longitude": 16.75, + "start_latitude": 7.25, + "end_latitude": 2.5, + }, + ), ) def test_result(self, forcing, tmp_path, sample_shape): expected = PCRGlobWBForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=str(sample_shape), - precipitationNC='pcrglobwb_pr.nc', - temperatureNC='pcrglobwb_tas.nc' + precipitationNC="pcrglobwb_pr.nc", + temperatureNC="pcrglobwb_tas.nc", ) assert forcing == expected + # TODO test if recipe was generated correctlu # TODO test if yaml was written diff --git a/tests/forcing/test_wflow.py b/tests/forcing/test_wflow.py index f68a6fd4..734ab27c 100644 --- a/tests/forcing/test_wflow.py +++ b/tests/forcing/test_wflow.py @@ -14,16 +14,14 @@ def mock_recipe_run(monkeypatch, tmp_path): data = {} class MockTaskOutput: - fake_forcing_path = str(tmp_path / 'wflow_forcing.nc') - data_files = ( - DataFile(fake_forcing_path), - ) + fake_forcing_path = str(tmp_path / "wflow_forcing.nc") + data_files = (DataFile(fake_forcing_path),) def mock_run(self): """Store recipe for inspection and return dummy output.""" nonlocal data - data['data_during_run'] = self.data - return {'wflow_daily/script': MockTaskOutput()} + data["data_during_run"] = self.data + return {"wflow_daily/script": MockTaskOutput()} monkeypatch.setattr(Recipe, "run", mock_run) return data @@ -33,88 +31,116 @@ class TestGenerateWithExtractRegion: @pytest.fixture def reference_recipe(self): return { - 'diagnostics': { - 'wflow_daily': { - 'additional_datasets': [{'dataset': 'ERA5', - 'project': 'OBS6', - 'tier': 3, - 'type': 'reanaly', - 'version': 1}], - 'description': 'WFlow input preprocessor for ' - 'daily data', - 'scripts': {'script': {'basin': 'Rhine', - 'dem_file': 'wflow_parameterset/meuse/staticmaps/wflow_dem.map', - 'regrid': 'area_weighted', - 'script': 'hydrology/wflow.py'}}, - 'variables': {'orog': {'mip': 'fx', - 'preprocessor': 'rough_cutout'}, - 'pr': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'rough_cutout', - 'start_year': 1989}, - 'psl': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'rough_cutout', - 'start_year': 1989}, - 'rsds': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'rough_cutout', - 'start_year': 1989}, - 'rsdt': {'end_year': 1999, - 'mip': 'CFday', - 'preprocessor': 'rough_cutout', - 'start_year': 1989}, - 'tas': {'end_year': 1999, - 'mip': 'day', - 'preprocessor': 'rough_cutout', - 'start_year': 1989}}}}, - 'documentation': {'authors': ['kalverla_peter', - 'camphuijsen_jaro', - 'alidoost_sarah', - 'aerts_jerom', - 'andela_bouwe'], - 'description': 'Pre-processes climate data for the WFlow hydrological model.\n', - 'projects': ['ewatercycle'], - 'references': ['acknow_project']}, - 'preprocessors': {'rough_cutout': {'extract_region': {'end_latitude': 2.5, - 'end_longitude': 16.75, - 'start_latitude': 7.25, - 'start_longitude': 10} - } - } + "diagnostics": { + "wflow_daily": { + "additional_datasets": [ + { + "dataset": "ERA5", + "project": "OBS6", + "tier": 3, + "type": "reanaly", + "version": 1, + } + ], + "description": "WFlow input preprocessor for " "daily data", + "scripts": { + "script": { + "basin": "Rhine", + "dem_file": "wflow_parameterset/meuse/staticmaps/wflow_dem.map", # noqa: E501 + "regrid": "area_weighted", + "script": "hydrology/wflow.py", + } + }, + "variables": { + "orog": {"mip": "fx", "preprocessor": "rough_cutout"}, + "pr": { + "end_year": 1999, + "mip": "day", + "preprocessor": "rough_cutout", + "start_year": 1989, + }, + "psl": { + "end_year": 1999, + "mip": "day", + "preprocessor": "rough_cutout", + "start_year": 1989, + }, + "rsds": { + "end_year": 1999, + "mip": "day", + "preprocessor": "rough_cutout", + "start_year": 1989, + }, + "rsdt": { + "end_year": 1999, + "mip": "CFday", + "preprocessor": "rough_cutout", + "start_year": 1989, + }, + "tas": { + "end_year": 1999, + "mip": "day", + "preprocessor": "rough_cutout", + "start_year": 1989, + }, + }, + } + }, + "documentation": { + "authors": [ + "kalverla_peter", + "camphuijsen_jaro", + "alidoost_sarah", + "aerts_jerom", + "andela_bouwe", + ], + "description": "Pre-processes climate data for the WFlow hydrological model.\n", # noqa: E501 + "projects": ["ewatercycle"], + "references": ["acknow_project"], + }, + "preprocessors": { + "rough_cutout": { + "extract_region": { + "end_latitude": 2.5, + "end_longitude": 16.75, + "start_latitude": 7.25, + "start_longitude": 10, + } + } + }, } @pytest.fixture def forcing(self, mock_recipe_run, sample_shape): return generate( - target_model='wflow', - dataset='ERA5', - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', + target_model="wflow", + dataset="ERA5", + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", shape=sample_shape, model_specific_options=dict( - dem_file='wflow_parameterset/meuse/staticmaps/wflow_dem.map', + dem_file="wflow_parameterset/meuse/staticmaps/wflow_dem.map", extract_region={ - 'start_longitude': 10, - 'end_longitude': 16.75, - 'start_latitude': 7.25, - 'end_latitude': 2.5, - } - ) + "start_longitude": 10, + "end_longitude": 16.75, + "start_latitude": 7.25, + "end_latitude": 2.5, + }, + ), ) def test_result(self, forcing, tmp_path, sample_shape): expected = WflowForcing( directory=str(tmp_path), - start_time='1989-01-02T00:00:00Z', - end_time='1999-01-02T00:00:00Z', - shape = str(sample_shape), - netcdfinput='wflow_forcing.nc' + start_time="1989-01-02T00:00:00Z", + end_time="1999-01-02T00:00:00Z", + shape=str(sample_shape), + netcdfinput="wflow_forcing.nc", ) assert forcing == expected def test_recipe_configured(self, forcing, mock_recipe_run, reference_recipe): - assert mock_recipe_run['data_during_run'] == reference_recipe + assert mock_recipe_run["data_during_run"] == reference_recipe def test_saved_yaml(self, forcing, tmp_path): saved_forcing = load(tmp_path) diff --git a/tests/models/test_abstract.py b/tests/models/test_abstract.py index 77a47c89..878ba724 100644 --- a/tests/models/test_abstract.py +++ b/tests/models/test_abstract.py @@ -1,8 +1,8 @@ import logging import weakref +from datetime import datetime, timezone from typing import Any, Iterable, Tuple from unittest.mock import patch -from datetime import datetime, timezone import numpy as np import pytest @@ -18,25 +18,25 @@ @pytest.fixture def setup_config(tmp_path): - CFG['parameterset_dir'] = tmp_path - CFG['ewatercycle_config'] = tmp_path / 'ewatercycle.yaml' + CFG["parameterset_dir"] = tmp_path + CFG["ewatercycle_config"] = tmp_path / "ewatercycle.yaml" yield CFG - CFG['ewatercycle_config'] = DEFAULT_CONFIG + CFG["ewatercycle_config"] = DEFAULT_CONFIG CFG.reload() class MockedModel(AbstractModel): - available_versions = ('0.4.2',) + available_versions = ("0.4.2",) - def __init__(self, version: str = '0.4.2', parameter_set: ParameterSet = None): + def __init__(self, version: str = "0.4.2", parameter_set: ParameterSet = None): super().__init__(version, parameter_set) def setup(self, *args, **kwargs) -> Tuple[str, str]: - if 'bmi' in kwargs: + if "bmi" in kwargs: # sub-class of AbstractModel should construct bmi # using grpc4bmi Docker or Singularity client - self.bmi = kwargs['bmi'] - return 'foobar.cfg', '.' + self.bmi = kwargs["bmi"] + return "foobar.cfg", "." def get_value_as_xarray(self, name: str) -> xr.DataArray: return xr.DataArray( @@ -44,28 +44,31 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: coords={ "latitude": [42.25, 42.21], "longitude": [-99.83, -99.32], - "time": '2014-09-06'}, + "time": "2014-09-06", + }, dims=["longitude", "latitude"], - name='Temperature', + name="Temperature", attrs=dict(units="degC"), ) - def _coords_to_indices(self, name: str, lat: Iterable[float], lon: Iterable[float]) -> Iterable[int]: + def _coords_to_indices( + self, name: str, lat: Iterable[float], lon: Iterable[float] + ) -> Iterable[int]: return np.array([0]) @property def parameters(self) -> Iterable[Tuple[str, Any]]: - return [('area', 42)] + return [("area", 42)] @pytest.fixture -@patch('basic_modeling_interface.Bmi') +@patch("basic_modeling_interface.Bmi") def bmi(MockedBmi): mocked_bmi = MockedBmi() mocked_bmi.get_start_time.return_value = 42.0 mocked_bmi.get_current_time.return_value = 43.0 mocked_bmi.get_end_time.return_value = 44.0 - mocked_bmi.get_time_units.return_value = 'days since 1970-01-01 00:00:00.0 00:00' + mocked_bmi.get_time_units.return_value = "days since 1970-01-01 00:00:00.0 00:00" return mocked_bmi @@ -78,29 +81,32 @@ def model(bmi: Bmi): def test_construct(): with pytest.raises(TypeError) as excinfo: - AbstractModel(version='0.4.2') + AbstractModel(version="0.4.2") msg = str(excinfo.value) assert "Can't instantiate abstract class" in msg - assert 'setup' in msg - assert 'parameters' in msg + assert "setup" in msg + assert "parameters" in msg def test_construct_with_unsupported_version(): with pytest.raises(ValueError) as excinfo: - MockedModel(version='1.2.3') + MockedModel(version="1.2.3") - assert "Supplied version 1.2.3 is not supported by this model. Available versions are ('0.4.2',)." in str(excinfo.value) + assert ( + "Supplied version 1.2.3 is not supported by this model. " + "Available versions are ('0.4.2',)." in str(excinfo.value) + ) def test_setup(model): result = model.setup() - expected = 'foobar.cfg', '.' + expected = "foobar.cfg", "." assert result == expected def test_initialize(model: MockedModel, bmi): - config_file = 'foobar.cfg' + config_file = "foobar.cfg" model.initialize(config_file) bmi.initialize.assert_called_once_with(config_file) @@ -131,7 +137,7 @@ def test_get_value(bmi, model: MockedModel): expected = np.array([[1.0, 2.0], [3.0, 4.0]]) bmi.get_value.return_value = expected - value = model.get_value('discharge') + value = model.get_value("discharge") assert_array_equal(value, expected) @@ -140,23 +146,23 @@ def test_get_value_at_coords(bmi, model: MockedModel): expected = np.array([1.0]) bmi.get_value_at_indices.return_value = expected - value = model.get_value_at_coords('discharge', [-99.83], [42.25]) + value = model.get_value_at_coords("discharge", [-99.83], [42.25]) assert_array_equal(value, expected) def test_set_value(model: MockedModel, bmi): value = np.array([1.0, 2.0]) - model.set_value('precipitation', value) + model.set_value("precipitation", value) - bmi.set_value.assert_called_once_with('precipitation', value) + bmi.set_value.assert_called_once_with("precipitation", value) def test_set_value_at_coords(model: MockedModel, bmi): value = np.array([1.0]) - model.set_value_at_coords('precipitation', [-99.83], [42.25], value) + model.set_value_at_coords("precipitation", [-99.83], [42.25], value) - bmi.set_value_at_indices.assert_called_once_with('precipitation', [0], value) + bmi.set_value_at_indices.assert_called_once_with("precipitation", [0], value) def test_start_time(model: MockedModel): @@ -180,7 +186,7 @@ def test_time(model: MockedModel): def test_time_units(model: MockedModel): units = model.time_units - assert units == 'days since 1970-01-01 00:00:00.0 00:00' + assert units == "days since 1970-01-01 00:00:00.0 00:00" def test_time_step(bmi, model: MockedModel): @@ -192,11 +198,11 @@ def test_time_step(bmi, model: MockedModel): def test_output_var_names(bmi, model: MockedModel): - bmi.get_output_var_names.return_value = ('discharge',) + bmi.get_output_var_names.return_value = ("discharge",) names = model.output_var_names - assert names == ('discharge',) + assert names == ("discharge",) def test_get_value_as_xarray(model: MockedModel): @@ -205,9 +211,10 @@ def test_get_value_as_xarray(model: MockedModel): coords={ "latitude": [42.25, 42.21], "longitude": [-99.83, -99.32], - "time": '2014-09-06'}, + "time": "2014-09-06", + }, dims=["longitude", "latitude"], - name='Temperature', + name="Temperature", attrs=dict(units="degC"), ) @@ -219,49 +226,49 @@ def test_get_value_as_xarray(model: MockedModel): def start_time_as_isostr(model: MockedModel): actual = model.start_time_as_isostr - expected = '1970-02-12T00:00:00Z' + expected = "1970-02-12T00:00:00Z" assert expected == actual def end_time_as_isostr(model: MockedModel): actual = model.end_time_as_isostr - expected = '1970-02-14T00:00:00Z' + expected = "1970-02-14T00:00:00Z" assert expected == actual def time_as_isostr(model: MockedModel): actual = model.time_as_isostr - expected = '1970-02-13T00:00:00Z' + expected = "1970-02-13T00:00:00Z" assert expected == actual def start_time_as_datetime(model: MockedModel): actual = model.start_time_as_isostr - expected = datetime(1970, 2, 12, tzinfo=timezone.utc) + expected = datetime(1970, 2, 12, tzinfo=timezone.utc) assert expected == actual def end_time_as_datetime(model: MockedModel): actual = model.end_time_as_isostr - expected = datetime(1970, 2, 14, tzinfo=timezone.utc) + expected = datetime(1970, 2, 14, tzinfo=timezone.utc) assert expected == actual def time_as_datetime(model: MockedModel): actual = model.time_as_isostr - expected = datetime(1970, 2, 13, tzinfo=timezone.utc) + expected = datetime(1970, 2, 13, tzinfo=timezone.utc) assert expected == actual def test_delete_model_resets_bmi(): - - class Object(): + class Object: """Target for weakref finalizer.""" + pass # Cannot use the `model` fixture, it doesn't play well with weakref @@ -277,53 +284,53 @@ class Object(): class TestCheckParameterSet: def test_correct_version(self, setup_config): ps = ParameterSet( - name='justatest', - directory='justatest', - config='justatest/config.ini', - target_model='mockedmodel', # == lowered class name - supported_model_versions={'0.4.2'} + name="justatest", + directory="justatest", + config="justatest/config.ini", + target_model="mockedmodel", # == lowered class name + supported_model_versions={"0.4.2"}, ) m = MockedModel(parameter_set=ps) assert m.parameter_set == ps def test_wrong_model(self, setup_config): ps = ParameterSet( - name='justatest', - directory='justatest', - config='justatest/config.ini', - target_model='wrongmodel', - supported_model_versions={'0.4.2'} + name="justatest", + directory="justatest", + config="justatest/config.ini", + target_model="wrongmodel", + supported_model_versions={"0.4.2"}, ) with pytest.raises(ValueError) as excinfo: MockedModel(parameter_set=ps) - expected = 'Parameter set has wrong target model' + expected = "Parameter set has wrong target model" assert expected in str(excinfo.value) def test_any_version(self, caplog, setup_config): ps = ParameterSet( - name='justatest', - directory='justatest', - config='justatest/config.ini', - target_model='mockedmodel', # == lowered class name - supported_model_versions=set() + name="justatest", + directory="justatest", + config="justatest/config.ini", + target_model="mockedmodel", # == lowered class name + supported_model_versions=set(), ) with caplog.at_level(logging.INFO): MockedModel(parameter_set=ps) - expected = 'is not explicitly listed in the supported model versions' + expected = "is not explicitly listed in the supported model versions" assert expected in caplog.text def test_unsupported_version(self, setup_config): ps = ParameterSet( - name='justatest', - directory='justatest', - config='justatest/config.ini', - target_model='mockedmodel', - supported_model_versions={'1.2.3'} + name="justatest", + directory="justatest", + config="justatest/config.ini", + target_model="mockedmodel", + supported_model_versions={"1.2.3"}, ) with pytest.raises(ValueError) as excinfo: MockedModel(parameter_set=ps) - expected = 'Parameter set is not compatible with version' + expected = "Parameter set is not compatible with version" assert expected in str(excinfo.value) diff --git a/tests/models/test_lisflood.py b/tests/models/test_lisflood.py index 77c50627..68391f18 100644 --- a/tests/models/test_lisflood.py +++ b/tests/models/test_lisflood.py @@ -66,9 +66,7 @@ def generate_forcing(self, tmp_path, parameterset: ParameterSet): @pytest.fixture def model(self, parameterset, generate_forcing): forcing = generate_forcing - m = Lisflood( - version="20.10", parameter_set=parameterset, forcing=forcing - ) + m = Lisflood(version="20.10", parameter_set=parameterset, forcing=forcing) yield m if m.bmi: # Clean up container @@ -119,20 +117,14 @@ def test_setup(self, model_with_setup, tmp_path): } assert find_values_in_xml(_cfg.config, "StepStart") == {"1"} assert find_values_in_xml(_cfg.config, "StepEnd") == {"11688"} - assert find_values_in_xml(_cfg.config, "PathMeteo") == { - f"{tmp_path}/forcing" - } + assert find_values_in_xml(_cfg.config, "PathMeteo") == {f"{tmp_path}/forcing"} assert find_values_in_xml(_cfg.config, "PathOut") == {str(config_dir)} - assert find_values_in_xml(_cfg.config, "IrrigationEfficiency") == { - "0.8" - } + assert find_values_in_xml(_cfg.config, "IrrigationEfficiency") == {"0.8"} assert find_values_in_xml(_cfg.config, "MaskMap") == { "$(PathMaps)/masksmall.map", "$(MaskMap)", } - assert find_values_in_xml(_cfg.config, "PrefixPrecipitation") == { - "mytp" - } + assert find_values_in_xml(_cfg.config, "PrefixPrecipitation") == {"mytp"} assert find_values_in_xml(_cfg.config, "PrefixTavg") == {"myta"} assert find_values_in_xml(_cfg.config, "PrefixE0") == {"mye0"} assert find_values_in_xml(_cfg.config, "PrefixES0") == {"es0"} @@ -181,9 +173,7 @@ def model(self, tmp_path, parameterset, generate_forcing): mask_file_in_ps = parameterset.directory / "maps/mask.map" shutil.copy(mask_file_in_ps, mask_dir / "mask.map") forcing = generate_forcing - m = Lisflood( - version="20.10", parameter_set=parameterset, forcing=forcing - ) + m = Lisflood(version="20.10", parameter_set=parameterset, forcing=forcing) yield m if m.bmi: # Clean up container @@ -193,12 +183,8 @@ def model(self, tmp_path, parameterset, generate_forcing): def model_with_setup(self, tmp_path, mocked_config, model: Lisflood): with patch.object( BmiClientSingularity, "__init__", return_value=None - ) as mocked_constructor, patch( - "datetime.datetime" - ) as mocked_datetime: - mocked_datetime.now.return_value = datetime( - 2021, 1, 2, 3, 4, 5 - ) + ) as mocked_constructor, patch("datetime.datetime") as mocked_datetime: + mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) config_file, config_dir = model.setup( MaskMap=str(tmp_path / "custommask/mask.map") ) @@ -210,9 +196,7 @@ def test_setup(self, model_with_setup, tmp_path): # Check setup returns expected_cfg_dir = CFG["output_dir"] / "lisflood_20210102_030405" assert config_dir == str(expected_cfg_dir) - assert config_file == str( - expected_cfg_dir / "lisflood_setting.xml" - ) + assert config_file == str(expected_cfg_dir / "lisflood_setting.xml") # Check container started mocked_constructor.assert_called_once_with( @@ -235,9 +219,7 @@ def test_setup(self, model_with_setup, tmp_path): assert find_values_in_xml(_cfg.config, "PathMeteo") == { f"{tmp_path}/forcing" } - assert find_values_in_xml(_cfg.config, "PathOut") == { - str(config_dir) - } + assert find_values_in_xml(_cfg.config, "PathOut") == {str(config_dir)} assert find_values_in_xml(_cfg.config, "IrrigationEfficiency") == { "0.75", "$(IrrigationEfficiency)", @@ -245,9 +227,7 @@ def test_setup(self, model_with_setup, tmp_path): assert find_values_in_xml(_cfg.config, "MaskMap") == { f"{tmp_path}/custommask/mask" } - assert find_values_in_xml(_cfg.config, "PrefixPrecipitation") == { - "mytp" - } + assert find_values_in_xml(_cfg.config, "PrefixPrecipitation") == {"mytp"} assert find_values_in_xml(_cfg.config, "PrefixTavg") == {"myta"} assert find_values_in_xml(_cfg.config, "PrefixE0") == {"mye0"} assert find_values_in_xml(_cfg.config, "PrefixES0") == {"es0"} diff --git a/tests/models/test_marrmotm01.py b/tests/models/test_marrmotm01.py index a40099ba..e9fa6109 100644 --- a/tests/models/test_marrmotm01.py +++ b/tests/models/test_marrmotm01.py @@ -10,13 +10,13 @@ from ewatercycle import CFG from ewatercycle.forcing import load_foreign -from ewatercycle.models.marrmot import Solver, MarrmotM01 +from ewatercycle.models.marrmot import MarrmotM01, Solver @pytest.fixture def mocked_config(tmp_path): - CFG['output_dir'] = tmp_path - CFG['container_engine'] = 'docker' + CFG["output_dir"] = tmp_path + CFG["container_engine"] = "docker" class TestWithDefaultsAndExampleData: @@ -26,13 +26,13 @@ def forcing_file(self, sample_marrmot_forcing_file): @pytest.fixture def generate_forcing(self, forcing_file): - forcing = load_foreign('marrmot', - directory=str(Path(forcing_file).parent), - start_time='1989-01-01T00:00:00Z', - end_time='1992-12-31T00:00:00Z', - forcing_info={ - 'forcing_file': str(Path(forcing_file).name) - }) + forcing = load_foreign( + "marrmot", + directory=str(Path(forcing_file).parent), + start_time="1989-01-01T00:00:00Z", + end_time="1992-12-31T00:00:00Z", + forcing_info={"forcing_file": str(Path(forcing_file).name)}, + ) return forcing @pytest.fixture @@ -45,16 +45,15 @@ def model(self, generate_forcing, mocked_config): @pytest.fixture def model_with_setup(self, model: MarrmotM01): - with patch('datetime.datetime') as mocked_datetime: + with patch("datetime.datetime") as mocked_datetime: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) cfg_file, cfg_dir = model.setup() return model, cfg_file, cfg_dir - def test_str(self, model, forcing_file): actual = str(model) - expected = "\n".join( + expected = "\n".join( [ "eWaterCycle MarrmotM01", "-------------------", @@ -69,32 +68,33 @@ def test_str(self, model, forcing_file): f" directory={str(Path(forcing_file).parent)}", " shape=None", " forcing_file=BMI_testcase_m01_BuffaloRiver_TN_USA.mat", - ]) + ] + ) assert actual == expected def test_parameters(self, model): expected = [ - ('maximum_soil_moisture_storage', 10.0), - ('initial_soil_moisture_storage', 5.0), - ('solver', Solver()), - ('start time', '1989-01-01T00:00:00Z'), - ('end time', '1992-12-31T00:00:00Z'), + ("maximum_soil_moisture_storage", 10.0), + ("initial_soil_moisture_storage", 5.0), + ("solver", Solver()), + ("start time", "1989-01-01T00:00:00Z"), + ("end time", "1992-12-31T00:00:00Z"), ] assert model.parameters == expected def test_setup(self, model_with_setup, forcing_file): model, cfg_file, cfg_dir = model_with_setup - expected_cfg_dir = CFG['output_dir'] / 'marrmot_20210102_030405' + expected_cfg_dir = CFG["output_dir"] / "marrmot_20210102_030405" assert cfg_dir == str(expected_cfg_dir) - assert cfg_file == str(expected_cfg_dir / 'marrmot-m01_config.mat') + assert cfg_file == str(expected_cfg_dir / "marrmot-m01_config.mat") assert model.bmi actual = loadmat(str(cfg_file)) expected_forcing = loadmat(forcing_file) - assert actual['model_name'] == "m_01_collie1_1p_1s" - assert_almost_equal(actual['time_start'], expected_forcing['time_start']) - assert_almost_equal(actual['time_end'], expected_forcing['time_end']) + assert actual["model_name"] == "m_01_collie1_1p_1s" + assert_almost_equal(actual["time_start"], expected_forcing["time_start"]) + assert_almost_equal(actual["time_end"], expected_forcing["time_end"]) # TODO compare forcings # assert_almost_equal(actual['forcing'], expected_forcing['forcing']) # TODO assert solver @@ -103,11 +103,11 @@ def test_setup(self, model_with_setup, forcing_file): def test_parameters_after_setup(self, model_with_setup): model = model_with_setup[0] expected = [ - ('maximum_soil_moisture_storage', 10.0), - ('initial_soil_moisture_storage', 5.0), - ('solver', Solver()), - ('start time', '1989-01-01T00:00:00Z'), - ('end time', '1992-12-31T00:00:00Z'), + ("maximum_soil_moisture_storage", 10.0), + ("initial_soil_moisture_storage", 5.0), + ("solver", Solver()), + ("start time", "1989-01-01T00:00:00Z"), + ("end time", "1992-12-31T00:00:00Z"), ] assert model.parameters == expected @@ -116,32 +116,28 @@ def test_get_value_as_xarray(self, model_with_setup): model.initialize(cfg_file) model.update() - actual = model.get_value_as_xarray('flux_out_Q') + actual = model.get_value_as_xarray("flux_out_Q") expected = xr.DataArray( data=[[11.91879913]], coords={ "longitude": [87.49], "latitude": [35.29], - "time": datetime(1989, 1, 2, tzinfo=timezone.utc) + "time": datetime(1989, 1, 2, tzinfo=timezone.utc), }, dims=["latitude", "longitude"], - name='flux_out_Q', - attrs={"units": 'mm day'}, + name="flux_out_Q", + attrs={"units": "mm day"}, ) assert_allclose(actual, expected) def test_setup_with_own_cfg_dir(self, tmp_path, mocked_config, model: MarrmotM01): - cfg_file, cfg_dir = model.setup( - cfg_dir=str(tmp_path) - ) + cfg_file, cfg_dir = model.setup(cfg_dir=str(tmp_path)) assert cfg_dir == str(tmp_path) def test_setup_create_cfg_dir(self, tmp_path, mocked_config, model: MarrmotM01): - work_dir = tmp_path / 'output' - cfg_file, cfg_dir = model.setup( - cfg_dir=str(work_dir) - ) + work_dir = tmp_path / "output" + cfg_file, cfg_dir = model.setup(cfg_dir=str(work_dir)) assert cfg_dir == str(work_dir) @@ -152,13 +148,13 @@ def forcing_file(self, sample_marrmot_forcing_file): @pytest.fixture def generate_forcing(self, forcing_file): - forcing = load_foreign('marrmot', - directory=str(Path(forcing_file).parent), - start_time='1989-01-01T00:00:00Z', - end_time='1992-12-31T00:00:00Z', - forcing_info={ - 'forcing_file': str(Path(forcing_file).name) - }) + forcing = load_foreign( + "marrmot", + directory=str(Path(forcing_file).parent), + start_time="1989-01-01T00:00:00Z", + end_time="1992-12-31T00:00:00Z", + forcing_info={"forcing_file": str(Path(forcing_file).name)}, + ) return forcing @pytest.fixture @@ -171,30 +167,30 @@ def model(self, generate_forcing, mocked_config): @pytest.fixture def model_with_setup(self, model: MarrmotM01): - with patch('datetime.datetime') as mocked_datetime: + with patch("datetime.datetime") as mocked_datetime: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) cfg_file, cfg_dir = model.setup( maximum_soil_moisture_storage=1234, initial_soil_moisture_storage=4321, - start_time='1990-01-01T00:00:00Z', - end_time='1991-12-31T00:00:00Z', + start_time="1990-01-01T00:00:00Z", + end_time="1991-12-31T00:00:00Z", ) return model, cfg_file, cfg_dir def test_setup(self, model_with_setup): model, cfg_file, cfg_dir = model_with_setup - expected_cfg_dir = CFG['output_dir'] / 'marrmot_20210102_030405' + expected_cfg_dir = CFG["output_dir"] / "marrmot_20210102_030405" assert cfg_dir == str(expected_cfg_dir) - assert cfg_file == str(expected_cfg_dir / 'marrmot-m01_config.mat') + assert cfg_file == str(expected_cfg_dir / "marrmot-m01_config.mat") assert model.bmi actual = loadmat(str(cfg_file)) - assert actual['model_name'] == "m_01_collie1_1p_1s" - assert actual['parameters'] == [[1234]] - assert actual['store_ini'] == [[4321]] - assert_almost_equal(actual['time_start'], [[1990, 1, 1, 0, 0, 0]]) - assert_almost_equal(actual['time_end'], [[1991, 12, 31, 0, 0, 0]]) + assert actual["model_name"] == "m_01_collie1_1p_1s" + assert actual["parameters"] == [[1234]] + assert actual["store_ini"] == [[4321]] + assert_almost_equal(actual["time_start"], [[1990, 1, 1, 0, 0, 0]]) + assert_almost_equal(actual["time_end"], [[1991, 12, 31, 0, 0, 0]]) class TestWithDatesOutsideRangeSetupAndExampleData: @@ -204,13 +200,13 @@ def forcing_file(self, sample_marrmot_forcing_file): @pytest.fixture def generate_forcing(self, forcing_file): - forcing = load_foreign('marrmot', - directory=str(Path(forcing_file).parent), - start_time='1989-01-01T00:00:00Z', - end_time='1992-12-31T00:00:00Z', - forcing_info={ - 'forcing_file': str(Path(forcing_file).name) - }) + forcing = load_foreign( + "marrmot", + directory=str(Path(forcing_file).parent), + start_time="1989-01-01T00:00:00Z", + end_time="1992-12-31T00:00:00Z", + forcing_info={"forcing_file": str(Path(forcing_file).name)}, + ) return forcing @pytest.fixture @@ -224,13 +220,13 @@ def model(self, generate_forcing, mocked_config): def test_setup_with_earlystart(self, model: MarrmotM01): with pytest.raises(ValueError) as excinfo: model.setup( - start_time='1980-01-01T00:00:00Z', + start_time="1980-01-01T00:00:00Z", ) - assert 'start_time outside forcing time range' in str(excinfo.value) + assert "start_time outside forcing time range" in str(excinfo.value) def test_setup_with_lateend(self, model: MarrmotM01): with pytest.raises(ValueError) as excinfo: model.setup( - end_time='2000-01-01T00:00:00Z', + end_time="2000-01-01T00:00:00Z", ) - assert 'end_time outside forcing time range' in str(excinfo.value) + assert "end_time outside forcing time range" in str(excinfo.value) diff --git a/tests/models/test_marrmotm14.py b/tests/models/test_marrmotm14.py index 9f75cdc4..82c33fe5 100644 --- a/tests/models/test_marrmotm14.py +++ b/tests/models/test_marrmotm14.py @@ -10,13 +10,13 @@ from ewatercycle import CFG from ewatercycle.forcing import load_foreign -from ewatercycle.models.marrmot import Solver, MarrmotM14 +from ewatercycle.models.marrmot import MarrmotM14, Solver @pytest.fixture def mocked_config(tmp_path): - CFG['output_dir'] = tmp_path - CFG['container_engine'] = 'docker' + CFG["output_dir"] = tmp_path + CFG["container_engine"] = "docker" class TestWithDefaultsAndExampleData: @@ -24,13 +24,13 @@ class TestWithDefaultsAndExampleData: def generate_forcing(self): # Downloaded from # https://github.com/wknoben/MARRMoT/blob/master/BMI/Config/BMI_testcase_m01_BuffaloRiver_TN_USA.mat - forcing = load_foreign('marrmot', - directory=f'{Path(__file__).parent}/data', - start_time='1989-01-01T00:00:00Z', - end_time='1992-12-31T00:00:00Z', - forcing_info={ - 'forcing_file': 'BMI_testcase_m01_BuffaloRiver_TN_USA.mat' - }) + forcing = load_foreign( + "marrmot", + directory=f"{Path(__file__).parent}/data", + start_time="1989-01-01T00:00:00Z", + end_time="1992-12-31T00:00:00Z", + forcing_info={"forcing_file": "BMI_testcase_m01_BuffaloRiver_TN_USA.mat"}, + ) return forcing @pytest.fixture @@ -44,7 +44,7 @@ def model(self, generate_forcing, mocked_config): @pytest.fixture def model_with_setup(self, model: MarrmotM14): - with patch('datetime.datetime') as mocked_datetime: + with patch("datetime.datetime") as mocked_datetime: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) cfg_file, cfg_dir = model.setup() @@ -52,18 +52,18 @@ def model_with_setup(self, model: MarrmotM14): def test_parameters(self, model): expected = [ - ('maximum_soil_moisture_storage', 1000.0), - ('threshold_flow_generation_evap_change', 0.5), - ('leakage_saturated_zone_flow_coefficient', 0.5), - ('zero_deficit_base_flow_speed', 100.0), - ('baseflow_coefficient', 0.5), - ('gamma_distribution_chi_parameter', 4.25), - ('gamma_distribution_phi_parameter', 2.5), - ('initial_upper_zone_storage', 900.0), - ('initial_saturated_zone_storage', 900.0), - ('solver', Solver()), - ('start time', '1989-01-01T00:00:00Z'), - ('end time', '1992-12-31T00:00:00Z'), + ("maximum_soil_moisture_storage", 1000.0), + ("threshold_flow_generation_evap_change", 0.5), + ("leakage_saturated_zone_flow_coefficient", 0.5), + ("zero_deficit_base_flow_speed", 100.0), + ("baseflow_coefficient", 0.5), + ("gamma_distribution_chi_parameter", 4.25), + ("gamma_distribution_phi_parameter", 2.5), + ("initial_upper_zone_storage", 900.0), + ("initial_saturated_zone_storage", 900.0), + ("solver", Solver()), + ("start time", "1989-01-01T00:00:00Z"), + ("end time", "1992-12-31T00:00:00Z"), ] assert model.parameters == expected @@ -71,16 +71,18 @@ def test_setup(self, model_with_setup): model, cfg_file, cfg_dir = model_with_setup actual = loadmat(str(cfg_file)) - forcing_file = f'{Path(__file__).parent}/data/BMI_testcase_m01_BuffaloRiver_TN_USA.mat' + forcing_file = ( + f"{Path(__file__).parent}/data/BMI_testcase_m01_BuffaloRiver_TN_USA.mat" + ) expected_forcing = loadmat(forcing_file) - expected_cfg_dir = CFG['output_dir'] / 'marrmot_20210102_030405' + expected_cfg_dir = CFG["output_dir"] / "marrmot_20210102_030405" assert cfg_dir == str(expected_cfg_dir) - assert cfg_file == str(expected_cfg_dir / 'marrmot-m14_config.mat') + assert cfg_file == str(expected_cfg_dir / "marrmot-m14_config.mat") assert model.bmi - assert actual['model_name'] == "m_14_topmodel_7p_2s" - assert_almost_equal(actual['time_start'], expected_forcing['time_start']) - assert_almost_equal(actual['time_end'], expected_forcing['time_end']) + assert actual["model_name"] == "m_14_topmodel_7p_2s" + assert_almost_equal(actual["time_start"], expected_forcing["time_start"]) + assert_almost_equal(actual["time_end"], expected_forcing["time_end"]) # TODO compare forcings # assert_almost_equal(actual['forcing'], expected_forcing['forcing']) # TODO assert solver @@ -89,18 +91,18 @@ def test_setup(self, model_with_setup): def test_parameters_after_setup(self, model_with_setup): model = model_with_setup[0] expected = [ - ('maximum_soil_moisture_storage', 1000.0), - ('threshold_flow_generation_evap_change', 0.5), - ('leakage_saturated_zone_flow_coefficient', 0.5), - ('zero_deficit_base_flow_speed', 100.0), - ('baseflow_coefficient', 0.5), - ('gamma_distribution_chi_parameter', 4.25), - ('gamma_distribution_phi_parameter', 2.5), - ('initial_upper_zone_storage', 900.0), - ('initial_saturated_zone_storage', 900.0), - ('solver', Solver()), - ('start time', '1989-01-01T00:00:00Z'), - ('end time', '1992-12-31T00:00:00Z'), + ("maximum_soil_moisture_storage", 1000.0), + ("threshold_flow_generation_evap_change", 0.5), + ("leakage_saturated_zone_flow_coefficient", 0.5), + ("zero_deficit_base_flow_speed", 100.0), + ("baseflow_coefficient", 0.5), + ("gamma_distribution_chi_parameter", 4.25), + ("gamma_distribution_phi_parameter", 2.5), + ("initial_upper_zone_storage", 900.0), + ("initial_saturated_zone_storage", 900.0), + ("solver", Solver()), + ("start time", "1989-01-01T00:00:00Z"), + ("end time", "1992-12-31T00:00:00Z"), ] assert model.parameters == expected @@ -109,32 +111,28 @@ def test_get_value_as_xarray(self, model_with_setup): model.initialize(cfg_file) model.update() - actual = model.get_value_as_xarray('flux_out_Q') + actual = model.get_value_as_xarray("flux_out_Q") expected = xr.DataArray( data=[[0.529399]], coords={ "longitude": [87.49], "latitude": [35.29], - "time": datetime(1989, 1, 2, tzinfo=timezone.utc) + "time": datetime(1989, 1, 2, tzinfo=timezone.utc), }, dims=["latitude", "longitude"], - name='flux_out_Q', - attrs={"units": 'mm day'}, + name="flux_out_Q", + attrs={"units": "mm day"}, ) assert_allclose(actual, expected) def test_setup_with_own_cfg_dir(self, tmp_path, mocked_config, model: MarrmotM14): - cfg_file, cfg_dir = model.setup( - cfg_dir=str(tmp_path) - ) + cfg_file, cfg_dir = model.setup(cfg_dir=str(tmp_path)) assert cfg_dir == str(tmp_path) def test_setup_create_cfg_dir(self, tmp_path, mocked_config, model: MarrmotM14): - work_dir = tmp_path / 'output' - cfg_file, cfg_dir = model.setup( - cfg_dir=str(work_dir) - ) + work_dir = tmp_path / "output" + cfg_file, cfg_dir = model.setup(cfg_dir=str(work_dir)) assert cfg_dir == str(work_dir) @@ -143,13 +141,13 @@ class TestWithCustomSetupAndExampleData: def generate_forcing(self): # Downloaded from # https://github.com/wknoben/MARRMoT/blob/master/BMI/Config/BMI_testcase_m01_BuffaloRiver_TN_USA.mat - forcing = load_foreign('marrmot', - directory=f'{Path(__file__).parent}/data', - start_time='1989-01-01T00:00:00Z', - end_time='1992-12-31T00:00:00Z', - forcing_info={ - 'forcing_file': 'BMI_testcase_m01_BuffaloRiver_TN_USA.mat' - }) + forcing = load_foreign( + "marrmot", + directory=f"{Path(__file__).parent}/data", + start_time="1989-01-01T00:00:00Z", + end_time="1992-12-31T00:00:00Z", + forcing_info={"forcing_file": "BMI_testcase_m01_BuffaloRiver_TN_USA.mat"}, + ) return forcing @pytest.fixture @@ -163,14 +161,14 @@ def model(self, generate_forcing, mocked_config): @pytest.fixture def model_with_setup(self, model: MarrmotM14): - with patch('datetime.datetime') as mocked_datetime: + with patch("datetime.datetime") as mocked_datetime: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) cfg_file, cfg_dir = model.setup( maximum_soil_moisture_storage=1234, initial_upper_zone_storage=4321, - start_time='1990-01-01T00:00:00Z', - end_time='1991-12-31T00:00:00Z', + start_time="1990-01-01T00:00:00Z", + end_time="1991-12-31T00:00:00Z", ) return model, cfg_file, cfg_dir @@ -179,15 +177,17 @@ def test_setup(self, model_with_setup): actual = loadmat(str(cfg_file)) - expected_cfg_dir = CFG['output_dir'] / 'marrmot_20210102_030405' + expected_cfg_dir = CFG["output_dir"] / "marrmot_20210102_030405" assert cfg_dir == str(expected_cfg_dir) - assert cfg_file == str(expected_cfg_dir / 'marrmot-m14_config.mat') + assert cfg_file == str(expected_cfg_dir / "marrmot-m14_config.mat") assert model.bmi - assert actual['model_name'] == "m_14_topmodel_7p_2s" - assert_array_equal(actual['parameters'], [[1234.0, 0.5, 0.5, 100.0, 0.5, 4.25, 2.5]]) - assert_array_equal(actual['store_ini'], [[4321, 900]]) - assert_almost_equal(actual['time_start'], [[1990, 1, 1, 0, 0, 0]]) - assert_almost_equal(actual['time_end'], [[1991, 12, 31, 0, 0, 0]]) + assert actual["model_name"] == "m_14_topmodel_7p_2s" + assert_array_equal( + actual["parameters"], [[1234.0, 0.5, 0.5, 100.0, 0.5, 4.25, 2.5]] + ) + assert_array_equal(actual["store_ini"], [[4321, 900]]) + assert_almost_equal(actual["time_start"], [[1990, 1, 1, 0, 0, 0]]) + assert_almost_equal(actual["time_end"], [[1991, 12, 31, 0, 0, 0]]) class TestWithDatesOutsideRangeSetupAndExampleData: @@ -195,13 +195,13 @@ class TestWithDatesOutsideRangeSetupAndExampleData: def generate_forcing(self): # Downloaded from # https://github.com/wknoben/MARRMoT/blob/master/BMI/Config/BMI_testcase_m01_BuffaloRiver_TN_USA.mat - forcing = load_foreign('marrmot', - directory=f'{Path(__file__).parent}/data', - start_time='1989-01-01T00:00:00Z', - end_time='1992-12-31T00:00:00Z', - forcing_info={ - 'forcing_file': 'BMI_testcase_m01_BuffaloRiver_TN_USA.mat' - }) + forcing = load_foreign( + "marrmot", + directory=f"{Path(__file__).parent}/data", + start_time="1989-01-01T00:00:00Z", + end_time="1992-12-31T00:00:00Z", + forcing_info={"forcing_file": "BMI_testcase_m01_BuffaloRiver_TN_USA.mat"}, + ) return forcing @pytest.fixture @@ -216,13 +216,13 @@ def model(self, generate_forcing, mocked_config): def test_setup_with_earlystart(self, model: MarrmotM14): with pytest.raises(ValueError) as excinfo: model.setup( - start_time='1980-01-01T00:00:00Z', + start_time="1980-01-01T00:00:00Z", ) - assert 'start_time outside forcing time range' in str(excinfo.value) + assert "start_time outside forcing time range" in str(excinfo.value) def test_setup_with_lateend(self, model: MarrmotM14): with pytest.raises(ValueError) as excinfo: model.setup( - end_time='2000-01-01T00:00:00Z', + end_time="2000-01-01T00:00:00Z", ) - assert 'end_time outside forcing time range' in str(excinfo.value) + assert "end_time outside forcing time range" in str(excinfo.value) diff --git a/tests/models/test_pcrglobwb.py b/tests/models/test_pcrglobwb.py index cf02c45c..09383dc1 100644 --- a/tests/models/test_pcrglobwb.py +++ b/tests/models/test_pcrglobwb.py @@ -52,9 +52,7 @@ def mocked_config(tmp_path): @pytest.fixture def parameter_set(mocked_config): - example_parameter_set = example_parameter_sets()[ - "pcrglobwb_rhinemeuse_30min" - ] + example_parameter_set = example_parameter_sets()["pcrglobwb_rhinemeuse_30min"] example_parameter_set.download() example_parameter_set.to_config() return example_parameter_set @@ -88,9 +86,9 @@ def initialized_model(model): def test_setup(model): - with patch.object( - BmiClientSingularity, "__init__", return_value=None - ), patch("datetime.datetime") as mocked_datetime: + with patch.object(BmiClientSingularity, "__init__", return_value=None), patch( + "datetime.datetime" + ) as mocked_datetime: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) cfg_file, cfg_dir = model.setup() @@ -101,23 +99,25 @@ def test_setup(model): def test_setup_withtimeoutexception(model, tmp_path): - with patch.object(BmiClientSingularity, "__init__", side_effect=FutureTimeoutError()), patch( - "datetime.datetime" - ) as mocked_datetime, pytest.raises(ValueError) as excinfo: + with patch.object( + BmiClientSingularity, "__init__", side_effect=FutureTimeoutError() + ), patch("datetime.datetime") as mocked_datetime, pytest.raises( + ValueError + ) as excinfo: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) model.setup() msg = str(excinfo.value) - assert 'docker pull ewatercycle/pcrg-grpc4bmi:setters' in msg - sif = tmp_path / 'ewatercycle-pcrg-grpc4bmi_setters.sif' + assert "docker pull ewatercycle/pcrg-grpc4bmi:setters" in msg + sif = tmp_path / "ewatercycle-pcrg-grpc4bmi_setters.sif" assert f"build {sif} docker://ewatercycle/pcrg-grpc4bmi:setters" in msg def test_setup_with_custom_cfg_dir(model, tmp_path): my_cfg_dir = str(tmp_path / "mycfgdir") - with patch.object( - BmiClientSingularity, "__init__", return_value=None - ), patch("datetime.datetime") as mocked_datetime: + with patch.object(BmiClientSingularity, "__init__", return_value=None), patch( + "datetime.datetime" + ) as mocked_datetime: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) cfg_file, cfg_dir = model.setup(cfg_dir=my_cfg_dir) @@ -133,8 +133,7 @@ def test_get_value_as_coords(initialized_model, caplog): result = model.get_value_at_coords("discharge", lon=[5.2], lat=[46.8]) msg = ( - "Requested point was lon: 5.2, lat: 46.8;" - " closest grid point is 5.00, 47.00." + "Requested point was lon: 5.2, lat: 46.8;" " closest grid point is 5.00, 47.00." ) assert msg in caplog.text diff --git a/tests/models/test_wflow.py b/tests/models/test_wflow.py index 694fd8f2..af4c35ee 100644 --- a/tests/models/test_wflow.py +++ b/tests/models/test_wflow.py @@ -109,10 +109,7 @@ def test_constructor_adds_api_riverrunoff(parameter_set, caplog): "Config file from parameter set is missing RiverRunoff option in API section" in caplog.text ) - assert ( - "added it with value '2, m/s option'" - in caplog.text - ) + assert "added it with value '2, m/s option'" in caplog.text def test_str(model, tmp_path): @@ -152,21 +149,21 @@ def test_setup(model): # Check content of config file cfg = CaseConfigParser() cfg.read(expected_cfg_file) - assert ( - cfg.get("API", "RiverRunoff") == "2, m/s" - ) + assert cfg.get("API", "RiverRunoff") == "2, m/s" def test_setup_withtimeoutexception(model, tmp_path): - with patch.object(BmiClientSingularity, "__init__", side_effect=FutureTimeoutError()), patch( - "datetime.datetime" - ) as mocked_datetime, pytest.raises(ValueError) as excinfo: + with patch.object( + BmiClientSingularity, "__init__", side_effect=FutureTimeoutError() + ), patch("datetime.datetime") as mocked_datetime, pytest.raises( + ValueError + ) as excinfo: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) model.setup() msg = str(excinfo.value) - assert 'docker pull ewatercycle/wflow-grpc4bmi:2020.1.1' in msg - sif = tmp_path / 'ewatercycle-wflow-grpc4bmi_2020.1.1.sif' + assert "docker pull ewatercycle/wflow-grpc4bmi:2020.1.1" in msg + sif = tmp_path / "ewatercycle-wflow-grpc4bmi_2020.1.1.sif" assert f"build {sif} docker://ewatercycle/wflow-grpc4bmi:2020.1.1" in msg @@ -189,9 +186,7 @@ def test_get_value_as_coords(initialized_model, caplog): with caplog.at_level(logging.DEBUG): result = model.get_value_at_coords("discharge", lon=[5.2], lat=[46.8]) - msg = ( - "Requested point was lon: 5.2, lat: 46.8; closest grid point is 5.00, 47.00." - ) + msg = "Requested point was lon: 5.2, lat: 46.8; closest grid point is 5.00, 47.00." assert msg in caplog.text assert result == np.array([1.0]) diff --git a/tests/observation/test_grdc.py b/tests/observation/test_grdc.py index b3db020a..21682bea 100644 --- a/tests/observation/test_grdc.py +++ b/tests/observation/test_grdc.py @@ -1,8 +1,8 @@ from datetime import datetime -import pandas as pd -import pytest import numpy as np +import pandas as pd +import pytest from pandas.testing import assert_frame_equal from ewatercycle import CFG @@ -11,9 +11,9 @@ @pytest.fixture def sample_grdc_file(tmp_path): - fn = tmp_path / '42424242_Q_Day.Cmd.txt' + fn = tmp_path / "42424242_Q_Day.Cmd.txt" # Sample with fictive data, but with same structure as real file - s = '''# Title: GRDC STATION DATA FILE + body = """# Title: GRDC STATION DATA FILE # -------------- # Format: DOS-ASCII # Field delimiter: ; @@ -51,9 +51,9 @@ def sample_grdc_file(tmp_path): YYYY-MM-DD;hh:mm; Value 2000-01-01;--:--; 123.000 2000-01-02;--:--; 456.000 -2000-01-03;--:--; -999.000''' - with open(fn, 'w', encoding='cp1252') as f: - f.write(s) +2000-01-03;--:--; -999.000""" # noqa: E800 + with fn.open("w", encoding="cp1252") as f: + f.write(body) return fn @@ -61,84 +61,88 @@ def sample_grdc_file(tmp_path): def expected_results(tmp_path, sample_grdc_file): data = pd.DataFrame( - {'streamflow': [123., 456., np.NaN]}, - index = [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)], + {"streamflow": [123.0, 456.0, np.NaN]}, + index=[datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)], ) - data.index.rename('time', inplace=True) + data.index.rename("time", inplace=True) metadata = { - 'altitude_masl': 8.0, - 'country_code': 'NA', - 'dataSetContent': 'MEAN DAILY DISCHARGE (Q)', - 'file_generation_date': '2000-02-02', - 'grdc_catchment_area_in_km2': 4242.0, - 'grdc_file_name': str(tmp_path / sample_grdc_file), - 'grdc_latitude_in_arc_degree': 52.356154, - 'grdc_longitude_in_arc_degree': 4.955153, - 'id_from_grdc': 42424242, - 'last_update': '2000-02-01', - 'no_of_years': 1, - 'nrMeasurements': 'NA', - 'river_name': 'SOME RIVER', - 'station_name': 'SOME', - 'time_series': '2000-01 - 2000-01', - 'units': 'm³/s', - 'UserEndTime': '2000-02-01T00:00Z', - 'UserStartTime': '2000-01-01T00:00Z', - 'nrMissingData': 1, - } + "altitude_masl": 8.0, + "country_code": "NA", + "dataSetContent": "MEAN DAILY DISCHARGE (Q)", + "file_generation_date": "2000-02-02", + "grdc_catchment_area_in_km2": 4242.0, + "grdc_file_name": str(tmp_path / sample_grdc_file), + "grdc_latitude_in_arc_degree": 52.356154, + "grdc_longitude_in_arc_degree": 4.955153, + "id_from_grdc": 42424242, + "last_update": "2000-02-01", + "no_of_years": 1, + "nrMeasurements": 3, + "river_name": "SOME RIVER", + "station_name": "SOME", + "time_series": "2000-01 - 2000-01", + "units": "m³/s", + "UserEndTime": "2000-02-01T00:00Z", + "UserStartTime": "2000-01-01T00:00Z", + "nrMissingData": 1, + } return data, metadata def test_get_grdc_data_with_datahome(tmp_path, expected_results): expected_data, expected_metadata = expected_results - result_data, result_metadata = get_grdc_data('42424242', '2000-01-01T00:00Z', '2000-02-01T00:00Z', data_home=str(tmp_path)) + result_data, result_metadata = get_grdc_data( + "42424242", "2000-01-01T00:00Z", "2000-02-01T00:00Z", data_home=str(tmp_path) + ) assert_frame_equal(result_data, expected_data) assert result_metadata == expected_metadata -def test_get_grdc_data_with_CFG(expected_results, tmp_path): - CFG['grdc_location'] = str(tmp_path) +def test_get_grdc_data_with_cfg(expected_results, tmp_path): + CFG["grdc_location"] = str(tmp_path) expected_data, expected_metadata = expected_results - result_data, result_metadata = get_grdc_data('42424242', '2000-01-01T00:00Z', '2000-02-01T00:00Z') + result_data, result_metadata = get_grdc_data( + "42424242", "2000-01-01T00:00Z", "2000-02-01T00:00Z" + ) assert_frame_equal(result_data, expected_data) assert result_metadata == expected_metadata def test_get_grdc_data_without_path(): - CFG['grdc_location'] = None - with pytest.raises(ValueError) as excinfo: - get_grdc_data('42424242', '2000-01-01T00:00Z', '2000-02-01T00:00Z') + CFG["grdc_location"] = None + with pytest.raises(ValueError, match=r"Provide the grdc path") as excinfo: + get_grdc_data("42424242", "2000-01-01T00:00Z", "2000-02-01T00:00Z") msg = str(excinfo.value) - print(msg) - assert 'data_home' in msg - assert 'grdc_location' in msg + assert "data_home" in msg + assert "grdc_location" in msg def test_get_grdc_data_wrong_path(tmp_path): - CFG['grdc_location'] = f'{tmp_path}_data' + CFG["grdc_location"] = f"{tmp_path}_data" - with pytest.raises(ValueError) as excinfo: - get_grdc_data('42424242', '2000-01-01T00:00Z', '2000-02-01T00:00Z') - msg = str(excinfo.value) - print(msg) - assert 'directory' in msg + with pytest.raises(ValueError, match=r"The grdc directory .* does not exist!"): + get_grdc_data("42424242", "2000-01-01T00:00Z", "2000-02-01T00:00Z") def test_get_grdc_data_without_file(tmp_path): - with pytest.raises(ValueError) as excinfo: - get_grdc_data('42424243', '2000-01-01T00:00Z', '2000-02-01T00:00Z', data_home=str(tmp_path)) - msg = str(excinfo.value) - print(msg) - assert 'file' in msg + with pytest.raises(ValueError, match="The grdc file .* does not exist!"): + get_grdc_data( + "42424243", + "2000-01-01T00:00Z", + "2000-02-01T00:00Z", + data_home=str(tmp_path), + ) def test_get_grdc_dat_custom_column_name(expected_results, tmp_path): - CFG['grdc_location'] = str(tmp_path) - result_data, result_metadata = get_grdc_data('42424242', '2000-01-01T00:00Z', '2000-02-01T00:00Z', column='observation') + CFG["grdc_location"] = str(tmp_path) + result_data, result_metadata = get_grdc_data( + "42424242", "2000-01-01T00:00Z", "2000-02-01T00:00Z", column="observation" + ) expected_default_data, expected_metadata = expected_results - expected_data = expected_default_data.rename(columns={'streamflow': 'observation'}) + expected_data = expected_default_data.rename(columns={"streamflow": "observation"}) assert_frame_equal(result_data, expected_data) assert result_metadata == expected_metadata diff --git a/tests/parameter_sets/__init__.py b/tests/parameter_sets/__init__.py index 24421595..a3d5013c 100644 --- a/tests/parameter_sets/__init__.py +++ b/tests/parameter_sets/__init__.py @@ -7,7 +7,9 @@ def test_download_example_parameter_sets(tmp_path): download_example_parameter_sets() assert (tmp_path / "parameters" / "pcrglobwb_rhinemeuse_30min").exists() - assert (tmp_path / "parameters/pcrglobwb_rhinemeuse_30min/setup_natural_test.ini").exists() + assert ( + tmp_path / "parameters/pcrglobwb_rhinemeuse_30min/setup_natural_test.ini" + ).exists() # TODO test for the case where ewatercycle.yaml cfg is not writable diff --git a/tests/parameter_sets/test_default_parameterset.py b/tests/parameter_sets/test_default_parameterset.py index d850a1a7..19c6fe60 100644 --- a/tests/parameter_sets/test_default_parameterset.py +++ b/tests/parameter_sets/test_default_parameterset.py @@ -9,15 +9,15 @@ class TestDefaults: @pytest.fixture def mocked_config(self, tmp_path): - CFG['parameterset_dir'] = tmp_path - config = tmp_path / 'mymockedconfig.ini' - config.write_text('Something') + CFG["parameterset_dir"] = tmp_path + config = tmp_path / "mymockedconfig.ini" + config.write_text("Something") return config @pytest.fixture def parameter_set(self, tmp_path, mocked_config: Path): return ParameterSet( - name='justatest', + name="justatest", directory=str(tmp_path), config=mocked_config.name, ) @@ -45,8 +45,8 @@ def test_repr(self, parameter_set: ParameterSet, tmp_path): def test_str(self, parameter_set: ParameterSet, tmp_path): expected = ( - 'Parameter set\n' - '-------------\n' + "Parameter set\n" + "-------------\n" "name=justatest\n" f"directory={str(tmp_path)}\n" f"config={str(tmp_path)}/mymockedconfig.ini\n" @@ -60,18 +60,18 @@ def test_str(self, parameter_set: ParameterSet, tmp_path): class TestOutsideCFG: @pytest.fixture def mocked_config(self, tmp_path): - CFG['parameterset_dir'] = tmp_path / 'parameter-sets' - config = tmp_path / 'mymockedconfig.ini' - config.write_text('Something') + CFG["parameterset_dir"] = tmp_path / "parameter-sets" + config = tmp_path / "mymockedconfig.ini" + config.write_text("Something") return config @pytest.fixture def parameter_set(self, tmp_path, mocked_config: Path): return ParameterSet( - name='justatest', - directory=str(tmp_path / 'my-parameter-set'), + name="justatest", + directory=str(tmp_path / "my-parameter-set"), config=mocked_config.name, ) def test_directory(self, parameter_set: ParameterSet, tmp_path): - assert parameter_set.directory == tmp_path / 'my-parameter-set' + assert parameter_set.directory == tmp_path / "my-parameter-set" diff --git a/tests/parameter_sets/test_example.py b/tests/parameter_sets/test_example.py index 99087dc2..a658e71d 100644 --- a/tests/parameter_sets/test_example.py +++ b/tests/parameter_sets/test_example.py @@ -1,5 +1,5 @@ import logging -from unittest.mock import patch, Mock +from unittest.mock import Mock, patch import pytest @@ -9,7 +9,7 @@ @pytest.fixture def setup_config(tmp_path): - CFG['parameterset_dir'] = tmp_path + CFG["parameterset_dir"] = tmp_path yield CFG # Rollback changes made to CFG by tests CFG.reload() @@ -18,63 +18,70 @@ def setup_config(tmp_path): @pytest.fixture def example(setup_config): return ExampleParameterSet( - name='firstexample', - config_url='https://github.com/mymodelorg/mymodelrepo/raw/master/mymodelexample/config.ini', - datafiles_url='https://github.com/mymodelorg/mymodelrepo/trunk/mymodelexample', - directory='mymodelexample', - config='mymodelexample/config.ini', - supported_model_versions={'0.4.2'}, + name="firstexample", + config_url="https://github.com/mymodelorg/mymodelrepo/raw/master/mymodelexample/config.ini", # noqa: E501 + datafiles_url="https://github.com/mymodelorg/mymodelrepo/trunk/mymodelexample", + directory="mymodelexample", + config="mymodelexample/config.ini", + supported_model_versions={"0.4.2"}, ) def test_to_config(example): example.to_config() - assert 'firstexample' in CFG['parameter_sets'] + assert "firstexample" in CFG["parameter_sets"] expected = dict( - doi='N/A', - target_model='generic', - directory='mymodelexample', - config='mymodelexample/config.ini', - supported_model_versions={'0.4.2'}, + doi="N/A", + target_model="generic", + directory="mymodelexample", + config="mymodelexample/config.ini", + supported_model_versions={"0.4.2"}, ) - assert CFG['parameter_sets']['firstexample'] == expected + assert CFG["parameter_sets"]["firstexample"] == expected -@patch('urllib.request.urlopen') -@patch('subprocess.check_call') +@patch("urllib.request.urlopen") +@patch("subprocess.check_call") def test_download(mock_check_call, mock_urlopen, example, tmp_path): - ps_dir = tmp_path / 'mymodelexample' + ps_dir = tmp_path / "mymodelexample" r = Mock() - r.read.return_value = b'somecontent' + r.read.return_value = b"somecontent" mock_urlopen.return_value = r mock_check_call.side_effect = lambda _: ps_dir.mkdir() example.download() - mock_urlopen.assert_called_once_with('https://github.com/mymodelorg/mymodelrepo/raw/master/mymodelexample/config.ini') - mock_check_call.assert_called_once_with([ - 'svn', 'export', - 'https://github.com/mymodelorg/mymodelrepo/trunk/mymodelexample', - ps_dir - ]) - assert (ps_dir / 'config.ini').read_text() == 'somecontent' + mock_urlopen.assert_called_once_with( + "https://github.com/mymodelorg/mymodelrepo/raw/master/mymodelexample/config.ini" + ) + mock_check_call.assert_called_once_with( + [ + "svn", + "export", + "https://github.com/mymodelorg/mymodelrepo/trunk/mymodelexample", + ps_dir, + ] + ) + assert (ps_dir / "config.ini").read_text() == "somecontent" def test_download_already_exists(example, tmp_path): - ps_dir = tmp_path / 'mymodelexample' + ps_dir = tmp_path / "mymodelexample" ps_dir.mkdir() with pytest.raises(ValueError) as excinfo: example.download() - assert 'already exists, will not overwrite.' in str(excinfo.value) + assert "already exists, will not overwrite." in str(excinfo.value) -@patch('urllib.request.urlopen') -@patch('subprocess.check_call') -def test_download_already_exists_but_skipped(mock_check_call, mock_urlopen, example, tmp_path, caplog): - ps_dir = tmp_path / 'mymodelexample' +@patch("urllib.request.urlopen") +@patch("subprocess.check_call") +def test_download_already_exists_but_skipped( + mock_check_call, mock_urlopen, example, tmp_path, caplog +): + ps_dir = tmp_path / "mymodelexample" ps_dir.mkdir() with caplog.at_level(logging.INFO): @@ -83,4 +90,4 @@ def test_download_already_exists_but_skipped(mock_check_call, mock_urlopen, exam mock_urlopen.assert_not_called() mock_check_call.assert_not_called() - assert 'already exists, skipping download.' in caplog.text + assert "already exists, skipping download." in caplog.text diff --git a/tests/test_analysis.py b/tests/test_analysis.py index bd6c3117..a8143ab7 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -1,14 +1,15 @@ -from ewatercycle.analysis import hydrograph -from matplotlib.testing.decorators import image_comparison -import pandas as pd import numpy as np +import pandas as pd +from matplotlib.testing.decorators import image_comparison + +from ewatercycle.analysis import hydrograph @image_comparison( - baseline_images=['hydrograph'], + baseline_images=["hydrograph"], remove_text=True, - extensions=['png'], - savefig_kwarg={'bbox_inches':'tight'}, + extensions=["png"], + savefig_kwarg={"bbox_inches": "tight"}, ) def test_hydrograph(): ntime = 300 @@ -18,19 +19,19 @@ def test_hydrograph(): np.random.seed(20210416) discharge = { - 'discharge_a': pd.Series(np.linspace(0, 2, ntime), index=dti), - 'discharge_b': pd.Series(3*np.random.random(ntime)**2, index=dti), - 'discharge_c': pd.Series(2*np.random.random(ntime)**2, index=dti), - 'reference': pd.Series(np.random.random(ntime)**2, index=dti), + "discharge_a": pd.Series(np.linspace(0, 2, ntime), index=dti), + "discharge_b": pd.Series(3 * np.random.random(ntime) ** 2, index=dti), + "discharge_c": pd.Series(2 * np.random.random(ntime) ** 2, index=dti), + "reference": pd.Series(np.random.random(ntime) ** 2, index=dti), } df = pd.DataFrame(discharge) precipitation = { - 'precipitation_a': pd.Series(np.random.random(ntime)/20, index=dti), - 'precipitation_b': pd.Series(np.random.random(ntime)/30, index=dti), + "precipitation_a": pd.Series(np.random.random(ntime) / 20, index=dti), + "precipitation_b": pd.Series(np.random.random(ntime) / 30, index=dti), } df_pr = pd.DataFrame(precipitation) - hydrograph(df, reference='reference', precipitation=df_pr) + hydrograph(df, reference="reference", precipitation=df_pr) diff --git a/tests/test_config.py b/tests/test_config.py index 2ee7eab1..3fec590b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -3,13 +3,13 @@ """Tests for the ewatercycle_parametersetdb module. """ -from ewatercycle.parametersetdb.config import fetch, YamlConfig +from ewatercycle.parametersetdb.config import YamlConfig, fetch def test_fetch(yaml_config_url): result = fetch(yaml_config_url) - assert 'PEQ_Hupsel.dat' in result + assert "PEQ_Hupsel.dat" in result class TestYamlConfig: @@ -20,8 +20,8 @@ def test_construct_from_data_url(self, yaml_config_url, yaml_config): def test_save(self, tmpdir, yaml_config_url): conf = YamlConfig(yaml_config_url) - fn = tmpdir.join('myconfig.yml') + fn = tmpdir.join("myconfig.yml") conf.save(str(fn)) - assert 'PEQ_Hupsel.dat' in fn.read() + assert "PEQ_Hupsel.dat" in fn.read() diff --git a/tests/test_datafiles.py b/tests/test_datafiles.py index dd34726f..118e72ae 100644 --- a/tests/test_datafiles.py +++ b/tests/test_datafiles.py @@ -2,14 +2,14 @@ def test_SymlinkCopier(tmp_path): - source_dir = tmp_path / 'source' + source_dir = tmp_path / "source" source_dir.mkdir() - source_forcings = source_dir / 'forcings.csv' - source_forcings.write_text('dummy') + source_forcings = source_dir / "forcings.csv" + source_forcings.write_text("dummy") copier = SymlinkCopier(str(source_dir)) - target_dir = tmp_path / 'target' + target_dir = tmp_path / "target" copier.save(target_dir) - target_forcings = target_dir / 'forcings.csv' - assert target_forcings.read_text() == 'dummy' + target_forcings = target_dir / "forcings.csv" + assert target_forcings.read_text() == "dummy" diff --git a/tests/test_parameter_sets.py b/tests/test_parameter_sets.py index f0846895..d2952e49 100644 --- a/tests/test_parameter_sets.py +++ b/tests/test_parameter_sets.py @@ -4,97 +4,104 @@ from ewatercycle import CFG from ewatercycle.config import DEFAULT_CONFIG -from ewatercycle.parameter_sets import available_parameter_sets, get_parameter_set, example_parameter_sets, \ - download_example_parameter_sets, ExampleParameterSet +from ewatercycle.parameter_sets import ( + ExampleParameterSet, + available_parameter_sets, + download_example_parameter_sets, + example_parameter_sets, + get_parameter_set, +) @pytest.fixture def setup_config(tmp_path): - CFG['parameterset_dir'] = tmp_path - CFG['ewatercycle_config'] = tmp_path / 'ewatercycle.yaml' + CFG["parameterset_dir"] = tmp_path + CFG["ewatercycle_config"] = tmp_path / "ewatercycle.yaml" yield CFG - CFG['ewatercycle_config'] = DEFAULT_CONFIG + CFG["ewatercycle_config"] = DEFAULT_CONFIG CFG.reload() @pytest.fixture def mocked_parameterset_dir(setup_config, tmp_path): - ps1_dir = tmp_path / 'ps1' + ps1_dir = tmp_path / "ps1" ps1_dir.mkdir() - config1 = ps1_dir / 'mymockedconfig1.ini' - config1.write_text('Something') - ps2_dir = tmp_path / 'ps2' + config1 = ps1_dir / "mymockedconfig1.ini" + config1.write_text("Something") + ps2_dir = tmp_path / "ps2" ps2_dir.mkdir() - config2 = ps2_dir / 'mymockedconfig2.ini' - config2.write_text('Something else') - CFG['parameter_sets'] = { - 'ps1': { - 'directory': str(ps1_dir), - 'config': str(config1.relative_to(tmp_path)), - 'target_model': 'generic', - 'doi': 'somedoi1' + config2 = ps2_dir / "mymockedconfig2.ini" + config2.write_text("Something else") + CFG["parameter_sets"] = { + "ps1": { + "directory": str(ps1_dir), + "config": str(config1.relative_to(tmp_path)), + "target_model": "generic", + "doi": "somedoi1", }, - 'ps2': { - 'directory': str(ps2_dir), - 'config': str(config2.relative_to(tmp_path)), - 'target_model': 'generic', - 'doi': 'somedoi2' + "ps2": { + "directory": str(ps2_dir), + "config": str(config2.relative_to(tmp_path)), + "target_model": "generic", + "doi": "somedoi2", + }, + "ps3": { + "directory": str(tmp_path / "ps3"), + "config": "unavailable_config_file", + "target_model": "generic", + "doi": "somedoi3", }, - 'ps3': { - 'directory': str(tmp_path / 'ps3'), - 'config': 'unavailable_config_file', - 'target_model': 'generic', - 'doi': 'somedoi3' - } } class TestAvailableParameterSets: def test_filled(self, mocked_parameterset_dir): - names = available_parameter_sets('generic') - assert set(names) == {'ps1', 'ps2'} # ps3 is filtered due to not being available + names = available_parameter_sets("generic") + assert set(names) == { + "ps1", + "ps2", + } # ps3 is filtered due to not being available def test_no_config(self, tmp_path): # Load default config shipped with package - CFG['ewatercycle_config'] = DEFAULT_CONFIG + CFG["ewatercycle_config"] = DEFAULT_CONFIG CFG.reload() with pytest.raises(ValueError) as excinfo: available_parameter_sets() - assert 'No configuration file found' in str(excinfo.value) + assert "No configuration file found" in str(excinfo.value) def test_no_sets_in_config(self, setup_config): with pytest.raises(ValueError) as excinfo: available_parameter_sets() - assert 'No parameter sets defined in' in str(excinfo.value) + assert "No parameter sets defined in" in str(excinfo.value) def test_no_sets_for_model(self, mocked_parameterset_dir): with pytest.raises(ValueError) as excinfo: - available_parameter_sets('somemodel') + available_parameter_sets("somemodel") - assert 'No parameter sets defined for somemodel model in' in str(excinfo.value) + assert "No parameter sets defined for somemodel model in" in str(excinfo.value) class TestGetParameterSet: - def test_valid(self, mocked_parameterset_dir, tmp_path): - actual = get_parameter_set('ps1') + actual = get_parameter_set("ps1") - assert actual.name == 'ps1' - assert actual.directory == tmp_path / 'ps1' - assert actual.config == tmp_path / 'ps1' / 'mymockedconfig1.ini' - assert actual.doi == 'somedoi1' - assert actual.target_model == 'generic' + assert actual.name == "ps1" + assert actual.directory == tmp_path / "ps1" + assert actual.config == tmp_path / "ps1" / "mymockedconfig1.ini" + assert actual.doi == "somedoi1" + assert actual.target_model == "generic" def test_unknown(self, mocked_parameterset_dir): with pytest.raises(KeyError): - get_parameter_set('ps9999') + get_parameter_set("ps9999") def test_unavailable(self, mocked_parameterset_dir): with pytest.raises(ValueError): - get_parameter_set('ps3') + get_parameter_set("ps3") def test_example_parameter_sets(setup_config): @@ -104,10 +111,10 @@ def test_example_parameter_sets(setup_config): assert name == examples[name].name -@patch.object(ExampleParameterSet, 'download') +@patch.object(ExampleParameterSet, "download") def test_download_example_parameter_sets(mocked_download, setup_config, tmp_path): download_example_parameter_sets() assert mocked_download.call_count > 0 - assert CFG['ewatercycle_config'].read_text() == CFG.dump_to_yaml() - assert len(CFG['parameter_sets']) > 0 + assert CFG["ewatercycle_config"].read_text() == CFG.dump_to_yaml() + assert len(CFG["parameter_sets"]) > 0 diff --git a/tests/test_parameterset.py b/tests/test_parameterset.py index 7fb65ff9..1b26ab18 100644 --- a/tests/test_parameterset.py +++ b/tests/test_parameterset.py @@ -1,14 +1,16 @@ # -*- coding: utf-8 -*- from unittest import mock -from ewatercycle.parametersetdb import build_from_urls, ParameterSet +from ewatercycle.parametersetdb import ParameterSet, build_from_urls from ewatercycle.parametersetdb.datafiles import SubversionCopier def test_build_from_urls(yaml_config_url, yaml_config): pset = build_from_urls( - config_format='yaml', config_url=yaml_config_url, - datafiles_format='svn', datafiles_url='http://example.com', + config_format="yaml", + config_url=yaml_config_url, + datafiles_format="svn", + datafiles_url="http://example.com", ) assert isinstance(pset.df, SubversionCopier) @@ -17,17 +19,17 @@ def test_build_from_urls(yaml_config_url, yaml_config): class TestParameterSet: def test_save_config(self, sample_parameterset: ParameterSet, tmpdir): - fn = tmpdir.join('myconfig.yml') + fn = tmpdir.join("myconfig.yml") sample_parameterset.save_config(str(fn)) - assert 'PEQ_Hupsel.dat' in fn.read() + assert "PEQ_Hupsel.dat" in fn.read() - @mock.patch('subprocess.check_call') + @mock.patch("subprocess.check_call") def test_save_datafiles(self, mock_check_call, sample_parameterset: ParameterSet): - fn = '/somewhere/adirectory' + fn = "/somewhere/adirectory" sample_parameterset.save_datafiles(fn) - expected_args = ['svn', 'export', sample_parameterset.df.source, fn] + expected_args = ["svn", "export", sample_parameterset.df.source, fn] mock_check_call.assert_called_once_with(expected_args)