diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index ec0228e2..00000000 --- a/.coveragerc +++ /dev/null @@ -1,11 +0,0 @@ -[run] -omit = - *example* - -[report] -exclude_lines = - pragma: no cover - def __repr__ - raise NotImplementedError - if __name__ == .__main__.: - def parse_args diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..c724163b --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,115 @@ +name: Continuous Integration + +on: + push: + branches: + - "master" + tags: + - "*" + pull_request: + branches: + - "master" + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +env: + # needed by coveralls + GITHUB_TOKEN: ${{ secrets.COVERALLS_TOKEN }} + +jobs: + source_check: + name: source check + runs-on: ubuntu-latest + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install black 'isort[colors]<6' + pip install --editable . + + - name: black check + run: | + python -m black --check --diff --color . + + - name: isort check + run: | + python -m isort --check --diff --color . + + build_sdist: + name: sdist on ${{ matrix.os }} with py ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: ['3.7', '3.8', '3.9', '3.10'] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: '0' + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build coveralls>=3.0.0 + pip install --editable .[test] + + - name: Run tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python -m pytest --cov spotpy --cov-report term-missing -v tests/ + python -m coveralls --service=github + + - name: Build sdist + run: | + python -m build + + - uses: actions/upload-artifact@v2 + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.9' + with: + path: dist + + upload_to_pypi: + needs: [build_sdist] + runs-on: ubuntu-latest + + steps: + - uses: actions/download-artifact@v2 + with: + name: artifact + path: dist + + - name: Publish to Test PyPI + # only if working on master + if: github.ref == 'refs/heads/master' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_TEST_TOKEN }} + repository_url: https://test.pypi.org/legacy/ + skip_existing: true + + - name: Publish to PyPI + # only if tagged + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml deleted file mode 100644 index 30acd6e4..00000000 --- a/.github/workflows/python-app.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python application - -on: - push: - branches: [ "master" ] - pull_request: - branches: [ "master" ] - -permissions: - contents: read - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest numpy matplotlib pandas scipy click deap docutils numba tables pathos - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml deleted file mode 100644 index ec703542..00000000 --- a/.github/workflows/python-publish.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Upload Python Package - -on: - release: - types: [published] - -permissions: - contents: read - -jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build - - name: Build package - run: python -m build - - name: Publish package - uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore index e7aa88d8..4cbaa380 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,8 @@ __pycache__/ #other files *.out *.png -*.csv +test_DDS.csv +./*.csv site/* #mkdocs.yml @@ -32,6 +33,9 @@ wheels/ .installed.cfg *.egg +# generated version file +src/spotpy/_version.py + # Python Unittest tool `py.test` .pytest_cache/* @@ -98,6 +102,8 @@ ENV/ .spyderproject # PyCharm .idea/ +# VSCode +.vscode/ # Rope project settings .ropeproject @@ -108,12 +114,12 @@ ENV/ .settings/ # hymod example -spotpy/examples/hymod/* +src/spotpy/examples/hymod/* # Test Cases -spotpy/hymod/testHymod/* -spotpy/hymod/RunsWithNewHymod/* -spotpy/hymod/likelihood_test/* +src/spotpy/hymod/testHymod/* +src/spotpy/hymod/RunsWithNewHymod/* +src/spotpy/hymod/likelihood_test/* # a doc folder for some notices _doc diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index eda4af49..00000000 --- a/.travis.yml +++ /dev/null @@ -1,59 +0,0 @@ -language: python -python: - #- "2.7" - - "3.7" -# - "3.8" #Errors on Travis, build times out because no output was received... - -# Uses cache to build faster -cache: pip - -# this ubuntu distribution has sure the libopenmpi-dev packages available -dist: bionic - -# command to install dependencies -install: - - sudo apt-get update - - sudo apt install libopenmpi-dev openmpi-bin - - pip install mpi4py - - python setup.py install - - pip install coveralls - - pip install pathos - - pip install numpy - - pip install matplotlib - - pip install pandas - - pip install scipy - - pip install click - - pip install deap - # Need to force pytest-cov to v2.6 as current version (2.6.1) is deprecated and pytest:3.10.1 - - pip install pytest==3.10.1 pytest-pep8 pytest-cov==2.6 - # Use docutils to generate html describe - - pip install docutils - - pip install numba - - pip install tables - - -script: - - py.test tests/test_* --cov spotpy --cov-report term-missing -v -# - mpirun -c 4 python spotpy/examples/tutorial_parallel_computing_hymod.py 100 - - pip uninstall spotpy -y - - -jobs: - include: - - stage: Coveralls - name: Report Coveralls - after_success: python3 -m coveralls - -# Publish spotpy on PyPi if taged -deploy: - provider: pypi - distributions: "sdist bdist_wheel" - on: - # In this case I want to deploy only when a tag is present... - tags: true - # ... and when tag is on master and respects the form "v0.0.0" - branch: - - master - - /v?(\d+\.)?(\d+\.)?(\*|\d+)$/ - user: thouska - password : ${deploying} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ad11f64f..510bcf0a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ There are 4 main ways of contributing to the SPOTPY project (in descending order * Adding new or improved functionality to the existing codebase * Fixing outstanding issues (bugs) with the existing codebase. They range from low-level software bugs to higher-level design problems. -* Contributing or improving the documentation (`docs`) or examples (`spotpy/examples`) +* Contributing or improving the documentation (`docs`) or examples (`src/spotpy/examples`) * Submitting issues related to bugs or desired enhancements # Opening issues @@ -76,7 +76,7 @@ We recommended that your contribution complies with the following guidelines bef * Please prefix the title of incomplete contributions with `[WIP]` (to indicate a work in progress). WIPs may be useful to (1) indicate you are working on something to avoid duplicated work, (2) request broad review of functionality or API, or (3) seek collaborators. -* When adding additional functionality, you may want to provide at least one example script in the ``spotpy/examples/`` folder. Have a look at other examples for reference. Examples should demonstrate why the new functionality is useful in practice and, if possible, compare it to other methods available in SPOTPY. +* When adding additional functionality, you may want to provide at least one example script in the ``src/spotpy/examples/`` folder. Have a look at other examples for reference. Examples should demonstrate why the new functionality is useful in practice and, if possible, compare it to other methods available in SPOTPY. You can also check for common programming errors with the following tools: @@ -111,7 +111,7 @@ tools: ## Style guide -Follow [TensorFlow's style guide](https://www.tensorflow.org/versions/master/how_tos/style_guide.html) or the [Google style guide](https://google.github.io/styleguide/pyguide.html) for writing code, which more or less follows PEP 8. +Try to make sure to follow the [Black style guide](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html) and [isort](https://pypi.org/project/isort/) for code submissions. -#### This guide was derived from the [scikit-learn guide to contributing](https://github.com/scikit-learn/scikit-learn/blob/master/CONTRIBUTING.md) +#### This guide was derived from the [scikit-learn guide to contributing](https://github.com/scikit-learn/scikit-learn/blob/master/CONTRIBUTING.md) and adopted to SPOTPY. diff --git a/MANIFEST.in b/MANIFEST.in index e5706096..f2da5cec 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,6 @@ -include README.md -include MANIFEST.in -include setup.py -include setup.cfg -recursive-include spotpy/examples/cmf_data * -recursive-include spotpy/examples/hymod_exe * -recursive-include spotpy/examples/hymod_unix * -recursive-include spotpy/examples/hymod_python * -include LICENSE \ No newline at end of file +prune ** +graft src/spotpy +graft tests +graft tutorials +include LICENSE README.md pyproject.toml setup.py +global-exclude __pycache__ *.py[co] diff --git a/README.md b/README.md index c4212bd9..3da2ac3a 100644 --- a/README.md +++ b/README.md @@ -25,13 +25,13 @@ A Statistical Parameter Optimization Tool for Python Purpose ================= -SPOTPY is a Python framework that enables the use of Computational optimization techniques for calibration, uncertainty +SPOTPY is a Python framework that enables the use of Computational optimization techniques for calibration, uncertainty and sensitivity analysis techniques of almost every (environmental-) model. The package is puplished in the open source journal PLoS One: -Houska, T., Kraft, P., Chamorro-Chavez, A. and Breuer, L.: SPOTting Model Parameters Using a Ready-Made Python Package, PLoS ONE, +Houska, T., Kraft, P., Chamorro-Chavez, A. and Breuer, L.: SPOTting Model Parameters Using a Ready-Made Python Package, PLoS ONE, 10(12), e0145180, doi:[10.1371/journal.pone.0145180](http://journals.plos.org/plosone/article?id=10.1371%2Fjournal.pone.0145180 "SPOTting Model Parameters Using a Ready-Made Python Package"), 2015 - -The simplicity and flexibility enables the use and test of different + +The simplicity and flexibility enables the use and test of different algorithms of almost any model, without the need of complex codes:: sampler = spotpy.algorithms.sceua(model_setup()) # Initialize your model with a setup file @@ -44,17 +44,17 @@ algorithms of almost any model, without the need of complex codes:: Features ================= -Complex algorithms bring complex tasks to link them with a model. -We want to make this task as easy as possible. +Complex algorithms bring complex tasks to link them with a model. +We want to make this task as easy as possible. Some features you can use with the SPOTPY package are: -* Fitting models to evaluation data with different algorithms. - Available algorithms are: - +* Fitting models to evaluation data with different algorithms. + Available algorithms are: + * Monte Carlo (`MC`) * Markov-Chain Monte-Carlo (`MCMC`) * Maximum Likelihood Estimation (`MLE`) - * Latin-Hypercube Sampling (`LHS`) + * Latin-Hypercube Sampling (`LHS`) * Simulated Annealing (`SA`) * Shuffled Complex Evolution Algorithm (`SCE-UA`) * Differential Evolution Markov Chain Algorithm (`DE-MCz`) @@ -97,7 +97,7 @@ Some features you can use with the SPOTPY package are: * Skewness * compare percentiles of discharge -* Prebuild parameter distribution functions: +* Prebuild parameter distribution functions: * Uniform * Normal @@ -112,7 +112,7 @@ Some features you can use with the SPOTPY package are: of a model. * Multi-objective support - + * MPI support for fast parallel computing * A progress bar monitoring the sampling loops. Enables you to plan your coffee brakes. @@ -181,7 +181,7 @@ Patches/enhancements/new algorithms and any other contributions to this package 1. Fork it ( http://github.com/thouska/spotpy/fork ) 2. Create your feature branch (``git checkout -b my-new-feature``) 3. Add your modifications -4. Add short summary of your modifications on ``CHANGELOG.rst`` +4. Add short summary of your modifications on ``CHANGELOG.md`` 5. Commit your changes (``git commit -m "Add some feature"``) 6. Push to the branch (``git push origin my-new-feature``) 7. Create new Pull Request diff --git a/README.rst b/README.rst deleted file mode 100644 index eac7aa9c..00000000 --- a/README.rst +++ /dev/null @@ -1,190 +0,0 @@ -SPOTPY -======== -**A Statistical Parameter Optimization Tool for Python** - -SPOTPY is a Python tool that enables the use of Computational optimization techniques for calibration, uncertainty -and sensitivity analysis techniques of almost every (environmental-) model. The package is puplished in the open source journal PLoS One - -Houska, T, Kraft, P, Chamorro-Chavez, A and Breuer, L; `SPOTting Model Parameters Using a Ready-Made Python Package `_; PLoS ONE; 2015 - -The simplicity and flexibility enables the use and test of different -algorithms without the need of complex codes:: - - sampler = spotpy.algorithms.sceua(model_setup()) # Initialize your model with a setup file - sampler.sample(10000) # Run the model - results = sampler.getdata() # Load the results - spotpy.analyser.plot_parametertrace(results) # Show the results - - -Features --------- - -Complex formal Bayesian informal Bayesian and non-Bayesian algorithms bring complex tasks to link them with a given model. -We want to make this task as easy as possible. Some features you can use with the SPOTPY package are: - -* Fitting models to evaluation data with different algorithms. - Available algorithms are: - - * Monte Carlo (`MC`) - * Markov-Chain Monte-Carlo (`MCMC`) - * Maximum Likelihood Estimation (`MLE`) - * Latin-Hypercube Sampling (`LHS`) - * Simulated Annealing (`SA`) - * Shuffled Complex Evolution Algorithm (`SCE-UA`) - * Differential Evolution Markov Chain Algorithm (`DE-MCz`) - * Differential Evolution Adaptive Metropolis Algorithm (`DREAM`) - * RObust Parameter Estimation (`ROPE`) - * Fourier Amplitude Sensitivity Test (`FAST`) - * Artificial Bee Colony (`ABC`) - * Fitness Scaled Chaotic Artificial Bee Colony (`FSCABC`) - * Dynamically Dimensioned Search algorithm (`DDS`) - * Pareto Archived - Dynamicallly Dimensioned Search algorithm (`PA-DDS`) - * Fast and Elitist Multiobjective Genetic Algorithm (`NSGA-II`) - -* Wide range of objective functions (also known as loss function, fitness function or energy function) to validate the sampled results. Available functions are - - * Bias - * Procentual Bias (`PBias`) - * Nash-Sutcliffe (`NSE`) - * logarithmic Nash-Sutcliffe (`logNSE`) - * logarithmic probability (`logp`) - * Correlation Coefficient (`r`) - * Coefficient of Determination (`r^2`) - * Mean Squared Error (`MSE`) - * Root Mean Squared Error (`RMSE`) - * Mean Absolute Error (`MAE`) - * Relative Root Mean Squared Error (`RRMSE`) - * Agreement Index (`AI`) - * Covariance, Decomposed MSE (`dMSE`) - * Kling-Gupta Efficiency (`KGE`) - * Non parametric Kling-Gupta Efficiency (`KGE_non_parametric`) - -* Wide range of likelihood functions to validate the sampled results: - - * logLikelihood - * Gaussian Likelihood to account for Measurement Errors - * Gaussian Likelihood to account for Heteroscedasticity - * Likelihood to accounr for Autocorrelation - * Generalized Likelihood Function - * Lapacian Likelihood - * Skewed Student Likelihood assuming homoscedasticity - * Skewed Student Likelihood assuming heteroscedasticity - * Skewed Student Likelihood assuming heteroscedasticity and Autocorrelation - * Noisy ABC Gaussian Likelihood - * ABC Boxcar Likelihood - * Limits Of Acceptability - * Inverse Error Variance Shaping Factor - * Nash Sutcliffe Efficiency Shaping Factor - * Exponential Transform Shaping Factor - * Sum of Absolute Error Residuals - -* Wide range of hydrological signatures functions to validate the sampled results: - - * Slope - * Flooding/Drought events - * Flood/Drought frequency - * Flood/Drought duration - * Flood/Drought variance - * Mean flow - * Median flow - * Skewness - * compare percentiles of discharge - -* Prebuild parameter distribution functions: - - * Uniform - * Normal - * logNormal - * Chisquare - * Exponential - * Gamma - * Wald - * Weilbull - -* Wide range to adapt algorithms to perform uncertainty-, sensitivity analysis or calibration - of a model. - -* Multi-objective support - -* MPI support for fast parallel computing - -* A progress bar monitoring the sampling loops. Enables you to plan your coffee brakes. - -* Use of NumPy functions as often as possible. This makes your coffee brakes short. - -* Different databases solutions: `ram` storage for fast sampling a simple , `csv` tables - the save solution for long duration samplings and a `sql` database for larger projects. - -* Automatic best run selecting and plotting - -* Parameter trace plotting - -* Parameter interaction plot including the Gaussian-kde function - -* Regression analysis between simulation and evaluation data - -* Posterior distribution plot - -* Convergence diagnostics with Gelman-Rubin and the Geweke plot - - -Documentation -------------- - -Documentation is available at ``__ - - -Install -------- - -Classical Python options exist to install SPOTPY: - -From PyPi: - - pip install spotpy - -From Conda-Forge: - - conda config --add channels conda-forge - conda config --set channel_priority strict - conda install spotpy - -From [Source](https://pypi.python.org/pypi/spotpy): - - python setup.py install - - -Papers citing SPOTPY --------------------- -See `Google Scholar `__ for a continuously updated list. - - -Support -------- - -* Feel free to contact the authors of this tool for any support questions. - -* If you use this package for a scientific research paper, please `cite `_ SPOTPY. - -* Please report any bug through mail or GitHub: https://github.com/thouska/spotpy. - -* If you want to share your code with others, you are welcome to do this through GitHub: https://github.com/thouska/spotpy. - - -Contributing ------------- -Patches/enhancements/new algorithms and any other contributions to this package are very welcome! - -1. Fork it ( http://github.com/thouska/spotpy/fork ) -2. Create your feature branch (``git checkout -b my-new-feature``) -3. Add your modifications -4. Add short summary of your modifications on ``CHANGELOG.rst`` -5. Commit your changes (``git commit -m "Add some feature"``) -6. Push to the branch (``git push origin my-new-feature``) -7. Create new Pull Request - - -Getting started ---------------- - -Have a look at https://github.com/thouska/spotpy/tree/master/spotpy/examples and https://spotpy.readthedocs.io/en/latest/getting_started/ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..b51fd8ed --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,127 @@ +[build-system] +requires = [ + "setuptools>=63", + "setuptools_scm[toml]>=6.4", +] +build-backend = "setuptools.build_meta" + +[project] +requires-python = ">=3.7" +name = "spotpy" +description = "A Statistical Parameter Optimization Tool." +authors = [ + {name = "Tobias Houska", email = "tobias.houska@umwelt.uni-giessen.de"}, + {name = "Philipp Kraft"}, + {name = "Alejandro Chamorro-Chavez"}, + {name = "Lutz Breuer"}, + {name = "Sebastian Müller", email = "sebastian.mueller@ufz.de"}, +] +maintainers = [ + {name = "Tobias Houska", email = "tobias.houska@umwelt.uni-giessen.de"}, +] +readme = "README.md" +license = {file = "LICENSE"} +dynamic = ["version"] +keywords = [ + "Monte Carlo", + "MCMC", + "MLE", + "SCE-UA", + "Simulated Annealing", + "DE-MCz", + "DREAM", + "ROPE", + "Artifical Bee Colony", + "DDS", + "PA-DDS", + "NSGAii", + "Uncertainty", + "Calibration", + "Model", + "Signatures", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: End Users/Desktop", + "Intended Audience :: Science/Research", + "Intended Audience :: Education", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Operating System :: Unix", + "Operating System :: Microsoft", + "Operating System :: MacOS", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Hydrology", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Scientific/Engineering :: Physics", + "Topic :: Utilities", +] +dependencies = [ + "numpy>=1.14.5", + "scipy>=1.5.0", +] + +[project.optional-dependencies] +plotting = [ + "pandas>=1", + "matplotlib>=3", +] +test = [ + "pytest-cov>=3", + "numba", + "pathos", + "matplotlib", + "click", + "pandas", + "tables", + "docutils", +] + +[project.urls] +Changelog = "https://github.com/thouska/spotpy/blob/master/CHANGELOG.md" +Conda-Forge = "https://anaconda.org/conda-forge/spotpy" +Documentation = "https://spotpy.readthedocs.io" +Homepage = "https://github.com/thouska/spotpy" +Source = "https://github.com/thouska/spotpy" +Tracker = "https://github.com/thouska/spotpy/issues" + +[tool.setuptools_scm] +write_to = "src/spotpy/_version.py" +write_to_template = "__version__ = '{version}'" +local_scheme = "no-local-version" +fallback_version = "0.0.0.dev0" +tag_regex = "^(?:[\\w-]+-)?[vV]?\\.?(?P\\d+(?:\\.\\d+){0,2}[^\\+]*)(?:\\+.*)?$" + +[tool.isort] +profile = "black" +multi_line_output = 3 + +[tool.black] +exclude = "_version.py" +target-version = [ + "py36", + "py37", + "py38", +] + +[tool.coverage] + [tool.coverage.run] + source = ["spotpy"] + + [tool.coverage.report] + exclude_lines = [ + "pragma: no cover", + "if __name__ == .__main__.:", + "def __repr__", + "def __str__", + "raise NotImplementedError", + "def parse_args", + ] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index cc639349..00000000 --- a/setup.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[wheel] -python-tag = py27 - -[metadata] -description-file = README.md - diff --git a/setup.py b/setup.py index cb7cbf0e..7f1a1763 100644 --- a/setup.py +++ b/setup.py @@ -1,31 +1,4 @@ -# Copyright (c) 2015, Tobias Houska +from setuptools import setup -from setuptools import setup, find_packages -import os - -setup( - name = 'spotpy', - version = '1.5.15', - description = 'A Statistical Parameter Optimization Tool', - long_description=open(os.path.join(os.path.dirname(__file__), - "README.rst")).read(), - author = 'Tobias Houska, Philipp Kraft, Alejandro Chamorro-Chavez and Lutz Breuer', - author_email = 'tobias.houska@umwelt.uni-giessen.de', - url = 'https://spotpy.readthedocs.io/en/latest/', - license = 'MIT', - install_requires=[ - 'scipy', 'numpy', 'pandas'], - packages=find_packages(exclude=["tests*", "docs*"]), - keywords = 'Monte Carlo, MCMC, MLE, SCE-UA, Simulated Annealing, DE-MCz, DREAM, ROPE, Artifical Bee Colony, DDS, PA-DDS, NSGAii, Uncertainty, Calibration, Model, Signatures', - classifiers = [ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Natural Language :: English', - 'License :: OSI Approved :: MIT License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Topic :: Software Development :: Libraries :: Python Modules'], - ) +if __name__ == "__main__": + setup() diff --git a/spotpy/__init__.py b/spotpy/__init__.py deleted file mode 100644 index c1360f70..00000000 --- a/spotpy/__init__.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright (c) 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -:paper: Houska, T., Kraft, P., Chamorro-Chavez, A. and Breuer, L.: -SPOTting Model Parameters Using a Ready-Made Python Package, -PLoS ONE, 10(12), e0145180, doi:10.1371/journal.pone.0145180, 2015. - -This package enables the comprehensive use of different Bayesian and Heuristic calibration -techniques in one Framework. It comes along with an algorithms folder for the -sampling and an analyser class for the plotting of results by the sampling. - -:dependencies: - Numpy >1.8 (http://www.numpy.org/) - - Scipy >1.5 (https://pypi.org/project/scipy/) - - Pandas >0.13 (optional) (http://pandas.pydata.org/) - - Matplotlib >1.4 (optional) (http://matplotlib.org/) - - CMF (optional) (http://fb09-pasig.umwelt.uni-giessen.de:8000/) - - mpi4py (optional) (http://mpi4py.scipy.org/) - - pathos (optional) (https://pypi.python.org/pypi/pathos/) - - sqlite3 (optional) (https://pypi.python.org/pypi/sqlite3/) - - numba (optional) (https://pypi.python.org/pypi/numba/) - - :help: For specific questions, try to use the documentation website at: - https://spotpy.readthedocs.io/en/latest/ - -For general things about parameter optimization techniques have a look at: -https://github.com/CamDavidsonPilon/Probabilistic-Programming-and-Bayesian-Methods-for-Hackers/tree/master/ - -Please cite our paper, if you are using SPOTPY. -''' -from . import database # Writes the results of the sampler in a user defined output file -from . import algorithms # Contains all the different algorithms implemented in SPOTPY -from . import parameter # Contains different distributions to describe the prior information for every model parameter -from . import analyser # Contains some examples to analyse the results of the different algorithms -from . import objectivefunctions # Quantifies goodness of fit between simulation and evaluation data with objective functions -from . import likelihoods # Quantifies goodness of fit between simulation and evaluation data with likelihood functions -from . import examples # Contains tutorials how to use SPOTPY -from . import describe # Contains some helper functions to describe samplers and set-ups -from .hydrology import signatures # Quantifies goodness of fit between simulation and evaluation data with hydrological signatures - -__version__ = '1.5.15' \ No newline at end of file diff --git a/spotpy/algorithms/__init__.py b/spotpy/algorithms/__init__.py deleted file mode 100644 index 2de3ea62..00000000 --- a/spotpy/algorithms/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright (c) 2015 by Tobias Houska - -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -:paper: Houska, T., Kraft, P., Chamorro-Chavez, A. and Breuer, L.: -SPOTting Model Parameters Using a Ready-Made Python Package, -PLoS ONE, 10(12), e0145180, doi:10.1371/journal.pone.0145180, 2015. - -Imports the different algorithms from this package. -To reduce dependencies, one may select here just the needed algorithm. -''' - - - - -from ._algorithm import _algorithm -from .demcz import demcz # Differential Evolution Markov Chain -from .lhs import lhs # Latin Hypercube Sampling -from .mcmc import mcmc # Metropolis Markov Chain Monte Carlo -from .mle import mle # Maximum Likelihood Estimation -from .mc import mc # Monte Carlo -from .sceua import sceua # Shuffled Complex Evolution -from .sa import sa # Simulated annealing -from .rope import rope # RObust Parameter Estimation -from .fast import fast # Fourier Amplitude Sensitivity Test -from .abc import abc # Artificial Bee Colony -from .fscabc import fscabc # Fitness Scaling Artificial Bee Colony -from .dream import dream # DiffeRential Evolution Adaptive Metropolis -from .nsgaii import NSGAII # A Fast and Elitist Multiobjective Genetic Algorithm: NSGA-II -from .list_sampler import list_sampler # Samples from given spotpy database -from .dds import dds # Dynamically Dimensioned Search algorithm -from .padds import padds # Pareto Archived - Dynamicallly Dimensioned Search algorithm \ No newline at end of file diff --git a/spotpy/algorithms/dream.py b/spotpy/algorithms/dream.py deleted file mode 100644 index 1cfbc7d5..00000000 --- a/spotpy/algorithms/dream.py +++ /dev/null @@ -1,395 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright (c) 2018 by Tobias Houska -This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). -:author: Tobias Houska and Motjaba Sadegh -''' - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from . import _algorithm -import numpy as np -import random -import time - - -class dream(_algorithm): - """ - Implements the DiffeRential Evolution Adaptive Metropolis (DREAM) algorithhm - based on: - Vrugt, J. A. (2016) Markov chain Monte Carlo simulation using the DREAM software package. - """ - - def __init__(self, *args, **kwargs): - """ - Input - ---------- - spot_setup: class - model: function - Should be callable with a parameter combination of the parameter-function - and return an list of simulation results (as long as evaluation list) - parameter: function - When called, it should return a random parameter combination. Which can - be e.g. uniform or Gaussian - objectivefunction: function - Should return the objectivefunction for a given list of a model simulation and - observation. - evaluation: function - Should return the true values as return by the model. - - dbname: str - * Name of the database where parameter, objectivefunction value and simulation results will be saved. - - dbformat: str - * ram: fast suited for short sampling time. no file will be created and results are saved in an array. - * csv: A csv file will be created, which you can import afterwards. - - parallel: str - * seq: Sequentiel sampling (default): Normal iterations on one core of your cpu. - * mpi: Message Passing Interface: Parallel computing on cluster pcs (recommended for unix os). - - save_sim: boolean - * True: Simulation results will be saved - * False: Simulation results will not be saved - """ - - kwargs['optimization_direction'] = 'maximize' - kwargs['algorithm_name'] = 'DiffeRential Evolution Adaptive Metropolis (DREAM) algorithm' - super(dream, self).__init__(*args, **kwargs) - - def check_par_validity_bound(self, par): - if len(par) == len(self.min_bound) and len(par) == len(self.max_bound): - for i in range(len(par)): - if par[i] < self.min_bound[i]: - par[i] = self.min_bound[i] - if par[i] > self.max_bound[i]: - par[i] = self.max_bound[i] - else: - print('ERROR: Bounds have not the same lenghts as Parameterarray') - return par - - def get_regular_startingpoint(self,nChains): - randompar=self.parameter()['random'] - for i in range(1000): - randompar=np.column_stack((randompar,self.parameter()['random'])) - startpoints = [] - for j in range(nChains): - startpoints.append(np.percentile(randompar,(j+1)/float(nChains+1)*100,axis=1))#,np.amax(randompar,axis=1) - startpoints = np.array(startpoints) - for k in range(len(randompar)): - random.shuffle(startpoints[:, k]) - return startpoints - - def check_par_validity_reflect(self, par): - if len(par) == len(self.min_bound) and len(par) == len(self.max_bound): - for i in range(len(par)): - if par[i] < self.min_bound[i]: - par[i] = self.min_bound[i] + (self.min_bound[i]- par[i]) - elif par[i] > self.max_bound[i]: - par[i] = self.max_bound[i] - (par[i] - self.max_bound[i]) - - # Postprocessing if reflecting jumped out of bounds - for i in range(len(par)): - if par[i] < self.min_bound[i]: - par[i] = self.min_bound[i] - if par[i] > self.max_bound[i]: - par[i] = self.max_bound[i] - else: - print('ERROR: Bounds have not the same lenghts as Parameterarray') - return par - - def _get_gamma(self,newN, nchain_pairs): - #N = Number of parameters - p = np.random.uniform(low=0,high=1) - if p >= 0.2: - # d_star is the dimension of subspace of parameters to jump - d_star = newN.count(True) - gamma = 2.38/np.sqrt(2*nchain_pairs*d_star) # /self.gammalevel - else: - gamma = 1 - return gamma - - def get_other_random_chains(self,cur_chain, nchain_pairs): - chain_pairs = [] - selectable_chain = list(range(self.nChains)) - selectable_chain.remove(cur_chain) - - for i in range(nchain_pairs): - pair_ith = random.sample(selectable_chain, 2) - chain_pairs.append(pair_ith) - for chain in pair_ith: - selectable_chain.remove(chain) - - return chain_pairs - - def get_new_proposal_vector(self,cur_chain,newN,c): - nchain_pairs = random.randint(1, self.delta) - gamma = self._get_gamma(newN,nchain_pairs) - chain_pairs = self.get_other_random_chains(cur_chain, nchain_pairs) - new_parameterset=[] - #position = self.chain_samples-1#self.nChains*self.chain_samples+self.chain_samples+cur_chain-1 - cur_par_set = list(self.bestpar[cur_chain][self.nChainruns[cur_chain]-1]) - random_par_sets1 = [] # contain all random_par_set1 - random_par_sets2 = [] # contain all random_par_set2 - - for i in range(nchain_pairs): - random_chain1 = chain_pairs[i][0] - random_chain2 = chain_pairs[i][1] - random_par_set1 = list( - self.bestpar[random_chain1][self.nChainruns[random_chain1]-1]) - random_par_set2 = list( - self.bestpar[random_chain2][self.nChainruns[random_chain2]-1]) - random_par_sets1.append(random_par_set1) - random_par_sets2.append(random_par_set2) - - random_par_set1 = [sum(i) for i in zip(*random_par_sets1)] # sum all random_par_set1 - random_par_set2 = [sum(i) for i in zip(*random_par_sets2)] # sum all random_par_set2 - - for i in range(self.N):#Go through parameters - - if newN[i] == True: - lambda_ = np.random.uniform(-c,c) - new_parameterset.append(cur_par_set[i] + (1.0+lambda_)*gamma*np.array( - random_par_set1[i]-random_par_set2[i]) + np.random.normal(0,self.eps)) - else: - new_parameterset.append(cur_par_set[i]) - - new_parameter=self.check_par_validity_reflect(new_parameterset) - #new_parameter=self.check_par_validity_bound(new_parameterset) - return new_parameter - -# new_par = np.random.normal(loc=old_par, scale=self.stepsizes) -# new_par = self.check_par_validity_reflect(new_par) -# return new_par - - def update_mcmc_status(self,par,like,sim,cur_chain): - self.bestpar[cur_chain][self.nChainruns[cur_chain]]=list(par) - self.bestlike[cur_chain]=like - self.bestsim[cur_chain]=list(sim) - - def get_r_hat(self, parameter_array): - """ - Based on some fancy mathlab code, it return an array [R_stat, MR_stat] - :param parameter_array: 3 dim array of parameter estimation sets - :type parameter_array: list - :return: [R_stat, MR_stat] - :rtype: list - """ - n, d, N = parameter_array.shape - - # Use only the last 50% of each chain (vrugt 2009), that means only the half of "d". Cause "d" ist the count - # of the repetition and we use the d/2 to d of those values which are already not NAN - whereIsNoNAN = np.logical_not(np.isnan(parameter_array)) - - alreadyToNum = np.sum(whereIsNoNAN[0, :, 0]) - - if alreadyToNum > 3: - parameter_array = parameter_array[:, int(np.floor(alreadyToNum / 2)): alreadyToNum, :] - else: - # the later functions need some data to work right, so we use in this case 100% of NON NAN values - parameter_array = parameter_array[:, 0: alreadyToNum, :] - - # I made a big confusion with d, n and N, I figured it out by tests - - if n > 3: - - mean_chains = np.zeros((n, N)) - for i in range(n): - for j in range(N): - mean_chains[i, j] = np.nanmean(parameter_array[i, :, j]) - - B_uni = np.zeros(N) - for i in range(N): - B_uni[i] = d * np.nanvar(mean_chains[:, i], - - ddof=1) # make numpy Mathalab like: https://stackoverflow.com/a/27600240/5885054 - - var_chains = np.zeros((n, N)) - for i in range(n): - for j in range(N): - var_chains[i, j] = np.nanvar(parameter_array[i, :, j], ddof=1) - - W_uni = np.zeros(N) - for i in range(N): - W_uni[i] = np.mean(var_chains[:, i]) - - sigma2 = ((d - 1) / d) * W_uni + (1 / d) * B_uni - - whichW_UNIIsNull = W_uni == 0.0 - W_uni[whichW_UNIIsNull] = np.random.uniform(0.1,1,1) - - R_stat = np.sqrt((n + 1) / n * (np.divide(sigma2, W_uni)) - (d - 1) / (n * d)) - - -# W_mult = 0 -# for ii in range(n): -# W_mult = W_mult + np.cov(np.nan_to_num(np.transpose(parameter_array[ii, :, :])), ddof=1) -# -# W_mult = W_mult / n + 2e-52 * np.eye(N) -# -# # Note that numpy.cov() considers its input data matrix to have observations in each column, -# # and variables in each row, so to get numpy.cov() to return what other packages do, -# # you have to pass the transpose of the data matrix to numpy.cov(). -# # https://stats.stackexchange.com/a/263508/168054 -# -# B_mult = np.cov(np.nan_to_num(np.transpose(mean_chains))) + 2e-52 * np.eye(N) # 2e-52 avoids problems with eig if var = 0 -# M = np.linalg.lstsq(W_mult, B_mult) -# R = np.max(np.abs(np.linalg.eigvals(M[0]))) -# MR_stat = np.sqrt((n + 1) / n * R + (d - 1) / d) - return R_stat#[R_stat, MR_stat] - - def sample(self, repetitions,nChains=7, nCr=3, delta=3, c=0.1, eps=10e-6, convergence_limit=1.2, runs_after_convergence=100,acceptance_test_option=6): - self.set_repetiton(repetitions) - print('Starting the DREAM algotrithm with '+str(repetitions)+ ' repetitions...') - if nChains <2*delta+1: - print('Please use at least n=2*delta+1 chains!') - return None - # Prepare storing MCMC chain as array of arrays. - # define stepsize of MCMC. - self.repetitions = int(repetitions) - self.nChains = int(nChains) - self.delta = delta - #Ensure initialisation of chains and database - self.burnIn = self.nChains - self.stepsizes = self.parameter()['step'] # array of stepsizes - self.nr_of_pars = len(self.stepsizes) - self.gammalevel=1 - starttime = time.time() - intervaltime = starttime - # Metropolis-Hastings iterations. - self.bestpar=np.array([[[np.nan]*self.nr_of_pars]*self.repetitions]*self.nChains) - #[0]->chain #[0][0]->parameter #[0][0][0]->repetitons - self.bestlike=[[-np.inf]]*self.nChains - self.bestsim=[[np.nan]]*self.nChains - self.accepted=np.zeros(self.nChains) - self.nChainruns=[0]*self.nChains - self.min_bound, self.max_bound = self.parameter( - )['minbound'], self.parameter()['maxbound'] - - #firstcall = True - - print('Initialize ', self.nChains, ' chain(s)...') - self.iter=0 - #for i in range(10): - startpoints = self.get_regular_startingpoint(nChains) - #param_generator = ((curChain,list(self.parameter()['random'])) for curChain in range(int(self.nChains))) #TODO: Start with regular interval raster - param_generator = ((curChain,list(startpoints[curChain])) for curChain in range(int(self.nChains))) #TODO: Start with regular interval raster - for curChain,par,sim in self.repeat(param_generator): - like = self.postprocessing(self.iter, par, sim, chains=curChain) - self.update_mcmc_status(par,like,sim,curChain) - self.iter+=1 - self.nChainruns[curChain] +=1 - - - print('Beginn of Random Walk') - convergence = False - #Walf through chains - self.r_hats=[] - self.eps = eps - self.CR = [] - for i in range(nCr): - self.CR.append((i+1)/nCr) - self.N = len(self.parameter()['random']) - nrN=1 - newN = [True]*self.N - while self.iter < self.repetitions: - param_generator = ((curChain,self.get_new_proposal_vector(curChain,newN,c)) for curChain in range(int(self.nChains))) - for cChain,par,sim in self.repeat(param_generator): - pCr = np.random.randint(0,nCr) - ids=[] - for i in range(self.N): - ids.append(np.random.uniform(low=0,high=1)) - newN = [] - nrN = 0 - for i in range(len(ids)): - if ids[i] < self.CR[pCr]: - newN.append(True) - nrN+=1 - else: - newN.append(False) - if nrN == 0: - ids=[np.random.randint(0,self.N)] - nrN=1 - like = self.postprocessing(self.iter, par, sim, chains=cChain) - - # set a option which type of comparision should be choose: - metro_opt=acceptance_test_option - - if metro_opt == 1: - logMetropHastRatio = like/self.bestlike[cChain] - - elif metro_opt == 2 or metro_opt == 4: - logMetropHastRatio = np.exp(like - self.bestlike[cChain]) - - elif metro_opt == 3: - # SSR probability evaluation - # nrN is defined in this loop so it will increase every step - logMetropHastRatio = (like / self.bestlike[cChain]) ** (-nrN * (1 + self._get_gamma(nrN)) / 2) - - elif metro_opt == 5: - # SSR probability evaluation, but now weighted with mesurement error - # Note that measurement error is single number --> homoscedastic; variance can be taken out of sum sign - # SIGMA will be calculated from the orginal data - Sigma = np.mean(np.array(self.evaluation)*0.1) - logMetropHastRatio = np.exp(-0.5 * (-like + self.bestlike[cChain])/ (Sigma ** 2)) # signs are different because we write -SSR - - elif metro_opt == 6: # SSR probability evaluation, but now weighted with mesurement error - # Note that measurement error is a vector --> heteroscedastic; variance within sum sign -- see CompDensity.m - logMetropHastRatio = np.exp(-0.5 * (-like + self.bestlike[cChain])) # signs are different because we write -SSR - - u = np.random.uniform(low=0.0, high=1) - - if logMetropHastRatio>u: - self.update_mcmc_status(par,like,sim,cChain) - self.accepted[cChain] += 1 # monitor acceptance - - else: - self.update_mcmc_status(self.bestpar[cChain][self.nChainruns[cChain]-1],self.bestlike[cChain],self.bestsim[cChain],cChain) - - if self.status.stop: - self.iter = self.repetitions - print('Stopping samplig') - break - self.iter+=1 - self.nChainruns[cChain] +=1 - - - r_hat = self.get_r_hat(self.bestpar) - self.r_hats.append(r_hat) - # Refresh progressbar every two seconds - acttime = time.time() - if acttime - intervaltime >= 2 and self.iter >=2 and self.nChainruns[-1] >=3: - text = "Acceptance rates [%] =" +str(np.around((self.accepted)/float(((self.iter-self.burnIn)/self.nChains)),decimals=4)*100).strip('array([])') - print(text) - text = "Convergence rates =" +str(np.around((r_hat),decimals=4)).strip('array([])') - print(text) - intervaltime = time.time() - - if (np.array(r_hat) < convergence_limit).all() and not convergence and self.nChainruns[-1] >=5: - #Stop sampling - print('#############') - print('Convergence has been achieved after '+str(self.iter)+' of '+str(self.repetitions)+' runs! Finally, '+str(runs_after_convergence)+' runs will be additionally sampled to form the posterior distribution') - print('#############') - self.repetitions = self.iter + runs_after_convergence - self.set_repetiton(self.repetitions) - #self.iter =self.repetitions - runs_after_convergence - convergence=True - - self.final_call() - - - #try: - # self.datawriter.finalize() - #except AttributeError: # Happens if no database was assigned - # pass - #print('End of sampling') - #text = '%i of %i (best like=%g)' % ( - # self.status.rep, repetitions, self.status.objectivefunction) - #print(text) - #print('Best parameter set') - #print(self.status.params) - #text = 'Duration:' + str(round((acttime - starttime), 2)) + ' s' - #print(text) - return self.r_hats diff --git a/spotpy/algorithms/nsgaii.py b/spotpy/algorithms/nsgaii.py deleted file mode 100644 index 1d6391fb..00000000 --- a/spotpy/algorithms/nsgaii.py +++ /dev/null @@ -1,405 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright (c) 2018 by Tobias Houska -This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). -:author: Iacopo Ferrario - -This file contains the NSGA-II Algorithm implemented for SPOTPY based on: -- K. Deb, A. Pratap, S. Agarwal and T. Meyarivan, "A fast and elitist multiobjective genetic algorithm: -NSGA-II," in IEEE Transactions on Evolutionary Computation, vol. 6, no. 2, pp. 182-197, Apr 2002. -doi: 10.1109/4235.996017 -URL: http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=996017&isnumber=21497 -- http://www.cs.colostate.edu/~genitor/MiscPubs/tutorial.pdf (Mutation and Crossover Algorithm) -- http://www.tik.ee.ethz.ch/file/6c0e384dceb283cd4301339a895b72b8/TIK-Report11.pdf (Tournament Selection) -''' - -import numpy as np -import math -from spotpy.algorithms import _algorithm -import copy - - - -class TournamentSelection: - - def __init__(self,pressure = 2): - self.pressure = pressure - - def calc(self,pop_rank): - - n_select = len(pop_rank) - n_random = n_select * self.pressure #n_select * n_parents * pressure - - n_perms = math.ceil(n_random / len(pop_rank)) - - P = random_permuations(n_perms, len(pop_rank))[:n_random] - - P = np.reshape(P, (n_select, self.pressure)) - - n_tournament,_ = P.shape - - ret = np.full(n_tournament,-1,dtype=np.int) - - for i in range(n_tournament): - a,b = P[i] - - if pop_rank[a] < pop_rank[b]: - ret[i] = a - else: - ret[i] = b - - return ret - - -def random_permuations(n, l): - perms = [] - for _ in range(n): - perms.append(np.random.permutation(l)) - return np.concatenate(perms) - - -class Crossover: - - def __init__(self,crossProb=0.9): - - self.crossProbThreshold = crossProb - - def calc(self,pop,n_var): - - n_pop = pop.shape[0] - crossProbability = np.random.random((n_pop)) - do_cross = crossProbability < self.crossProbThreshold - R = np.random.randint(0,n_pop,(n_pop,2)) - parents = R[do_cross] - crossPoint = np.random.randint(1,n_var,parents.shape[0]) - d = pop[parents,:] - child = [] - for i in range(parents.shape[0]): - child.append(np.concatenate([d[i,0,:crossPoint[i]],d[i,1,crossPoint[i]:]])) - child = np.vstack(child) - pop[do_cross,:] = child - return pop - - - - -class PolynomialMutation: - - def __init__(self,prob_mut,eta_mut): - - self.prob_mut = prob_mut - self.eta_mut = eta_mut - - def calc(self,x,xl,xu): - - X = copy.deepcopy(x) - Y = np.full(X.shape,np.inf) - - do_mutation = np.random.random(X.shape) < self.prob_mut - - m = np.sum(np.sum(do_mutation)) - - Y[:,:] = X - - xl = np.repeat(xl[None,:],X.shape[0],axis=0)[do_mutation] #selecting who is mutating - xu = np.repeat(xu[None,:],X.shape[0],axis=0)[do_mutation] - - X = X[do_mutation] - - delta1 = (X - xl) / (xu - xl) - delta2 = (xu - X) / (xu -xl) - - mut_pow = 1.0/(self.eta_mut + 1.0) - - rand = np.random.random(X.shape) - mask = rand <= 0.5 - mask_not = np.logical_not(mask) - - deltaq = np.zeros(X.shape) - - xy = 1.0 - delta1 - val = 2.0 * rand + (1.0 - 2.0 * rand) * (np.power(xy, (self.eta_mut + 1.0))) - d = np.power(val, mut_pow) - 1.0 - deltaq[mask] = d[mask] - - xy = 1.0 - delta2 - val = 2.0 * (1.0 - rand) + 2.0 * (rand - 0.5) * (np.power(xy, (self.eta_mut + 1.0))) - d = 1.0 - (np.power(val, mut_pow)) - deltaq[mask_not] = d[mask_not] - - ret = X + deltaq * (xu - xl) - ret[ret < xl] = xl[ret < xl] - ret[ret > xu] = xu[ret > xu] - - Y[do_mutation] = ret - - return Y - - - - - - -class NSGAII(_algorithm): - """ - Implements the "Fast and Elitist Multiobjective Genetic Algorithm: NSGA-II - by Kalyanmoy Deb, Associate Member, IEEE, Amrit Pratap, Sameer Agarwal, and T. Meyarivan - - """ - - def __init__(self, *args, **kwargs): - """ - Input - ---------- - spot_setup: class - model: function - Should be callable with a parameter combination of the parameter-function - and return an list of simulation results (as long as evaluation list) - parameter: function - When called, it should return a random parameter combination. Which can - be e.g. uniform or Gaussian - objectivefunction: function - Should return the objectivefunction for a given list of a model simulation and - observation. - evaluation: function - Should return the true values as return by the model. - - dbname: str - * Name of the database where parameter, objectivefunction value and simulation results will be saved. - - dbformat: str - * ram: fast suited for short sampling time. no file will be created and results are saved in an array. - * csv: A csv file will be created, which you can import afterwards. - - parallel: str - * seq: Sequentiel sampling (default): Normal iterations on one core of your cpu. - * mpi: Message Passing Interface: Parallel computing on cluster pcs (recommended for unix os). - - save_sim: boolean - * True: Simulation results will be saved - * False: Simulation results will not be saved - """ - self._return_all_likes=True #alloes multi-objective calibration - kwargs['optimization_direction'] = 'minimize' - kwargs['algorithm_name'] = 'Fast and Elitist Multiobjective Genetic Algorithm: NSGA-II' - super(NSGAII, self).__init__(*args, **kwargs) - - - def fastSort(self,x): - n = x.shape[0] - S = np.zeros((n,n),dtype=bool) - Np = np.zeros(n) - - - for i in range(n): - for j in range(n): - S[i,j] = self.dominates(x[i,:],x[j,:]) - - nDom = np.sum(S,axis=0) # the n solutions that dominates i - Np[nDom == 0] = 1 # if i == 0, i is non-dominated, set i rank to 1, i belongs to first non-dominated front - k = 1 - # loop over pareto fronts - while np.sum(Np == 0) > 0: - l = np.arange(n)[Np==k] # first non-dominated front - for i in l: # loop over the non-dominated front - nDom[S[i,:]] = nDom[S[i,:]] -1 # reduce by 1 the rank of the solutions that i dominates - k += 1 - # now nDom has been reduced by 1, so the next non-dominated front will be nDom == 0 - # and Np == 0 ensure that we don't pass over the first ranked non-dom solutions - Np[(nDom == 0) & (Np == 0) ] = k - - return Np.astype(int) - - - - def dominates(self,a,b): - if len(a.shape) >1: - ret = (np.sum(a <= b,axis =1) == a.shape[1]) & (np.sum(a < b,axis=1) >0) - else: - ret = (np.sum(a <= b) == len(a)) & (np.sum(a < b) >0) - return ret - - - def crowdDist(self,x): - n = x.shape[0] - - nobj = x.shape[1] - - dist = np.zeros(n) - - ord = np.argsort(x,axis=0) - - X = x[ord,range(nobj)] - - dist = np.vstack([X,np.full(nobj,np.inf)]) - np.vstack([np.full(nobj,-np.inf),X]) - - norm = np.max(X,axis=0) - np.min(X,axis=0) - dist_to_last,dist_to_next = dist, np.copy(dist) - dist_to_last,dist_to_next = dist_to_last[:-1]/norm ,dist_to_next[1:]/norm - J = np.argsort(ord,axis=0) - ret = np.sum(dist_to_last[J, np.arange(nobj)] + dist_to_next[J, np.arange(nobj)], axis=1) / nobj - - return ret - - def crowdDist2(self,x): - n = x.shape[0] - - dist = np.zeros(n) - - for obj in range(x.shape[1]): - ord = np.argsort(x[:,obj]) - dist[ord[[0,-1]]] = np.inf - - norm = np.max(x[:,obj]) - np.min(x[:,obj]) - - for i in range(1,n-1): - dist[i] = dist[ord[i]] + (x[ord[i+1],obj] - x[ord[i-1],obj])/norm - - return dist - - - - - def sample(self, generations, n_obj, n_pop = None, - selection = TournamentSelection(pressure=2), - crossover = Crossover(crossProb=0.9), - mutation = PolynomialMutation(prob_mut=0.25,eta_mut=30)): - - self.n_obj = n_obj - self.selection = selection - self.crossover = crossover - self.mutation = mutation - - self.n_pop = n_pop - self.generations= generations - self.set_repetiton(self.generations*self.n_pop) - self.skip_duplicates = True # False does not work yet - - Pt = np.vstack([self.parameter()['random'] for i in range(self.n_pop)]) - - #Burn-in - #TODO: I would suggest to make the burin-in sample indiviudual for each cpu-core in case of parallel usage, compare dream.py, but not sure if this is defined in the publication - # evaluate population - param_generator = ((i,Pt[i,:]) for i in range(self.n_pop)) - - ret = list(self.repeat(param_generator)) - - Of = [] - for p in range(self.n_pop): - index,parameters,simulation_results = ret[p] - Of.append(self.postprocessing(0, parameters, simulation_results, chains=p)) - Of = np.vstack(Of) - - - nonDomRank = self.fastSort(Of) - - crDist = np.empty(self.n_pop) - for rk in range(1,np.max(nonDomRank)+1): - crDist[nonDomRank == rk] = self.crowdDist(Of[nonDomRank ==rk,:]) - - - # sorting - - rank = np.lexsort((-crDist,nonDomRank)) - Ptsort = Pt[rank] - Ofsort = Of[rank] - - Of_parent = Ofsort[:,:] - Pt_parent = Ptsort[:,:] - - # selection - - offsprings = self.selection.calc(pop_rank = rank) - - Qt = Ptsort[offsprings,:] - - - # crossover - try: - n_var = self.setup.n_var - except AttributeError: - n_var = len(parameters) - - - Qt = self.crossover.calc(pop = Qt,n_var = n_var) - - # mutation - self.min_bound, self.max_bound = self.parameter()['minbound'], self.parameter()['maxbound'] - self.varminbound = np.array([]) - self.varmaxbound = np.array([]) - for i in range(len(self.min_bound)): - self.varminbound = np.append(self.varminbound,self.min_bound[i]) - self.varmaxbound = np.append(self.varmaxbound,self.max_bound[i]) - - Qt = self.mutation.calc(x = Qt,xl = self.varminbound,xu = self.varmaxbound) - - for igen in range(1,self.generations - 1): - - Rt = np.vstack([Pt_parent,Qt]) - - - if self.skip_duplicates: - - # evaluate population - param_generator = ((i,Qt[i,:]) for i in range(self.n_pop)) - - ret = list(self.repeat(param_generator)) - - Of = [] - for p in range(self.n_pop): - index, parameters,simulation_results = ret[p] - Of.append(self.postprocessing(igen, parameters, simulation_results, chains=p)) - Of = np.vstack(Of) - - Of = np.vstack([Of_parent ,Of]) - - nonDomRank = self.fastSort(Of) - - crDist = np.empty(len(Of)) - for rk in range(1,np.max(nonDomRank)+1): - crDist[nonDomRank == rk] = self.crowdDist(Of[nonDomRank ==rk,:]) - else: - - n_pop_combined = self.n_pop *2 - # evaluate population - param_generator = ((i,Rt[i,:]) for i in range( n_pop_combined )) - - - #import pdb;pdb.set_trace() - ret = list(self.repeat(param_generator)) - - Of = [] - for p in range(n_pop_combined): - index, parameters,simulation_results = ret[p] - Of.append(self.postprocessing(igen, parameters, simulation_results, chains=p)) - Of = np.vstack(Of) - - nonDomRank = self.fastSort(Of) - - crDist = np.empty(n_pop_combined) - for rk in range(1,np.max(nonDomRank)+1): - crDist[nonDomRank == rk] = self.crowdDist(Of[nonDomRank ==rk,:]) - - - # sorting - rank = np.lexsort((-crDist,nonDomRank))[:self.n_pop] - Ptsort = Rt[rank] - Ofsort = Of[rank] - - Pt_parent = Ptsort[:,:] - - Of_parent = Ofsort[:,:] - - - # selection - offsprings = self.selection.calc(pop_rank = rank) - - Qt = Ptsort[offsprings,:] - # crossover - Qt = self.crossover.calc(pop = Qt,n_var = n_var) - # mutation - Qt = self.mutation.calc(x = Qt,xl = self.varminbound,xu =self.varmaxbound) - - - - self.final_call() diff --git a/spotpy/cli.py b/spotpy/cli.py deleted file mode 100644 index 04ad2807..00000000 --- a/spotpy/cli.py +++ /dev/null @@ -1,97 +0,0 @@ -from __future__ import division, print_function, unicode_literals - -from . import algorithms, database, describe -import click -import inspect -import io -import os - - -def get_config_from_file(): - """ - Gets the spotpy configuration from a config file 'spotpy.conf'. - - Example: - - sampler = mc - dbtype = csv - parallel = seq - # This is a comment - runs = 10 - """ - config = {} - if os.path.exists('spotpy.conf'): - with io.open('spotpy.conf') as f: - for line in f: - if not line.strip().startswith('#'): - try: - k, v = line.split('=', 1) - config[k.strip()] = v.strip() - except ValueError: - pass - return config - - -def get_sampler_from_string(sampler_name): - return getattr(algorithms, sampler_name) - - -def make_type_from_module(module, *exclude): - - def use(cl): - # Check if class name starts with an exclusion term - return inspect.isclass(cl) and not any([cl.__name__.startswith(ex) for ex in ('_', ) + exclude]) - members = inspect.getmembers(module, use) - return click.Choice([n for n, m in members if not n.startswith('_')]) - - -@click.group(context_settings=dict(help_option_names=['-h', '--help'])) -def cli(): - pass - - -@cli.command() -@click.pass_context -@click.option('--sampler', '-s', type=make_type_from_module(algorithms), default='mc', - help='Select the spotpy sampler') -@click.option('--dbformat', type=click.Choice(database.__dir__()), default='ram', - help='The type of the database') -@click.option('--dbname', type=click.STRING, help='The name of the database, leave open for ram') -@click.option('--parallel', '-p', type=click.Choice(['seq', 'mpc', 'mpi']), default='seq', - help='Parallelization: seq = no parallelization, mpi = MPI (for clusters), mpc = multiprocessing') -@click.option('--runs', '-n', type=click.INT, default=1, help='Number of runs') -@click.option('--config', '-c', is_flag=True, - help='Print only the configuration, can be used to create a config file with your_model.py > spotpy.conf') -def run(ctx, **kwargs): - """ - Runs a sampler for automatic calibration - """ - setup = ctx.obj - if kwargs.pop('config', None): - click.echo('\n'.join('{} = {}'.format(k, v) for k, v in kwargs.items())) - else: - sampler_name = kwargs.pop('sampler') - sampler_class = get_sampler_from_string(sampler_name) - runs = kwargs.pop('runs') - sampler = sampler_class(setup, **kwargs) - sampler.sample(runs) - - -@cli.command() -@click.pass_context -def gui(ctx): - """ - Shows a GUI for manual calibration - """ - from spotpy.gui.mpl import GUI - setup = ctx.obj - gui = GUI(setup) - gui.show() - - -def main(setup): - # Prevent help text from wrapping - cli.help = '\b\n' + describe.setup(setup).replace('\n\n', '\n\b\n') - config = get_config_from_file() - cli(obj=setup, auto_envvar_prefix='SPOTPY', default_map=config) - diff --git a/spotpy/database/sql.py b/spotpy/database/sql.py deleted file mode 100644 index f103f3be..00000000 --- a/spotpy/database/sql.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -import numpy as np -import sqlite3 -import sys -from .base import database - -if sys.version_info[0] >= 3: - unicode = str - - -class PickalableSWIG: - def __setstate__(self, state): - self.__init__(*state['args']) - def __getstate__(self): - return {'args': self.args} - - -class PickalableSQL3Connect(sqlite3.Connection, PickalableSWIG): - def __init__(self, *args,**kwargs): - self.args = args - sqlite3.Connection.__init__(self,*args,**kwargs) - - -class PickalableSQL3Cursor(sqlite3.Cursor, PickalableSWIG): - def __init__(self, *args,**kwargs): - self.args = args - sqlite3.Cursor.__init__(self,*args,**kwargs) - - - -class sql(database): - """ - This class saves the process in the working storage. It can be used if - safety matters. - """ - - def __init__(self, *args, **kwargs): - import os - # init base class - super(sql, self).__init__(*args, **kwargs) - # Create a open file, which needs to be closed after the sampling - try: - os.remove(self.dbname + '.db') - except: - pass - - self.db = PickalableSQL3Connect(self.dbname + '.db') - self.db_cursor = PickalableSQL3Cursor(self.db) - # Create Table - # self.db_cursor.execute('''CREATE TABLE IF NOT EXISTS '''+self.dbname+''' - # (like1 real, parx real, pary real, simulation1 real, chain int)''') - self.db_cursor.execute('''CREATE TABLE IF NOT EXISTS ''' + self.dbname + ''' - (''' + ' real ,'.join(self.header) + ''')''') - - def save(self, objectivefunction, parameterlist, simulations=None, chains=1): - coll = (self.dim_dict['like'](objectivefunction) + - self.dim_dict['par'](parameterlist) + - self.dim_dict['simulation'](simulations) + - [chains]) - # Apply rounding of floats - coll = map(self.db_precision, coll) - self.db_cursor.execute( - "INSERT INTO " + self.dbname + " VALUES (" + '"' + str('","'.join(map(str, coll))) + '"' + ")") - - self.db.commit() - - def finalize(self): - self.db.close() - - def getdata(self): - self.db = PickalableSQL3Connect(self.dbname + '.db') - self.db_cursor = PickalableSQL3Cursor(self.db) - - if sys.version_info[0] >= 3: - headers = [(row[1], ">> spotpy.describe.sampler(sampler) ->>> spotpy.describe.setup(model) -""" -from __future__ import division, absolute_import, unicode_literals -import sys -from .parameter import get_parameters_from_setup -from .algorithms._algorithm import _algorithm -if sys.version_info[0] >= 3: - from inspect import getdoc as _getdoc - unicode = str -else: - def _getdoc(obj): - u = obj.__doc__ - try: - return u'\n'.join(l.strip() for l in u.split(u'\n') if l.strip()) - except UnicodeDecodeError: - raise AssertionError( - '{}: Docstring uses unicode but {} misses the line ``from __future__ import unicode_literals``' - .format(obj, type(obj).__module__) - ) - -try: - from docutils.core import publish_string -except ImportError: - publish_string = None - - -def describe(obj): - """ - Returns a long string description of a sampler with its model - :param obj: A sampler - :return: str - """ - return 'Sampler:\n--------\n{}\n\nModel:\n------\n{}'.format(sampler(obj), setup(obj.setup)) - - -def sampler(obj): - """ - Returns a string representation of the sampler. - By design, it is rather verbose and returns a - large multiline description - :return: - """ - cname = unicode(type(obj).__name__) - s = [cname, '=' * len(cname), _getdoc(obj), - ' db format: ' + obj.dbformat, - ' db name: ' + obj.dbname, - ' save simulation: ' + str(obj.save_sim), - ' parallel: ' + type(obj.repeat).__module__.split('.')[-1]] - return '\n'.join(s) - - -def setup(obj): - """ - Describes a spotpy setup using its class name, docstring and parameters - :param obj: A spotpy compatible model setup - :return: A describing string - """ - # Get class name - cname = unicode(type(obj).__name__) - # Add doc string - mdoc = _getdoc(obj).strip('\n').replace('\r', '\n') - # Get parameters from class - params = '\n'.join(' - {p}'.format(p=unicode(p)) for p in get_parameters_from_setup(obj)) - parts = [cname, '=' * len(cname), mdoc, 'Parameters:', '-' * 11, params] - return '\n'.join(parts) - - -if sys.version_info > (3, 5): - - from pathlib import Path - import webbrowser - - - - class rst: - """ - Creates a reStructuredText description of a sampler or a setup - - Usage: - >>>description = spotpy.describe.rst(sampler) - >>>print(description) # Prints the rst source text - >>># Add additional text section - >>>description.append('#. One idea' + '\n' + '#. Another one.', title='Ideas', titlelevel=2) - >>>description.append_image('media/image.png') - >>>print(description.as_html()) # Create html - >>>description.show_in_browser() - """ - - caption_characters = '=-#~*+^' - - def __init__(self, setup_or_sampler): - """ - Creates a reStructuredText description of a sampler or a setup - :param setup_or_sampler: Either a spotpy.algorithm sampler or a spotpy setup - """ - if isinstance(setup_or_sampler, _algorithm): - self.setup = setup_or_sampler.setup - self.sampler = setup_or_sampler - self.rst_text = [self._sampler_text()] - else: - self.setup = setup_or_sampler - self.sampler = None - self.rst_text = [] - - if self.setup: - self.rst_text.append(self._setup_text()) - - def append(self, text='', title=None, titlelevel=1): - """ - Appends additional descriptions in reStructured text to the generated text - :param text: The rst text to add - :param title: A title for the text - :param titlelevel: The level of the section (0->h1.title, 1->h1, 2->h2, etc.) - :return: - """ - res = '\n' - if title: - res += rst._as_rst_caption(title, titlelevel) - self.rst_text.append(res + text) - - def append_image(self, imgpath, **kwargs): - """ - Links an image to the output - :param imgpath: Path to the image (must be found from the http server) - :param kwargs: Any keyword with value is translated in rst as `:keyword: value` - and added to the image description - - >>>description.append_image('https://img.shields.io/travis/thouska/spotpy/master.svg', - ... target='https://github.com/thouska', - ... width='200px') - """ - rst = '.. image:: {}'.format(imgpath) - for k, v in kwargs.items(): - rst += '\n :{}: {}'.format(k, v) - rst += '\n' - self.append(rst) - - def append_math(self, latex): - """ - Appends a block equation to the output - :param latex: Latex formula - """ - rst = '.. math::\n' - rst += ' ' + latex + '\n' - self.append(rst) - - def __str__(self): - return '\n'.join(self.rst_text) - - @classmethod - def _as_rst_caption(cls, s, level=1): - """ - Marks text as a section caption - :param s: String to be marked as caption - :param level: Caption level 0-6, translates to 0=h1.title, 1=h1, 2=h2, etc. - :return: The string as rst caption - """ - return s + '\n' + cls.caption_characters[level] * len(s) + '\n\n' - - css = """ - body, table, div, p, dl { - font-family: Lucida Grande, Verdana, Geneva, Arial, sans-serif; - font-size: 16px; - } - li>p { - margin: 0px; - } - /* @group Heading Levels */ - h1.title { - background-color: #fff; - color: #0040A0; - text-align: left; - font-size: 200%; - border: solid 2px #1f6992; - } - h1 { - background-color: #1f6992; - color: #fff; - padding: .2em .5em; - font-size: 150%; - } - h2 { - background-color: #cde; - color: #000; - padding: .2em .5em; - border-bottom: solid 2px #1f6992; - font-size: 120%; - } - - h3 { - font-size: 100%; - border-bottom: solid 2px #0040A0; - } - div.line { - font-family: "Lucida Console", "Lucida Sans Typewriter","DejaVu Sans Mono",monospace; - font-size: 100%; - } - - img { - max-width: 720px; - } - - """ - - def as_html(self, css=None): - """ - Converts the generated reStructuredText as html5 - - :css: A string containing a cascading style sheet. If None, the default css is used - :return: The html document as string - """ - if publish_string is None: - raise NotImplementedError('The docutils package needs to be installed') - args = {'input_encoding': 'unicode', - 'output_encoding': 'unicode'} - res = publish_string(source=str(self), - writer_name='html5', - settings_overrides=args) - style_idx = res.index('') - css = css or self.css - # Include css - res = res[:style_idx] + css + res[style_idx:] - return res - - def show_in_browser(self, filename=None, css=None): - """ - Writes the content as html to disk and opens a browser showing the result - - :param filename: The html filename, if None use .html - :param css: A style string, if None the default style is used - """ - html = self.as_html(css).replace('unicode', 'utf-8') - fn = filename or type(self.setup).__name__ + '.html' - path = Path(fn).absolute() - path.write_text(html, encoding='utf-8') - webbrowser.open_new_tab(path.as_uri()) - - def _sampler_text(self): - """ - Generates the rst for the sampler - :return: - """ - obj = self.sampler - cname = rst._as_rst_caption(type(obj).__name__, 0) - s = [ - '- **db format:** ' + obj.dbformat, - '- **db name:** ' + obj.dbname, - '- **save simulation:** ' + str(obj.save_sim), - '- **parallel:** ' + type(obj.repeat).__module__.split('.')[-1], - '', '' - ] - return cname + _getdoc(obj).strip('\n') + '\n\n' + '\n'.join(s) - - def _setup_text(self): - """ - Generates the rst for the setup - :return: - """ - # Get class name - obj = self.setup - cname = rst._as_rst_caption(type(obj).__name__, 0) - # Add doc string - mdoc = _getdoc(obj).strip('\n').replace('\r', '\n') + '\n\n' - # Get parameters from class - param_caption = rst._as_rst_caption('Parameters', 1) - params = '\n'.join('#. **{p.name}:** {p}'.format(p=p) for p in get_parameters_from_setup(obj)) - return cname + mdoc + param_caption + params - diff --git a/spotpy/examples/3dplot.py b/spotpy/examples/3dplot.py deleted file mode 100644 index febe635e..00000000 --- a/spotpy/examples/3dplot.py +++ /dev/null @@ -1,59 +0,0 @@ -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -This file shows how to make 3d surface plots. -''' -import spotpy - -from mpl_toolkits.mplot3d import Axes3D -from mpl_toolkits.mplot3d.art3d import Poly3DCollection, Line3DCollection -from matplotlib import cm -from matplotlib.ticker import LinearLocator, FormatStrFormatter -import matplotlib.pyplot as plt -from numpy import * - - -fig = plt.figure(figsize=(10,10)) -ax = fig.gca(projection='3d') -# -# Plot Rosenbrock surface -X = arange(-30, 30, 0.05) -Y = arange(-30, 30, 0.05) -X, Y = meshgrid(X, Y) - -#from spot_setup_rosenbrock import spot_setup -#from spot_setup_griewank import spot_setup -from spotpy.examples.spot_setup_ackley import spot_setup - -Z = np.zeros(X.shape) -for i in range(X.shape[0]): - for j in range(X.shape[1]): - sim=spot_setup().simulation([X[i,j],Y[i,j]]) - like=spotpy.objectivefunctions.rmse(sim,[0]) - Z[i,j] = like - - -surf_Rosen = ax.plot_surface(X, Y, Z,rstride=5,linewidth=0, cmap=cm.rainbow) -ax.set_xlabel('x') -ax.set_ylabel('y') -ax.set_zlabel('RMSE') -plt.tight_layout() -plt.savefig('Griewank3d.tif',dpi=300) - - -#surf_Rosen = ax.plot_surface(X_Rosen, Y_Rosen, Z_Rosen, rstride=1, cstride=1, -# cmap=cm.coolwarm, linewidth=0, antialiased=False, alpha = 0.3) - -# Adjust axes -#ax.set_zlim(0, 600) -#ax.zaxis.set_major_locator(LinearLocator(5)) -#ax.zaxis.set_major_formatter(FormatStrFormatter('%.0f')) - -# Report minimum -#print 'Minimum location', v0_ori, '\nMinimum value', Rosenbrock(v0_ori), '\nNumber of function evaluations', f_evals - -# Render plot -plt.show() \ No newline at end of file diff --git a/spotpy/examples/dds/dds_parallel_plot.py b/spotpy/examples/dds/dds_parallel_plot.py deleted file mode 100644 index cd6a85e8..00000000 --- a/spotpy/examples/dds/dds_parallel_plot.py +++ /dev/null @@ -1,51 +0,0 @@ -import numpy as np - -import matplotlib.pylab as plt -import json -import matplotlib as mp - -data_normalizer = mp.colors.Normalize() -color_map = mp.colors.LinearSegmentedColormap( - "my_map", - { - "red": [(0, 1.0, 1.0), - (1.0, .5, .5)], - "green": [(0, 0.5, 0.5), - (1.0, 0, 0)], - "blue": [(0, 0.50, 0.5), - (1.0, 0, 0)] - } -) - - -def autolabel(ax, rects): - """ - Attach a text label above each bar displaying its height - """ - for rect in rects: - height = rect.get_height() - ax.text(rect.get_x() + rect.get_width() / 2., 1.05 * height, - '%f' % height, - ha='center', va='bottom') - - -def subplot(data, name, ylabel): - fig = plt.figure(figsize=(20, 6)) - ax = plt.subplot(111) - rep_labels = [str(j) for j in reps] - x_pos = [i for i, _ in enumerate(rep_labels)] - X = np.arange(len(data)) - ax_plot = ax.bar(x_pos, data, color=color_map(data_normalizer(data)), width=0.45) - - plt.xticks(x_pos, rep_labels) - plt.xlabel("Repetitions") - plt.ylabel(ylabel) - - autolabel(ax, ax_plot) - plt.savefig(name + ".png") - - -parallel_data = json.loads('{"dds_duration": [1.1293659210205078, 3.254117250442505, 4.888171672821045, 18.719818592071533, 34.56907820701599, 169.47716689109802, 337.86882615089417, 1644.955144405365, 3348.948029756546], "rep": [30, 500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000], "dds_like": [-8384.884435178812, -8269.480874403698, -8268.453892284442, -8268.51195094138, -8269.65509041187, -8268.1421690868, -8267.791798085422, -8267.79178644684, -8268.141980514703]}') -reps = parallel_data["rep"] -subplot(parallel_data["dds_duration"], "DDS_PARALLEL_DURATION_all", "Duration of Run in Seconds") -subplot(parallel_data["dds_like"], "DDS_PARALLEL_OBJECTIVEFUNCTION_all", "Best Objective Function Value") diff --git a/spotpy/examples/getting_started.py b/spotpy/examples/getting_started.py deleted file mode 100644 index 89971d4c..00000000 --- a/spotpy/examples/getting_started.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -This class holds the example code from the getting_started web-documention. -''' -from __future__ import print_function, division, absolute_import, unicode_literals -# Getting started - -#To start your experience with SPOT you need to have SPOT installed. Please see the [Installation chapter](index.md) for further details. -#To use SPOT we have to import it and use one of the pre-build examples: -import spotpy # Load the SPOT package into your working storage -from spotpy.examples.spot_setup_rosenbrock import spot_setup # Import the two dimensional Rosenbrock example - - -#The example comes along with parameter boundaries, the Rosenbrock function, the optimal value of the function and RMSE as a likelihood. -#So we can directly start to analyse the Rosenbrock function with one of the algorithms. We start with a simple Monte Carlo sampling: -if __name__ == '__main__': - # Give Monte Carlo algorithm the example setup and saves results in a RosenMC.csv file - #spot_setup.slow = True - sampler = spotpy.algorithms.mc(spot_setup(), dbname='RosenMC', dbformat='ram') - - #Now we can sample with the implemented Monte Carlo algortihm: - sampler.sample(10000) # Sample 100.000 parameter combinations - results=sampler.getdata() - #Now we want to have a look at the results. First we want to know, what the algorithm has done during the 10.000 iterations: - #spot.analyser.plot_parametertrace(results) # Use the analyser to show the parameter trace - spotpy.analyser.plot_parameterInteraction(results) - posterior=spotpy.analyser.get_posterior(results) - spotpy.analyser.plot_parameterInteraction(posterior) - #spotpy.analyser.plot_posterior_parametertrace(results, threshold=0.9) - - print(spotpy.analyser.get_best_parameterset(results)) \ No newline at end of file diff --git a/spotpy/examples/gui_hymod.py b/spotpy/examples/gui_hymod.py deleted file mode 100644 index d0f7f9b3..00000000 --- a/spotpy/examples/gui_hymod.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -Shows the usage of the matplotlib GUI - -Needs at least Python 3.5 -""" - -from __future__ import division, print_function, unicode_literals - - -import spotpy -from spotpy.gui.mpl import GUI -from spotpy.examples.spot_setup_hymod_python import spot_setup -from spotpy.objectivefunctions import rmse - -if __name__ == '__main__': - setup_class=spot_setup(rmse) - - #Select number of maximum allowed repetitions - rep=10000 - - # Create the SCE-UA sampler of spotpy, alt_objfun is set to None to force SPOTPY - # to jump into the def objectivefunction in the spot_setup class (default is - # spotpy.objectivefunctions.rmse) - sampler=spotpy.algorithms.sceua(setup_class, dbname='SCEUA_hymod', dbformat='csv', alt_objfun=None) - - #Start the sampler, one can specify ngs, kstop, peps and pcento id desired - #sampler.sample(rep,ngs=10, kstop=50, peps=0.1, pcento=0.1) - - gui = GUI(spot_setup()) - gui.show() diff --git a/spotpy/examples/hymod_unix/__init__.py b/spotpy/examples/hymod_unix/__init__.py deleted file mode 100644 index c91096bc..00000000 --- a/spotpy/examples/hymod_unix/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright (c) 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -:paper: Houska, T., Kraft, P., Chamorro-Chavez, A. and Breuer, L.: -SPOTting Model Parameters Using a Ready-Made Python Package, -PLoS ONE, 10(12), e0145180, doi:10.1371/journal.pone.0145180, 2015. - -This package enables the comprehensive use of different Bayesian and Heuristic calibration -techniques in one Framework. It comes along with an algorithms folder for the -sampling and an analyser class for the plotting of results by the sampling. - -:dependencies: - Numpy >1.8 (http://www.numpy.org/) - - Pandas >0.13 (optional) (http://pandas.pydata.org/) - - Matplotlib >1.4 (optional) (http://matplotlib.org/) - - CMF (optional) (http://fb09-pasig.umwelt.uni-giessen.de:8000/) - - mpi4py (optional) (http://mpi4py.scipy.org/) - - pathos (optional) (https://pypi.python.org/pypi/pathos/) - - sqlite3 (optional) (https://pypi.python.org/pypi/sqlite3/) - - :help: For specific questions, try to use the documentation website at: - http://fb09-pasig.umwelt.uni-giessen.de/spotpy/ - -For general things about parameter optimization techniques have a look at: -https://github.com/CamDavidsonPilon/Probabilistic-Programming-and-Bayesian-Methods-for-Hackers/tree/master/ - -Please cite our paper, if you are using SPOTPY. -''' diff --git a/spotpy/examples/spot_setup_ackley.py b/spotpy/examples/spot_setup_ackley.py deleted file mode 100644 index b1b3212c..00000000 --- a/spotpy/examples/spot_setup_ackley.py +++ /dev/null @@ -1,41 +0,0 @@ -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -This example implements the Ackley function into SPOT. -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -import numpy as np -import spotpy - -class spot_setup(object): - def __init__(self,dim=30): - self.dim=dim - self.params = [] - for i in range(self.dim): - self.params.append(spotpy.parameter.Uniform(str(i),-32.768,32.768,2.5,-20.0)) - - def parameters(self): - return spotpy.parameter.generate(self.params) - - def simulation(self, vector): - firstSum = 0.0 - secondSum = 0.0 - for c in range(len(vector)): - firstSum += c**2.0 - secondSum += np.cos(2.0*np.pi*vector[c]) - n = float(len(vector)) - return [-20.0*np.exp(-0.2*np.sqrt(firstSum/n)) - np.exp(secondSum/n) + 20 + np.e] - - def evaluation(self): - observations=[0] - return observations - - def objectivefunction(self, simulation,evaluation): - objectivefunction= -spotpy.objectivefunctions.rmse(evaluation = evaluation, simulation = simulation) - return objectivefunction \ No newline at end of file diff --git a/spotpy/examples/spot_setup_dds.py b/spotpy/examples/spot_setup_dds.py deleted file mode 100644 index e39e85dd..00000000 --- a/spotpy/examples/spot_setup_dds.py +++ /dev/null @@ -1,98 +0,0 @@ -import spotpy -from spotpy.parameter import Uniform -from spotpy.objectivefunctions import rmse -import numpy as np - - -def ackley10(vector): - length = len(vector) - sum1 = 0 - sum2 = 0 - for i in range(length): - sum1 = sum1 + vector[i] ** 2 - sum2 = sum2 + np.cos(2 * np.pi * vector[i]) - return -1*(-20 * np.exp(-0.2 * (sum1 / length) ** 0.5) - np.exp(sum2 / length)) - - -def griewank10(vector): - sum1 = 0 - term2 = 1 - term3 = 1 - - for i in range(len(vector)): - sum1 = sum1 + (vector[i] ** 2) / 4000 - term2 = term2 * np.cos(vector[i] / (i + 1) ** 0.5) - - return -1*(sum1 - term2 + term3) - - -class spot_setup(object): - """ - Setup for a simple example to run DDS Algorithm - """ - - def __init__(self): - self.params = None - self.objfunc = None - - def _objfunc_switcher(self, name): - """ - Set new parameter and objective function while setup is instanced in a test case - :param name: function name which overwrites initial objective function - :return: - """ - - if name == "ackley": - self.objfunc = ackley10 - self.params = [Uniform(str(j), -2, 2, 1.5, 3.0, -2, 2, doc=str(j) + ' value of Rosenbrock function') - for j in range(10)] - elif name == "griewank": - self.objfunc = griewank10 - self.params = [Uniform('d' + str(j), -500, 700, 1.5, 3.0, -500, 700, - doc=str(j) + 'distinc parameter within a boundary', as_int=True) - for j in range(2)] + [Uniform('c' + str(j), -500, 700, 1.5, 3.0, -500, 700, - doc=str(j) + 'continuous parameter within a boundary') - for j in range(8)] - elif name == "cmf_style": - self.objfunc = ackley10 - self.params = [Uniform(.5, 5., optguess=1.5, doc='saturated depth at beginning'), - Uniform(.001, .8, optguess=.1, doc='porosity of matrix [m3 Pores / m3 Soil]'), - Uniform(1., 240., optguess=10., - doc='ssaturated conductivity of macropores [m/day]'), - Uniform(.0001, .5, optguess=.05, doc='macropore fraction [m3/m3]'), - Uniform(.005, 1., optguess=.05, - doc='mean distance between the macropores [m]'), - Uniform(0., 1., optguess=0., - doc='water content when matric potential pointing towards -infinity'), - Uniform(.5, 1., optguess=.99, - doc='wetness above which the parabolic extrapolation is used instead of VGM'), - Uniform(0., 50, optguess=.1, - doc='exchange rate [1/day] for macropore-matrix-exchange')] - - def parameters(self): - if self.params is None: - self.params = [ - Uniform("0", -10, 10, 1.5, 3.0, -10, 10, doc='x value of Rosenbrock function'), - Uniform("1", -10, 10, 1.5, 3.0, -10, 10, doc='y value of Rosenbrock function'), - Uniform("z", -10, 10, 1.5, 3.0, -10, 10, doc='z value of Rosenbrock function')] - return spotpy.parameter.generate(self.params) - - def simulation(self, vector): - x = np.array(vector) - # simulations = [sum(100.0 * (x[1:] - x[:-1] ** 2.0) ** 2.0 + (1 - x[:-1]) ** 2.0)] - simulations = x * np.random.rand(len(vector)) - #simulations = x * np.sum(vector) - return simulations - - def evaluation(self): - # observations = [0] - observations = [2, 3, 4] - return observations - - def objectivefunction(self, simulation, evaluation, params): - - if self.objfunc is None: - return -1*rmse(evaluation, simulation) - else: - pars, names = params - return self.objfunc(pars) diff --git a/spotpy/examples/spot_setup_dtlz1.py b/spotpy/examples/spot_setup_dtlz1.py deleted file mode 100644 index b2d99746..00000000 --- a/spotpy/examples/spot_setup_dtlz1.py +++ /dev/null @@ -1,61 +0,0 @@ - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -import numpy as np -import spotpy -import time - -def g1(x,k): - return 100*( k + np.sum(np.square(x - 0.5) - np.cos(20*np.pi*(x -0.5)), axis=1)) - - -def dtlz1(x,n_var,n_obj): - - - k = n_var - n_obj + 1 - - X, X_M = x[:, :n_obj - 1], x[:, n_obj - 1:] - g = g1(X_M,k) - - f = [] - for i in range(0,n_obj): - _f = 0.5 * (1 + g) - _f *= np.prod(X[:, :X.shape[1] -i],axis=1) - if i> 0: - _f *= 1 - X[:,X.shape[1] -i] - f.append(_f) - - return f - - - - - - -class spot_setup(object): - def __init__(self,n_var=5,n_obj=3): - self.n_var=n_var - self.n_obj = n_obj - - self.params = [] - for i in range(self.n_var): - self.params.append(spotpy.parameter.Uniform(str(i),0,1)) - def parameters(self): - return spotpy.parameter.generate(self.params) - - def simulation(self, vector): - vars = np.array(vector)[None] - sim = dtlz1(vars,n_var=self.n_var,n_obj=self.n_obj) - #time.sleep(0.1) - return sim - def evaluation(self): - observations=[0]*self.n_obj - return observations - - def objectivefunction(self, simulation,evaluation): - obj = [] - for i,f in enumerate(simulation): - obj.append(spotpy.objectivefunctions.mae(evaluation = [evaluation[i]], simulation = [simulation[i]])) - return obj diff --git a/spotpy/examples/spot_setup_griewank.py b/spotpy/examples/spot_setup_griewank.py deleted file mode 100644 index fad0618c..00000000 --- a/spotpy/examples/spot_setup_griewank.py +++ /dev/null @@ -1,45 +0,0 @@ -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -This example implements the Griewank function into SPOT. -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -import numpy as np -import spotpy - -class spot_setup(object): - def __init__(self, dim = 2): - self.dim = dim - self.params = [] - for i in range(self.dim): - self.params.append(spotpy.parameter.Uniform(str(i), -20, 20, 2, 4.0)) - - def parameters(self): - return spotpy.parameter.generate(self.params) - - - def simulation(self, vector): - n = len(vector) - fr = 4000 - s = 0 - p = 1 - for j in range(n): - s = s + vector[j]**2 - for j in range(n): - p = p * np.cos(vector[j] / np.sqrt(j+1)) - simulation = [s / fr - p + 1] - return simulation - - def evaluation(self): - observations = [0] - return observations - - def objectivefunction(self, simulation,evaluation): - objectivefunction= -spotpy.objectivefunctions.rmse(evaluation = evaluation, simulation = simulation) - return objectivefunction \ No newline at end of file diff --git a/spotpy/examples/spot_setup_hymod_exe.py b/spotpy/examples/spot_setup_hymod_exe.py deleted file mode 100644 index c4acc7c5..00000000 --- a/spotpy/examples/spot_setup_hymod_exe.py +++ /dev/null @@ -1,92 +0,0 @@ -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). -:author: Tobias Houska -This example implements the external hydrological model HYMOD into SPOTPY. -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -import numpy as np -import spotpy -import os -from distutils.dir_util import copy_tree, remove_tree -#from shutil import rmtree -import sys - -class spot_setup(object): - def __init__(self,parallel='seq'): - - self.params = [spotpy.parameter.Uniform('cmax',low=1.0 , high=500, optguess=412.33), - spotpy.parameter.Uniform('bexp',low=0.1 , high=2.0, optguess=0.1725), - spotpy.parameter.Uniform('alpha',low=0.1 , high=0.99, optguess=0.8127), - spotpy.parameter.Uniform('Ks',low=0.0 , high=0.10, optguess=0.0404), - spotpy.parameter.Uniform('Kq',low=0.1 , high=0.99, optguess=0.5592)] - - self.curdir = os.getcwd() - self.owd = os.path.realpath(__file__)+os.sep+'..' - self.hymod_path = self.owd+os.sep+'hymod_exe' - self.evals = list(np.genfromtxt(self.hymod_path+os.sep+'bound.txt',skip_header=65)[:,3])[:730] - self.Factor = 1944 * (1000 * 1000 ) / (1000 * 60 * 60 * 24) - self.parallel = parallel - - def parameters(self): - return spotpy.parameter.generate(self.params) - - def simulation(self,x): - - if self.parallel == 'seq': - call = '' - elif self.parallel == 'mpi': - #Running n parallel, care has to be taken when files are read or written - #Therefor we check the ID of the current computer core - call = str(int(os.environ['OMPI_COMM_WORLD_RANK'])+2) - #And generate a new folder with all underlying files - copy_tree(self.hymod_path, self.hymod_path+call) - - elif self.parallel == 'mpc': - #Running n parallel, care has to be taken when files are read or written - #Therefor we check the ID of the current computer core - call =str(os.getpid()) - #And generate a new folder with all underlying files - copy_tree(self.hymod_path, self.hymod_path+call) - else: - raise 'No call variable was assigned' - - os.chdir(self.hymod_path+call) - try: - params = open('Param.in','w') - - for i in range(len(x)): - if i == len(x): - params.write(str(round(x[i],5))) - else: - params.write(str(round(x[i],5))+' ') - params.close() - os.system('HYMODsilent.exe') - - #try: - SimRR = open('Q.out', 'r') - - simulations=[] - for i in range(64): - SimRR.readline() - for i in range(730): - val= SimRR.readline() - simulations.append(float(val)*self.Factor) - SimRR.close() - except: - 'Model has failed' - simulations=[np.nan]*795 #Assign bad values - model might have crashed - os.chdir(self.curdir) - if self.parallel == 'mpi' or self.parallel == 'mpc': - remove_tree(self.hymod_path+call) - return simulations - - def evaluation(self): - return self.evals - - def objectivefunction(self,simulation,evaluation, params=None): - like = spotpy.objectivefunctions.nashsutcliffe(evaluation,simulation) # Just an example, please choose an appropriate objective function depending on the used algorithm - return like diff --git a/spotpy/examples/spot_setup_hymod_python.py b/spotpy/examples/spot_setup_hymod_python.py deleted file mode 100644 index 38f6d6a1..00000000 --- a/spotpy/examples/spot_setup_hymod_python.py +++ /dev/null @@ -1,78 +0,0 @@ -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -This example implements the python version of hymod into SPOTPY. -''' - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from spotpy.parameter import Uniform -from spotpy.objectivefunctions import rmse -from spotpy.examples.hymod_python.hymod import hymod -import os - -class spot_setup(object): - cmax = Uniform(low=1.0 , high=500, optguess=412.33) - bexp = Uniform(low=0.1 , high=2.0, optguess=0.1725) - alpha = Uniform(low=0.1 , high=0.99, optguess=0.8127) - Ks = Uniform(low=0.001 , high=0.10, optguess=0.0404) - Kq = Uniform(low=0.1 , high=0.99, optguess=0.5592) - #fake1 =spotpy.parameter.Uniform(low=0.1 , high=10, optguess=0.5592) - #fake2 =spotpy.parameter.Uniform(low=0.1 , high=10, optguess=0.5592) - - def __init__(self, obj_func=None): - #Just a way to keep this example flexible and applicable to various examples - self.obj_func = obj_func - #Transform [mm/day] into [l s-1], where 1.783 is the catchment area - self.Factor = 1.783 * 1000 * 1000 / (60 * 60 * 24) - #Load Observation data from file - self.PET,self.Precip = [], [] - self.date,self.trueObs = [], [] - #Find Path to Hymod on users system - self.owd = os.path.dirname(os.path.realpath(__file__)) - self.hymod_path = self.owd+os.sep+'hymod_python' - climatefile = open(self.hymod_path+os.sep+'hymod_input.csv', 'r') - headerline = climatefile.readline()[:-1] - - #Read model forcing in working storage (this is done only ones) - if ';' in headerline: - self.delimiter = ';' - else: - self.delimiter = ',' - self.header = headerline.split(self.delimiter) - for line in climatefile: - values = line.strip().split(self.delimiter) - self.date.append(str(values[0])) - self.Precip.append(float(values[1])) - self.PET.append(float(values[2])) - self.trueObs.append(float(values[3])) - climatefile.close() - - def simulation(self,x): - #Here the model is actualy startet with one paramter combination - data = hymod(self.Precip, self.PET, x[0], x[1], x[2], x[3], x[4]) - sim=[] - for val in data: - sim.append(val*self.Factor) - #The first year of simulation data is ignored (warm-up) - return sim[366:] - - def evaluation(self): - return self.trueObs[366:] - - def objectivefunction(self,simulation,evaluation, params=None): - #SPOTPY expects to get one or multiple values back, - #that define the performance of the model run - if not self.obj_func: - # This is used if not overwritten by user - like = rmse(evaluation,simulation) - else: - #Way to ensure flexible spot setup class - like = self.obj_func(evaluation,simulation) - return like \ No newline at end of file diff --git a/spotpy/examples/spot_setup_hymod_python_pareto.py b/spotpy/examples/spot_setup_hymod_python_pareto.py deleted file mode 100644 index b4e97b31..00000000 --- a/spotpy/examples/spot_setup_hymod_python_pareto.py +++ /dev/null @@ -1,66 +0,0 @@ -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -This example implements the python version of hymod into SPOTPY. -''' - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import spotpy -from spotpy.examples.hymod_python.hymod import hymod -import os -import numpy as np - -class spot_setup(object): - cmax = spotpy.parameter.Uniform(low=1.0 , high=500, optguess=412.33) - bexp = spotpy.parameter.Uniform(low=0.1 , high=2.0, optguess=0.1725) - alpha = spotpy.parameter.Uniform(low=0.1 , high=0.99, optguess=0.8127) - Ks = spotpy.parameter.Uniform(low=0.0 , high=0.10, optguess=0.0404) - Kq = spotpy.parameter.Uniform(low=0.1 , high=0.99, optguess=0.5592) - - def __init__(self): - #Transform [mm/day] into [l s-1], where 1.783 is the catchment area - self.Factor = 1.783 * 1000 * 1000 / (60 * 60 * 24) - #Load Observation data from file - self.PET,self.Precip = [], [] - self.date,self.trueObs = [], [] - self.owd = os.path.dirname(os.path.realpath(__file__)) - self.hymod_path = self.owd+os.sep+'hymod_python' - climatefile = open(self.hymod_path+os.sep+'hymod_input.csv', 'r') - headerline = climatefile.readline()[:-1] - - if ';' in headerline: - self.delimiter = ';' - else: - self.delimiter = ',' - self.header = headerline.split(self.delimiter) - for line in climatefile: - values = line.strip().split(self.delimiter) - self.date.append(str(values[0])) - self.Precip.append(float(values[1])) - self.PET.append(float(values[2])) - self.trueObs.append(float(values[3])) - - climatefile.close() - - - def simulation(self,x): - data = hymod(self.Precip, self.PET, x[0], x[1], x[2], x[3], x[4]) - sim=[] - for val in data: - sim.append(val*self.Factor) - return sim[366:] - - def evaluation(self): - return self.trueObs[366:] - - def objectivefunction(self,simulation,evaluation, params=None): - return [-abs(spotpy.objectivefunctions.bias(evaluation, simulation)), - spotpy.objectivefunctions.rsquared(evaluation, simulation), - spotpy.objectivefunctions.nashsutcliffe(evaluation, simulation)] \ No newline at end of file diff --git a/spotpy/examples/spot_setup_hymod_unix.py b/spotpy/examples/spot_setup_hymod_unix.py deleted file mode 100644 index 71745003..00000000 --- a/spotpy/examples/spot_setup_hymod_unix.py +++ /dev/null @@ -1,101 +0,0 @@ -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -This example implements the external hydrological model HYMOD into SPOTPY. -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -import numpy as np -try: - import spotpy -except ImportError: - import sys - sys.path.append(".") - import spotpy -import os -import multiprocessing as mp -from distutils.dir_util import copy_tree, remove_tree -#from shutil import rmtree -import sys - -class spot_setup(object): - def __init__(self,parallel='seq'): - - self.params = [spotpy.parameter.Uniform('cmax',low=1.0 , high=500, optguess=412.33), - spotpy.parameter.Uniform('bexp',low=0.1 , high=2.0, optguess=0.1725), - spotpy.parameter.Uniform('alpha',low=0.1 , high=0.99, optguess=0.8127), - spotpy.parameter.Uniform('Ks',low=0.0 , high=0.10, optguess=0.0404), - spotpy.parameter.Uniform('Kq',low=0.1 , high=0.99, optguess=0.5592)] - - self.curdir = os.getcwd() - self.owd = os.path.dirname(os.path.realpath(__file__)) - self.hymod_path = self.owd+os.sep+'hymod_unix' - self.evals = list(np.genfromtxt(self.hymod_path+os.sep+'bound.txt',skip_header=65)[:,3])[:730] - self.Factor = 1944 * (1000 * 1000 ) / (1000 * 60 * 60 * 24) - self.parallel = parallel - - def parameters(self): - return spotpy.parameter.generate(self.params) - - def simulation(self,x): - - if self.parallel == 'seq': - call = '' - elif self.parallel == 'mpi': - #Running n parallel, care has to be taken when files are read or written - #Therefor we check the ID of the current computer core - call = str(int(os.environ['OMPI_COMM_WORLD_RANK'])+2) - #And generate a new folder with all underlying files - copy_tree(self.hymod_path, self.hymod_path+call) - - elif self.parallel == 'mpc': - #Running n parallel, care has to be taken when files are read or written - #Therefor we check the ID of the current computer core - call =str(os.getpid()) - #And generate a new folder with all underlying files - copy_tree(self.hymod_path, self.hymod_path+call) - else: - raise 'No call variable was assigned' - - os.chdir(self.hymod_path+call) - try: - params = open('Param.in','w') - - for i in range(len(x)): - if i == len(x): - params.write(str(round(x[i],5))) - else: - params.write(str(round(x[i],5))+' ') - params.close() - - os.system('./hymod_%s.%s' % (sys.version_info.major, sys.version_info.minor)) - - SimRR = open('Q.out', 'r') - - simulations=[] - for i in range(64): - SimRR.readline() - for i in range(730): - val= SimRR.readline() - simulations.append(float(val)*self.Factor) - SimRR.close() - except: - 'Model has failed' - simulations=[np.nan]*795 #Assign bad values - model might have crashed - os.chdir(self.curdir) - if self.parallel == 'mpi' or self.parallel == 'mpc': - remove_tree(self.hymod_path+call) - return simulations - - def evaluation(self): - return self.evals - - def objectivefunction(self,simulation,evaluation, params=None): - like = spotpy.objectivefunctions.nashsutcliffe(evaluation,simulation) # Just an example, please choose an appropriate objective function depending on the used algorithm - return like - diff --git a/spotpy/examples/spot_setup_rosenbrock.py b/spotpy/examples/spot_setup_rosenbrock.py deleted file mode 100644 index 99632b05..00000000 --- a/spotpy/examples/spot_setup_rosenbrock.py +++ /dev/null @@ -1,49 +0,0 @@ -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Optimization Tool for Python (SPOTPY). -:author: Tobias Houska - -This example implements the Rosenbrock function into a SPOTPY class. -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import numpy as np -from spotpy.parameter import Uniform -from spotpy.objectivefunctions import rmse - -class spot_setup(object): - """ - A 3 dimensional implementation of the Rosenbrock function - - Result at (1,1,1) is 0. - """ - x = Uniform(-10, 10, 1.5, 3.0, -10, 10, doc='x value of Rosenbrock function') - y = Uniform(-10, 10, 1.5, 3.0, -10, 10, doc='y value of Rosenbrock function') - z = Uniform(-10, 10, 1.5, 3.0, -10, 10, doc='z value of Rosenbrock function') - - def __init__(self,obj_func=None): - self.obj_func = obj_func - - def simulation(self, vector): - x=np.array(vector) - simulations= [sum(100.0 * (x[1:] - x[:-1] ** 2.0) ** 2.0 + (1 - x[:-1]) ** 2.0)] - return simulations - - def evaluation(self): - observations = [0] - return observations - - def objectivefunction(self, simulation, evaluation, params=None): - - #SPOTPY expects to get one or multiple values back, - #that define the performence of the model run - if not self.obj_func: - # This is used if not overwritten by user - like = rmse(evaluation,simulation) - else: - #Way to ensure on flexible spot setup class - like = self.obj_func(evaluation,simulation) - return like diff --git a/spotpy/examples/spot_setup_standardnormal.py b/spotpy/examples/spot_setup_standardnormal.py deleted file mode 100644 index bffc601c..00000000 --- a/spotpy/examples/spot_setup_standardnormal.py +++ /dev/null @@ -1,37 +0,0 @@ -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -This example implements the Standard Normal function into SPOT. -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -import numpy as np -import spotpy - - -class spot_setup(object): - def __init__(self,mean=0,std=1): - self.params = [spotpy.parameter.Uniform('x',-5,5,1.5,3.0) - ] - self.mean=mean - self.std=std - - def parameters(self): - return spotpy.parameter.generate(self.params) - - def simulation(self,x): - simulations= (1.0/(self.std*np.sqrt(2*np.pi)))**((-1.0/2.0)*(((x-self.mean)/self.std)**2)) - return simulations - - def evaluation(self): - observations = [0] - return observations - - def objectivefunction(self, simulation,evaluation): - objectivefunction = -spotpy.objectivefunctions.rmse(evaluation = evaluation,simulation = simulation) - return objectivefunction \ No newline at end of file diff --git a/spotpy/examples/tutorial_Parameterlist_iterator.py b/spotpy/examples/tutorial_Parameterlist_iterator.py deleted file mode 100644 index 724bed36..00000000 --- a/spotpy/examples/tutorial_Parameterlist_iterator.py +++ /dev/null @@ -1,45 +0,0 @@ -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -This example implements the Rosenbrock function into SPOT. -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -import numpy as np -import spotpy - - -class spot_setup(object): - - def __init__(self): - self.slow = 1000 - self.params = [spotpy.parameter.List('x',[1,2,3,4,5,6,7,8,9,0]), #Give possible x values as a List - spotpy.parameter.List('y',[0,1,2,5,6,7,8,9,0,1]) #Give possible y values as a List - - ] - def parameters(self): - return spotpy.parameter.generate(self.params) - - def simulation(self,vector): - x=np.array(vector) - for i in range(self.slow): - s = np.sin(i) - simulations= [sum(100.0*(x[1:] - x[:-1]**2.0)**2.0 + (1 - x[:-1])**2.0)] - return simulations - - def evaluation(self): - observations=[0] - return observations - - def objectivefunction(self,simulation,evaluation): - objectivefunction = -spotpy.objectivefunctions.rmse(evaluation,simulation) - return objectivefunction - - -sampler=spotpy.algorithms.mc(spot_setup(),dbname='Iterator_example', dbformat='csv') #Parameter lists can be sampled with MC -sampler.sample(10) #Choose equaly or less repetitions as you have parameters in your List diff --git a/spotpy/examples/tutorial_ackley.py b/spotpy/examples/tutorial_ackley.py deleted file mode 100644 index 98b01206..00000000 --- a/spotpy/examples/tutorial_ackley.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -This class holds the example code from the ackley tutorial web-documention. -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -import spotpy -from spotpy.examples.spot_setup_ackley import spot_setup - - -#Create samplers for every algorithm: -results=[] -spot_setup=spot_setup() -rep=5000 - -sampler=spotpy.algorithms.mc(spot_setup, dbname='ackleyMC', dbformat='csv') -sampler.sample(rep) -results.append(sampler.getdata()) -# -sampler=spotpy.algorithms.lhs(spot_setup, dbname='ackleyLHS', dbformat='csv') -sampler.sample(rep) -results.append(sampler.getdata()) - -sampler=spotpy.algorithms.mle(spot_setup, dbname='ackleyMLE', dbformat='csv') -sampler.sample(rep) -results.append(sampler.getdata()) - -sampler=spotpy.algorithms.mcmc(spot_setup, dbname='ackleyMCMC', dbformat='csv') -sampler.sample(rep) -results.append(sampler.getdata()) - -sampler=spotpy.algorithms.sceua(spot_setup, dbname='ackleySCEUA', dbformat='csv') -sampler.sample(rep,ngs=2) -results.append(sampler.getdata()) - -sampler=spotpy.algorithms.sa(spot_setup, dbname='ackleySA', dbformat='csv') -sampler.sample(rep) -results.append(sampler.getdata()) - -sampler=spotpy.algorithms.demcz(spot_setup, dbname='ackleyDEMCz', dbformat='csv') -sampler.sample(rep,nChains=30) -results.append(sampler.getdata()) -# -sampler=spotpy.algorithms.rope(spot_setup, dbname='ackleyROPE', dbformat='csv') -sampler.sample(rep) -results.append(sampler.getdata()) - - - -algorithms=['MC','LHS','MLE','MCMC','SCEUA','SA','DEMCz','ROPE'] -results=[] -for algorithm in algorithms: - results.append(spotpy.analyser.load_csv_results('ackley'+algorithm)) - - -evaluation = spot_setup.evaluation() - -spotpy.analyser.plot_objectivefunctiontraces(results,evaluation,algorithms) - - diff --git a/spotpy/examples/tutorial_dream_hymod.py b/spotpy/examples/tutorial_dream_hymod.py deleted file mode 100644 index 792eb07e..00000000 --- a/spotpy/examples/tutorial_dream_hymod.py +++ /dev/null @@ -1,101 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -This class holds example code how to use the dream algorithm -''' - -import numpy as np -import spotpy -#from spotpy.examples.spot_setup_hymod_exe import spot_setup -from spotpy.examples.spot_setup_hymod_python import spot_setup -import matplotlib.pyplot as plt -from spotpy.likelihoods import gaussianLikelihoodMeasErrorOut as GausianLike -from spotpy.analyser import plot_parameter_trace -from spotpy.analyser import plot_posterior_parameter_histogram -if __name__ == "__main__": - parallel ='seq' - # Initialize the Hymod example (will only work on Windows systems) - #spot_setup=spot_setup(parallel=parallel) - spot_setup=spot_setup(GausianLike) - - # Create the Dream sampler of spotpy, alt_objfun is set to None to force SPOTPY - # to jump into the def objectivefunction in the spot_setup class (default is - # spotpy.objectivefunctions.log_p) - - #Select number of maximum repetitions - rep=5000 - - # Select seven chains and set the Gelman-Rubin convergence limit - delta = 3 - nChains = 7 - convergence_limit = 1.2 - - # Other possible settings to modify the DREAM algorithm, for details see Vrugt (2016) - c = 0.1 - nCr = 3 - eps = 10e-6 - runs_after_convergence = 100 - acceptance_test_option = 6 - - sampler=spotpy.algorithms.dream(spot_setup, dbname='DREAM_hymod', dbformat='csv', parallel=parallel) - r_hat = sampler.sample(rep, nChains, nCr, delta, c, eps, convergence_limit, - runs_after_convergence,acceptance_test_option) - - - - - # Load the results gained with the dream sampler, stored in DREAM_hymod.csv - results = spotpy.analyser.load_csv_results('DREAM_hymod') - # Get fields with simulation data - fields=[word for word in results.dtype.names if word.startswith('sim')] - - - # Example plot to show remaining parameter uncertainty # - fig= plt.figure(figsize=(9,6)) - ax = plt.subplot(1,1,1) - q5,q25,q75,q95=[],[],[],[] - for field in fields: - q5.append(np.percentile(results[field][-100:-1],2.5))# ALl 100 runs after convergence - q95.append(np.percentile(results[field][-100:-1],97.5))# ALl 100 runs after convergence - ax.plot(q5,color='dimgrey',linestyle='solid') - ax.plot(q95,color='dimgrey',linestyle='solid') - ax.fill_between(np.arange(0,len(q5),1),list(q5),list(q95),facecolor='dimgrey',zorder=0, - linewidth=0,label='simulation uncertainty') - ax.plot(spot_setup.evaluation(),color='red', markersize=2,label='data') - ax.set_ylim(-50,450) - ax.set_xlim(0,729) - ax.set_ylabel('Discharge [l s-1]') - ax.set_xlabel('Days') - ax.legend() - fig.savefig('DREAM_simulation_uncertainty_Hymod.png',dpi=150) - ######################################################### - - - # Example plot to show the convergence ################# - spotpy.analyser.plot_gelman_rubin(results, r_hat, fig_name='DREAM_r_hat.png') - ######################################################## - - - - - # Example plot to show the parameter distribution ###### - parameters = spotpy.parameter.get_parameters_array(spot_setup) - - fig, ax = plt.subplots(nrows=5, ncols=2) - fig.set_figheight(9) - fig.set_figwidth(9) - for par_id in range(len(parameters)): - plot_parameter_trace(ax[par_id][0], results, parameters[par_id]) - plot_posterior_parameter_histogram(ax[par_id][1], results, parameters[par_id]) - - ax[-1][0].set_xlabel('Iterations') - ax[-1][1].set_xlabel('Parameter range') - - plt.show() - fig.tight_layout() - fig.savefig('DREAM_parameter_uncertainty_Hymod.png',dpi=300) - ####################################################### \ No newline at end of file diff --git a/spotpy/examples/tutorial_griewank.py b/spotpy/examples/tutorial_griewank.py deleted file mode 100644 index b28d84a1..00000000 --- a/spotpy/examples/tutorial_griewank.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -This class holds the example code from the Griewank tutorial web-documention. -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -import spotpy -from spotpy.examples.spot_setup_griewank import spot_setup - - -#Create samplers for every algorithm: -results=[] -spot_setup=spot_setup() -rep=5000 - -sampler=spotpy.algorithms.mc(spot_setup, dbname='GriewankMC', dbformat='csv') -sampler.sample(rep) -results.append(sampler.getdata()) - -sampler=spotpy.algorithms.lhs(spot_setup, dbname='GriewankLHS', dbformat='csv') -sampler.sample(rep) -results.append(sampler.getdata()) - -sampler=spotpy.algorithms.mle(spot_setup, dbname='GriewankMLE', dbformat='csv') -sampler.sample(rep) -results.append(sampler.getdata()) - -sampler=spotpy.algorithms.mcmc(spot_setup, dbname='GriewankMCMC', dbformat='csv') -sampler.sample(rep) -results.append(sampler.getdata()) - -sampler=spotpy.algorithms.sceua(spot_setup, dbname='GriewankSCEUA', dbformat='csv') -sampler.sample(rep,ngs=4) -results.append(sampler.getdata()) - -sampler=spotpy.algorithms.sa(spot_setup, dbname='GriewankSA', dbformat='csv') -sampler.sample(rep) -results.append(sampler.getdata()) - -sampler=spotpy.algorithms.demcz(spot_setup, dbname='GriewankDEMCz', dbformat='csv') -sampler.sample(rep,nChains=4) -results.append(sampler.getdata()) - -sampler=spotpy.algorithms.rope(spot_setup, dbname='GriewankROPE', dbformat='csv') -sampler.sample(rep) -results.append(sampler.getdata()) - - - -algorithms=['MC','LHS','MLE','MCMC','SCEUA','SA','DEMCz','ROPE'] -#results=[] -#for algorithm in algorithms: -# results.append(spot.analyser.load_csv_results('Griewank'+algorithm)) - - -evaluation = spot_setup.evaluation() - -spotpy.analyser.plot_heatmap_griewank(results,algorithms) diff --git a/spotpy/examples/tutorial_likelihood.py b/spotpy/examples/tutorial_likelihood.py deleted file mode 100644 index a0b9c4c3..00000000 --- a/spotpy/examples/tutorial_likelihood.py +++ /dev/null @@ -1,362 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright (c) 2017 by Benjamin Manns -This file is part of Statistical Parameter Estimation Tool (SPOTPY). -:author: Tobias Houska, Benjamin Manns - -This code shows how to use the likelihood framework and present all existing function. -''' - -import numpy as np -import spotpy - -# First we use all available likelihood functions just alone. The pydoc of every function tells, if we can add a -# parameter `param` to the function which includes model parameter. The `param` must be None or a tuple with values -# and names. If `param` is None, the needed values are calculated by the function itself. - -data, comparedata = np.random.normal(1500, 2530, 20), np.random.normal(15, 25, 20) - -l = spotpy.likelihoods.logLikelihood(data, comparedata) -print("logLikelihood: " + str(l)) - -l = spotpy.likelihoods.gaussianLikelihoodMeasErrorOut(data, comparedata) -print("gaussianLikelihoodMeasErrorOut: " + str(l)) - -l = spotpy.likelihoods.gaussianLikelihoodHomoHeteroDataError(data, comparedata) -print("gaussianLikelihoodHomoHeteroDataError: " + str(l)) - -# Here are examples where functions get `params` -l = spotpy.likelihoods.LikelihoodAR1NoC(data, comparedata, params=([0.98], ["likelihood_phi"])) -print("LikelihoodAR1NoC: " + str(l)) - -l = spotpy.likelihoods.LikelihoodAR1WithC(data, comparedata) -print("LikelihoodAR1WithC: " + str(l)) - -l = spotpy.likelihoods.generalizedLikelihoodFunction(data, comparedata, params= -([np.random.uniform(-0.99, 1, 1), np.random.uniform(0.1, 10, 1), np.random.uniform(0, 1, 1), np.random.uniform(0, 1, 0), - np.random.uniform(0, 0.99, 1), np.random.uniform(0, 100, 1)], - ["likelihood_beta", "likelihood_xi", "likelihood_sigma0", "likelihood_sigma1", "likelihood_phi1", "likelihood_muh"])) -print("generalizedLikelihoodFunction: " + str(l)) - -l = spotpy.likelihoods.LaplacianLikelihood(data, comparedata) -print("LaplacianLikelihood: " + str(l)) - -l = spotpy.likelihoods.SkewedStudentLikelihoodHomoscedastic(data, comparedata) -print("SkewedStudentLikelihoodHomoscedastic: " + str(l)) - -l = spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedastic(data, comparedata) -print("SkewedStudentLikelihoodHeteroscedastic: " + str(l)) - -l = spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel(data, comparedata, params=( - [np.random.uniform(2.01, 100, 1), np.random.uniform(0.01, 100, 1), np.random.uniform(-.99, .99, 1)], - ["likelihood_nu", "likelihood_kappa", "likelihood_phi"])) - -print("SkewedStudentLikelihoodHeteroscedasticAdvancedARModel: " + str(l)) - -l = spotpy.likelihoods.NoisyABCGaussianLikelihood(data, comparedata) -print("NoisyABCGaussianLikelihood: " + str(l)) - -l = spotpy.likelihoods.ABCBoxcarLikelihood(data, comparedata) -print("ABCBoxcarLikelihood: " + str(l)) - -l = spotpy.likelihoods.LimitsOfAcceptability(data, comparedata) -print("LimitsOfAcceptability: " + str(l)) - -l = spotpy.likelihoods.InverseErrorVarianceShapingFactor(data, comparedata) -print("inverseErrorVarianceShapingFactor: " + str(l)) - -l = spotpy.likelihoods.ExponentialTransformErrVarShapingFactor(data, comparedata) -print("inverseErrorVarianceShapingFactor: " + str(l)) - -l = spotpy.likelihoods.NashSutcliffeEfficiencyShapingFactor(data, comparedata) -print("NashSutcliffeEfficiencyShapingFactor: " + str(l)) - -l = spotpy.likelihoods.sumOfAbsoluteErrorResiduals(data, comparedata) -print("sumOfAbsoluteErrorResiduals: " + str(l)) - - -# We also can use the likelihood functions in an algorithmus. We will need a setup class like this - - -class spot_setup_gauss(object): - def __init__(self): - self.params = [ - # Original mean: 12, sd:23 - spotpy.parameter.Uniform('mean', -20, 20, 2, 3.0, -20, 20), - spotpy.parameter.Uniform('sd', 1, 30, 2, 3.01, 1, 30), - - # Some likelihood function need additional parameter, look them up in the documentation - # spotpy.parameter.Uniform('likelihood_nu', 2.01, 100, 1.5, 3.0, -10, 10), - # spotpy.parameter.Uniform('likelihood_kappa', 0.01, 100, 1.5, 3.0, -10, 10), - #spotpy.parameter.Uniform('likelihood_phi', -.99, .99, 0.1, 0.1, 0.1, 0.1), - - # spotpy.parameter.Uniform('likelihood_beta', -.99, .99, 1.5, 3.0, -10, 10), - # spotpy.parameter.Uniform('likelihood_xsi', 0.11, 10, 1.5, 3.0, -10, 10), - # spotpy.parameter.Uniform('likelihood_sigma0', 0, 1, 1.5, 3.0, -10, 10), - - # spotpy.parameter.Uniform('likelihood_sigma1', 0, 1, 1.5, 3.0, -10, 10), - # spotpy.parameter.Uniform('likelihood_phi1', 0, .99, 1.5, 3.0, -10, 10), - # spotpy.parameter.Uniform('likelihood_muh', 0, 100, 1.5, 3.0, -10, 10) - - ] - - def parameters(self): - return spotpy.parameter.generate(self.params) - - def simulation(self, vector): - # x = np.random.randint(-100, 100, size=201) - # simulations= [sum(100.0 * (x[1:] - x[:-1] **2.0) **2.0 + (1 - x[:-1]) **2.0)] - import subprocess - - import json - - output = subprocess.check_output( - "/Library/Frameworks/R.framework/Versions/3.1/Resources/Rscript /Users/Karlson/Documents/03_PRIVAT/Privat_UNI/Uni_JOB/Job_AGRAR/develop/spotpy/spotpy/likelihood_test/myR_2.R " + str( - vector[0])+" "+str(vector[1]) , shell=True) - #print("this parameters: "+str(vector[0])+" "+str(vector[1])) - - output = output.decode("utf-8") - back = json.loads(output) - - simulations = back - - return simulations - - def evaluation(self): - - # Gauss with mean=12 and sd=23 - observations = [1.537678, 7.615278, 33.54329, 12.09963, 24.69595, 7.033905, 21.30595, -17.77526, -7.09708, - 36.80745, 10.68426, -42.7048, -21.01126, -6.314566, 38.01058, -15.79536, -17.69119, -4.482229, - 12.30351, -30.54512, 8.468925, 27.44369, 37.20623, -8.753253, 39.40037, 29.03273, 0.5257918, - 25.98343, -16.09876, 6.430084, 4.755722, -10.38204, 7.97673, -37.55442, 58.04988, 20.41361, - 32.13943, 30.37884, 6.898094, 13.32948, -14.5311, 25.0606, 25.81364, 25.82836, 47.70208, - 31.1919, 24.74743, 18.21143, 10.67086, 47.29963, 40.3718, 39.21012, 6.774497, -4.244702, - 13.45878, 33.80645, 15.64674, -6.277918, 36.83417, 23.13524, 11.85227, 31.38894, 13.00289, - 6.47117, 19.31257, -11.23233, 21.07677, 14.96549, 9.952789, 23.54167, 46.7991, 47.64822, - 7.33875, 17.64916, 38.79842, 11.75935, 17.70734, 15.64669, -6.890646, 3.710825, 44.42125, - -20.1855, 24.32393, 56.55909, 33.02915, 5.173076, -24.00348, 16.62815, -21.64049, 18.2159, - 41.69109, -31.26055, 36.9492, 2.780838, -4.519057, -24.71357, 20.63503, 17.08391, 26.23503, - 12.82442, 22.13652, 21.21188, 47.99579, 44.52914, -0.5511025, 55.47107, -15.12694, 2.884632, - 7.361032, 12.66143, 37.38807, 53.63648, 9.114074, 12.68311, -6.890102, 32.40405, 22.93079, - 1.498509, 22.68785, 29.71565, 21.42051, -9.961459, -10.22352, -28.16017, 14.14882, 9.64758, - -5.821728, -21.93086, 19.94631, 16.29195, 28.87528, 25.81239, 52.44341, 5.229822, -17.92572, - 11.85504, 17.21691, 17.19854, -6.37061, 16.1524, -13.08297, 13.45471, 9.43481, -2.177022, - 17.46416, 2.647446, 16.77834, 20.77827, 49.37553, 8.435563, -13.85352, 17.06572, 5.550149, - 2.674943, -21.95848, 11.82037, 30.51478, 8.334891, -17.1576, -16.82652, 50.31279, -31.05799, - 52.69635, 22.11049, -43.32149, -13.5348, -5.125771, 1.801732, 19.30368, 14.94216, -19.32855, - 20.75345, 21.03398, 5.430688, -3.163607, 10.1321, -12.9058, -13.77746, 25.02506, -7.187246, - 39.93147, 5.330449, 6.705344, -16.47149, -34.20934, 28.66261, -6.420032, 4.682751, -9.622858, - 17.95173, 3.316576, 14.6763, 13.84716, -20.52151, 24.35037, 28.57057, 18.17082, 26.14141, - 72.05923, -12.29775,26.83472] - - return observations - - def objectivefunction(self, simulation=simulation, evaluation=evaluation, params=None): - # Some functions do not need a `param` attribute, you will see that in the documentation or if an error occur. - # objectivefunction = spotpy.likelihoods.LimitsOfAcceptability(evaluation, simulation,params=params) - #objectivefunction = spotpy.likelihoods.NoisyABCGaussianLikelihood(evaluation, simulation) - #objectivefunction = spotpy.likelihoods.LimitsOfAcceptability(evaluation, simulation) - objectivefunction = spotpy.objectivefunctions.rmse(simulation=simulation,evaluation=evaluation) - #print(objectivefunction) - - return objectivefunction - -class spot_setup_wald(object): - def __init__(self): - self.params = [ - # See https://de.wikipedia.org/wiki/Inverse_Normalverteilung - # Original 12,23 - spotpy.parameter.Uniform('lambda', 1, 1000, 0.1, 3.0, -20, 20), - #spotpy.parameter.Uniform('nu', 1, 30, 2, 3.01, 1, 30), -] - - def parameters(self): - return spotpy.parameter.generate(self.params) - - def simulation(self, vector): - import subprocess - - import json - - output = subprocess.check_output( - "/Library/Frameworks/R.framework/Versions/3.1/Resources/Rscript /Users/Karlson/Documents/03_PRIVAT/Privat_UNI/Uni_JOB/Job_AGRAR/develop/spotpy/spotpy/likelihood_test/myR_WALD.R " + str(vector[0]), shell=True) - # print("this parameters: "+str(vector[0])+" "+str(vector[1])) - - output = output.decode("utf-8") - back = json.loads(output) - - simulations = back - - return simulations - - def evaluation(self): - # Wald distrubtion with lambda=42 (nu = 1 as it is original inverse Gauß) - observations = [0.8215101,1.050744,1.068614,0.9237615,1.134586,0.8342905,0.9203649,0.8423139,1.016296,0.819583,0.7727125,1.049373,0.9064652,1.129859,0.7390692,0.7807588,0.9512094,0.751157,0.8342608,0.9535379,0.8855571,0.8164966,0.9859118,0.9663425,0.9168434,1.096442,1.075291,0.7939873,0.8371087,0.8899696,0.8223036,0.9441274,1.251677,0.9946841,0.9688333,0.8584872,1.118507,1.092399,0.9389445,1.320034,1.05912,0.8073291,0.9718409,0.9993603,1.009801,1.191749,1.096261,0.9104541,1.135112,1.024141,0.68865,1.117724,1.071344,0.9730503,1.03879,0.9040554,1.226641,1.090904,0.9188659,0.9516232,1.111537,0.7868174,1.03979,0.8529991,1.546705,0.9973017,0.9056773,1.020306,0.8666091,0.8227436,1.107373,1.240635,0.8642053,1.012499,0.8189009,0.9112955,0.9133874,0.764895,0.9954879,1.016124,1.135945,1.210386,0.8935554,1.133396,0.8744451,1.27583,0.9399524,1.109192,1.024147,1.010473,0.823447,1.063746,1.587057,1.25963,1.075372,0.9058057,1.149925,0.8951753,0.8786255,0.7846421,1.089261,1.155204,0.9162714,1.091259,1.012252,0.9860885,1.000627,0.8831002,0.9736084,1.020061,1.099529,0.8503705,1.092692,1.018956,0.9597126,0.9760877,0.8305396,1.010439,0.8659965,1.216233,1.15933,0.8352535,1.086288,1.085791,1.215822,1.455505,0.8928623,1.227453,0.9778177,1.248284,0.678555,0.9379088,1.076307,1.081512,1.056103,0.9475012,1.11073,1.216543,1.409434,0.8973831,0.7879291,1.039925,0.9954887,0.8129037,1.088005,1.010168,0.9842995,1.034264,0.9122271,1.128363,1.331232,1.206762,1.134155,1.166505,1.154047,1.054108,1.07734,0.8397705,0.9748741,1.133629,0.9498966,0.9889976,1.023417,0.9424091,0.9424539,1.246194,0.9413468,1.15047,0.7332654,1.496362,0.828069,0.7696388,0.918564,0.8388578,0.9455839,0.8227491,0.9551339,0.963993,1.051606,1.013032,0.8144458,1.07049,1.029506,0.7699333,0.9409208,1.341655,1.023382,0.9868176,0.9950876,0.954334,0.957515,1.136036,1.265562,0.9722909,0.7632513,0.8805661,0.8904488,1.052702,1.036818,0.9569595,0.9428334] - - return observations - - def objectivefunction(self, simulation=simulation, evaluation=evaluation, params=None): - objectivefunction = spotpy.likelihoods.NoisyABCGaussianLikelihood(evaluation, simulation) - - return objectivefunction - -class spot_setup_ar_1_students_t_res(object): - def __init__(self): - self.params = [ - # For students - Skew - spotpy.parameter.Uniform('likelihood_kappa', 1, 1, 1, 1, 1, 1), - spotpy.parameter.Uniform('likelihood_phi', -0.99, 0.99, 0.1, 3.0, -0.99, 0.99), - spotpy.parameter.Uniform('likelihood_nu', 2.1, 100, 2, 3.01, 2.1, 10), - - ] - - def parameters(self): - return spotpy.parameter.generate(self.params) - - def simulation(self, vector): - import subprocess - - import json - # parameter 0:phi, 1:nu - output = subprocess.check_output( - "/Library/Frameworks/R.framework/Versions/3.1/Resources/Rscript /Users/Karlson/Documents/03_PRIVAT/Privat_UNI/Uni_JOB/Job_AGRAR/develop/spotpy/spotpy/likelihood_test/myR_AR1_Student-t-res.R " + str( - vector[0])+" "+str(vector[1]), shell=True) - #print("this parameters: "+str(vector[0])+" "+str(vector[1]) + " "+str(vector[2])) - - output = output.decode("utf-8") - back = json.loads(output) - - simulations = back - - return simulations - - def evaluation(self): - # AR1 with student-t distributed residuals with phi=42 and df=7 - observations = [-0.806554,-1.036565,-1.054924,1.207057,-0.2267287,-0.6241089,-0.9865154,0.004914617,0.5435943,-0.2529109,-0.220569,-1.085152,-1.446932,-0.2271839,-0.9695579,-1.042956,-1.677816,-0.3111386,0.2023657,1.681282,0.3311756,-2.011093,-0.05581758,1.792977,1.469434,-0.4426333,0.8043387,1.093684,1.99667,-0.1171755,0.5409204,-1.036451,-0.03801139,0.6125889,0.09342139,1.977853,0.5931005,2.947713,-1.51345,0.152891,-1.079158,-1.466529,-1.504046,2.181459,-1.966612,0.3770235,-0.7682167,-0.2559416,1.215604,0.9032672,0.7817497,-1.634914,-0.1532924,-1.329244,-3.425294,-0.9132035,-0.1814598,-0.3703962,0.8464317,0.3035075,-0.1755196,-0.7057517,-0.6808711,-1.221144,0.04788332,0.7737842,0.9854636,0.7845628,3.702611,3.388055,1.097186,0.672756,-0.3338004,0.2702849,-0.6202132,-2.228129,-0.7818483,0.155476,0.657133,0.5652424,0.0008205475,0.4045611,0.7796379,0.4325643,-0.354133,-1.166525,-0.9882422,0.4509809,-0.4848221,0.1715434,-1.196337,0.4085583,-0.04952907,1.035795,0.9496666,0.3619785,0.8666738,0.285121,-1.432747,-1.939233,-0.1684185,0.003022823,-0.4922612,0.5042833,0.09296572,-1.455734,-0.213669,1.063985,0.2004293,-1.649053,0.200737,-0.105006,-2.633298,-3.161435,-0.5550424,-1.713277,-0.5169007,-0.3834445,-0.2791567,1.262414,0.06332764,1.640421,0.3144967,-0.01392552,-1.151283,-1.268196,-1.880887,1.010745,0.2109186,2.752468,0.09242852,-2.878202,-1.477239,-2.243395,-0.6384147,-0.6950003,-1.686313,-0.6945071,1.603874,0.09509077,-0.09816484,1.150218,2.324576,1.504623,-1.063881,1.330131,2.611619,2.494136,0.6338899,1.004135,0.1381033,-0.360101,1.124663,1.78177,1.603268,1.325218,2.4939,2.153747,1.97724,1.18353,-0.7253327,0.823499,0.4020309,0.6603788,0.129381,-1.321761,1.939023,0.6186619,0.9888297,1.118437,0.2581724,1.385318,1.997055,-0.8354623,0.441386,-0.5992372,-2.595201,1.591507,-0.7960406,0.2250709,0.6408333,-0.06189073,0.103896,-0.4334433,0.1433027,-0.6207327,0.2989117,-0.2577758,-0.5267562,-3.398514,0.1546277,1.012887,0.06408349,1.258883,0.5752389,-1.694637,-0.7765886,2.41088,0.2273703,-1.2156,-1.232374] - - return observations - - def objectivefunction(self, simulation=simulation, evaluation=evaluation, params=None): - objectivefunction = spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel(evaluation, simulation,params=params) - if np.isinf(objectivefunction): - objectivefunction = 0.0 - print("Class log "+str(objectivefunction)) - return objectivefunction - - - -class spot_setup_ar_1_gauss_res(object): - def __init__(self): - self.params = [ - spotpy.parameter.Uniform('likelihood_phi', -0.99, 0.99, 0.2, 3.0, -0.99, 0.99), - ] - - def parameters(self): - return spotpy.parameter.generate(self.params) - - def simulation(self, vector): - import subprocess - - import json - # parameter 0:phi - output = subprocess.check_output( - "/Library/Frameworks/R.framework/Versions/3.1/Resources/Rscript /Users/Karlson/Documents/03_PRIVAT/Privat_UNI/Uni_JOB/Job_AGRAR/develop/spotpy/spotpy/likelihood_test/myR_AR1_Norm_res.R " + str( - vector[0]), shell=True) - print("this parameters: "+str(vector[0])) - - output = output.decode("utf-8") - back = json.loads(output) - - simulations = back - - return simulations - - def evaluation(self): - # AR1 with normal distributed residuals with phi=0.428 and mean=0,sd=1 - observations = [-2.522743,1.408725,1.946646,0.888204,0.7897667,0.6112302,2.290427,2.868624,1.556995,0.05254887,0.7461225,0.3719867,0.979471,-1.190781,0.4436523,2.236696,0.2605191,0.673389,0.7251472,0.6608128,-0.7443824,0.371268,0.02141081,0.4607711,-0.5507639,-1.005326,0.7367659,0.5269135,0.1166365,-0.5970824,-0.7521213,0.367761,0.1125972,-0.0795236,0.1165288,0.6486639,1.932282,1.033983,-0.9876154,0.1796362,-0.3983238,0.01101198,0.9050182,-0.6080171,0.08849208,-1.81438,-0.07913209,-0.009007132,-1.160989,-1.297887,1.419709,-1.066902,-1.270009,-1.030414,-2.38863,-0.2904009,0.7489411,-0.1846965,-1.433198,-1.145359,-0.04856701,1.087533,0.7987545,0.641762,-0.378111,-1.019192,0.2837018,2.259854,0.4158938,1.451425,0.9710148,1.325311,0.04831358,-0.2373003,0.09663009,-2.557514,-0.9230433,-0.6250428,-0.6359625,-0.2501693,1.096061,1.564296,1.956924,-0.4511307,-0.521957,-0.3384552,-0.3905848,-1.05168,1.266555,1.835193,1.168423,-0.5542428,-0.7080179,0.8867539,0.9273763,0.9104679,1.761588,2.650327,0.549592,1.152438,0.1226201,-0.1466213,0.6201685,-0.244967,1.728424,-0.1991486,0.4715499,2.541504,2.055189,0.5441279,1.075167,0.6381296,0.7177508,1.106246,-0.6729018,-2.086599,-1.199925,0.7879157,0.01633025,-0.5845763,0.4181293,-2.16246,-2.197336,-2.209761,-1.856266,-1.021362,0.6899624,-0.898831,0.3702167,1.202066,0.5307163,-1.183152,-1.305882,-1.34188,0.9997525,-1.611335,0.4367853,1.24461,2.474933,0.9325948,2.022836,-0.6440014,-1.253051,0.2687869,-1.139736,0.1336643,-1.383847,0.3793314,-0.7788819,0.8646879,1.433118,1.026648,1.31162,-0.8718095,0.3493136,-0.2196879,0.3182434,-0.7072177,0.6959091,-0.5620885,0.8712038,0.6010974,-0.007788187,-0.1008797,-0.5404524,-0.6134115,2.287364,-0.04623299,-1.30409,0.6175855,-1.475111,0.2202588,0.5428336,0.3996769,0.1243578,0.3912388,1.346204,0.6947638,-0.5677999,0.3445474,-0.7952659,1.144519,1.850942,-0.312677,-1.347769,-0.6379291,0.1777509,0.1736661,-0.6718341,0.8210482,-0.4401946,0.9218458,-0.2263964,0.2356263,-0.6045727,0.124017,0.6440486,1.095587,0.4844389,-0.4179212,1.385794] - - return observations - - def objectivefunction(self, simulation=simulation, evaluation=evaluation, params=None): - objectivefunction = spotpy.likelihoods.LikelihoodAR1NoC(evaluation, simulation,params=params) - - return objectivefunction - - - - -class spot_setup_generalizedGauss(object): - def __init__(self): - self.params = [ - - spotpy.parameter.Uniform('likelihood_phi1', 0.01, 0.99, 3.0, 3.0, 0.01, 0.99), - spotpy.parameter.Uniform('likelihood_xi', 0.1, 10, 3, 3, 0.1, 10), - spotpy.parameter.Uniform('likelihood_beta', 0.1, 0.99, 3, 3.0, -0.99, 0.99), # positive - simulation in R says it! - - - spotpy.parameter.Uniform('likelihood_sigma0', 0, 1, 0.1, 3.0, 0, 1), - spotpy.parameter.Uniform('likelihood_sigma1', 0, 1, 0.1, 3.0, 0, 1), - - spotpy.parameter.Uniform('likelihood_muh', 0, 100, 0.1, 3.0, 0, 100), - ] - - def parameters(self): - return spotpy.parameter.generate(self.params) - - def simulation(self, vector): - import subprocess - - import json - # we need a skew exponential power to add xi and beta in it which is the model for the error residuals of AR1 Model - - output = subprocess.check_output( - "/Library/Frameworks/R.framework/Versions/3.1/Resources/Rscript /Users/Karlson/Documents/03_PRIVAT/Privat_UNI/Uni_JOB/Job_AGRAR/develop/spotpy/spotpy/likelihood_test/myR_AR1_SEP_res.R " + str( - vector[0]) + " " + str(vector[1])+ " "+str(vector[2]), shell=True) - # print("this parameters: "+str(vector[0])+" "+str(vector[1]) + " "+str(vector[2])) - - output = output.decode("utf-8") - back = json.loads(output) - if back[0] == "NA": - back = np.repeat(np.NAN,back.__len__()).tolist() - simulations = back - - return simulations - - def evaluation(self): - # AR1 with SEP distribution phi=0.142 xsi=3, beta=9.5 - # [ 0.99 1.28841626 0.99 0.43366888 0.38087079 72.72585542] - observations = [1.367689,-0.320067,-0.04381581,0.6023338,0.9038274,1.034441,-0.3475758,-0.6766884,0.375266,0.3902351,1.41773,1.146159,1.142029,-0.5467857,1.132456,0.05771065,-0.01329709,1.245674,1.262945,0.5637976,-0.6106627,-0.1347206,0.4439383,-0.1191365,0.6781304,-0.4293178,0.1856715,0.4008803,1.34489,0.9124905,1.237749,0.5098399,0.8364595,-0.4464507,0.6631744,0.2039722,-0.05081068,0.7299973,0.8854515,1.180466,-0.4876658,0.7830223,-0.4316994,1.099141,0.5340687,0.8495034,0.8779076,0.6735508,-0.3102846,0.2900948,-0.05825545,-0.941212,1.025862,0.7281562,-0.361788,0.6388547,1.038316,1.343974,0.8034503,1.39158,0.5718842,0.4621339,0.828369,1.091347,0.9504174,1.100152,0.4411185,1.178236,1.528249,1.311347,1.011896,0.6851925,1.102152,1.191884,-0.3198258,1.023772,1.021118,0.351345,-0.7778747,0.3130401,0.8374449,-0.2165474,0.6511311,0.9294736,1.007714,1.124968,1.122693,1.053253,1.064944,-0.3810931,-0.7520672,0.07500417,-0.6589652,0.9858736,-0.3338579,0.5976432,1.065922,-0.1717056,-0.7935736,-0.2154963,0.1094597,0.9271599,0.8882699,-0.204626,-0.1153957,-0.03153619,1.145353,0.1135476,-0.0652023,-0.3510398,-0.8471455,-0.7796421,-0.2307928,-0.2594656,0.8092929,0.4113968,-0.188539,1.19418,0.8070983,0.9118222,1.071649,1.051291,-0.8035766,0.8092788,-0.1163294,-0.02733921,-0.6852544,-0.408636,-0.08736997,0.4578535,0.7799243,0.4268271,0.5604364,-0.1452668,-0.1654945,-0.483941,0.1326935,0.4563893,0.7192259,0.7154398,1.120117,0.3798121,-0.1878339,-0.5358535,-0.5510031,0.0233894,0.07423701,0.9234318,-0.1600513,1.372747,0.6790618,0.8772782,-0.1664986,0.9622479,0.9873338,0.2296364,-1.002397,-0.2306121,-0.1446204,0.31194,-0.1342179,0.08709208,-0.2634807,-0.5770442,0.5588156,-0.4229277,-0.8920537,-0.3130578,0.9963966,-0.1462349,0.2177117,1.019684,0.3005968,0.7721734,-0.1104061,-0.7366346,0.9962065,0.4035172,1.175904,1.103031,-0.742134,-0.3189378,0.5614889,0.4403444,-0.6407969,-0.4805289,0.2666954,0.8946856,0.200042,0.700875,-0.3121022,-0.4439033,-0.7696692,0.6370263,1.367421,0.8487393,0.1497969,-0.4690384,1.088799,0.0210073,0.3703306] - - return observations - - def objectivefunction(self, simulation=simulation, evaluation=evaluation, params=None): - objectivefunction = spotpy.likelihoods.generalizedLikelihoodFunction(evaluation,simulation,params=params) - if np.isinf(objectivefunction): - objectivefunction = 0.0 - print("Class log " + str(objectivefunction)) - return objectivefunction - - - - -# And now we can start an algorithm to find the best possible data - - -results = [] -spot_setup = spot_setup_gauss() -rep = 1000 - -# TODO alt_objfun immer wieder anschalten sonst Trick 17! -sampler = spotpy.algorithms.lhs(spot_setup, dbname='RosenMC', dbformat='csv', alt_objfun=None) -sampler.sample(rep) -results.append(sampler.getdata()) - -import pickle - -pickle.dump(results, open('mle_LimitsOfAcceptability_v2.p', 'wb')) - -# from matplotlib import pyplot as plt -# plt.plot(results) -# plt.show() -# plt.plot(results['like1']) -# plt.show() diff --git a/spotpy/examples/tutorial_nsgaii_dltz1.py b/spotpy/examples/tutorial_nsgaii_dltz1.py deleted file mode 100644 index 96031b5b..00000000 --- a/spotpy/examples/tutorial_nsgaii_dltz1.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import spotpy -from spotpy.examples.spot_setup_dtlz1 import spot_setup -import numpy as np -import sys -import matplotlib.pyplot as plt -from mpl_toolkits.mplot3d import Axes3D -import pandas as pd - - -if __name__ == "__main__": - #Create samplers for every algorithm: - results=[] - n_obj = 3 - spot_setup=spot_setup(n_var=5, n_obj=n_obj) - generations=10 - n_pop = 30 - skip_duplicates = False - - sampler=spotpy.algorithms.NSGAII(spot_setup=spot_setup, - dbname='NSGA2', - dbformat='csv', - save_sim=True) - sampler.sample(generations,n_obj=3, n_pop=n_pop,skip_duplicates=skip_duplicates) - - - last = None - first = None - - # output calibration - - df = pd.read_csv("NSGA2.csv") - - df["like3"] = df.like3 * -1 - - - if last: - df = df.iloc[-last:,:] - elif first: - df = df.iloc[:first,:] - else: - pass - - - - # plot objective functions - fig = plt.figure() - for i,name in enumerate(df.columns[:n_obj]): - ax = fig.add_subplot(n_obj,1,i +1) - df.loc[::5,name].plot(lw=0.5,figsize=(18,8),ax = ax,color="black") - plt.title(name) - plt.show() - - - - x,y,z = df.iloc[-n_pop:,0],df.iloc[-n_pop:,1],df.iloc[-n_pop:,2] - fig = plt.figure() - ax = fig.add_subplot(111, projection='3d') - ax.scatter(x,y,z,marker="o") - ax.set_xlabel("x") - ax.set_ylabel("y") - ax.set_zlabel("z") - plt.show() - - # plot parameters - fig = plt.figure() - for i,name in enumerate(df.columns[n_obj:8]): - ax = fig.add_subplot(5,1,i +1) - df.loc[:,name].plot(lw=0.5,figsize=(18,8),ax = ax,color="black") - plt.title(name) - plt.show() - diff --git a/spotpy/examples/tutorial_padds_hymod.py b/spotpy/examples/tutorial_padds_hymod.py deleted file mode 100644 index 1b90071d..00000000 --- a/spotpy/examples/tutorial_padds_hymod.py +++ /dev/null @@ -1,144 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -This class holds example code how to use the dream algorithm -''' - -import numpy as np -try: - import spotpy -except ImportError: - import sys - sys.path.append(".") - import spotpy - -from spotpy.examples.spot_setup_hymod_python import spot_setup -import matplotlib.pyplot as plt - - -def multi_obj_func(evaluation, simulation): - #used to overwrite objective function in hymod example - like1 = abs(spotpy.objectivefunctions.pbias(evaluation, simulation)) - like2 = spotpy.objectivefunctions.rmse(evaluation, simulation) - like3 = spotpy.objectivefunctions.rsquared(evaluation, simulation)*-1 - return np.array([like1, like2, like3]) - -if __name__ == "__main__": - parallel ='seq' # Runs everthing in sequential mode - np.random.seed(2000) # Makes the results reproduceable - - # Initialize the Hymod example - # In this case, we tell the setup which algorithm we want to use, so - # we can use this exmaple for different algorithms - spot_setup=spot_setup(multi_obj_func) - - #Select number of maximum allowed repetitions - rep=2000 - - - # Create the PADDS sampler of spotpy, alt_objfun is set to None to force SPOTPY - # to jump into the def objectivefunction in the spot_setup class (default is - # spotpy.objectivefunctions.rmse) - sampler=spotpy.algorithms.padds(spot_setup, dbname='padds_hymod', dbformat='csv') - - #Start the sampler, one can specify metric if desired - sampler.sample(rep,metric='ones') - - # Load the results gained with the sceua sampler, stored in padds_hymod.csv - #results = spotpy.analyser.load_csv_results('padds_hymod') - results = sampler.getdata() - - # from pprint import pprint - # #pprint(results) - # pprint(results['chain']) - - for likno in range(1,4): - fig_like1 = plt.figure(1,figsize=(9,5)) - plt.plot(results['like'+str(likno)]) - plt.show() - fig_like1.savefig('hymod_padds_objectivefunction_' + str(likno) + '.png', dpi=300) - - - fig, ax=plt.subplots(3) - for likenr in range(3): - ax[likenr].plot(results['like'+str(likenr+1)]) - ax[likenr].set_ylabel('like'+str(likenr+1)) - fig.savefig('hymod_padds_objectivefunction.png', dpi=300) - - - - - - # Example plot to show the parameter distribution ###### - def plot_parameter_trace(ax, results, parameter): - ax.plot(results['par'+parameter.name],'.') - ax.set_ylabel(parameter.name) - ax.set_ylim(parameter.minbound, parameter.maxbound) - - def plot_parameter_histogram(ax, results, parameter): - #chooses the last 20% of the sample - ax.hist(results['par'+parameter.name][int(len(results)*0.9):], - bins =np.linspace(parameter.minbound,parameter.maxbound,20)) - ax.set_ylabel('Density') - ax.set_xlim(parameter.minbound, parameter.maxbound) - - fig= plt.figure(2,figsize=(9,9)) - - ax1 = plt.subplot(5,2,1) - plot_parameter_trace(ax1, results, spot_setup.cmax) - - ax2 = plt.subplot(5,2,2) - plot_parameter_histogram(ax2,results, spot_setup.cmax) - - ax3 = plt.subplot(5,2,3) - plot_parameter_trace(ax3, results, spot_setup.bexp) - - ax4 = plt.subplot(5,2,4) - plot_parameter_histogram(ax4, results, spot_setup.bexp) - - ax5 = plt.subplot(5,2,5) - plot_parameter_trace(ax5, results, spot_setup.alpha) - - ax6 = plt.subplot(5,2,6) - plot_parameter_histogram(ax6, results, spot_setup.alpha) - - ax7 = plt.subplot(5,2,7) - plot_parameter_trace(ax7, results, spot_setup.Ks) - - ax8 = plt.subplot(5,2,8) - plot_parameter_histogram(ax8, results, spot_setup.Ks) - - ax9 = plt.subplot(5,2,9) - plot_parameter_trace(ax9, results, spot_setup.Kq) - ax9.set_xlabel('Iterations') - - ax10 = plt.subplot(5,2,10) - plot_parameter_histogram(ax10, results, spot_setup.Kq) - ax10.set_xlabel('Parameter range') - - plt.show() - fig.savefig('hymod_parameters.png',dpi=300) - - - # Example plot to show remaining parameter uncertainty # - fields=[word for word in results.dtype.names if word.startswith('sim')] - fig= plt.figure(3, figsize=(16,9)) - ax11 = plt.subplot(1,1,1) - q5,q25,q75,q95=[],[],[],[] - for field in fields: - q5.append(np.percentile(results[field][-100:-1],2.5)) - q95.append(np.percentile(results[field][-100:-1],97.5)) - ax11.plot(q5,color='dimgrey',linestyle='solid') - ax11.plot(q95,color='dimgrey',linestyle='solid') - ax11.fill_between(np.arange(0,len(q5),1),list(q5),list(q95),facecolor='dimgrey',zorder=0, - linewidth=0,label='parameter uncertainty') - ax11.plot(spot_setup.evaluation(),'r.',label='data') - ax11.set_ylim(-50,450) - ax11.set_xlim(0,729) - ax11.legend() - fig.savefig('python_hymod.png',dpi=300) - ######################################################### \ No newline at end of file diff --git a/spotpy/examples/tutorial_rosenbrock.py b/spotpy/examples/tutorial_rosenbrock.py deleted file mode 100644 index d4c8c36d..00000000 --- a/spotpy/examples/tutorial_rosenbrock.py +++ /dev/null @@ -1,112 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright (c) 2018 by Tobias Houska -This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). -:author: Tobias Houska - -This file holds the example code from the Rosenbrock tutorial web-documention. -''' - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -try: - import spotpy -except ImportError: - import sys - sys.path.append(".") - import spotpy - -from spotpy.examples.spot_setup_rosenbrock import spot_setup -from spotpy.describe import describe - - - -#Create samplers for every algorithm: -results=[] -rep=1000 -timeout=10 #Given in Seconds - -parallel = "seq" -dbformat = "csv" - -# Bayesian algorithms should be run with a likelihood function -bayesian_likelihood_func = spotpy.likelihoods.gaussianLikelihoodMeasErrorOut - - -sampler = spotpy.algorithms.mc(spot_setup(), - parallel=parallel, dbname='RosenMC', dbformat=dbformat, sim_timeout=timeout) -print(describe(sampler)) -sampler.sample(rep) -results.append(sampler.getdata()) - - -sampler = spotpy.algorithms.lhs(spot_setup(), - parallel=parallel, dbname='RosenLHS', dbformat=dbformat, sim_timeout=timeout) -sampler.sample(rep) -results.append(sampler.getdata()) - - -sampler=spotpy.algorithms.mle(spot_setup(obj_func=bayesian_likelihood_func), - parallel=parallel, dbname='RosenMLE', dbformat=dbformat, sim_timeout=timeout) -sampler.sample(rep) -results.append(sampler.getdata()) - - - -sampler=spotpy.algorithms.sceua(spot_setup(), - parallel=parallel, dbname='RosenSCEUA', dbformat=dbformat, sim_timeout=timeout) -sampler.sample(rep,ngs=4) -results.append(sampler.getdata()) - - -sampler=spotpy.algorithms.sa(spot_setup(obj_func=bayesian_likelihood_func), - parallel=parallel, dbname='RosenSA', dbformat=dbformat, sim_timeout=timeout) -sampler.sample(rep) -results.append(sampler.getdata()) - - - -sampler=spotpy.algorithms.rope(spot_setup(), - parallel=parallel, dbname='RosenROPE', dbformat=dbformat,sim_timeout=timeout) -sampler.sample(rep) -results.append(sampler.getdata()) - - -sampler=spotpy.algorithms.abc(spot_setup(), - parallel=parallel, dbname='RosenABC', dbformat=dbformat,sim_timeout=timeout) -sampler.sample(rep) -results.append(sampler.getdata()) - - -sampler=spotpy.algorithms.fscabc(spot_setup(), - parallel=parallel, dbname='RosenFSABC', dbformat=dbformat, sim_timeout=timeout) -sampler.sample(rep) -results.append(sampler.getdata()) - - -sampler=spotpy.algorithms.mcmc(spot_setup(obj_func=bayesian_likelihood_func), - parallel=parallel, dbname='RosenMCMC', dbformat=dbformat, sim_timeout=timeout) -sampler.sample(rep) -results.append(sampler.getdata()) - - -sampler=spotpy.algorithms.demcz(spot_setup(obj_func = bayesian_likelihood_func), - parallel=parallel, dbname='RosenDEMCz', dbformat=dbformat, sim_timeout=timeout) -sampler.sample(rep,nChains=4) -results.append(sampler.getdata()) - -sampler=spotpy.algorithms.dream(spot_setup(obj_func = bayesian_likelihood_func), - parallel=parallel, dbname='RosenDREAM', dbformat=dbformat, sim_timeout=timeout) -sampler.sample(rep,nChains=4) -results.append(sampler.getdata()) - - -print(results[0].dtype) # Check for Travis: Get the last sampled parameter for x -evaluation = spot_setup().evaluation() - -# Example how to plot the data -algorithms = ['mc','lhs','mle','sceua','sa','rope','abc','fscabc', 'mcmc', 'demcz', 'dream'] -spotpy.analyser.plot_parametertrace_algorithms(results, algorithms, spot_setup()) diff --git a/spotpy/examples/tutorial_sceua_hymod.py b/spotpy/examples/tutorial_sceua_hymod.py deleted file mode 100644 index abd2b380..00000000 --- a/spotpy/examples/tutorial_sceua_hymod.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright 2015 by Tobias Houska -This file is part of Statistical Parameter Estimation Tool (SPOTPY). - -:author: Tobias Houska - -This class holds example code how to use the dream algorithm -''' - -import numpy as np -import spotpy -from spotpy.examples.spot_setup_hymod_python import spot_setup -import matplotlib.pyplot as plt - - -if __name__ == "__main__": - parallel ='seq' # Runs everthing in sequential mode - np.random.seed(2000) # Makes the results reproduceable - - # Initialize the Hymod example - # In this case, we tell the setup which algorithm we want to use, so - # we can use this exmaple for different algorithms - spot_setup=spot_setup(spotpy.objectivefunctions.rmse) - - #Select number of maximum allowed repetitions - rep=5000 - sampler=spotpy.algorithms.sceua(spot_setup, dbname='SCEUA_hymod', dbformat='csv') - - #Start the sampler, one can specify ngs, kstop, peps and pcento id desired - sampler.sample(rep, ngs=7, kstop=3, peps=0.1, pcento=0.1) - - # Load the results gained with the sceua sampler, stored in SCEUA_hymod.csv - results = spotpy.analyser.load_csv_results('SCEUA_hymod') - - - #Plot how the objective function was minimized during sampling - fig= plt.figure(1,figsize=(9,6)) - plt.plot(results['like1']) - plt.show() - plt.ylabel('RMSE') - plt.xlabel('Iteration') - fig.savefig('SCEUA_objectivefunctiontrace.png',dpi=150) - - # Plot the best model run - #Find the run_id with the minimal objective function value - bestindex,bestobjf = spotpy.analyser.get_minlikeindex(results) - - # Select best model run - best_model_run = results[bestindex] - - #Filter results for simulation results - fields=[word for word in best_model_run.dtype.names if word.startswith('sim')] - best_simulation = list(best_model_run[fields]) - - fig= plt.figure(figsize=(9,6)) - ax = plt.subplot(1,1,1) - ax.plot(best_simulation,color='black',linestyle='solid', label='Best objf.='+str(bestobjf)) - ax.plot(spot_setup.evaluation(),'r.',markersize=3, label='Observation data') - plt.xlabel('Number of Observation Points') - plt.ylabel ('Discharge [l s-1]') - plt.legend(loc='upper right') - fig.savefig('SCEUA_best_modelrun.png',dpi=150) - diff --git a/spotpy/examples/tutorial_signatures.py b/spotpy/examples/tutorial_signatures.py deleted file mode 100644 index fb9f311d..00000000 --- a/spotpy/examples/tutorial_signatures.py +++ /dev/null @@ -1,147 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright (c) 2017 by Benjamin Manns -This file is part of Statistical Parameter Estimation Tool (SPOTPY). -:author: Benjamin Manns - -This code shows you, how to use the hydroligcal signatures. They can also be implemented in the def objective function. -''' - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from spotpy.examples.spot_setup_hymod import spot_setup -import spotpy.signatures as sig - -from pprint import pprint -print("INFO: For this example you need the folder >hymod< in the examples folder") - -spot_setup = spot_setup() -parameterset = spot_setup.parameters()['random'] -simulation = spot_setup.simulation(parameterset) -observation = spot_setup.evaluation() - - -# Beispiele zum einfachen Copy & Paste - -print(sig.getMeanFlow(simulation, observation,mode="get_signature")) -print(sig.getMeanFlow(simulation, observation,mode="get_raw_data")) -print(sig.getMeanFlow(simulation, observation,mode="calc_Dev")) - -print(sig.getMedianFlow(simulation, observation,mode="get_signature")) -print(sig.getMedianFlow(simulation, observation,mode="get_raw_data")) -print(sig.getMedianFlow(simulation, observation,mode="calc_Dev")) - -print(sig.getSkewness(simulation, observation,mode="get_signature")) -print(sig.getSkewness(simulation, observation,mode="get_raw_data")) -print(sig.getSkewness(simulation, observation,mode="calc_Dev")) - -print(sig.getCoeffVariation(simulation, observation,mode="get_signature")) -print(sig.getCoeffVariation(simulation, observation,mode="get_raw_data")) -print(sig.getCoeffVariation(simulation, observation,mode="calc_Dev")) - -print(sig.getQ001(simulation, observation,mode="get_signature")) -print(sig.getQ001(simulation, observation,mode="get_raw_data")) -print(sig.getQ001(simulation, observation,mode="calc_Dev")) - -print(sig.getQ01(simulation, observation,mode="get_signature")) -print(sig.getQ01(simulation, observation,mode="get_raw_data")) -print(sig.getQ01(simulation, observation,mode="calc_Dev")) - -print(sig.getQ1(simulation, observation,mode="get_signature")) -print(sig.getQ1(simulation, observation,mode="get_raw_data")) -print(sig.getQ1(simulation, observation,mode="calc_Dev")) - -print(sig.getQ5(simulation, observation,mode="get_signature")) -print(sig.getQ5(simulation, observation,mode="get_raw_data")) -print(sig.getQ5(simulation, observation,mode="calc_Dev")) - -print(sig.getQ10(simulation, observation,mode="get_signature")) -print(sig.getQ10(simulation, observation,mode="get_raw_data")) -print(sig.getQ10(simulation, observation,mode="calc_Dev")) - -print(sig.getQ20(simulation, observation,mode="get_signature")) -print(sig.getQ20(simulation, observation,mode="get_raw_data")) -print(sig.getQ20(simulation, observation,mode="calc_Dev")) - -print(sig.getQ85(simulation, observation,mode="get_signature")) -print(sig.getQ85(simulation, observation,mode="get_raw_data")) -print(sig.getQ85(simulation, observation,mode="calc_Dev")) - -print(sig.getQ95(simulation, observation,mode="get_signature")) -print(sig.getQ95(simulation, observation,mode="get_raw_data")) -print(sig.getQ95(simulation, observation,mode="calc_Dev")) - -print(sig.getQ99(simulation, observation,mode="get_signature")) -print(sig.getQ99(simulation, observation,mode="get_raw_data")) -print(sig.getQ99(simulation, observation,mode="calc_Dev")) - -print(sig.getSlopeFDC(simulation, observation,mode="get_signature")) -print(sig.getSlopeFDC(simulation, observation,mode="get_raw_data")) -print(sig.getSlopeFDC(simulation, observation,mode="calc_Dev")) - -try: - import pandas as pd - import matplotlib.pyplot as plt - timespanlen = simulation.__len__() - ddd = pd.date_range("2015-01-01 11:00", freq="5min",periods=timespanlen) - dd_daily = pd.date_range("2015-05-01", periods=timespanlen) - - print(sig.getAverageFloodOverflowPerSection(simulation, observation,mode="get_signature", datetime_series=dd_daily,threshold_value=1)) - print(sig.getAverageFloodOverflowPerSection(simulation, observation,mode="get_raw_data", datetime_series=dd_daily,threshold_value=1)) - print(sig.getAverageFloodOverflowPerSection(simulation, observation,mode="calc_Dev", datetime_series=dd_daily,threshold_value=1)) - - print(sig.getAverageFloodFrequencyPerSection(simulation, observation,datetime_series=dd_daily,threshold_value=1,mode="get_signature")) - print(sig.getAverageFloodFrequencyPerSection(simulation, observation,datetime_series=dd_daily,threshold_value=1,mode="get_raw_data")) - - # If you want to plot the raw data, simple do: - vals = sig.getAverageFloodFrequencyPerSection(simulation, observation,datetime_series=dd_daily,threshold_value=1,mode="get_raw_data") - vals.plot() - plt.show() - - print(sig.getAverageFloodFrequencyPerSection(simulation, observation,datetime_series=dd_daily,threshold_value=1,mode="calc_Dev")) - - print(sig.getAverageFloodDuration(simulation, observation,datetime_series=dd_daily,threshold_value=3,mode="get_signature")) - print(sig.getAverageFloodDuration(simulation, observation,datetime_series=dd_daily,threshold_value=3,mode="get_raw_data")) - print(sig.getAverageFloodDuration(simulation, observation,datetime_series=dd_daily,threshold_value=3,mode="calc_Dev")) - - print(sig.getAverageBaseflowUnderflowPerSection(simulation, observation,datetime_series=dd_daily,threshold_value=4,mode="get_signature")) - print(sig.getAverageBaseflowUnderflowPerSection(simulation, observation,datetime_series=dd_daily,threshold_value=4,mode="get_raw_data")) - print(sig.getAverageBaseflowUnderflowPerSection(simulation, observation,datetime_series=dd_daily,threshold_value=4,mode="calc_Dev")) - - print(sig.getAverageBaseflowFrequencyPerSection(simulation, observation,datetime_series=dd_daily,threshold_value=3,mode="get_signature")) - print(sig.getAverageBaseflowFrequencyPerSection(simulation, observation,datetime_series=ddd,threshold_value=5,mode="get_raw_data")) - print(sig.getAverageBaseflowFrequencyPerSection(simulation, observation,datetime_series=dd_daily,threshold_value=3,mode="calc_Dev")) - - print(sig.getAverageBaseflowDuration(simulation, observation, datetime_series=dd_daily, threshold_value=3,mode="get_signature")) - print(sig.getAverageBaseflowDuration(simulation, observation, datetime_series=ddd, threshold_value=5,mode="get_raw_data")) - print(sig.getAverageBaseflowDuration(simulation, observation, datetime_series=dd_daily, threshold_value=3,mode="calc_Dev")) - - - - print(sig.getFloodFrequency(simulation, observation,datetime_series=pd.date_range("2015-05-01", periods=timespanlen),threshold_value=3,mode="get_signature")) - print(sig.getFloodFrequency(simulation, observation,datetime_series=pd.date_range("2015-05-01", periods=timespanlen),threshold_value=3,mode="get_raw_data")) - print(sig.getFloodFrequency(simulation, observation,datetime_series=pd.date_range("2015-05-01", periods=timespanlen),threshold_value=3,mode="calc_Dev")) - - print(sig.getBaseflowFrequency(simulation, observation,datetime_series=pd.date_range("2015-05-01", periods=timespanlen),threshold_value=3,mode="get_signature")) - print(sig.getBaseflowFrequency(simulation, observation,datetime_series=pd.date_range("2015-05-01", freq="5min", periods=timespanlen),threshold_value=3,mode="get_raw_data")) - print(sig.getBaseflowFrequency(simulation, observation,datetime_series=pd.date_range("2015-05-01", periods=timespanlen),threshold_value=3,mode="calc_Dev")) - - print(sig.getLowFlowVar(simulation, observation, datetime_series=pd.date_range("2015-05-01", periods=timespanlen),mode="get_signature")) - print(sig.getLowFlowVar(simulation, observation, datetime_series=pd.date_range("2015-05-01", periods=timespanlen),mode="get_raw_data")) - print(sig.getLowFlowVar(simulation, observation, datetime_series=pd.date_range("2015-05-01", periods=timespanlen),mode="calc_Dev")) - - print(sig.getHighFlowVar(simulation, observation, datetime_series=pd.date_range("2015-05-01", periods=timespanlen),mode="get_signature")) - print(sig.getHighFlowVar(simulation, observation, datetime_series=pd.date_range("2015-05-01", periods=timespanlen),mode="get_raw_data")) - print(sig.getHighFlowVar(simulation, observation, datetime_series=pd.date_range("2015-05-01", periods=timespanlen),mode="calc_Dev")) - - print(sig.getBaseflowIndex(simulation, observation, datetime_series=pd.date_range("2015-05-01", periods=timespanlen),mode="get_signature")) - print(sig.getBaseflowIndex(simulation, observation, datetime_series=pd.date_range("2015-05-01", periods=timespanlen),mode="get_raw_data")) - print(sig.getBaseflowIndex(simulation, observation, datetime_series=pd.date_range("2015-05-01", periods=timespanlen),mode="calc_Dev")) - -except ImportError: - print('Please install Pandas to use these signature functions') - - - diff --git a/src/spotpy/__init__.py b/src/spotpy/__init__.py new file mode 100644 index 00000000..22b920e6 --- /dev/null +++ b/src/spotpy/__init__.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +""" +Copyright (c) 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +:paper: Houska, T., Kraft, P., Chamorro-Chavez, A. and Breuer, L.: +SPOTting Model Parameters Using a Ready-Made Python Package, +PLoS ONE, 10(12), e0145180, doi:10.1371/journal.pone.0145180, 2015. + +This package enables the comprehensive use of different Bayesian and Heuristic calibration +techniques in one Framework. It comes along with an algorithms folder for the +sampling and an analyser class for the plotting of results by the sampling. + +:dependencies: - Numpy >1.14.5 (http://www.numpy.org/) + - Scipy >1.5 (https://pypi.org/project/scipy/) + - Pandas >1.0 (optional) (http://pandas.pydata.org/) + - Matplotlib >3.0 (optional) (http://matplotlib.org/) + - CMF (optional) (http://fb09-pasig.umwelt.uni-giessen.de:8000/) + - mpi4py (optional) (http://mpi4py.scipy.org/) + - pathos (optional) (https://pypi.python.org/pypi/pathos/) + - sqlite3 (optional) (https://pypi.python.org/pypi/sqlite3/) + - numba (optional) (https://pypi.python.org/pypi/numba/) + + :help: For specific questions, try to use the documentation website at: + https://spotpy.readthedocs.io/en/latest/ + +For general things about parameter optimization techniques have a look at: +https://github.com/CamDavidsonPilon/Probabilistic-Programming-and-Bayesian-Methods-for-Hackers/tree/master/ + +Please cite our paper, if you are using SPOTPY. +""" +from . import algorithms # Contains all the different algorithms implemented in SPOTPY +from . import ( + analyser, # Contains some examples to analyse the results of the different algorithms +) +from . import ( + database, # Writes the results of the sampler in a user defined output file +) +from . import ( + describe, # Contains some helper functions to describe samplers and set-ups +) +from . import examples # Contains tutorials how to use SPOTPY +from . import ( + likelihoods, # Quantifies goodness of fit between simulation and evaluation data with likelihood functions +) +from . import ( + objectivefunctions, # Quantifies goodness of fit between simulation and evaluation data with objective functions +) +from . import ( + parameter, # Contains different distributions to describe the prior information for every model parameter +) +from .hydrology import ( + signatures, # Quantifies goodness of fit between simulation and evaluation data with hydrological signatures +) + +try: + from ._version import __version__ +except ModuleNotFoundError: # pragma: no cover + # package is not installed + __version__ = "0.0.0.dev0" diff --git a/src/spotpy/algorithms/__init__.py b/src/spotpy/algorithms/__init__.py new file mode 100644 index 00000000..427b32a5 --- /dev/null +++ b/src/spotpy/algorithms/__init__.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Copyright (c) 2015 by Tobias Houska + +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +:paper: Houska, T., Kraft, P., Chamorro-Chavez, A. and Breuer, L.: +SPOTting Model Parameters Using a Ready-Made Python Package, +PLoS ONE, 10(12), e0145180, doi:10.1371/journal.pone.0145180, 2015. + +Imports the different algorithms from this package. +To reduce dependencies, one may select here just the needed algorithm. +""" + + +from ._algorithm import _algorithm +from .abc import abc # Artificial Bee Colony +from .dds import dds # Dynamically Dimensioned Search algorithm +from .demcz import demcz # Differential Evolution Markov Chain +from .dream import dream # DiffeRential Evolution Adaptive Metropolis +from .fast import fast # Fourier Amplitude Sensitivity Test +from .fscabc import fscabc # Fitness Scaling Artificial Bee Colony +from .lhs import lhs # Latin Hypercube Sampling +from .list_sampler import list_sampler # Samples from given spotpy database +from .mc import mc # Monte Carlo +from .mcmc import mcmc # Metropolis Markov Chain Monte Carlo +from .mle import mle # Maximum Likelihood Estimation +from .nsgaii import ( + NSGAII, # A Fast and Elitist Multiobjective Genetic Algorithm: NSGA-II +) +from .padds import padds # Pareto Archived - Dynamicallly Dimensioned Search algorithm +from .rope import rope # RObust Parameter Estimation +from .sa import sa # Simulated annealing +from .sceua import sceua # Shuffled Complex Evolution diff --git a/spotpy/algorithms/_algorithm.py b/src/spotpy/algorithms/_algorithm.py similarity index 61% rename from spotpy/algorithms/_algorithm.py rename to src/spotpy/algorithms/_algorithm.py index 5e59186c..22890910 100644 --- a/spotpy/algorithms/_algorithm.py +++ b/src/spotpy/algorithms/_algorithm.py @@ -1,21 +1,17 @@ -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska This file holds the standards for every algorithm. -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from spotpy import database -from spotpy import parameter -import numpy as np -import time -import threading +""" import random +import threading +import time + +import numpy as np +from spotpy import database, parameter try: from queue import Queue @@ -30,7 +26,7 @@ class _RunStatistic(object): """ - this class checks for each run if the objectivefunction got better and holds the + this class checks for each run if the objectivefunction got better and holds the best parameter set. Every _algorithm has an object of this class as status. Usage: @@ -39,27 +35,29 @@ class _RunStatistic(object): """ def __init__(self, repetitions, algorithm_name, optimization_direction, parnames): - self.optimization_direction = optimization_direction #grid, mazimize, minimize - print('Initializing the ',algorithm_name,' with ',repetitions,' repetitions') - if optimization_direction == 'minimize': + self.optimization_direction = optimization_direction # grid, mazimize, minimize + print( + "Initializing the ", algorithm_name, " with ", repetitions, " repetitions" + ) + if optimization_direction == "minimize": self.compare = self.minimizer - print('The objective function will be minimized') - if optimization_direction == 'maximize': + print("The objective function will be minimized") + if optimization_direction == "maximize": self.compare = self.maximizer - print('The objective function will be maximized') - if optimization_direction == 'grid': + print("The objective function will be maximized") + if optimization_direction == "grid": self.compare = self.grid self.rep = 0 self.parnames = parnames - self.parameters= len(parnames) - self.params_min = [np.nan]*self.parameters - self.params_max = [np.nan]*self.parameters + self.parameters = len(parnames) + self.params_min = [np.nan] * self.parameters + self.params_max = [np.nan] * self.parameters self.objectivefunction_min = 1e308 self.objectivefunction_max = -1e308 self.starttime = time.time() self.last_print = time.time() - + self.repetitions = repetitions self.stop = False @@ -81,17 +79,15 @@ def grid(self, objval, params): self.objectivefunction_max = objval self.params_max = list(params) - def __call__(self, objectivefunction, params, block_print=False): - self.rep+=1 - if type(objectivefunction) == type([]): #TODO: change to iterable + self.rep += 1 + if type(objectivefunction) == type([]): # TODO: change to iterable self.compare(objectivefunction[0], params) elif type(objectivefunction) == type(np.array([])): pass else: self.compare(objectivefunction, params) - if self.rep == self.repetitions: self.stop = True @@ -104,60 +100,83 @@ def print_status(self): # Refresh progressbar every two second if acttime - self.last_print >= 2: avg_time_per_run = (acttime - self.starttime) / (self.rep + 1) - timestr = time.strftime("%H:%M:%S", time.gmtime(round(avg_time_per_run * (self.repetitions - (self.rep + 1))))) - if self.optimization_direction == 'minimize': - text = '%i of %i, minimal objective function=%g, time remaining: %s' % ( - self.rep, self.repetitions, self.objectivefunction_min, timestr) - - if self.optimization_direction == 'maximize': - text = '%i of %i, maximal objective function=%g, time remaining: %s' % ( - self.rep, self.repetitions, self.objectivefunction_max, timestr) - - if self.optimization_direction == 'grid': - text = '%i of %i, min objf=%g, max objf=%g, time remaining: %s' % ( - self.rep, self.repetitions, self.objectivefunction_min, self.objectivefunction_max, timestr) + timestr = time.strftime( + "%H:%M:%S", + time.gmtime( + round(avg_time_per_run * (self.repetitions - (self.rep + 1))) + ), + ) + if self.optimization_direction == "minimize": + text = "%i of %i, minimal objective function=%g, time remaining: %s" % ( + self.rep, + self.repetitions, + self.objectivefunction_min, + timestr, + ) + + if self.optimization_direction == "maximize": + text = "%i of %i, maximal objective function=%g, time remaining: %s" % ( + self.rep, + self.repetitions, + self.objectivefunction_max, + timestr, + ) + + if self.optimization_direction == "grid": + text = "%i of %i, min objf=%g, max objf=%g, time remaining: %s" % ( + self.rep, + self.repetitions, + self.objectivefunction_min, + self.objectivefunction_max, + timestr, + ) print(text) self.last_print = time.time() def print_status_final(self): - print('\n*** Final SPOTPY summary ***') - print('Total Duration: ' + str(round((time.time() - self.starttime), 2)) + ' seconds') - print('Total Repetitions:', self.rep) - - if self.optimization_direction == 'minimize': - print('Minimal objective value: %g' % (self.objectivefunction_min)) - print('Corresponding parameter setting:') + print("\n*** Final SPOTPY summary ***") + print( + "Total Duration: " + + str(round((time.time() - self.starttime), 2)) + + " seconds" + ) + print("Total Repetitions:", self.rep) + + if self.optimization_direction == "minimize": + print("Minimal objective value: %g" % (self.objectivefunction_min)) + print("Corresponding parameter setting:") for i in range(self.parameters): - text = '%s: %g' % (self.parnames[i], self.params_min[i]) + text = "%s: %g" % (self.parnames[i], self.params_min[i]) print(text) - if self.optimization_direction == 'maximize': - print('Maximal objective value: %g' % (self.objectivefunction_max)) - print('Corresponding parameter setting:') + if self.optimization_direction == "maximize": + print("Maximal objective value: %g" % (self.objectivefunction_max)) + print("Corresponding parameter setting:") for i in range(self.parameters): - text = '%s: %g' % (self.parnames[i], self.params_max[i]) + text = "%s: %g" % (self.parnames[i], self.params_max[i]) print(text) - if self.optimization_direction == 'grid': - print('Minimal objective value: %g' % (self.objectivefunction_min)) - print('Corresponding parameter setting:') + if self.optimization_direction == "grid": + print("Minimal objective value: %g" % (self.objectivefunction_min)) + print("Corresponding parameter setting:") for i in range(self.parameters): - text = '%s: %g' % (self.parnames[i], self.params_min[i]) + text = "%s: %g" % (self.parnames[i], self.params_min[i]) print(text) - print('Maximal objective value: %g' % (self.objectivefunction_max)) - print('Corresponding parameter setting:') + print("Maximal objective value: %g" % (self.objectivefunction_max)) + print("Corresponding parameter setting:") for i in range(self.parameters): - text = '%s: %g' % (self.parnames[i], self.params_max[i]) + text = "%s: %g" % (self.parnames[i], self.params_max[i]) print(text) - print('******************************\n') - + print("******************************\n") def __repr__(self): - return 'Min objectivefunction: %g \n Max objectivefunction: %g' % ( - self.objectivefunction_min, self.objectivefunction_max) + return "Min objectivefunction: %g \n Max objectivefunction: %g" % ( + self.objectivefunction_min, + self.objectivefunction_max, + ) class _algorithm(object): @@ -167,24 +186,24 @@ class _algorithm(object): Input ---------- spot_setup: class - model: function - Should be callable with a parameter combination of the parameter-function + model: function + Should be callable with a parameter combination of the parameter-function and return an list of simulation results (as long as evaluation list) parameter: function - When called, it should return a random parameter combination. Which can + When called, it should return a random parameter combination. Which can be e.g. uniform or Gaussian - objectivefunction: function - Should return the objectivefunction for a given list of a model simulation and + objectivefunction: function + Should return the objectivefunction for a given list of a model simulation and observation. evaluation: function Should return the true values as return by the model. dbname: str - Name of the database where parameter, objectivefunction value and simulation + Name of the database where parameter, objectivefunction value and simulation results will be saved. dbformat: str ram: fast suited for short sampling time. no file will be created and results are saved in an array. - csv: A csv file will be created, which you can import afterwards. + csv: A csv file will be created, which you can import afterwards. parallel: str seq: Sequentiel sampling (default): Normal iterations on one core of your cpu. mpc: Multi processing: Iterations on all available cores on your (single) pc @@ -203,27 +222,43 @@ class _algorithm(object): the algorithms uses the number in random_state as seed for numpy. This way stochastic processes can be reproduced. """ - _unaccepted_parameter_types = (parameter.List, ) - - def __init__(self, spot_setup, dbname=None, dbformat=None, dbinit=True, - dbappend=False, parallel='seq', save_sim=True, breakpoint=None, - backup_every_rep=100, save_threshold=-np.inf, db_precision=np.float32, - sim_timeout=None, random_state=None, optimization_direction='grid', algorithm_name=''): + _unaccepted_parameter_types = (parameter.List,) + + def __init__( + self, + spot_setup, + dbname=None, + dbformat=None, + dbinit=True, + dbappend=False, + parallel="seq", + save_sim=True, + breakpoint=None, + backup_every_rep=100, + save_threshold=-np.inf, + db_precision=np.float32, + sim_timeout=None, + random_state=None, + optimization_direction="grid", + algorithm_name="", + ): # Initialize the user defined setup class self.setup = spot_setup - param_info = parameter.get_parameters_array(self.setup, unaccepted_parameter_types=self._unaccepted_parameter_types) - self.all_params = param_info['random'] + param_info = parameter.get_parameters_array( + self.setup, unaccepted_parameter_types=self._unaccepted_parameter_types + ) + self.all_params = param_info["random"] self.constant_positions = parameter.get_constant_indices(spot_setup) if self.constant_positions: self.non_constant_positions = [] for i, val in enumerate(self.all_params): if self.all_params[i] not in self.constant_positions: self.non_constant_positions.append(i) - else: - self.non_constant_positions = np.arange(0,len(self.all_params)) + else: + self.non_constant_positions = np.arange(0, len(self.all_params)) self.parameter = self.get_parameters - self.parnames = param_info['name'] + self.parnames = param_info["name"] self.algorithm_name = algorithm_name # Create a type to hold the parameter values using a namedtuple self.partype = parameter.ParameterSet(param_info) @@ -231,8 +266,8 @@ def __init__(self, spot_setup, dbname=None, dbformat=None, dbinit=True, self.evaluation = self.setup.evaluation() self.save_sim = save_sim self.optimization_direction = optimization_direction - self.dbname = dbname or 'customDb' - self.dbformat = dbformat or 'ram' + self.dbname = dbname or "customDb" + self.dbformat = dbformat or "ram" self.db_precision = db_precision self.breakpoint = breakpoint self.backup_every_rep = backup_every_rep @@ -241,50 +276,52 @@ def __init__(self, spot_setup, dbname=None, dbformat=None, dbinit=True, # 'dbappend' used to append to the existing data base, after restart self.dbinit = dbinit self.dbappend = dbappend - + # Set the random state - if random_state is None: #ToDo: Have to discuss if these 3 lines are neccessary. + if ( + random_state is None + ): # ToDo: Have to discuss if these 3 lines are neccessary. random_state = np.random.randint(low=0, high=2**30) - np.random.seed(random_state) #Both numpy.random and random or used in spotpy - random.seed(random_state) #Both numpy.random and random or used in spotpy - + np.random.seed(random_state) # Both numpy.random and random or used in spotpy + random.seed(random_state) # Both numpy.random and random or used in spotpy # If value is not None a timeout will set so that the simulation will break after sim_timeout seconds without return a value self.sim_timeout = sim_timeout self.save_threshold = save_threshold - - self._return_all_likes = False #allows multi-objective calibration if set to True, is set by the algorithm - - if breakpoint == 'read' or breakpoint == 'readandwrite': - print('Reading backupfile') + + self._return_all_likes = False # allows multi-objective calibration if set to True, is set by the algorithm + + if breakpoint == "read" or breakpoint == "readandwrite": + print("Reading backupfile") try: - open(self.dbname+'.break') + open(self.dbname + ".break") except FileNotFoundError: - print('Backupfile not found') + print("Backupfile not found") self.dbappend = True # Now a repeater (ForEach-object) is loaded # A repeater is a convinent wrapper to repeat tasks # We have the same interface for sequential and for parallel tasks - if parallel == 'seq': + if parallel == "seq": from spotpy.parallel.sequential import ForEach - elif parallel == 'mpi': + elif parallel == "mpi": from spotpy.parallel.mpi import ForEach # MPC is based on pathos mutiprocessing and uses ordered map, so results are given back in the order # as the parameters are - elif parallel == 'mpc': + elif parallel == "mpc": from spotpy.parallel.mproc import ForEach # UMPC is based on pathos mutiprocessing and uses unordered map, so results are given back in the order # as the subprocesses are finished which may speed up the whole simulation process but is not recommended if # objective functions do their calculation based on the order of the data because the order of the result is chaotic # and randomized - elif parallel == 'umpc': + elif parallel == "umpc": from spotpy.parallel.umproc import ForEach else: raise ValueError( - "'%s' is not a valid keyword for parallel processing" % parallel) + "'%s' is not a valid keyword for parallel processing" % parallel + ) # This is the repeater for the model runs. The simulate method does the work # If you need different tasks, the repeater can be pushed into a "phase" using the @@ -295,15 +332,16 @@ def __init__(self, spot_setup, dbname=None, dbformat=None, dbinit=True, # method "save" needs to know whether objective function result is list or float, default is float self.like_struct_typ = type(1.1) - + def __str__(self): - return '{type}({mtype}())->{dbname}'.format( + return "{type}({mtype}())->{dbname}".format( type=type(self).__name__, mtype=type(self.setup).__name__, - dbname=self.dbname) + dbname=self.dbname, + ) def __repr__(self): - return '{type}()'.format(type=type(self).__name__) + return "{type}()".format(type=type(self).__name__) def get_parameters(self): """ @@ -313,8 +351,9 @@ def get_parameters(self): return pars[self.non_constant_positions] def set_repetiton(self, repetitions): - self.status = _RunStatistic(repetitions, self.algorithm_name, - self.optimization_direction, self.parnames) + self.status = _RunStatistic( + repetitions, self.algorithm_name, self.optimization_direction, self.parnames + ) # In MPI, this command will do nothing on the master process # but the worker processes are going to wait for jobs. # Hence the workers will only receive parameters for the @@ -329,20 +368,26 @@ def final_call(self): pass self.status.print_status_final() - def _init_database(self, like, randompar, simulations): if self.dbinit: - print('Initialize database...') - - self.datawriter = database.get_datawriter(self.dbformat, - self.dbname, self.parnames, like, randompar, simulations, - save_sim=self.save_sim, dbappend=self.dbappend, - dbinit=self.dbinit, db_precision=self.db_precision, - setup=self.setup) + print("Initialize database...") + + self.datawriter = database.get_datawriter( + self.dbformat, + self.dbname, + self.parnames, + like, + randompar, + simulations, + save_sim=self.save_sim, + dbappend=self.dbappend, + dbinit=self.dbinit, + db_precision=self.db_precision, + setup=self.setup, + ) self.dbinit = False - def __is_list_type(self, data): if type(data) == type: return data == list or data == type(np.array([])) @@ -354,50 +399,74 @@ def save(self, like, randompar, simulations, chains=1): self._init_database(like, randompar, simulations) # Test if like and the save threshold are float/list and compare accordingly if self.__is_list_type(like) and self.__is_list_type(self.save_threshold): - if all(i > j for i, j in zip(like, self.save_threshold)): #Compares list/list + if all( + i > j for i, j in zip(like, self.save_threshold) + ): # Compares list/list self.datawriter.save(like, randompar, simulations, chains=chains) - if (not self.__is_list_type(like)) and (not self.__is_list_type(self.save_threshold)): - if like>self.save_threshold: #Compares float/float + if (not self.__is_list_type(like)) and ( + not self.__is_list_type(self.save_threshold) + ): + if like > self.save_threshold: # Compares float/float self.datawriter.save(like, randompar, simulations, chains=chains) if self.__is_list_type(like) and (not self.__is_list_type(self.save_threshold)): - if like[0]>self.save_threshold: #Compares list/float + if like[0] > self.save_threshold: # Compares list/float self.datawriter.save(like, randompar, simulations, chains=chains) - if (not self.__is_list_type(like)) and self.__is_list_type(self.save_threshold): #Compares float/list + if (not self.__is_list_type(like)) and self.__is_list_type( + self.save_threshold + ): # Compares float/list if (like > self.save_threshold).all: self.datawriter.save(like, randompar, simulations, chains=chains) def read_breakdata(self, dbname): - ''' Read data from a pickle file if a breakpoint is set. - Reason: In case of incomplete optimizations, old data can be restored. ''' + """Read data from a pickle file if a breakpoint is set. + Reason: In case of incomplete optimizations, old data can be restored.""" import pickle - with open(dbname+'.break', 'rb') as breakfile: - work,backuptime,repos,obmin,obmax,pmin,pmax=pickle.load(breakfile) - self.status.starttime=self.status.starttime-backuptime - self.status.rep=repos - self.status.objectivefunction_min=obmin - self.status.objectivefunction_max=obmax - self.status.params_min=pmin - self.status.params_max=pmax + + with open(dbname + ".break", "rb") as breakfile: + work, backuptime, repos, obmin, obmax, pmin, pmax = pickle.load(breakfile) + self.status.starttime = self.status.starttime - backuptime + self.status.rep = repos + self.status.objectivefunction_min = obmin + self.status.objectivefunction_max = obmax + self.status.params_min = pmin + self.status.params_max = pmax return work def write_breakdata(self, dbname, work): - ''' Write data to a pickle file if a breakpoint has been set.''' + """Write data to a pickle file if a breakpoint has been set.""" import pickle - work=(work,self.status.last_print-self.status.starttime,self.status.rep,self.status.objectivefunction_min,self.status.objectivefunction_max,self.status.params_min,self.status.params_max) - with open(str(dbname)+'.break', 'wb') as breakfile: + + work = ( + work, + self.status.last_print - self.status.starttime, + self.status.rep, + self.status.objectivefunction_min, + self.status.objectivefunction_max, + self.status.params_min, + self.status.params_max, + ) + with open(str(dbname) + ".break", "wb") as breakfile: pickle.dump(work, breakfile) def getdata(self): return self.datawriter.getdata() def update_params(self, params): - #Add potential Constant parameters + # Add potential Constant parameters self.all_params[self.non_constant_positions] = params return self.all_params - - - def postprocessing(self, rep, params, simulation, chains=1, save_run=True, negativlike=False, block_print=False): # TODO: rep not necessaray - + + def postprocessing( + self, + rep, + params, + simulation, + chains=1, + save_run=True, + negativlike=False, + block_print=False, + ): # TODO: rep not necessaray + params = self.update_params(params) if negativlike is True: like = -self.getfitness(simulation=simulation, params=params) @@ -408,7 +477,7 @@ def postprocessing(self, rep, params, simulation, chains=1, save_run=True, negat # This is needed as some algorithms just want to know the fitness, # before they actually save the run in a database (e.g. sce-ua) - self.status(like,params,block_print=block_print) + self.status(like, params, block_print=block_print) if save_run is True and simulation is not None: self.save(like, params, simulations=simulation, chains=chains) if self._return_all_likes: @@ -417,34 +486,41 @@ def postprocessing(self, rep, params, simulation, chains=1, save_run=True, negat try: iter(like) return like[0] - except TypeError: # Happens if iter(like) fails, i.e. if like is just one value + except TypeError: # Happens if iter(like) fails, i.e. if like is just one value return like - def getfitness(self, simulation, params): """ Calls the user defined spot_setup objectivefunction """ try: - #print('Using parameters in fitness function') - return self.setup.objectivefunction(evaluation=self.evaluation, simulation=simulation, params = (params,self.parnames)) + # print('Using parameters in fitness function') + return self.setup.objectivefunction( + evaluation=self.evaluation, + simulation=simulation, + params=(params, self.parnames), + ) + + except TypeError: # Happens if the user does not allow to pass parameter in the spot_setup.objectivefunction + # print('Not using parameters in fitness function') + return self.setup.objectivefunction( + evaluation=self.evaluation, simulation=simulation + ) - except TypeError: # Happens if the user does not allow to pass parameter in the spot_setup.objectivefunction - #print('Not using parameters in fitness function') - return self.setup.objectivefunction(evaluation=self.evaluation, simulation=simulation) - def simulate(self, id_params_tuple): """This is a simple wrapper of the model, returning the result together with the run id and the parameters. This is needed, because some parallel things can mix up the ordering of runs """ id, params = id_params_tuple - self.all_params[self.non_constant_positions] = params #TODO: List parameters are not updated if not accepted for the algorithm, we may have to warn/error if list is given + self.all_params[ + self.non_constant_positions + ] = params # TODO: List parameters are not updated if not accepted for the algorithm, we may have to warn/error if list is given all_params = self.all_params if self.sim_timeout: # we need a layer to fetch returned data from a threaded process into a queue. - def model_layer(q,all_params): + def model_layer(q, all_params): # Call self.model with a namedtuple instead of another sequence q.put(self.setup.simulation(self.partype(*all_params))) diff --git a/spotpy/algorithms/abc.py b/src/spotpy/algorithms/abc.py similarity index 70% rename from spotpy/algorithms/abc.py rename to src/spotpy/algorithms/abc.py index be0d4fba..f0f67b85 100644 --- a/spotpy/algorithms/abc.py +++ b/src/spotpy/algorithms/abc.py @@ -1,18 +1,16 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Patrick Lauer -''' +""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from . import _algorithm -import numpy as np import random +import numpy as np + +from . import _algorithm + class abc(_algorithm): """ @@ -54,12 +52,13 @@ def __init__(self, *args, **kwargs): * True: Simulation results will be saved * False: Simulation results will not be saved """ - kwargs['optimization_direction'] = 'maximize' - kwargs['algorithm_name'] = 'Artificial Bee Colony (ABC) algorithm' + kwargs["optimization_direction"] = "maximize" + kwargs["algorithm_name"] = "Artificial Bee Colony (ABC) algorithm" super(abc, self).__init__(*args, **kwargs) - - def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, ownlimit=False, limit=24): + def sample( + self, repetitions, eb=48, a=(1 / 10), peps=0.0001, ownlimit=False, limit=24 + ): """ @@ -79,45 +78,49 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, ownlimit=False, li sets the limit """ self.set_repetiton(repetitions) - print('Starting the ABC algotrithm with '+str(repetitions)+ ' repetitions...') + print( + "Starting the ABC algotrithm with " + str(repetitions) + " repetitions..." + ) # Initialize ABC parameters: - randompar = self.parameter()['random'] + randompar = self.parameter()["random"] self.nopt = randompar.size random.seed() if ownlimit == True: self.limit = limit else: self.limit = eb - lb, ub = self.parameter()['minbound'], self.parameter()['maxbound'] + lb, ub = self.parameter()["minbound"], self.parameter()["maxbound"] # Initialization work = [] icall = 0 gnrng = 1e100 # Calculate the objective function - param_generator = ( - (rep, self.parameter()['random']) for rep in range(eb)) + param_generator = ((rep, self.parameter()["random"]) for rep in range(eb)) for rep, randompar, simulations in self.repeat(param_generator): # Calculate fitness - like = self.postprocessing(rep, randompar, simulations, chains = 1, negativlike=True) + like = self.postprocessing( + rep, randompar, simulations, chains=1, negativlike=True + ) c = 0 p = 0 work.append([like, randompar, like, randompar, c, p]) - icall +=1 + icall += 1 if self.status.stop: - print('Stopping sampling') + print("Stopping sampling") break while icall < repetitions and gnrng > peps: psum = 0 - # Employed bee phase + # Employed bee phase # Generate new input parameters for i, val in enumerate(work): k = i while k == i: k = random.randint(0, (eb - 1)) j = random.randint(0, (self.nopt - 1)) - work[i][3][j] = work[i][1][j] + \ - random.uniform(-a, a) * (work[i][1][j] - work[k][1][j]) + work[i][3][j] = work[i][1][j] + random.uniform(-a, a) * ( + work[i][1][j] - work[k][1][j] + ) if work[i][3][j] < lb[j]: work[i][3][j] = lb[j] if work[i][3][j] > ub[j]: @@ -127,25 +130,27 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, ownlimit=False, li param_generator = ((rep, work[rep][3]) for rep in range(eb)) for rep, randompar, simulations in self.repeat(param_generator): # Calculate fitness - clike = self.postprocessing(icall+eb, randompar, simulations, chains = 2, negativlike=True) + clike = self.postprocessing( + icall + eb, randompar, simulations, chains=2, negativlike=True + ) if clike > work[rep][0]: work[rep][1] = work[rep][3] work[rep][0] = clike work[rep][4] = 0 else: - work[rep][4] = work[rep][4] + 1 + work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - print('Stopping samplig') - break # Probability distribution for roulette wheel selection + print("Stopping samplig") + break # Probability distribution for roulette wheel selection bn = [] for i, val in enumerate(work): psum = psum + (1 / work[i][0]) for i, val in enumerate(work): - work[i][5] = ((1 / work[i][0]) / psum) + work[i][5] = (1 / work[i][0]) / psum bn.append(work[i][5]) bounds = np.cumsum(bn) - # Onlooker bee phase + # Onlooker bee phase # Roulette wheel selection for i, val in enumerate(work): pn = random.uniform(0, 1) @@ -157,14 +162,16 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, ownlimit=False, li z = t break j = random.randint(0, (self.nopt - 1)) - # Generate new input parameters + # Generate new input parameters try: - work[i][3][j] = work[z][1][j] + \ - random.uniform(-a, a) * (work[z][1][j] - work[k][1][j]) + work[i][3][j] = work[z][1][j] + random.uniform(-a, a) * ( + work[z][1][j] - work[k][1][j] + ) except UnboundLocalError: - z=0 - work[i][3][j] = work[z][1][j] + \ - random.uniform(-a, a) * (work[z][1][j] - work[k][1][j]) + z = 0 + work[i][3][j] = work[z][1][j] + random.uniform(-a, a) * ( + work[z][1][j] - work[k][1][j] + ) if work[i][3][j] < lb[j]: work[i][3][j] = lb[j] if work[i][3][j] > ub[j]: @@ -173,39 +180,43 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, ownlimit=False, li param_generator = ((rep, work[rep][3]) for rep in range(eb)) for rep, randompar, simulations in self.repeat(param_generator): # Calculate fitness - clike = self.postprocessing(icall+eb, randompar, simulations, chains = 3, negativlike=True) + clike = self.postprocessing( + icall + eb, randompar, simulations, chains=3, negativlike=True + ) if clike > work[rep][0]: work[rep][1] = work[rep][3] work[rep][0] = clike work[rep][4] = 0 else: - work[rep][4] = work[rep][4] + 1 + work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - print('Stopping samplig') + print("Stopping samplig") break - # Scout bee phase + # Scout bee phase for i, val in enumerate(work): if work[i][4] >= self.limit: - work[i][1] = self.parameter()['random'] + work[i][1] = self.parameter()["random"] work[i][4] = 0 - t, work[i][0], simulations = self.simulate( - (icall, work[i][1])) - clike = self.postprocessing(icall+eb, randompar, simulations, chains = 4, negativlike=True) + t, work[i][0], simulations = self.simulate((icall, work[i][1])) + clike = self.postprocessing( + icall + eb, randompar, simulations, chains=4, negativlike=True + ) work[i][0] = clike icall += 1 if self.status.stop: - print('Stopping samplig') + print("Stopping samplig") break gnrng = -self.status.objectivefunction_max if icall >= repetitions: - print('*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT') - print('ON THE MAXIMUM NUMBER OF TRIALS ') + print("*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT") + print("ON THE MAXIMUM NUMBER OF TRIALS ") print(repetitions) - print('HAS BEEN EXCEEDED.') + print("HAS BEEN EXCEEDED.") if gnrng < peps: print( - 'THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE AT RUN') + "THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE AT RUN" + ) print(icall) self.final_call() diff --git a/spotpy/algorithms/dds.py b/src/spotpy/algorithms/dds.py similarity index 70% rename from spotpy/algorithms/dds.py rename to src/spotpy/algorithms/dds.py index 1cd368b5..fe5c31d0 100644 --- a/spotpy/algorithms/dds.py +++ b/src/spotpy/algorithms/dds.py @@ -1,12 +1,14 @@ import numpy as np -from . import _algorithm + from spotpy.parameter import ParameterSet +from . import _algorithm + class DDSGenerator: """ - This class is used by the DDS algorithm to generate a new sample of parameters based on the current one. - Current parameter are exchanged in `ParameterSet` objects. + This class is used by the DDS algorithm to generate a new sample of parameters based on the current one. + Current parameter are exchanged in `ParameterSet` objects. """ def __init__(self, np_random): @@ -117,7 +119,9 @@ def neigh_value_discrete(self, s, s_min, s_max, r): s_new = s_max s_new = np.round(s_new) # New value must be integer - if s_new == s: # pick a number between s_max and s_min by a Uniform distribution + if ( + s_new == s + ): # pick a number between s_max and s_min by a Uniform distribution sample = s_min - 1 + np.ceil((s_max - s_min) * self.np_random.rand()) if sample < s: s_new = sample @@ -144,24 +148,24 @@ def neigh_value_mixed(self, x_curr, r, j, x_min, x_max): class dds(_algorithm): """ - Implements the Dynamically dimensioned search algorithm for computationally efficient watershed model - calibration - by - Tolson, B. A. and C. A. Shoemaker (2007), Dynamically dimensioned search algorithm for computationally efficient - watershed model calibration, Water Resources Research, 43, W01413, 10.1029/2005WR004723. - Asadzadeh, M. and B. A. Tolson (2013), Pareto archived dynamically dimensioned search with hypervolume-based - selection for multi-objective optimization, Engineering Optimization. 10.1080/0305215X.2012.748046. + Implements the Dynamically dimensioned search algorithm for computationally efficient watershed model + calibration + by + Tolson, B. A. and C. A. Shoemaker (2007), Dynamically dimensioned search algorithm for computationally efficient + watershed model calibration, Water Resources Research, 43, W01413, 10.1029/2005WR004723. + Asadzadeh, M. and B. A. Tolson (2013), Pareto archived dynamically dimensioned search with hypervolume-based + selection for multi-objective optimization, Engineering Optimization. 10.1080/0305215X.2012.748046. - http://www.civil.uwaterloo.ca/btolson/software.aspx + http://www.civil.uwaterloo.ca/btolson/software.aspx - Method: - "The DDS algorithm is a novel and simple stochastic single-solution based heuristic global search - algorithm that was developed for the purpose of finding good global solutions - (as opposed to globally optimal solutions) within a specified maximum function (or model) evaluation limit." - (Page 3) + Method: + "The DDS algorithm is a novel and simple stochastic single-solution based heuristic global search + algorithm that was developed for the purpose of finding good global solutions + (as opposed to globally optimal solutions) within a specified maximum function (or model) evaluation limit." + (Page 3) - The DDS algorithm is a simple greedy algorithm, always using the best solution (min or max) from the current - point of view. This may not lead to the global optimization. + The DDS algorithm is a simple greedy algorithm, always using the best solution (min or max) from the current + point of view. This may not lead to the global optimization. """ @@ -206,13 +210,13 @@ def __init__(self, *args, **kwargs): self.r = kwargs.pop("r") except KeyError: self.r = 0.2 # default value - kwargs['optimization_direction'] = 'maximize' - kwargs['algorithm_name'] = 'Dynamically Dimensioned Search (DDS) algorithm' + kwargs["optimization_direction"] = "maximize" + kwargs["algorithm_name"] = "Dynamically Dimensioned Search (DDS) algorithm" super(dds, self).__init__(*args, **kwargs) self.np_random = np.random - #self.status.params_max = ParameterSet(self.parameter()) + # self.status.params_max = ParameterSet(self.parameter()) # self.generator_repetitions will be set in `sample` and is needed to generate a # generator which sends back actual parameter s_test @@ -230,7 +234,9 @@ def get_next_x_curr(self): """ # We need to shift position and length of the sampling process for rep in range(self.generator_repetitions): - yield rep, self.calculate_next_s_test(self.params_max, rep, self.generator_repetitions, self.r) + yield rep, self.calculate_next_s_test( + self.params_max, rep, self.generator_repetitions, self.r + ) def sample(self, repetitions, trials=1, x_initial=np.array([])): """ @@ -271,34 +277,51 @@ def sample(self, repetitions, trials=1, x_initial=np.array([])): debug_results = [] self.set_repetiton(repetitions) - self.min_bound, self.max_bound = self.parameter( - )['minbound'], self.parameter()['maxbound'] - print('Starting the DDS algotrithm with '+str(repetitions)+ ' repetitions...') - - number_of_parameters = self.status.parameters # number_of_parameters is the amount of parameters + self.min_bound, self.max_bound = ( + self.parameter()["minbound"], + self.parameter()["maxbound"], + ) + print( + "Starting the DDS algotrithm with " + str(repetitions) + " repetitions..." + ) + + number_of_parameters = ( + self.status.parameters + ) # number_of_parameters is the amount of parameters if len(x_initial) == 0: - initial_iterations = np.int(np.max([5, round(0.005 * repetitions)])) + initial_iterations = int(np.max([5, round(0.005 * repetitions)])) elif len(x_initial) != number_of_parameters: - raise ValueError("User specified 'x_initial' has not the same length as available parameters") + raise ValueError( + "User specified 'x_initial' has not the same length as available parameters" + ) else: initial_iterations = 1 x_initial = np.array(x_initial) - if not (np.all(x_initial <= self.max_bound) and np.all( - x_initial >= self.min_bound)): - raise ValueError("User specified 'x_initial' but the values are not within the parameter range") + if not ( + np.all(x_initial <= self.max_bound) + and np.all(x_initial >= self.min_bound) + ): + raise ValueError( + "User specified 'x_initial' but the values are not within the parameter range" + ) # Users can define trial runs in within "repetition" times the algorithm will be executed for trial in range(trials): - #objectivefunction_max = -1e308 + # objectivefunction_max = -1e308 params_max = x_initial # repitionno_best saves on which iteration the best parameter configuration has been found repitionno_best = initial_iterations # needed to initialize variable and avoid code failure when small # iterations - params_max, repetions_left, objectivefunction_max = self.calc_initial_para_configuration(initial_iterations, trial, - repetitions, x_initial) + ( + params_max, + repetions_left, + objectivefunction_max, + ) = self.calc_initial_para_configuration( + initial_iterations, trial, repetitions, x_initial + ) params_max = self.fix_status_params_format(params_max) - trial_best_value = list(params_max)#self.status.params_max.copy() - + trial_best_value = list(params_max) # self.status.params_max.copy() + # important to set this field `generator_repetitions` so that # method `get_next_s_test` can generate exact parameters self.generator_repetitions = repetions_left @@ -311,9 +334,20 @@ def sample(self, repetitions, trials=1, x_initial=np.array([])): self.params_max = list(x_curr) self.params_max = self.fix_status_params_format(self.params_max) - print('Best solution found has obj function value of ' + str(objectivefunction_max) + ' at ' - + str(repitionno_best) + '\n\n') - debug_results.append({"sbest": self.params_max, "trial_initial": trial_best_value,"objfunc_val": objectivefunction_max}) + print( + "Best solution found has obj function value of " + + str(objectivefunction_max) + + " at " + + str(repitionno_best) + + "\n\n" + ) + debug_results.append( + { + "sbest": self.params_max, + "trial_initial": trial_best_value, + "objfunc_val": objectivefunction_max, + } + ) self.final_call() return debug_results @@ -322,8 +356,10 @@ def fix_status_params_format(self, params_max): start_params.set_by_array([j for j in params_max]) return start_params - def calc_initial_para_configuration(self, initial_iterations, trial, repetitions, x_initial): - #max_bound, min_bound = self.status.params_max.maxbound, self.status.params_max.minbound + def calc_initial_para_configuration( + self, initial_iterations, trial, repetitions, x_initial + ): + # max_bound, min_bound = self.status.params_max.maxbound, self.status.params_max.minbound parameter_bound_range = self.max_bound - self.min_bound number_of_parameters = len(parameter_bound_range) discrete_flag = ParameterSet(self.parameter()).as_int @@ -334,34 +370,59 @@ def calc_initial_para_configuration(self, initial_iterations, trial, repetitions # by trying which randomized generated input matches best # initial_iterations is the number of function evaluations to initialize the DDS algorithm solution if initial_iterations > 1: - print('Finding best starting point for trial ' + str(trial + 1) + ' using ' + str( - initial_iterations) + ' random samples.') - repetions_left = repetitions - initial_iterations # use this to reduce number of fevals in DDS loop + print( + "Finding best starting point for trial " + + str(trial + 1) + + " using " + + str(initial_iterations) + + " random samples." + ) + repetions_left = ( + repetitions - initial_iterations + ) # use this to reduce number of fevals in DDS loop if repetions_left <= 0: - raise ValueError('# Initialization samples >= Max # function evaluations.') + raise ValueError( + "# Initialization samples >= Max # function evaluations." + ) starting_generator = ( - (rep, [self.np_random.randint(np.int(self.min_bound[j]), np.int(self.max_bound[j]) + 1) if - discrete_flag[j] else self.min_bound[j] + parameter_bound_range[j] * self.np_random.rand() - for j in - range(number_of_parameters)]) for rep in range(int(initial_iterations))) + ( + rep, + [ + self.np_random.randint( + int(self.min_bound[j]), int(self.max_bound[j]) + 1 + ) + if discrete_flag[j] + else self.min_bound[j] + + parameter_bound_range[j] * self.np_random.rand() + for j in range(number_of_parameters) + ], + ) + for rep in range(int(initial_iterations)) + ) for rep, x_curr, simulations in self.repeat(starting_generator): - like = self.postprocessing(rep, x_curr, simulations) # get obj function value + like = self.postprocessing( + rep, x_curr, simulations + ) # get obj function value # status setting update if like > objectivefunction_max: objectivefunction_max = like - params_max = list(x_curr) + params_max = list(x_curr) params_max = self.fix_status_params_format(params_max) else: # now initial_iterations=1, using a user supplied initial solution. Calculate obj func value. - repetions_left = repetitions - 1 # use this to reduce number of fevals in DDS loop - rep, x_test_param, simulations = self.simulate((0, x_initial)) # get from the inputs + repetions_left = ( + repetitions - 1 + ) # use this to reduce number of fevals in DDS loop + rep, x_test_param, simulations = self.simulate( + (0, x_initial) + ) # get from the inputs like = self.postprocessing(rep, x_test_param, simulations) if like > objectivefunction_max: - objectivefunction_max = like - params_max = list(x_test_param) - params_max = self.fix_status_params_format(params_max) + objectivefunction_max = like + params_max = list(x_test_param) + params_max = self.fix_status_params_format(params_max) return params_max, repetions_left, objectivefunction_max def calculate_next_s_test(self, previous_x_curr, rep, rep_limit, r): @@ -383,22 +444,35 @@ def calculate_next_s_test(self, previous_x_curr, rep, rep_limit, r): :return: next parameter set """ amount_params = len(previous_x_curr) - new_x_curr = previous_x_curr.copy() # define new_x_curr initially as current (previous_x_curr for greedy) + new_x_curr = ( + previous_x_curr.copy() + ) # define new_x_curr initially as current (previous_x_curr for greedy) randompar = self.np_random.rand(amount_params) probability_neighborhood = 1.0 - np.log(rep + 1) / np.log(rep_limit) dvn_count = 0 # counter for how many decision variables vary in neighbour for j in range(amount_params): - if randompar[j] < probability_neighborhood: # then j th DV selected to vary in neighbour + if ( + randompar[j] < probability_neighborhood + ): # then j th DV selected to vary in neighbour dvn_count = dvn_count + 1 - new_value = self.dds_generator.neigh_value_mixed(previous_x_curr, r, j, self.min_bound[j],self.max_bound[j]) + new_value = self.dds_generator.neigh_value_mixed( + previous_x_curr, r, j, self.min_bound[j], self.max_bound[j] + ) new_x_curr[j] = new_value # change relevant dec var value in x_curr if dvn_count == 0: # no DVs selected at random, so select ONE - dec_var = np.int(np.ceil(amount_params * self.np_random.rand())) - 1 - new_value = self.dds_generator.neigh_value_mixed(previous_x_curr, r, dec_var, self.min_bound[dec_var], - self.max_bound[dec_var]) - new_x_curr[dec_var] = new_value # change relevant decision variable value in s_test + dec_var = int(np.ceil(amount_params * self.np_random.rand())) - 1 + new_value = self.dds_generator.neigh_value_mixed( + previous_x_curr, + r, + dec_var, + self.min_bound[dec_var], + self.max_bound[dec_var], + ) + new_x_curr[ + dec_var + ] = new_value # change relevant decision variable value in s_test return new_x_curr diff --git a/spotpy/algorithms/demcz.py b/src/spotpy/algorithms/demcz.py similarity index 71% rename from spotpy/algorithms/demcz.py rename to src/spotpy/algorithms/demcz.py index d325613a..b3fef8fa 100644 --- a/spotpy/algorithms/demcz.py +++ b/src/spotpy/algorithms/demcz.py @@ -1,17 +1,14 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska -''' +""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from . import _algorithm import numpy as np +from . import _algorithm + class demcz(_algorithm): """ @@ -60,7 +57,6 @@ class demcz(_algorithm): WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. """ - def __init__(self, *args, **kwargs): """ Input @@ -77,25 +73,27 @@ def __init__(self, *args, **kwargs): observation. evaluation: function Should return the true values as return by the model. - + dbname: str * Name of the database where parameter, objectivefunction value and simulation results will be saved. - + dbformat: str * ram: fast suited for short sampling time. no file will be created and results are saved in an array. * csv: A csv file will be created, which you can import afterwards. - + parallel: str * seq: Sequentiel sampling (default): Normal iterations on one core of your cpu. * mpc: Multi processing: Iterations on all available cores on your cpu (recommended for windows os). * mpi: Message Passing Interface: Parallel computing on cluster pcs (recommended for unix os). - + save_sim: boolean * True: Simulation results will be saved * False: Simulationt results will not be saved - """ - kwargs['optimization_direction'] = 'maximize' - kwargs['algorithm_name'] = 'Differential Evolution Markov Chain (DE-MC) algorithm' + """ + kwargs["optimization_direction"] = "maximize" + kwargs[ + "algorithm_name" + ] = "Differential Evolution Markov Chain (DE-MC) algorithm" super(demcz, self).__init__(*args, **kwargs) def check_par_validity(self, par): @@ -106,20 +104,30 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - print('ERROR Bounds have not the same lenghts as Parameterarray') + print("ERROR Bounds have not the same lenghts as Parameterarray") return par - - def sample(self, repetitions, nChains=3, burnIn=100, thin=1, - convergenceCriteria=.8, variables_of_interest=None, - DEpairs=2, adaptationRate='auto', eps=5e-2, - mConvergence=True, mAccept=True): + + def sample( + self, + repetitions, + nChains=3, + burnIn=100, + thin=1, + convergenceCriteria=0.8, + variables_of_interest=None, + DEpairs=2, + adaptationRate="auto", + eps=5e-2, + mConvergence=True, + mAccept=True, + ): """ Samples from a posterior distribution using DREAM. Parameters ---------- repetitions : int - number of draws from the sample distribution to be returned + number of draws from the sample distribution to be returned nChains : int number of different chains to employ burnInSize : int @@ -133,17 +141,21 @@ def sample(self, repetitions, nChains=3, burnIn=100, thin=1, ------- None : None """ - - self.set_repetiton(repetitions) - print('Starting the DEMCz algotrithm with '+str(repetitions)+ ' repetitions...') - self.min_bound, self.max_bound = self.parameter( - )['minbound'], self.parameter()['maxbound'] + self.set_repetiton(repetitions) + print( + "Starting the DEMCz algotrithm with " + str(repetitions) + " repetitions..." + ) + + self.min_bound, self.max_bound = ( + self.parameter()["minbound"], + self.parameter()["maxbound"], + ) repetitions = int(repetitions / nChains) ndraw_max = repetitions * nChains maxChainDraws = int(ndraw_max / nChains) - dimensions = len(self.parameter()['random']) + dimensions = len(self.parameter()["random"]) # minbound,maxbound=self.find_min_max() # select variables if necessary @@ -154,15 +166,15 @@ def sample(self, repetitions, nChains=3, burnIn=100, thin=1, else: slices = [slice(None, None)] - # make a list of starting chains that at least span the dimension space # in this case it will be of size 2*dim nSeedIterations = max(int(np.ceil(dimensions * 2 / nChains)), 2) # init a simulationhistory instance - history = _SimulationHistory(maxChainDraws + nSeedIterations, - nChains, dimensions) - history.add_group('interest', slices) + history = _SimulationHistory( + maxChainDraws + nSeedIterations, nChains, dimensions + ) + history.add_group("interest", slices) ### BURN_IN burnInpar = [np.zeros((nChains, dimensions))] * nSeedIterations @@ -171,15 +183,15 @@ def sample(self, repetitions, nChains=3, burnIn=100, thin=1, simulationlist = [] old_like = np.empty(nChains) param_generator = ( - (rep, self.parameter()['random']) for rep in range(int(nChains))) - + (rep, self.parameter()["random"]) for rep in range(int(nChains)) + ) for rep, vector, simulations in self.repeat(param_generator): burnInpar[i][rep] = vector likelist = self.postprocessing(i, vector, simulations, chains=rep) simulationlist.append(simulations) self._logPs.append(likelist) - old_like[rep] = likelist + old_like[rep] = likelist burnInpar[i][rep] = vector if self.status.stop: break @@ -188,16 +200,16 @@ def sample(self, repetitions, nChains=3, burnIn=100, thin=1, gamma = None self.accepts_ratio = 0.000001 - + # initilize the convergence diagnostic object grConvergence = _GRConvergence() covConvergence = _CovarianceConvergence() - + # get the starting log objectivefunction and position for each of the # chains currentVectors = burnInpar[-1] currentLogPs = self._logPs[-1] - + # 2)now loop through and sample cur_iter = 0 accepts_ratio_weighting = 1 - np.exp(-1.0 / 30) @@ -206,114 +218,142 @@ def sample(self, repetitions, nChains=3, burnIn=100, thin=1, # 1) we have not drawn enough samples to satisfy the minimum number of iterations # 2) or any of the dimensions have not converged # 3) and we have not done more than the maximum number of iterations - + while cur_iter < maxChainDraws: print(cur_iter, burnIn) if cur_iter == burnIn: - print('starting') + print("starting") history.start_sampling() - + # every5th iteration allow a big jump if np.random.randint(5) == 0.0: gamma = np.array([1.0]) else: gamma = np.array([2.38 / np.sqrt(2 * DEpairs * dimensions)]) - + if cur_iter >= burnIn: proposalVectors = _dream_proposals( - currentVectors, history, dimensions, nChains, DEpairs, gamma, .05, eps) + currentVectors, + history, + dimensions, + nChains, + DEpairs, + gamma, + 0.05, + eps, + ) for i in range(len(proposalVectors)): - proposalVectors[i] = self.check_par_validity( - proposalVectors[i]) + proposalVectors[i] = self.check_par_validity(proposalVectors[i]) # print proposalVectors else: proposalVectors = [] for i in range(nChains): - proposalVectors.append(self.parameter()['random']) - proposalVectors[i] = self.check_par_validity( - proposalVectors[i]) - - # if self.bounds_ok(minbound,maxbound,proposalVectors,nChains): + proposalVectors.append(self.parameter()["random"]) + proposalVectors[i] = self.check_par_validity(proposalVectors[i]) + + # if self.bounds_ok(minbound,maxbound,proposalVectors,nChains): proposalLogPs = [] old_simulationlist = simulationlist old_likelist = self._logPs[-1] new_simulationlist = [] new_likelist = [] - + param_generator = ( - (rep, list(proposalVectors[rep])) for rep in range(int(nChains))) + (rep, list(proposalVectors[rep])) for rep in range(int(nChains)) + ) for rep, vector, simulations in self.repeat(param_generator): new_simulationlist.append(simulations) - like = self.postprocessing(cur_iter+nSeedIterations, list(vector), simulations, chains=rep) + like = self.postprocessing( + cur_iter + nSeedIterations, + list(vector), + simulations, + chains=rep, + ) self._logPs.append(like) new_likelist.append(like) proposalLogPs.append(like) if self.status.stop: cur_iter = maxChainDraws break - + if not self.status.stop: # apply the metrop decision to decide whether to accept or reject # each chain proposal decisions, acceptance = self._metropolis_hastings( - currentLogPs, proposalLogPs, nChains) - self._update_accepts_ratio(accepts_ratio_weighting, acceptance) + currentLogPs, proposalLogPs, nChains + ) + self._update_accepts_ratio(accepts_ratio_weighting, acceptance) # choose from list of possible choices if 1d_decision is True at # specific index, else use default choice # np.choose(1d_decision[:,None], (list of possible choices, default # choice) - save_likes=[] - save_pars=[] - save_sims=[] + save_likes = [] + save_pars = [] + save_sims = [] for curchain in range(nChains): if decisions[curchain]: - save_likes.append(float(new_likelist[curchain])) - old_like[curchain]=float(new_likelist[curchain]) - save_pars.append(proposalVectors[curchain]) - save_sims.append(new_simulationlist[curchain]) + save_likes.append(float(new_likelist[curchain])) + old_like[curchain] = float(new_likelist[curchain]) + save_pars.append(proposalVectors[curchain]) + save_sims.append(new_simulationlist[curchain]) else: - save_likes.append(old_like[curchain]) - save_pars.append(currentVectors[curchain]) - save_sims.append(old_simulationlist[curchain]) - + save_likes.append(old_like[curchain]) + save_pars.append(currentVectors[curchain]) + save_sims.append(old_simulationlist[curchain]) + currentVectors = np.choose( - decisions[:, np.newaxis], (currentVectors, proposalVectors)) + decisions[:, np.newaxis], (currentVectors, proposalVectors) + ) currentLogPs = np.choose(decisions, (currentLogPs, proposalLogPs)) - - simulationlist = [[new_simulationlist, old_simulationlist][ - int(x)][ix] for ix, x in enumerate(decisions)] - + + simulationlist = [ + [new_simulationlist, old_simulationlist][int(x)][ix] + for ix, x in enumerate(decisions) + ] + likelist = list( - np.choose(decisions[:, np.newaxis], (new_likelist, old_likelist))) - + np.choose( + decisions[:, np.newaxis], (new_likelist, old_likelist) + ) + ) + # we only want to recalculate convergence criteria when we are past # the burn in period - + if cur_iter % thin == 0: - + historyStartMovementRate = adaptationRate # try to adapt more when the acceptance rate is low and less # when it is high - if adaptationRate == 'auto': + if adaptationRate == "auto": historyStartMovementRate = min( - (.234 / self.accepts_ratio) * .5, .95) - - history.record( - currentVectors, currentLogPs, historyStartMovementRate, grConvergence=grConvergence.R) + (0.234 / self.accepts_ratio) * 0.5, 0.95 + ) - - if history.nsamples > 0 and cur_iter > lastRecalculation * 1.1 and history.nsequence_histories > dimensions: + history.record( + currentVectors, + currentLogPs, + historyStartMovementRate, + grConvergence=grConvergence.R, + ) + + if ( + history.nsamples > 0 + and cur_iter > lastRecalculation * 1.1 + and history.nsequence_histories > dimensions + ): lastRecalculation = cur_iter grConvergence.update(history) - covConvergence.update(history, 'all') - covConvergence.update(history, 'interest') + covConvergence.update(history, "all") + covConvergence.update(history, "interest") if all(grConvergence.R < convergenceCriteria): cur_iter = maxChainDraws print( - 'All chains fullfil the convergence criteria. Sampling stopped.') - cur_iter+=1 - + "All chains fullfil the convergence criteria. Sampling stopped." + ) + cur_iter += 1 + # 3) finalize # only make the second half of draws available because that's the only # part used by the convergence diagnostic @@ -322,23 +362,25 @@ def sample(self, repetitions, nChains=3, burnIn=100, thin=1, self.iter = cur_iter self.burnIn = burnIn self.R = grConvergence.R - text = 'Gelman Rubin R=' + str(self.R) + text = "Gelman Rubin R=" + str(self.R) print(text) self.status.rep = self.status.repetitions self.final_call() - def _update_accepts_ratio(self, weighting, acceptances): - self.accepts_ratio = weighting * \ - np.mean(acceptances) + (1 - weighting) * self.accepts_ratio + self.accepts_ratio = ( + weighting * np.mean(acceptances) + (1 - weighting) * self.accepts_ratio + ) - def _metropolis_hastings(self, currentLogPs, proposalLogPs, nChains, - jumpLogP=0, reverseJumpLogP=0): + def _metropolis_hastings( + self, currentLogPs, proposalLogPs, nChains, jumpLogP=0, reverseJumpLogP=0 + ): """ makes a decision about whether the proposed vector should be accepted """ - logMetropHastRatio = (np.array( - proposalLogPs) - np.array(currentLogPs)) # + (reverseJumpLogP - jumpLogP) + logMetropHastRatio = np.array(proposalLogPs) - np.array( + currentLogPs + ) # + (reverseJumpLogP - jumpLogP) decision = np.log(np.random.uniform(size=nChains)) < logMetropHastRatio # replace values which ecoures an overflow for e^x with 100 @@ -350,13 +392,11 @@ def _metropolis_hastings(self, currentLogPs, proposalLogPs, nChains, class _SimulationHistory(object): - group_indicies = {'all': slice(None, None)} + group_indicies = {"all": slice(None, None)} def __init__(self, maxChainDraws, nChains, dimensions): - self._combined_history = np.zeros( - (nChains * maxChainDraws, dimensions)) - self._sequence_histories = np.zeros( - (nChains, dimensions, maxChainDraws)) + self._combined_history = np.zeros((nChains * maxChainDraws, dimensions)) + self._sequence_histories = np.zeros((nChains, dimensions, maxChainDraws)) self._logPSequences = np.zeros((nChains, maxChainDraws)) self._logPHistory = np.zeros(nChains * maxChainDraws) self.r_hat = [] * dimensions @@ -380,17 +420,23 @@ def record(self, vectors, logPs, increment, grConvergence=None): self._record(vectors, logPs, increment, grConvergence) else: for i in range(vectors.shape[2]): - self._record( - vectors[:, :, i], logPs[:, i], increment, grConvergence) + self._record(vectors[:, :, i], logPs[:, i], increment, grConvergence) def _record(self, vectors, logPs, increment, grConvergence): self._sequence_histories[:, :, self.relevantHistoryEnd] = vectors - self._combined_history[(self.relevantHistoryEnd * self._nChains):( - self.relevantHistoryEnd * self._nChains + self._nChains), :] = vectors + self._combined_history[ + (self.relevantHistoryEnd * self._nChains) : ( + self.relevantHistoryEnd * self._nChains + self._nChains + ), + :, + ] = vectors self._logPSequences[:, self.relevantHistoryEnd] = logPs - self._logPHistory[(self.relevantHistoryEnd * self._nChains): - (self.relevantHistoryEnd * self._nChains + self._nChains)] = logPs + self._logPHistory[ + (self.relevantHistoryEnd * self._nChains) : ( + self.relevantHistoryEnd * self._nChains + self._nChains + ) + ] = logPs self.relevantHistoryEnd += 1 if np.isnan(increment): self.relevantHistoryStart += 0 @@ -398,16 +444,19 @@ def _record(self, vectors, logPs, increment, grConvergence): self.relevantHistoryStart += increment self.r_hat.append(grConvergence) - def start_sampling(self): self._sampling_start = self.relevantHistoryEnd @property def sequence_histories(self): - return self.group_sequence_histories('all') + return self.group_sequence_histories("all") def group_sequence_histories(self, name): - return self._sequence_histories[:, self.group_indicies[name], int(np.ceil(self.relevantHistoryStart)):self.relevantHistoryEnd] + return self._sequence_histories[ + :, + self.group_indicies[name], + int(np.ceil(self.relevantHistoryStart)) : self.relevantHistoryEnd, + ] @property def nsequence_histories(self): @@ -415,10 +464,15 @@ def nsequence_histories(self): @property def combined_history(self): - return self.group_combined_history('all') + return self.group_combined_history("all") def group_combined_history(self, name): - return self._combined_history[(int(np.ceil(self.relevantHistoryStart)) * self._nChains):(self.relevantHistoryEnd * self._nChains), self.group_indicies[name]] + return self._combined_history[ + (int(np.ceil(self.relevantHistoryStart)) * self._nChains) : ( + self.relevantHistoryEnd * self._nChains + ), + self.group_indicies[name], + ] @property def ncombined_history(self): @@ -426,13 +480,15 @@ def ncombined_history(self): @property def samples(self): - return self.group_samples('all') + return self.group_samples("all") def group_samples(self, name): if self._sampling_start < np.Inf: start = int( - max(np.ceil(self.relevantHistoryStart), self._sampling_start) * self._nChains) - end = (self.relevantHistoryEnd * self._nChains) + max(np.ceil(self.relevantHistoryStart), self._sampling_start) + * self._nChains + ) + end = self.relevantHistoryEnd * self._nChains else: start = 0 end = 0 @@ -444,7 +500,11 @@ def nsamples(self): @property def combined_history_logps(self): - return self._logPHistory[(np.ceil(self.relevantHistoryStart) * self._nChains):(self.relevantHistoryEnd * self._nChains)] + return self._logPHistory[ + (np.ceil(self.relevantHistoryStart) * self._nChains) : ( + self.relevantHistoryEnd * self._nChains + ) + ] def _random_no_replace(sampleSize, populationSize, numSamples): @@ -492,21 +552,21 @@ def rv(relevantHistory): covariance2 = np.cov(relevantHistory[midpoint:end, :].transpose()) _eigenvalues1, _eigenvectors1 = _eigen(covariance1) - basis1 = (np.sqrt(_eigenvalues1)[np.newaxis, :] * _eigenvectors1) + basis1 = np.sqrt(_eigenvalues1)[np.newaxis, :] * _eigenvectors1 _eigenvalues2, _eigenvectors2 = _eigen(covariance2) - basis2 = (np.sqrt(_eigenvalues2)[np.newaxis, :] * _eigenvectors2) + basis2 = np.sqrt(_eigenvalues2)[np.newaxis, :] * _eigenvectors2 # project the second basis onto the first basis try: projection = np.dot(np.linalg.inv(basis1), basis2) except np.linalg.linalg.LinAlgError: - projection = (np.array(basis1) * np.nan) - print('Exception happend!') + projection = np.array(basis1) * np.nan + print("Exception happend!") # find the releative size in each of the basis1 directions - return np.log(np.sum(projection**2, axis=0)**.5) + return np.log(np.sum(projection**2, axis=0) ** 0.5) def _eigen(a, n=-1): @@ -525,7 +585,9 @@ def _eigen(a, n=-1): return _eigenvalues[indicies[0:n]], _eigenvectors[:, indicies[0:n]] -def _dream_proposals(currentVectors, history, dimensions, nChains, DEpairs, gamma, jitter, eps): +def _dream_proposals( + currentVectors, history, dimensions, nChains, DEpairs, gamma, jitter, eps +): """ generates and returns proposal vectors given the current states """ @@ -540,18 +602,20 @@ def _dream_proposals(currentVectors, history, dimensions, nChains, DEpairs, gamm # makes sure we have already selected the current chain so it is not replaced # this ensures that the the two chosen chains cannot be the same as the # chain for which the jump is - chains += (chains >= currentIndex) + chains += chains >= currentIndex - chainDifferences = (np.sum(combined_history[chains[:, 0:DEpairs], :], axis=1) - - np.sum(combined_history[chains[:, DEpairs:(DEpairs * 2)], :], axis=1)) + chainDifferences = np.sum( + combined_history[chains[:, 0:DEpairs], :], axis=1 + ) - np.sum(combined_history[chains[:, DEpairs : (DEpairs * 2)], :], axis=1) e = np.random.normal(0, jitter, (nChains, dimensions)) # could replace eps with 1e-6 here E = np.random.normal(0, eps, (nChains, dimensions)) - proposalVectors = currentVectors + \ - (1 + e) * gamma[:, np.newaxis] * chainDifferences + E + proposalVectors = ( + currentVectors + (1 + e) * gamma[:, np.newaxis] * chainDifferences + E + ) return proposalVectors @@ -599,44 +663,48 @@ def update(self, history): betweenChainVariances = np.var(means, axis=0) * N - varEstimate = (1 - 1.0 / N) * withinChainVariances + \ - (1.0 / N) * betweenChainVariances + varEstimate = (1 - 1.0 / N) * withinChainVariances + ( + 1.0 / N + ) * betweenChainVariances if np.abs(withinChainVariances).all() > 0: self._R = np.sqrt(varEstimate / withinChainVariances) else: - self._R = np.random.uniform(0.1,0.9,withinChainVariances.__len__()) + self._R = np.random.uniform(0.1, 0.9, withinChainVariances.__len__()) # self._W can be a float or an array, in both cases I want to exclude 0.0 values try: if self._W != 0.0: - if (withinChainVariances / self._W).all() <=0.0: - self._WChange = np.abs(np.log(1.1) ** .5) + if (withinChainVariances / self._W).all() <= 0.0: + self._WChange = np.abs(np.log(1.1) ** 0.5) else: - self._WChange = np.abs(np.log(withinChainVariances / self._W)**.5) + self._WChange = np.abs( + np.log(withinChainVariances / self._W) ** 0.5 + ) else: - self._WChange = np.abs(np.log(1.1) ** .5) + self._WChange = np.abs(np.log(1.1) ** 0.5) except ValueError: if self._W.all() != 0.0: if (withinChainVariances / self._W).all() <= 0.0: - self._WChange = np.abs(np.log(1.1) ** .5) + self._WChange = np.abs(np.log(1.1) ** 0.5) else: # log of values less then 1 gives a negative number, where I replace these values to get square working tmp_WChDV = varEstimate / self._V tmp_WChDV[tmp_WChDV < 1.0] = np.random.uniform(1, 5, 1) - self._WChange = np.abs(np.log(tmp_WChDV) ** .5) + self._WChange = np.abs(np.log(tmp_WChDV) ** 0.5) else: - self._WChange = np.abs(np.log(1.1) ** .5) - + self._WChange = np.abs(np.log(1.1) ** 0.5) self._W = withinChainVariances - if (varEstimate / self._V).any() <= 0.0 or np.isnan(varEstimate / self._V).any(): - self._VChange = np.abs(np.log(1.1) ** .5) + if (varEstimate / self._V).any() <= 0.0 or np.isnan( + varEstimate / self._V + ).any(): + self._VChange = np.abs(np.log(1.1) ** 0.5) else: # log of values less then 1 gives a negative number, where I replace these values to get square working tmp_EsDV = varEstimate / self._V - tmp_EsDV[tmp_EsDV<1.0]=np.random.uniform(1,5,1) - self._VChange = np.abs(np.log(tmp_EsDV) ** .5) + tmp_EsDV[tmp_EsDV < 1.0] = np.random.uniform(1, 5, 1) + self._VChange = np.abs(np.log(tmp_EsDV) ** 0.5) self._V = varEstimate diff --git a/src/spotpy/algorithms/dream.py b/src/spotpy/algorithms/dream.py new file mode 100644 index 00000000..8305796f --- /dev/null +++ b/src/spotpy/algorithms/dream.py @@ -0,0 +1,469 @@ +# -*- coding: utf-8 -*- +""" +Copyright (c) 2018 by Tobias Houska +This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). +:author: Tobias Houska and Motjaba Sadegh +""" + +import random +import time + +import numpy as np + +from . import _algorithm + + +class dream(_algorithm): + """ + Implements the DiffeRential Evolution Adaptive Metropolis (DREAM) algorithhm + based on: + Vrugt, J. A. (2016) Markov chain Monte Carlo simulation using the DREAM software package. + """ + + def __init__(self, *args, **kwargs): + """ + Input + ---------- + spot_setup: class + model: function + Should be callable with a parameter combination of the parameter-function + and return an list of simulation results (as long as evaluation list) + parameter: function + When called, it should return a random parameter combination. Which can + be e.g. uniform or Gaussian + objectivefunction: function + Should return the objectivefunction for a given list of a model simulation and + observation. + evaluation: function + Should return the true values as return by the model. + + dbname: str + * Name of the database where parameter, objectivefunction value and simulation results will be saved. + + dbformat: str + * ram: fast suited for short sampling time. no file will be created and results are saved in an array. + * csv: A csv file will be created, which you can import afterwards. + + parallel: str + * seq: Sequentiel sampling (default): Normal iterations on one core of your cpu. + * mpi: Message Passing Interface: Parallel computing on cluster pcs (recommended for unix os). + + save_sim: boolean + * True: Simulation results will be saved + * False: Simulation results will not be saved + """ + + kwargs["optimization_direction"] = "maximize" + kwargs[ + "algorithm_name" + ] = "DiffeRential Evolution Adaptive Metropolis (DREAM) algorithm" + super(dream, self).__init__(*args, **kwargs) + + def check_par_validity_bound(self, par): + if len(par) == len(self.min_bound) and len(par) == len(self.max_bound): + for i in range(len(par)): + if par[i] < self.min_bound[i]: + par[i] = self.min_bound[i] + if par[i] > self.max_bound[i]: + par[i] = self.max_bound[i] + else: + print("ERROR: Bounds have not the same lenghts as Parameterarray") + return par + + def get_regular_startingpoint(self, nChains): + randompar = self.parameter()["random"] + for i in range(1000): + randompar = np.column_stack((randompar, self.parameter()["random"])) + startpoints = [] + for j in range(nChains): + startpoints.append( + np.percentile(randompar, (j + 1) / float(nChains + 1) * 100, axis=1) + ) # ,np.amax(randompar,axis=1) + startpoints = np.array(startpoints) + for k in range(len(randompar)): + random.shuffle(startpoints[:, k]) + return startpoints + + def check_par_validity_reflect(self, par): + if len(par) == len(self.min_bound) and len(par) == len(self.max_bound): + for i in range(len(par)): + if par[i] < self.min_bound[i]: + par[i] = self.min_bound[i] + (self.min_bound[i] - par[i]) + elif par[i] > self.max_bound[i]: + par[i] = self.max_bound[i] - (par[i] - self.max_bound[i]) + + # Postprocessing if reflecting jumped out of bounds + for i in range(len(par)): + if par[i] < self.min_bound[i]: + par[i] = self.min_bound[i] + if par[i] > self.max_bound[i]: + par[i] = self.max_bound[i] + else: + print("ERROR: Bounds have not the same lenghts as Parameterarray") + return par + + def _get_gamma(self, newN, nchain_pairs): + # N = Number of parameters + p = np.random.uniform(low=0, high=1) + if p >= 0.2: + # d_star is the dimension of subspace of parameters to jump + d_star = newN.count(True) + gamma = 2.38 / np.sqrt(2 * nchain_pairs * d_star) # /self.gammalevel + else: + gamma = 1 + return gamma + + def get_other_random_chains(self, cur_chain, nchain_pairs): + chain_pairs = [] + selectable_chain = list(range(self.nChains)) + selectable_chain.remove(cur_chain) + + for i in range(nchain_pairs): + pair_ith = random.sample(selectable_chain, 2) + chain_pairs.append(pair_ith) + for chain in pair_ith: + selectable_chain.remove(chain) + + return chain_pairs + + def get_new_proposal_vector(self, cur_chain, newN, c): + nchain_pairs = random.randint(1, self.delta) + gamma = self._get_gamma(newN, nchain_pairs) + chain_pairs = self.get_other_random_chains(cur_chain, nchain_pairs) + new_parameterset = [] + # position = self.chain_samples-1#self.nChains*self.chain_samples+self.chain_samples+cur_chain-1 + cur_par_set = list(self.bestpar[cur_chain][self.nChainruns[cur_chain] - 1]) + random_par_sets1 = [] # contain all random_par_set1 + random_par_sets2 = [] # contain all random_par_set2 + + for i in range(nchain_pairs): + random_chain1 = chain_pairs[i][0] + random_chain2 = chain_pairs[i][1] + random_par_set1 = list( + self.bestpar[random_chain1][self.nChainruns[random_chain1] - 1] + ) + random_par_set2 = list( + self.bestpar[random_chain2][self.nChainruns[random_chain2] - 1] + ) + random_par_sets1.append(random_par_set1) + random_par_sets2.append(random_par_set2) + + random_par_set1 = [ + sum(i) for i in zip(*random_par_sets1) + ] # sum all random_par_set1 + random_par_set2 = [ + sum(i) for i in zip(*random_par_sets2) + ] # sum all random_par_set2 + + for i in range(self.N): # Go through parameters + + if newN[i] == True: + lambda_ = np.random.uniform(-c, c) + new_parameterset.append( + cur_par_set[i] + + (1.0 + lambda_) + * gamma + * np.array(random_par_set1[i] - random_par_set2[i]) + + np.random.normal(0, self.eps) + ) + else: + new_parameterset.append(cur_par_set[i]) + + new_parameter = self.check_par_validity_reflect(new_parameterset) + # new_parameter=self.check_par_validity_bound(new_parameterset) + return new_parameter + + # new_par = np.random.normal(loc=old_par, scale=self.stepsizes) + # new_par = self.check_par_validity_reflect(new_par) + # return new_par + + def update_mcmc_status(self, par, like, sim, cur_chain): + self.bestpar[cur_chain][self.nChainruns[cur_chain]] = list(par) + self.bestlike[cur_chain] = like + self.bestsim[cur_chain] = list(sim) + + def get_r_hat(self, parameter_array): + """ + Based on some fancy mathlab code, it return an array [R_stat, MR_stat] + :param parameter_array: 3 dim array of parameter estimation sets + :type parameter_array: list + :return: [R_stat, MR_stat] + :rtype: list + """ + n, d, N = parameter_array.shape + + # Use only the last 50% of each chain (vrugt 2009), that means only the half of "d". Cause "d" ist the count + # of the repetition and we use the d/2 to d of those values which are already not NAN + whereIsNoNAN = np.logical_not(np.isnan(parameter_array)) + + alreadyToNum = np.sum(whereIsNoNAN[0, :, 0]) + + if alreadyToNum > 3: + parameter_array = parameter_array[ + :, int(np.floor(alreadyToNum / 2)) : alreadyToNum, : + ] + else: + # the later functions need some data to work right, so we use in this case 100% of NON NAN values + parameter_array = parameter_array[:, 0:alreadyToNum, :] + + # I made a big confusion with d, n and N, I figured it out by tests + + if n > 3: + + mean_chains = np.zeros((n, N)) + for i in range(n): + for j in range(N): + mean_chains[i, j] = np.nanmean(parameter_array[i, :, j]) + + B_uni = np.zeros(N) + for i in range(N): + B_uni[i] = d * np.nanvar( + mean_chains[:, i], ddof=1 + ) # make numpy Mathalab like: https://stackoverflow.com/a/27600240/5885054 + + var_chains = np.zeros((n, N)) + for i in range(n): + for j in range(N): + var_chains[i, j] = np.nanvar(parameter_array[i, :, j], ddof=1) + + W_uni = np.zeros(N) + for i in range(N): + W_uni[i] = np.mean(var_chains[:, i]) + + sigma2 = ((d - 1) / d) * W_uni + (1 / d) * B_uni + + whichW_UNIIsNull = W_uni == 0.0 + W_uni[whichW_UNIIsNull] = np.random.uniform(0.1, 1, 1) + + R_stat = np.sqrt( + (n + 1) / n * (np.divide(sigma2, W_uni)) - (d - 1) / (n * d) + ) + + # W_mult = 0 + # for ii in range(n): + # W_mult = W_mult + np.cov(np.nan_to_num(np.transpose(parameter_array[ii, :, :])), ddof=1) + # + # W_mult = W_mult / n + 2e-52 * np.eye(N) + # + # # Note that numpy.cov() considers its input data matrix to have observations in each column, + # # and variables in each row, so to get numpy.cov() to return what other packages do, + # # you have to pass the transpose of the data matrix to numpy.cov(). + # # https://stats.stackexchange.com/a/263508/168054 + # + # B_mult = np.cov(np.nan_to_num(np.transpose(mean_chains))) + 2e-52 * np.eye(N) # 2e-52 avoids problems with eig if var = 0 + # M = np.linalg.lstsq(W_mult, B_mult) + # R = np.max(np.abs(np.linalg.eigvals(M[0]))) + # MR_stat = np.sqrt((n + 1) / n * R + (d - 1) / d) + return R_stat # [R_stat, MR_stat] + + def sample( + self, + repetitions, + nChains=7, + nCr=3, + delta=3, + c=0.1, + eps=10e-6, + convergence_limit=1.2, + runs_after_convergence=100, + acceptance_test_option=6, + ): + self.set_repetiton(repetitions) + print( + "Starting the DREAM algotrithm with " + str(repetitions) + " repetitions..." + ) + if nChains < 2 * delta + 1: + print("Please use at least n=2*delta+1 chains!") + return None + # Prepare storing MCMC chain as array of arrays. + # define stepsize of MCMC. + self.repetitions = int(repetitions) + self.nChains = int(nChains) + self.delta = delta + # Ensure initialisation of chains and database + self.burnIn = self.nChains + self.stepsizes = self.parameter()["step"] # array of stepsizes + self.nr_of_pars = len(self.stepsizes) + self.gammalevel = 1 + starttime = time.time() + intervaltime = starttime + # Metropolis-Hastings iterations. + self.bestpar = np.array( + [[[np.nan] * self.nr_of_pars] * self.repetitions] * self.nChains + ) + # [0]->chain #[0][0]->parameter #[0][0][0]->repetitons + self.bestlike = [[-np.inf]] * self.nChains + self.bestsim = [[np.nan]] * self.nChains + self.accepted = np.zeros(self.nChains) + self.nChainruns = [0] * self.nChains + self.min_bound, self.max_bound = ( + self.parameter()["minbound"], + self.parameter()["maxbound"], + ) + + # firstcall = True + + print("Initialize ", self.nChains, " chain(s)...") + self.iter = 0 + # for i in range(10): + startpoints = self.get_regular_startingpoint(nChains) + # param_generator = ((curChain,list(self.parameter()['random'])) for curChain in range(int(self.nChains))) #TODO: Start with regular interval raster + param_generator = ( + (curChain, list(startpoints[curChain])) + for curChain in range(int(self.nChains)) + ) # TODO: Start with regular interval raster + for curChain, par, sim in self.repeat(param_generator): + like = self.postprocessing(self.iter, par, sim, chains=curChain) + self.update_mcmc_status(par, like, sim, curChain) + self.iter += 1 + self.nChainruns[curChain] += 1 + + print("Beginn of Random Walk") + convergence = False + # Walf through chains + self.r_hats = [] + self.eps = eps + self.CR = [] + for i in range(nCr): + self.CR.append((i + 1) / nCr) + self.N = len(self.parameter()["random"]) + nrN = 1 + newN = [True] * self.N + while self.iter < self.repetitions: + param_generator = ( + (curChain, self.get_new_proposal_vector(curChain, newN, c)) + for curChain in range(int(self.nChains)) + ) + for cChain, par, sim in self.repeat(param_generator): + pCr = np.random.randint(0, nCr) + ids = [] + for i in range(self.N): + ids.append(np.random.uniform(low=0, high=1)) + newN = [] + nrN = 0 + for i in range(len(ids)): + if ids[i] < self.CR[pCr]: + newN.append(True) + nrN += 1 + else: + newN.append(False) + if nrN == 0: + ids = [np.random.randint(0, self.N)] + nrN = 1 + like = self.postprocessing(self.iter, par, sim, chains=cChain) + + # set a option which type of comparision should be choose: + metro_opt = acceptance_test_option + + if metro_opt == 1: + logMetropHastRatio = like / self.bestlike[cChain] + + elif metro_opt == 2 or metro_opt == 4: + logMetropHastRatio = np.exp(like - self.bestlike[cChain]) + + elif metro_opt == 3: + # SSR probability evaluation + # nrN is defined in this loop so it will increase every step + logMetropHastRatio = (like / self.bestlike[cChain]) ** ( + -nrN * (1 + self._get_gamma(nrN)) / 2 + ) + + elif metro_opt == 5: + # SSR probability evaluation, but now weighted with mesurement error + # Note that measurement error is single number --> homoscedastic; variance can be taken out of sum sign + # SIGMA will be calculated from the orginal data + Sigma = np.mean(np.array(self.evaluation) * 0.1) + logMetropHastRatio = np.exp( + -0.5 * (-like + self.bestlike[cChain]) / (Sigma**2) + ) # signs are different because we write -SSR + + elif ( + metro_opt == 6 + ): # SSR probability evaluation, but now weighted with mesurement error + # Note that measurement error is a vector --> heteroscedastic; variance within sum sign -- see CompDensity.m + logMetropHastRatio = np.exp( + -0.5 * (-like + self.bestlike[cChain]) + ) # signs are different because we write -SSR + + u = np.random.uniform(low=0.0, high=1) + + if logMetropHastRatio > u: + self.update_mcmc_status(par, like, sim, cChain) + self.accepted[cChain] += 1 # monitor acceptance + + else: + self.update_mcmc_status( + self.bestpar[cChain][self.nChainruns[cChain] - 1], + self.bestlike[cChain], + self.bestsim[cChain], + cChain, + ) + + if self.status.stop: + self.iter = self.repetitions + print("Stopping samplig") + break + self.iter += 1 + self.nChainruns[cChain] += 1 + + r_hat = self.get_r_hat(self.bestpar) + self.r_hats.append(r_hat) + # Refresh progressbar every two seconds + acttime = time.time() + if ( + acttime - intervaltime >= 2 + and self.iter >= 2 + and self.nChainruns[-1] >= 3 + ): + text = "Acceptance rates [%] =" + str( + np.around( + (self.accepted) + / float(((self.iter - self.burnIn) / self.nChains)), + decimals=4, + ) + * 100 + ).strip("array([])") + print(text) + text = "Convergence rates =" + str( + np.around((r_hat), decimals=4) + ).strip("array([])") + print(text) + intervaltime = time.time() + + if ( + (np.array(r_hat) < convergence_limit).all() + and not convergence + and self.nChainruns[-1] >= 5 + ): + # Stop sampling + print("#############") + print( + "Convergence has been achieved after " + + str(self.iter) + + " of " + + str(self.repetitions) + + " runs! Finally, " + + str(runs_after_convergence) + + " runs will be additionally sampled to form the posterior distribution" + ) + print("#############") + self.repetitions = self.iter + runs_after_convergence + self.set_repetiton(self.repetitions) + # self.iter =self.repetitions - runs_after_convergence + convergence = True + + self.final_call() + + # try: + # self.datawriter.finalize() + # except AttributeError: # Happens if no database was assigned + # pass + # print('End of sampling') + # text = '%i of %i (best like=%g)' % ( + # self.status.rep, repetitions, self.status.objectivefunction) + # print(text) + # print('Best parameter set') + # print(self.status.params) + # text = 'Duration:' + str(round((acttime - starttime), 2)) + ' s' + # print(text) + return self.r_hats diff --git a/spotpy/algorithms/fast.py b/src/spotpy/algorithms/fast.py similarity index 77% rename from spotpy/algorithms/fast.py rename to src/spotpy/algorithms/fast.py index 9556e2ec..143e369a 100644 --- a/spotpy/algorithms/fast.py +++ b/src/spotpy/algorithms/fast.py @@ -1,27 +1,25 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska and the SALib team -''' +""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from . import _algorithm -import numpy as np import math +import numpy as np + +from . import _algorithm + class fast(_algorithm): - ''' + """ Fourier Amplitude Sensitivity Test (FAST) - + This class holds the Fourier Amplitude Sensitivity Test (FAST) based on Cukier et al. (1973) and Saltelli et al. (1999): Cukier, R. I., Fortuin, C. M., Shuler, K. E., Petschek, A. G. and Schaibly, J. H.: Study of the sensitivity of coupled reaction systems to uncertainties in rate coefficients. I Theory, J. Chem. Phys., 59(8), 3873–3878, 1973. - + Saltelli, A., Tarantola, S. and Chan, K. P.-S.: A Quantitative Model-Independent Method for Global Sensitivity Analysis of Model Output, Technometrics, 41(1), 39–56, doi:10.1080/00401706.1999.10485594, 1999. The presented code is based on SALib @@ -29,52 +27,52 @@ class fast(_algorithm): The Sensitivity Analysis Library is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. The Sensitivity Analysis Library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with the Sensitivity Analysis Library. If not, see http://www.gnu.org/licenses/. - ''' + """ - def __init__(self, *args, **kwargs): - ''' + def __init__(self, *args, **kwargs): + """ Input ---------- spot_setup: class - model: function - Should be callable with a parameter combination of the parameter-function + model: function + Should be callable with a parameter combination of the parameter-function and return an list of simulation results (as long as evaluation list) parameter: function - When called, it should return a random parameter combination. Which can + When called, it should return a random parameter combination. Which can be e.g. uniform or Gaussian - objectivefunction: function - Should return the objectivefunction for a given list of a model simulation and + objectivefunction: function + Should return the objectivefunction for a given list of a model simulation and observation. evaluation: function Should return the true values as return by the model. - + dbname: str * Name of the database where parameter, objectivefunction value and simulation results will be saved. - + dbformat: str * ram: fast suited for short sampling time. no file will be created and results are saved in an array. - * csv: A csv file will be created, which you can import afterwards. - + * csv: A csv file will be created, which you can import afterwards. + parallel: str * seq: Sequentiel sampling (default): Normal iterations on one core of your cpu. * mpi: Message Passing Interface: Parallel computing on cluster pcs (recommended for unix os). - + save_sim: boolean *True: Simulation results will be saved *False: Simulation results will not be saved - ''' - kwargs['algorithm_name'] = 'Fourier Amplitude Sensitivity Test (FAST)' + """ + kwargs["algorithm_name"] = "Fourier Amplitude Sensitivity Test (FAST)" super(fast, self).__init__(*args, **kwargs) def scale_samples(self, params, bounds): - ''' + """ Rescales samples in 0-to-1 range to arbitrary bounds. Arguments: bounds - list of lists of dimensions num_params-by-2 params - numpy array of dimensions num_params-by-N, where N is the number of samples - ''' + """ # Check bounds are legal (upper bound is greater than lower bound) b = np.array(bounds) lower_bounds = b[:, 0] @@ -87,11 +85,11 @@ def scale_samples(self, params, bounds): # argument for the numpy ufunctions # The calculation is equivalent to: # sample * (upper_bound - lower_bound) + lower_bound - np.add(np.multiply(params, - (upper_bounds - lower_bounds), - out=params), - lower_bounds, - out=params) + np.add( + np.multiply(params, (upper_bounds - lower_bounds), out=params), + lower_bounds, + out=params, + ) def matrix(self, bounds, N, M=4): D = len(bounds) @@ -107,7 +105,7 @@ def matrix(self, bounds, N, M=4): # Discretization of the frequency space, s s = (2 * math.pi / N) * np.arange(N) - #s = math.pi / 2.0 * (2 * np.arange(1,N+1) - N-1) / N + # s = math.pi / 2.0 * (2 * np.arange(1,N+1) - N-1) / N # Transformation to get points in the X space X = np.empty([N * D, D]) @@ -124,8 +122,7 @@ def matrix(self, bounds, N, M=4): phi = 2 * math.pi * np.random.rand() for j in range(D): - g = 0.5 + (1 / math.pi) * \ - np.arcsin(np.sin(omega2[j] * s + phi)) + g = 0.5 + (1 / math.pi) * np.arcsin(np.sin(omega2[j] * s + phi)) X[l, j] = g self.scale_samples(X, bounds) @@ -140,16 +137,24 @@ def analyze(self, problem, Y, D, parnames, M=4, print_to_console=False): N = int(Y.size / D) elif Y.size > D: N = int(Y.size / D) - rest = Y.size - N*D - print(""" - We can not use """ + str(rest) + """ samples which was generated - of totaly """ + str(Y.size) + """ - """) + rest = Y.size - N * D + print( + """ + We can not use """ + + str(rest) + + """ samples which was generated + of totaly """ + + str(Y.size) + + """ + """ + ) else: - print(""" - Error: Number of samples in model output file must be a multiple of D, + print( + """ + Error: Number of samples in model output file must be a multiple of D, where D is the number of parameters in your parameter file. - """) + """ + ) exit() # Recreate the vector omega used in the sampling @@ -165,14 +170,13 @@ def analyze(self, problem, Y, D, parnames, M=4, print_to_console=False): # Calculate and Output the First and Total Order Values if print_to_console: print("Parameter First Total") - Si = dict((k, [None] * D) for k in ['S1', 'ST']) + Si = dict((k, [None] * D) for k in ["S1", "ST"]) for i in range(D): l = np.arange(i * N, (i + 1) * N) - Si['S1'][i] = self.compute_first_order(Y[l], N, M, omega[0]) - Si['ST'][i] = self.compute_total_order(Y[l], N, omega[0]) + Si["S1"][i] = self.compute_first_order(Y[l], N, M, omega[0]) + Si["ST"][i] = self.compute_total_order(Y[l], N, omega[0]) if print_to_console: - print("%s %f %f" % - (parnames[i], Si['S1'][i], Si['ST'][i])) + print("%s %f %f" % (parnames[i], Si["S1"][i], Si["ST"][i])) return Si def compute_first_order(self, outputs, N, M, omega): @@ -187,7 +191,7 @@ def compute_total_order(self, outputs, N, omega): Sp = np.power(np.absolute(f[np.arange(1, int((N + 1) / 2))]) / N, 2) V = 2 * np.sum(Sp) Dt = 2 * sum(Sp[np.arange(int(omega / 2))]) - return (1 - Dt / V) + return 1 - Dt / V def sample(self, repetitions, M=4): """ @@ -195,18 +199,19 @@ def sample(self, repetitions, M=4): Input ---------- - repetitions: int - Maximum number of runs. + repetitions: int + Maximum number of runs. """ self.set_repetiton(repetitions) - print('Starting the FAST algotrithm with '+str(repetitions)+ ' repetitions...') - print('Creating FAST Matrix') + print( + "Starting the FAST algotrithm with " + str(repetitions) + " repetitions..." + ) + print("Creating FAST Matrix") # Get the names of the parameters to analyse - names = self.parameter()['name'] + names = self.parameter()["name"] # Get the minimum and maximum value for each parameter from the # distribution - parmin, parmax = self.parameter()['minbound'], self.parameter()[ - 'maxbound'] + parmin, parmax = self.parameter()["minbound"], self.parameter()["maxbound"] # Create an Matrix to store the parameter sets N = int(math.ceil(float(repetitions) / float(len(parmin)))) @@ -214,29 +219,29 @@ def sample(self, repetitions, M=4): for i in range(len(parmin)): bounds.append([parmin[i], parmax[i]]) Matrix = self.matrix(bounds, N, M=M) - lastbackup=0 - if self.breakpoint == 'read' or self.breakpoint == 'readandwrite': + lastbackup = 0 + if self.breakpoint == "read" or self.breakpoint == "readandwrite": data_frombreak = self.read_breakdata(self.dbname) rep = data_frombreak[0] Matrix = data_frombreak[1] - param_generator = ( - (rep, Matrix[rep]) for rep in range(len(Matrix))) + param_generator = ((rep, Matrix[rep]) for rep in range(len(Matrix))) for rep, randompar, simulations in self.repeat(param_generator): # Calculate the objective function self.postprocessing(rep, randompar, simulations) - if self.breakpoint == 'write' or self.breakpoint == 'readandwrite': - if rep >= lastbackup+self.backup_every_rep: + if self.breakpoint == "write" or self.breakpoint == "readandwrite": + if rep >= lastbackup + self.backup_every_rep: work = (rep, Matrix[rep:]) self.write_breakdata(self.dbname, work) lastbackup = rep self.final_call() - - try: + + try: data = self.datawriter.getdata() # this is likely to crash if database does not assign name 'like1' Si = self.analyze( - bounds, data['like1'], len(bounds), names, M=M, print_to_console=True) + bounds, data["like1"], len(bounds), names, M=M, print_to_console=True + ) except AttributeError: # Happens if no database was assigned pass diff --git a/spotpy/algorithms/fscabc.py b/src/spotpy/algorithms/fscabc.py similarity index 73% rename from spotpy/algorithms/fscabc.py rename to src/spotpy/algorithms/fscabc.py index 1538f760..03aec53a 100644 --- a/spotpy/algorithms/fscabc.py +++ b/src/spotpy/algorithms/fscabc.py @@ -1,30 +1,28 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Patrick Lauer -''' +""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from ._algorithm import _algorithm -import numpy as np import random +import numpy as np + +from ._algorithm import _algorithm + class fscabc(_algorithm): """ - This class holds the Fitness Scaled Chaotic Artificial Bee Colony (FSCABC) algorithm, + This class holds the Fitness Scaled Chaotic Artificial Bee Colony (FSCABC) algorithm, based on: - - Yudong Zhang, Lenan Wu, and Shuihua Wang (2011). Magnetic Resonance Brain Image - Classification by an Improved Artificial Bee Colony Algorithm. + + Yudong Zhang, Lenan Wu, and Shuihua Wang (2011). Magnetic Resonance Brain Image + Classification by an Improved Artificial Bee Colony Algorithm. Progress In Electromagnetics Research - - Yudong Zhang, Lenan Wu, and Shuihua Wang (2013). - UCAV Path Planning by Fitness-Scaling Adaptive Chaotic Particle Swarm Optimization. + + Yudong Zhang, Lenan Wu, and Shuihua Wang (2013). + UCAV Path Planning by Fitness-Scaling Adaptive Chaotic Particle Swarm Optimization. Mathematical Problems in Engineering """ @@ -60,15 +58,16 @@ def __init__(self, *args, **kwargs): * True: Simulation results will be saved * False: Simulation results will not be saved """ - kwargs['optimization_direction'] = 'maximize' - kwargs['algorithm_name'] = 'Fitness Scaled Chaotic Artificial Bee Colony (FSCABC) algorithm' + kwargs["optimization_direction"] = "maximize" + kwargs[ + "algorithm_name" + ] = "Fitness Scaled Chaotic Artificial Bee Colony (FSCABC) algorithm" super(fscabc, self).__init__(*args, **kwargs) def mutate(self, r): x = 4 * r * (1 - r) return x - def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None): """ Parameters @@ -80,60 +79,65 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None a: float mutation factor peps: float - convergence criterion + convergence criterion kpow: float exponent for power scaling method limit: int sets the limit for scout bee phase """ self.set_repetiton(repetitions) - print('Starting the FSCABC algotrithm with '+str(repetitions)+ ' repetitions...') + print( + "Starting the FSCABC algotrithm with " + + str(repetitions) + + " repetitions..." + ) # Initialize FSCABC parameters: parset = self.parameter() - randompar = parset['random'] - lb, ub = parset['minbound'], parset['maxbound'] + randompar = parset["random"] + lb, ub = parset["minbound"], parset["maxbound"] self.nopt = randompar.size random.seed() - lastbackup=0 + lastbackup = 0 if limit == None: - self.limit = int(eb/2) + self.limit = int(eb / 2) else: self.limit = int(limit) # Generate chaos r = 0.25 while r == 0.25 or r == 0.5 or r == 0.75: r = random.random() - + icall = 0 gnrng = 1e100 - if self.breakpoint == 'read' or self.breakpoint == 'readandwrite': + if self.breakpoint == "read" or self.breakpoint == "readandwrite": data_frombreak = self.read_breakdata(self.dbname) icall = data_frombreak[0] work = data_frombreak[1] gnrng = data_frombreak[2] r = data_frombreak[3] # Here database needs to be reinvoked - elif self.breakpoint is None or self.breakpoint == 'write': + elif self.breakpoint is None or self.breakpoint == "write": # Initialization work = [] # Calculate the objective function - param_generator = ( - (rep, self.parameter()['random']) for rep in range(eb)) + param_generator = ((rep, self.parameter()["random"]) for rep in range(eb)) for rep, randompar, simulations in self.repeat(param_generator): # Calculate fitness - like = self.postprocessing(rep, randompar, simulations, negativlike=True) + like = self.postprocessing( + rep, randompar, simulations, negativlike=True + ) c = 0 p = 0 # (fit_x,x,fit_v,v,limit,normalized fitness) work.append([like, randompar, like, randompar, c, p]) - icall +=1 + icall += 1 if self.status.stop: - #icall = repetitions - print('Stopping samplig') + # icall = repetitions + print("Stopping samplig") break - #Bee Phases + # Bee Phases while icall < repetitions and gnrng > peps: # Employed bee phase # Generate new input parameters @@ -142,23 +146,26 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None while k == i: k = random.randint(0, (eb - 1)) j = random.randint(0, (self.nopt - 1)) - work[i][3][j] = work[i][1][j] + \ - random.uniform(-a, a) * (work[i][1][j] - work[k][1][j]) + work[i][3][j] = work[i][1][j] + random.uniform(-a, a) * ( + work[i][1][j] - work[k][1][j] + ) if work[i][3][j] < lb[j]: work[i][3][j] = lb[j] if work[i][3][j] > ub[j]: work[i][3][j] = ub[j] - ''' + """ #Scout bee phase if work[i][4] >= self.limit: work[i][3]=self.parameter()['random'] work[i][4]=0 - ''' + """ # Calculate the objective function param_generator = ((rep, work[rep][3]) for rep in range(eb)) for rep, randompar, simulations in self.repeat(param_generator): # Calculate fitness - clike = self.postprocessing(icall, randompar, simulations, chains=1, negativlike=True) + clike = self.postprocessing( + icall, randompar, simulations, chains=1, negativlike=True + ) if clike > work[rep][0]: work[rep][1] = work[rep][3] work[rep][0] = clike @@ -167,9 +174,9 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - print('Stopping samplig') + print("Stopping samplig") break - + # Fitness scaling bn = [] csum = 0 @@ -181,7 +188,7 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None work[i][5] = work[i][5] / csum bn.append(work[i][5]) bounds = np.cumsum(bn) - # Onlooker bee phase + # Onlooker bee phase # Roulette wheel selection for i, val in enumerate(work): pn = random.uniform(0, 1) @@ -193,9 +200,10 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None z = t break j = random.randint(0, (self.nopt - 1)) - # Generate new input parameters - work[i][3][j] = work[z][1][j] + \ - random.uniform(-a, a) * (work[z][1][j] - work[k][1][j]) + # Generate new input parameters + work[i][3][j] = work[z][1][j] + random.uniform(-a, a) * ( + work[z][1][j] - work[k][1][j] + ) if work[i][3][j] < lb[j]: work[i][3][j] = lb[j] if work[i][3][j] > ub[j]: @@ -204,7 +212,9 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None param_generator = ((rep, work[rep][3]) for rep in range(eb)) for rep, randompar, simulations in self.repeat(param_generator): # Calculate fitness - clike = self.postprocessing(icall, randompar, simulations, chains=2, negativlike=True) + clike = self.postprocessing( + icall, randompar, simulations, chains=2, negativlike=True + ) if clike > work[rep][0]: work[rep][1] = work[rep][3] work[rep][0] = clike @@ -213,37 +223,42 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - print('Stopping samplig') + print("Stopping samplig") break - # Scout bee phase + # Scout bee phase for i, val in enumerate(work): if work[i][4] >= self.limit: for g, bound in enumerate(ub): r = self.mutate(r) work[i][1][g] = lb[g] + r * (ub[g] - lb[g]) work[i][4] = 0 - t, work[i][0], simulations = self.simulate( - (icall, work[i][1])) - clike = self.postprocessing(icall, randompar, simulations, chains=3, negativlike=True) + t, work[i][0], simulations = self.simulate((icall, work[i][1])) + clike = self.postprocessing( + icall, randompar, simulations, chains=3, negativlike=True + ) work[i][0] = clike icall += 1 if self.status.stop: - print('Stopping samplig') + print("Stopping samplig") break gnrng = -self.status.objectivefunction_max - if self.breakpoint == 'write' or self.breakpoint == 'readandwrite'\ - and icall >= lastbackup+self.backup_every_rep: + if ( + self.breakpoint == "write" + or self.breakpoint == "readandwrite" + and icall >= lastbackup + self.backup_every_rep + ): savework = (icall, work, gnrng, r) self.write_breakdata(self.dbname, savework) lastbackup = icall if icall >= repetitions: - print('*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT') - print('ON THE MAXIMUM NUMBER OF TRIALS ') + print("*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT") + print("ON THE MAXIMUM NUMBER OF TRIALS ") print(repetitions) - print('HAS BEEN EXCEEDED.') + print("HAS BEEN EXCEEDED.") if gnrng < peps: print( - 'THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE') + "THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE" + ) self.final_call() diff --git a/spotpy/algorithms/lhs.py b/src/spotpy/algorithms/lhs.py similarity index 83% rename from spotpy/algorithms/lhs.py rename to src/spotpy/algorithms/lhs.py index d9ac0535..998857fd 100644 --- a/spotpy/algorithms/lhs.py +++ b/src/spotpy/algorithms/lhs.py @@ -5,14 +5,13 @@ :author: Tobias Houska """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from . import _algorithm -import numpy as np import random +import numpy as np + +from . import _algorithm + + class lhs(_algorithm): """ The Latin Hypercube algorithm generates random parameters from their respective @@ -51,7 +50,7 @@ def __init__(self, *args, **kwargs): * True: Simulation results will be saved * False: Simulation results will not be saved """ - kwargs['algorithm_name'] = 'Latin Hypercube Sampling (LHS)' + kwargs["algorithm_name"] = "Latin Hypercube Sampling (LHS)" super(lhs, self).__init__(*args, **kwargs) def sample(self, repetitions): @@ -62,16 +61,17 @@ def sample(self, repetitions): maximum number of function evaluations allowed during optimization """ self.set_repetiton(repetitions) - print('Starting the LHS algotrithm with '+str(repetitions)+ ' repetitions...') - print('Creating LatinHyperCube Matrix') + print( + "Starting the LHS algotrithm with " + str(repetitions) + " repetitions..." + ) + print("Creating LatinHyperCube Matrix") # Get the names of the parameters to analyse - names = self.parameter()['name'] + names = self.parameter()["name"] # Define the jump size between the parameter segment = 1 / float(repetitions) # Get the minimum and maximum value for each parameter from the # distribution - parmin, parmax = self.parameter()['minbound'], self.parameter()[ - 'maxbound'] + parmin, parmax = self.parameter()["minbound"], self.parameter()["maxbound"] # Create an matrx to store the parameter sets matrix = np.empty((repetitions, len(parmin))) @@ -85,9 +85,8 @@ def sample(self, repetitions): random.shuffle(matrix[:, i]) # A generator that produces the parameters - param_generator = ((rep, matrix[rep]) - for rep in range(int(repetitions))) + param_generator = ((rep, matrix[rep]) for rep in range(int(repetitions))) for rep, randompar, simulations in self.repeat(param_generator): - # A function that calculates the fitness of the run and the manages the database + # A function that calculates the fitness of the run and the manages the database self.postprocessing(rep, randompar, simulations) self.final_call() diff --git a/spotpy/algorithms/list_sampler.py b/src/spotpy/algorithms/list_sampler.py similarity index 87% rename from spotpy/algorithms/list_sampler.py rename to src/spotpy/algorithms/list_sampler.py index 3138ea43..8f386e2a 100644 --- a/spotpy/algorithms/list_sampler.py +++ b/src/spotpy/algorithms/list_sampler.py @@ -1,16 +1,20 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska -''' -from . import _algorithm +""" from .. import analyser +from . import _algorithm + + class list_sampler(_algorithm): """ This class holds the List sampler, which samples from a given spotpy database """ + _excluded_parameter_classes = () + def __init__(self, *args, **kwargs): """ Input @@ -43,7 +47,7 @@ def __init__(self, *args, **kwargs): * True: Simulation results will be saved * False: Simulation results will not be saved """ - kwargs['algorithm_name'] = 'List Sampler' + kwargs["algorithm_name"] = "List Sampler" super(list_sampler, self).__init__(*args, **kwargs) def sample(self, repetitions=None): @@ -56,18 +60,19 @@ def sample(self, repetitions=None): maximum number of function evaluations allowed during sampling If not given number if iterations will be determined based on given list """ - + parameters = analyser.load_csv_parameter_results(self.dbname) - self.dbname=self.dbname+'list' + self.dbname = self.dbname + "list" if not repetitions: - repetitions=len(parameters) + repetitions = len(parameters) self.set_repetiton(repetitions) - + # Initialization - print('Starting the List sampler with '+str(repetitions)+ ' repetitions...') - param_generator = ((rep, list(parameters[rep])) - for rep in range(int(repetitions))) + print("Starting the List sampler with " + str(repetitions) + " repetitions...") + param_generator = ( + (rep, list(parameters[rep])) for rep in range(int(repetitions)) + ) for rep, randompar, simulations in self.repeat(param_generator): - # A function that calculates the fitness of the run and the manages the database + # A function that calculates the fitness of the run and the manages the database self.postprocessing(rep, list(randompar), simulations) self.final_call() diff --git a/spotpy/algorithms/mc.py b/src/spotpy/algorithms/mc.py similarity index 84% rename from spotpy/algorithms/mc.py rename to src/spotpy/algorithms/mc.py index a22e853b..171ac155 100644 --- a/spotpy/algorithms/mc.py +++ b/src/spotpy/algorithms/mc.py @@ -5,18 +5,17 @@ :author: Tobias Houska """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from . import _algorithm + class mc(_algorithm): """ The Monte Carlo algorithm generates random parameters from their respective distribution functions. """ + _unaccepted_parameter_types = () + def __init__(self, *args, **kwargs): """ Input @@ -49,7 +48,7 @@ def __init__(self, *args, **kwargs): * True: Simulation results will be saved * False: Simulation results will not be saved """ - kwargs['algorithm_name'] = 'Monte Carlo (MC) sampler' + kwargs["algorithm_name"] = "Monte Carlo (MC) sampler" super(mc, self).__init__(*args, **kwargs) def sample(self, repetitions): @@ -58,16 +57,16 @@ def sample(self, repetitions): Input ---------- - repetitions: int + repetitions: int Maximum number of runs. """ self.set_repetiton(repetitions) - print('Starting the MC algorithm with {} repetitions...'.format(repetitions)) + print("Starting the MC algorithm with {} repetitions...".format(repetitions)) # A generator that produces parametersets if called - param_generator = ((rep, self.parameter()['random']) - for rep in range(int(repetitions))) + param_generator = ( + (rep, self.parameter()["random"]) for rep in range(int(repetitions)) + ) for rep, randompar, simulations in self.repeat(param_generator): - # A function that calculates the fitness of the run and the manages the database + # A function that calculates the fitness of the run and the manages the database self.postprocessing(rep, randompar, simulations) self.final_call() - diff --git a/spotpy/algorithms/mcmc.py b/src/spotpy/algorithms/mcmc.py similarity index 56% rename from spotpy/algorithms/mcmc.py rename to src/spotpy/algorithms/mcmc.py index 7682f68f..04078c68 100644 --- a/spotpy/algorithms/mcmc.py +++ b/src/spotpy/algorithms/mcmc.py @@ -1,23 +1,21 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska -''' +""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from . import _algorithm -import numpy as np import time - +import numpy as np + +from . import _algorithm + + class mcmc(_algorithm): """ This class holds the MarkovChainMonteCarlo (MCMC) algorithm, based on: - Metropolis, N., Rosenbluth, A. W., Rosenbluth, M. N., Teller, A. H. and Teller, E. (1953) + Metropolis, N., Rosenbluth, A. W., Rosenbluth, M. N., Teller, A. H. and Teller, E. (1953) Equation of state calculations by fast computing machines, J. Chem. Phys. """ @@ -53,8 +51,8 @@ def __init__(self, *args, **kwargs): * True: Simulation results will be saved * False: Simulation results will not be saved """ - kwargs['optimization_direction'] = 'maximize' - kwargs['algorithm_name'] = 'Markov Chain Monte Carlo (MCMC) sampler' + kwargs["optimization_direction"] = "maximize" + kwargs["algorithm_name"] = "Markov Chain Monte Carlo (MCMC) sampler" super(mcmc, self).__init__(*args, **kwargs) def check_par_validity(self, par): @@ -65,14 +63,14 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - print('ERROR: Bounds have not the same lenghts as Parameterarray') + print("ERROR: Bounds have not the same lenghts as Parameterarray") return par def check_par_validity_reflect(self, par): if len(par) == len(self.min_bound) and len(par) == len(self.max_bound): for i in range(len(par)): if par[i] < self.min_bound[i]: - par[i] = self.min_bound[i] + (self.min_bound[i]- par[i]) + par[i] = self.min_bound[i] + (self.min_bound[i] - par[i]) elif par[i] > self.max_bound[i]: par[i] = self.max_bound[i] - (par[i] - self.max_bound[i]) @@ -83,70 +81,92 @@ def check_par_validity_reflect(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - print('ERROR: Bounds have not the same lenghts as Parameterarray') + print("ERROR: Bounds have not the same lenghts as Parameterarray") return par - - def get_new_proposal_vector(self,old_par): + + def get_new_proposal_vector(self, old_par): new_par = np.random.normal(loc=old_par, scale=self.stepsizes) - #new_par = self.check_par_validity(new_par) + # new_par = self.check_par_validity(new_par) new_par = self.check_par_validity_reflect(new_par) return new_par - def update_mcmc_status(self,par,like,sim,cur_chain): - self.bestpar[cur_chain]=par - self.bestlike[cur_chain]=like - self.bestsim[cur_chain]=sim + def update_mcmc_status(self, par, like, sim, cur_chain): + self.bestpar[cur_chain] = par + self.bestlike[cur_chain] = like + self.bestsim[cur_chain] = sim - - def sample(self, repetitions,nChains=1): + def sample(self, repetitions, nChains=1): self.set_repetiton(repetitions) - print('Starting the MCMC algotrithm with '+str(repetitions)+ ' repetitions...') + print( + "Starting the MCMC algotrithm with " + str(repetitions) + " repetitions..." + ) # Prepare storing MCMC chain as array of arrays. self.nChains = int(nChains) - #Ensure initialisation of chains and database + # Ensure initialisation of chains and database self.burnIn = self.nChains - # define stepsize of MCMC. - self.stepsizes = self.parameter()['step'] # array of stepsizes + # define stepsize of MCMC. + self.stepsizes = self.parameter()["step"] # array of stepsizes # Metropolis-Hastings iterations. - self.bestpar=np.array([[np.nan]*len(self.stepsizes)]*self.nChains) - self.bestlike=[[-np.inf]]*self.nChains - self.bestsim=[[np.nan]]*self.nChains - self.accepted=np.zeros(self.nChains) - self.nChainruns=[[0]]*self.nChains - self.min_bound, self.max_bound = self.parameter( - )['minbound'], self.parameter()['maxbound'] - print('Initialize ', self.nChains, ' chain(s)...') - self.iter=0 - param_generator = ((curChain,self.parameter()['random']) for curChain in range(int(self.nChains))) - for curChain,randompar,simulations in self.repeat(param_generator): - # A function that calculates the fitness of the run and the manages the database - like = self.postprocessing(self.iter, randompar, simulations, chains=curChain) + self.bestpar = np.array([[np.nan] * len(self.stepsizes)] * self.nChains) + self.bestlike = [[-np.inf]] * self.nChains + self.bestsim = [[np.nan]] * self.nChains + self.accepted = np.zeros(self.nChains) + self.nChainruns = [[0]] * self.nChains + self.min_bound, self.max_bound = ( + self.parameter()["minbound"], + self.parameter()["maxbound"], + ) + print("Initialize ", self.nChains, " chain(s)...") + self.iter = 0 + param_generator = ( + (curChain, self.parameter()["random"]) + for curChain in range(int(self.nChains)) + ) + for curChain, randompar, simulations in self.repeat(param_generator): + # A function that calculates the fitness of the run and the manages the database + like = self.postprocessing( + self.iter, randompar, simulations, chains=curChain + ) self.update_mcmc_status(randompar, like, simulations, curChain) - self.iter+=1 + self.iter += 1 intervaltime = time.time() - print('Beginn of Random Walk') + print("Beginn of Random Walk") # Walk through chains while self.iter <= repetitions - self.burnIn: - param_generator = ((curChain,self.get_new_proposal_vector(self.bestpar[curChain])) for curChain in range(int(self.nChains))) - for cChain,randompar,simulations in self.repeat(param_generator): - # A function that calculates the fitness of the run and the manages the database - like = self.postprocessing(self.iter, randompar, simulations, chains=cChain) - logMetropHastRatio = np.abs(self.bestlike[cChain])/np.abs(like) + param_generator = ( + (curChain, self.get_new_proposal_vector(self.bestpar[curChain])) + for curChain in range(int(self.nChains)) + ) + for cChain, randompar, simulations in self.repeat(param_generator): + # A function that calculates the fitness of the run and the manages the database + like = self.postprocessing( + self.iter, randompar, simulations, chains=cChain + ) + logMetropHastRatio = np.abs(self.bestlike[cChain]) / np.abs(like) u = np.random.uniform(low=0.3, high=1) - if logMetropHastRatio>1.0 or logMetropHastRatio>u: - self.update_mcmc_status(randompar,like,simulations,cChain) + if logMetropHastRatio > 1.0 or logMetropHastRatio > u: + self.update_mcmc_status(randompar, like, simulations, cChain) self.accepted[cChain] += 1 # monitor acceptance - self.iter+=1 + self.iter += 1 # Progress bar acttime = time.time() - #Refresh MCMC progressbar every two second - if acttime - intervaltime >= 2 and self.iter >=2: - text = '%i of %i (best like=%g)' % ( - self.iter + self.burnIn, repetitions, self.status.objectivefunction_max) - text = "Acceptance rates [%] =" +str(np.around((self.accepted)/float(((self.iter-self.burnIn)/self.nChains)),decimals=4)*100).strip('array([])') + # Refresh MCMC progressbar every two second + if acttime - intervaltime >= 2 and self.iter >= 2: + text = "%i of %i (best like=%g)" % ( + self.iter + self.burnIn, + repetitions, + self.status.objectivefunction_max, + ) + text = "Acceptance rates [%] =" + str( + np.around( + (self.accepted) + / float(((self.iter - self.burnIn) / self.nChains)), + decimals=4, + ) + * 100 + ).strip("array([])") print(text) intervaltime = time.time() - self.final_call() - + self.final_call() diff --git a/spotpy/algorithms/mle.py b/src/spotpy/algorithms/mle.py similarity index 74% rename from spotpy/algorithms/mle.py rename to src/spotpy/algorithms/mle.py index 2a6fec48..62bc1a82 100644 --- a/spotpy/algorithms/mle.py +++ b/src/spotpy/algorithms/mle.py @@ -1,57 +1,56 @@ -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska -''' +""" -from . import _algorithm import numpy as np +from . import _algorithm class mle(_algorithm): """ - This class holds the Maximum Likelihood (MLE) algorithm, + This class holds the Maximum Likelihood (MLE) algorithm, based on a simple uphill method as presented by Houska et al (2015): - Houska, T., Kraft, P., Chamorro-Chavez, A. and Breuer, L. (2015) + Houska, T., Kraft, P., Chamorro-Chavez, A. and Breuer, L. (2015) SPOTting Model Parameters Using a Ready-Made Python Package, PLoS ONE. """ def __init__(self, *args, **kwargs): - ''' + """ Implements the Maximum Likelihood Estimation algorithm. - + Input ---------- spot_setup: class - model: function - Should be callable with a parameter combination of the parameter-function + model: function + Should be callable with a parameter combination of the parameter-function and return an list of simulation results (as long as evaluation list) parameter: function - When called, it should return a random parameter combination. Which can + When called, it should return a random parameter combination. Which can be e.g. uniform or Gaussian - objectivefunction: function - Should return the objectivefunction for a given list of a model simulation and + objectivefunction: function + Should return the objectivefunction for a given list of a model simulation and observation. evaluation: function Should return the true values as return by the model. - + dbname: str * Name of the database where parameter, objectivefunction value and simulation results will be saved. - + dbformat: str * ram: fast suited for short sampling time. no file will be created and results are saved in an array. - * csv: A csv file will be created, which you can import afterwards. - + * csv: A csv file will be created, which you can import afterwards. + save_sim: boolean * True: Simulation results will be saved * False: Simulation results will not be saved - ''' - kwargs['optimization_direction'] = 'maximize' - kwargs['algorithm_name'] = 'Maximum Likelihood Estimation (MLE) algorithm' + """ + kwargs["optimization_direction"] = "maximize" + kwargs["algorithm_name"] = "Maximum Likelihood Estimation (MLE) algorithm" super(mle, self).__init__(*args, **kwargs) - def check_par_validity(self, par): if len(par) == len(self.min_bound) and len(par) == len(self.max_bound): for i in range(len(par)): @@ -60,47 +59,50 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - print('ERROR Bounds have not the same lenghts as Parameterarray') + print("ERROR Bounds have not the same lenghts as Parameterarray") return par def sample(self, repetitions): self.set_repetiton(repetitions) - print('Starting the MLE algotrithm with '+str(repetitions)+ ' repetitions...') + print( + "Starting the MLE algotrithm with " + str(repetitions) + " repetitions..." + ) # Define stepsize of MLE - stepsizes = self.parameter()['step'] # array of stepsizes + stepsizes = self.parameter()["step"] # array of stepsizes accepted = 0.0 - self.min_bound, self.max_bound = self.parameter( - )['minbound'], self.parameter()['maxbound'] + self.min_bound, self.max_bound = ( + self.parameter()["minbound"], + self.parameter()["maxbound"], + ) # Metropolis-Hastings iterations. burnIn = int(repetitions / 10) likes = [] pars = [] sims = [] - print('burnIn...') + print("burnIn...") for i in range(burnIn): - randompar = self.parameter()['random'] + randompar = self.parameter()["random"] pars.append(randompar) _, _, simulations = self.simulate((i, randompar)) sims.append(simulations) like = self.postprocessing(i, randompar, simulations) likes.append(like) - old_like = max(likes) old_par = pars[likes.index(old_like)] - print('Beginn Random Walk') + print("Beginn Random Walk") for rep in range(repetitions - burnIn): # Suggest new candidate from Gaussian proposal distribution. # Use stepsize provided for every dimension. new_par = np.random.normal(loc=old_par, scale=stepsizes) new_par = self.check_par_validity(new_par) _, _, new_simulations = self.simulate((i, new_par)) - new_like = self.postprocessing(rep+burnIn, new_par, new_simulations) + new_like = self.postprocessing(rep + burnIn, new_par, new_simulations) # Accept new candidate in Monte-Carlo fashing. - if (new_like > old_like): + if new_like > old_like: accepted = accepted + 1.0 # monitor acceptance old_par = new_par old_like = new_like - #self.status(rep, new_like, new_par) + # self.status(rep, new_like, new_par) - self.final_call() + self.final_call() diff --git a/src/spotpy/algorithms/nsgaii.py b/src/spotpy/algorithms/nsgaii.py new file mode 100644 index 00000000..dbc1bcd5 --- /dev/null +++ b/src/spotpy/algorithms/nsgaii.py @@ -0,0 +1,417 @@ +# -*- coding: utf-8 -*- +""" +Copyright (c) 2018 by Tobias Houska +This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). +:author: Iacopo Ferrario + +This file contains the NSGA-II Algorithm implemented for SPOTPY based on: +- K. Deb, A. Pratap, S. Agarwal and T. Meyarivan, "A fast and elitist multiobjective genetic algorithm: +NSGA-II," in IEEE Transactions on Evolutionary Computation, vol. 6, no. 2, pp. 182-197, Apr 2002. +doi: 10.1109/4235.996017 +URL: http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=996017&isnumber=21497 +- http://www.cs.colostate.edu/~genitor/MiscPubs/tutorial.pdf (Mutation and Crossover Algorithm) +- http://www.tik.ee.ethz.ch/file/6c0e384dceb283cd4301339a895b72b8/TIK-Report11.pdf (Tournament Selection) +""" + +import copy +import math + +import numpy as np + +from spotpy.algorithms import _algorithm + + +class TournamentSelection: + def __init__(self, pressure=2): + self.pressure = pressure + + def calc(self, pop_rank): + + n_select = len(pop_rank) + n_random = n_select * self.pressure # n_select * n_parents * pressure + + n_perms = math.ceil(n_random / len(pop_rank)) + + P = random_permuations(n_perms, len(pop_rank))[:n_random] + + P = np.reshape(P, (n_select, self.pressure)) + + n_tournament, _ = P.shape + + ret = np.full(n_tournament, -1, dtype=int) + + for i in range(n_tournament): + a, b = P[i] + + if pop_rank[a] < pop_rank[b]: + ret[i] = a + else: + ret[i] = b + + return ret + + +def random_permuations(n, l): + perms = [] + for _ in range(n): + perms.append(np.random.permutation(l)) + return np.concatenate(perms) + + +class Crossover: + def __init__(self, crossProb=0.9): + + self.crossProbThreshold = crossProb + + def calc(self, pop, n_var): + + n_pop = pop.shape[0] + crossProbability = np.random.random((n_pop)) + do_cross = crossProbability < self.crossProbThreshold + R = np.random.randint(0, n_pop, (n_pop, 2)) + parents = R[do_cross] + crossPoint = np.random.randint(1, n_var, parents.shape[0]) + d = pop[parents, :] + child = [] + for i in range(parents.shape[0]): + child.append( + np.concatenate([d[i, 0, : crossPoint[i]], d[i, 1, crossPoint[i] :]]) + ) + child = np.vstack(child) + pop[do_cross, :] = child + return pop + + +class PolynomialMutation: + def __init__(self, prob_mut, eta_mut): + + self.prob_mut = prob_mut + self.eta_mut = eta_mut + + def calc(self, x, xl, xu): + + X = copy.deepcopy(x) + Y = np.full(X.shape, np.inf) + + do_mutation = np.random.random(X.shape) < self.prob_mut + + m = np.sum(np.sum(do_mutation)) + + Y[:, :] = X + + xl = np.repeat(xl[None, :], X.shape[0], axis=0)[ + do_mutation + ] # selecting who is mutating + xu = np.repeat(xu[None, :], X.shape[0], axis=0)[do_mutation] + + X = X[do_mutation] + + delta1 = (X - xl) / (xu - xl) + delta2 = (xu - X) / (xu - xl) + + mut_pow = 1.0 / (self.eta_mut + 1.0) + + rand = np.random.random(X.shape) + mask = rand <= 0.5 + mask_not = np.logical_not(mask) + + deltaq = np.zeros(X.shape) + + xy = 1.0 - delta1 + val = 2.0 * rand + (1.0 - 2.0 * rand) * (np.power(xy, (self.eta_mut + 1.0))) + d = np.power(val, mut_pow) - 1.0 + deltaq[mask] = d[mask] + + xy = 1.0 - delta2 + val = 2.0 * (1.0 - rand) + 2.0 * (rand - 0.5) * ( + np.power(xy, (self.eta_mut + 1.0)) + ) + d = 1.0 - (np.power(val, mut_pow)) + deltaq[mask_not] = d[mask_not] + + ret = X + deltaq * (xu - xl) + ret[ret < xl] = xl[ret < xl] + ret[ret > xu] = xu[ret > xu] + + Y[do_mutation] = ret + + return Y + + +class NSGAII(_algorithm): + """ + Implements the "Fast and Elitist Multiobjective Genetic Algorithm: NSGA-II + by Kalyanmoy Deb, Associate Member, IEEE, Amrit Pratap, Sameer Agarwal, and T. Meyarivan + + """ + + def __init__(self, *args, **kwargs): + """ + Input + ---------- + spot_setup: class + model: function + Should be callable with a parameter combination of the parameter-function + and return an list of simulation results (as long as evaluation list) + parameter: function + When called, it should return a random parameter combination. Which can + be e.g. uniform or Gaussian + objectivefunction: function + Should return the objectivefunction for a given list of a model simulation and + observation. + evaluation: function + Should return the true values as return by the model. + + dbname: str + * Name of the database where parameter, objectivefunction value and simulation results will be saved. + + dbformat: str + * ram: fast suited for short sampling time. no file will be created and results are saved in an array. + * csv: A csv file will be created, which you can import afterwards. + + parallel: str + * seq: Sequentiel sampling (default): Normal iterations on one core of your cpu. + * mpi: Message Passing Interface: Parallel computing on cluster pcs (recommended for unix os). + + save_sim: boolean + * True: Simulation results will be saved + * False: Simulation results will not be saved + """ + self._return_all_likes = True # alloes multi-objective calibration + kwargs["optimization_direction"] = "minimize" + kwargs[ + "algorithm_name" + ] = "Fast and Elitist Multiobjective Genetic Algorithm: NSGA-II" + super(NSGAII, self).__init__(*args, **kwargs) + + def fastSort(self, x): + n = x.shape[0] + S = np.zeros((n, n), dtype=bool) + Np = np.zeros(n) + + for i in range(n): + for j in range(n): + S[i, j] = self.dominates(x[i, :], x[j, :]) + + nDom = np.sum(S, axis=0) # the n solutions that dominates i + Np[ + nDom == 0 + ] = 1 # if i == 0, i is non-dominated, set i rank to 1, i belongs to first non-dominated front + k = 1 + # loop over pareto fronts + while np.sum(Np == 0) > 0: + l = np.arange(n)[Np == k] # first non-dominated front + for i in l: # loop over the non-dominated front + nDom[S[i, :]] = ( + nDom[S[i, :]] - 1 + ) # reduce by 1 the rank of the solutions that i dominates + k += 1 + # now nDom has been reduced by 1, so the next non-dominated front will be nDom == 0 + # and Np == 0 ensure that we don't pass over the first ranked non-dom solutions + Np[(nDom == 0) & (Np == 0)] = k + + return Np.astype(int) + + def dominates(self, a, b): + if len(a.shape) > 1: + ret = (np.sum(a <= b, axis=1) == a.shape[1]) & (np.sum(a < b, axis=1) > 0) + else: + ret = (np.sum(a <= b) == len(a)) & (np.sum(a < b) > 0) + return ret + + def crowdDist(self, x): + n = x.shape[0] + + nobj = x.shape[1] + + dist = np.zeros(n) + + ord = np.argsort(x, axis=0) + + X = x[ord, range(nobj)] + + dist = np.vstack([X, np.full(nobj, np.inf)]) - np.vstack( + [np.full(nobj, -np.inf), X] + ) + + norm = np.max(X, axis=0) - np.min(X, axis=0) + dist_to_last, dist_to_next = dist, np.copy(dist) + dist_to_last, dist_to_next = dist_to_last[:-1] / norm, dist_to_next[1:] / norm + J = np.argsort(ord, axis=0) + ret = ( + np.sum( + dist_to_last[J, np.arange(nobj)] + dist_to_next[J, np.arange(nobj)], + axis=1, + ) + / nobj + ) + + return ret + + def crowdDist2(self, x): + n = x.shape[0] + + dist = np.zeros(n) + + for obj in range(x.shape[1]): + ord = np.argsort(x[:, obj]) + dist[ord[[0, -1]]] = np.inf + + norm = np.max(x[:, obj]) - np.min(x[:, obj]) + + for i in range(1, n - 1): + dist[i] = ( + dist[ord[i]] + (x[ord[i + 1], obj] - x[ord[i - 1], obj]) / norm + ) + + return dist + + def sample( + self, + generations, + n_obj, + n_pop=None, + selection=TournamentSelection(pressure=2), + crossover=Crossover(crossProb=0.9), + mutation=PolynomialMutation(prob_mut=0.25, eta_mut=30), + ): + + self.n_obj = n_obj + self.selection = selection + self.crossover = crossover + self.mutation = mutation + + self.n_pop = n_pop + self.generations = generations + self.set_repetiton(self.generations * self.n_pop) + self.skip_duplicates = True # False does not work yet + + Pt = np.vstack([self.parameter()["random"] for i in range(self.n_pop)]) + + # Burn-in + # TODO: I would suggest to make the burin-in sample indiviudual for each cpu-core in case of parallel usage, compare dream.py, but not sure if this is defined in the publication + # evaluate population + param_generator = ((i, Pt[i, :]) for i in range(self.n_pop)) + + ret = list(self.repeat(param_generator)) + + Of = [] + for p in range(self.n_pop): + index, parameters, simulation_results = ret[p] + Of.append(self.postprocessing(0, parameters, simulation_results, chains=p)) + Of = np.vstack(Of) + + nonDomRank = self.fastSort(Of) + + crDist = np.empty(self.n_pop) + for rk in range(1, np.max(nonDomRank) + 1): + crDist[nonDomRank == rk] = self.crowdDist(Of[nonDomRank == rk, :]) + + # sorting + + rank = np.lexsort((-crDist, nonDomRank)) + Ptsort = Pt[rank] + Ofsort = Of[rank] + + Of_parent = Ofsort[:, :] + Pt_parent = Ptsort[:, :] + + # selection + + offsprings = self.selection.calc(pop_rank=rank) + + Qt = Ptsort[offsprings, :] + + # crossover + try: + n_var = self.setup.n_var + except AttributeError: + n_var = len(parameters) + + Qt = self.crossover.calc(pop=Qt, n_var=n_var) + + # mutation + self.min_bound, self.max_bound = ( + self.parameter()["minbound"], + self.parameter()["maxbound"], + ) + self.varminbound = np.array([]) + self.varmaxbound = np.array([]) + for i in range(len(self.min_bound)): + self.varminbound = np.append(self.varminbound, self.min_bound[i]) + self.varmaxbound = np.append(self.varmaxbound, self.max_bound[i]) + + Qt = self.mutation.calc(x=Qt, xl=self.varminbound, xu=self.varmaxbound) + + for igen in range(1, self.generations - 1): + + Rt = np.vstack([Pt_parent, Qt]) + + if self.skip_duplicates: + + # evaluate population + param_generator = ((i, Qt[i, :]) for i in range(self.n_pop)) + + ret = list(self.repeat(param_generator)) + + Of = [] + for p in range(self.n_pop): + index, parameters, simulation_results = ret[p] + Of.append( + self.postprocessing( + igen, parameters, simulation_results, chains=p + ) + ) + Of = np.vstack(Of) + + Of = np.vstack([Of_parent, Of]) + + nonDomRank = self.fastSort(Of) + + crDist = np.empty(len(Of)) + for rk in range(1, np.max(nonDomRank) + 1): + crDist[nonDomRank == rk] = self.crowdDist(Of[nonDomRank == rk, :]) + else: + + n_pop_combined = self.n_pop * 2 + # evaluate population + param_generator = ((i, Rt[i, :]) for i in range(n_pop_combined)) + + # import pdb;pdb.set_trace() + ret = list(self.repeat(param_generator)) + + Of = [] + for p in range(n_pop_combined): + index, parameters, simulation_results = ret[p] + Of.append( + self.postprocessing( + igen, parameters, simulation_results, chains=p + ) + ) + Of = np.vstack(Of) + + nonDomRank = self.fastSort(Of) + + crDist = np.empty(n_pop_combined) + for rk in range(1, np.max(nonDomRank) + 1): + crDist[nonDomRank == rk] = self.crowdDist(Of[nonDomRank == rk, :]) + + # sorting + rank = np.lexsort((-crDist, nonDomRank))[: self.n_pop] + Ptsort = Rt[rank] + Ofsort = Of[rank] + + Pt_parent = Ptsort[:, :] + + Of_parent = Ofsort[:, :] + + # selection + offsprings = self.selection.calc(pop_rank=rank) + + Qt = Ptsort[offsprings, :] + # crossover + Qt = self.crossover.calc(pop=Qt, n_var=n_var) + # mutation + Qt = self.mutation.calc(x=Qt, xl=self.varminbound, xu=self.varmaxbound) + + self.final_call() diff --git a/spotpy/algorithms/padds.py b/src/spotpy/algorithms/padds.py similarity index 72% rename from spotpy/algorithms/padds.py rename to src/spotpy/algorithms/padds.py index ab861efe..ce35b7e4 100644 --- a/spotpy/algorithms/padds.py +++ b/src/spotpy/algorithms/padds.py @@ -1,16 +1,19 @@ -import numpy as np +import copy from copy import deepcopy +import numpy as np +from scipy.spatial.qhull import ConvexHull, QhullError + from spotpy.algorithms.dds import DDSGenerator -from . import _algorithm from spotpy.parameter import ParameterSet -import copy -from scipy.spatial.qhull import ConvexHull, QhullError + +from . import _algorithm + class BestValue(object): """ - BestValue holds a parameter set and a best objective value, which is used by the PADDS Algorithm. - Updates are done within the algorithm + BestValue holds a parameter set and a best objective value, which is used by the PADDS Algorithm. + Updates are done within the algorithm """ def __init__(self, para_func, obj_value): @@ -19,15 +22,21 @@ def __init__(self, para_func, obj_value): self.best_obj_val = obj_value self.best_rep = 0 - def copy(self): to_copy = BestValue(self.parameters.copy(), self.best_obj_val) to_copy.best_rep = self.best_rep return to_copy def __str__(self): - return "BestValue(best_obj_val = " + str(self.best_obj_val) + ", best_rep = " + str(self.best_rep) + ", " \ - + str(self.parameters) + ")" + return ( + "BestValue(best_obj_val = " + + str(self.best_obj_val) + + ", best_rep = " + + str(self.best_rep) + + ", " + + str(self.parameters) + + ")" + ) def reset_rep(self): self.best_rep = 0 @@ -90,15 +99,16 @@ def __init__(self, *args, **kwargs): except KeyError: self.r = 0.2 # default value - self._return_all_likes=True #allows multi-objective calibration - kwargs['optimization_direction'] = 'minimize' - kwargs['algorithm_name'] = 'Pareto Archived Dynamically Dimensioned Search (PADDS) algorithm' + self._return_all_likes = True # allows multi-objective calibration + kwargs["optimization_direction"] = "minimize" + kwargs[ + "algorithm_name" + ] = "Pareto Archived Dynamically Dimensioned Search (PADDS) algorithm" super(padds, self).__init__(*args, **kwargs) self.np_random = np.random - self.best_value = BestValue(self.parameter, None) self.dds_generator = DDSGenerator(self.np_random) @@ -115,14 +125,14 @@ def __init__(self, *args, **kwargs): def _set_np_random(self, f_rand): self.np_random = f_rand - if hasattr(self,"hvc"): + if hasattr(self, "hvc"): self.hvc._set_np_random(f_rand) self.dds_generator.np_random = f_rand def roulette_wheel(self, metric): cumul_metric = np.cumsum(metric) probability = self.np_random.rand() * cumul_metric[-1] - levels = (cumul_metric >= probability) + levels = cumul_metric >= probability length = cumul_metric.shape[0] return np.array(range(length))[levels][0] @@ -132,36 +142,58 @@ def get_next_x_curr(self): """ # We need to shift position and length of the sampling process for rep in range(self.generator_repetitions): - if self.dominance_flag == -1: # if the last generated solution was dominated + if ( + self.dominance_flag == -1 + ): # if the last generated solution was dominated index = self.roulette_wheel(self.metric) - self.best_value.parameters, self.best_value.best_obj_val = self.pareto_front[index][1], self.pareto_front[index][0] + self.best_value.parameters, self.best_value.best_obj_val = ( + self.pareto_front[index][1], + self.pareto_front[index][0], + ) else: # otherwise use the last generated solution - self.best_value.parameters, self.best_value.best_obj_val = (self.parameter_current, self.obj_func_current) + self.best_value.parameters, self.best_value.best_obj_val = ( + self.parameter_current, + self.obj_func_current, + ) # # This line is needed to get an array of data converted into a parameter object self.best_value.fix_format() - yield rep, self.calculate_next_s_test(self.best_value.parameters, rep, self.generator_repetitions, self.r) + yield rep, self.calculate_next_s_test( + self.best_value.parameters, rep, self.generator_repetitions, self.r + ) def calculate_initial_parameterset(self, repetitions, initial_objs, initial_params): self.obj_func_current = np.array([0.0]) self.parameter_current = np.array([0.0] * self.number_of_parameters) - self.parameter_range = self.best_value.parameters.maxbound - self.best_value.parameters.minbound - self.pareto_front = np.array([[np.array([]), np.array([0] * self.number_of_parameters)]]) - #self.pareto_front = np.array([np.append([np.inf] * self.like_struct_len, [0] * self.number_of_parameters)]) - - if(len(initial_objs) != len(initial_params)): - raise ValueError("User specified 'initial_objs' and 'initial_params' have no equal length") + self.parameter_range = ( + self.best_value.parameters.maxbound - self.best_value.parameters.minbound + ) + self.pareto_front = np.array( + [[np.array([]), np.array([0] * self.number_of_parameters)]] + ) + # self.pareto_front = np.array([np.append([np.inf] * self.like_struct_len, [0] * self.number_of_parameters)]) + + if len(initial_objs) != len(initial_params): + raise ValueError( + "User specified 'initial_objs' and 'initial_params' have no equal length" + ) if len(initial_objs) == 0: - initial_iterations = np.int(np.max([5, round(0.005 * repetitions)])) + initial_iterations = int(np.max([5, round(0.005 * repetitions)])) self.calc_initial_pareto_front(initial_iterations) elif initial_params.shape[1] != self.number_of_parameters: - raise ValueError("User specified 'initial_params' has not the same length as available parameters") + raise ValueError( + "User specified 'initial_params' has not the same length as available parameters" + ) else: - if not (np.all(initial_params <= self.best_value.parameters.maxbound) and np.all( - initial_params >= self.best_value.parameters.minbound)): - raise ValueError("User specified 'initial_params' but the values are not within the parameter range") + if not ( + np.all(initial_params <= self.best_value.parameters.maxbound) + and np.all(initial_params >= self.best_value.parameters.minbound) + ): + raise ValueError( + "User specified 'initial_params' but the values are not within the parameter range" + ) initial_iterations = initial_params.shape[0] for i in range(initial_params.shape[0]): @@ -169,36 +201,64 @@ def calculate_initial_parameterset(self, repetitions, initial_objs, initial_para if len(initial_objs[i]) > 0: self.obj_func_current = np.array(initial_objs[i]) else: - self.obj_func_current = np.array(self.getfitness(simulation=[], params=self.parameter_current)) + self.obj_func_current = np.array( + self.getfitness(simulation=[], params=self.parameter_current) + ) if i == 0: # Initial value - self.pareto_front = np.array([[self.obj_func_current, self.parameter_current]]) + self.pareto_front = np.array( + [[self.obj_func_current, self.parameter_current]] + ) dominance_flag = 1 else: - self.pareto_front, dominance_flag = nd_check(self.pareto_front, self.obj_func_current, - self.parameter_current.copy()) + self.pareto_front, dominance_flag = nd_check( + self.pareto_front, + self.obj_func_current, + self.parameter_current.copy(), + ) self.dominance_flag = dominance_flag return initial_iterations, copy.deepcopy(self.parameter_current) - def sample(self, repetitions, trials=1, initial_objs=np.array([]), initial_params=np.array([]), metric="ones"): + def sample( + self, + repetitions, + trials=1, + initial_objs=np.array([]), + initial_params=np.array([]), + metric="ones", + ): # every iteration a map of all relevant values is stored, only for debug purpose. # Spotpy will not need this values. debug_results = [] - print('Starting the PADDS algotrithm with ' + str(repetitions) + ' repetitions...') - print('WARNING: THE PADDS algorithm as implemented in SPOTPY is in an beta stage and not ready for production use!') + print( + "Starting the PADDS algotrithm with " + str(repetitions) + " repetitions..." + ) + print( + "WARNING: THE PADDS algorithm as implemented in SPOTPY is in an beta stage and not ready for production use!" + ) self.set_repetiton(repetitions) - self.number_of_parameters = len(self.best_value.parameters) # number_of_parameters is the amount of parameters + self.number_of_parameters = len( + self.best_value.parameters + ) # number_of_parameters is the amount of parameters if metric == "hvc": self.hvc = HVC(np_random=self.np_random) - - self.min_bound, self.max_bound = self.parameter()['minbound'], self.parameter()['maxbound'] + + self.min_bound, self.max_bound = ( + self.parameter()["minbound"], + self.parameter()["maxbound"], + ) # Users can define trial runs in within "repetition" times the algorithm will be executed for trial in range(trials): self.best_value.best_obj_val = 1e-308 - repitionno_best, self.best_value.parameters = self.calculate_initial_parameterset(repetitions, initial_objs, initial_params) + ( + repitionno_best, + self.best_value.parameters, + ) = self.calculate_initial_parameterset( + repetitions, initial_objs, initial_params + ) repetions_left = repetitions - repitionno_best @@ -218,16 +278,33 @@ def sample(self, repetitions, trials=1, initial_objs=np.array([]), initial_param if num_imp == 0 and num_deg > 0: self.dominance_flag = -1 # New solution is dominated by its parents else: # Do dominance check only if new solution is not dominated by its parent - self.pareto_front, self.dominance_flag = nd_check(self.pareto_front, self.obj_func_current, x_curr.copy()) - if self.dominance_flag != -1: # means, that new parameter set is a new non-dominated solution + self.pareto_front, self.dominance_flag = nd_check( + self.pareto_front, self.obj_func_current, x_curr.copy() + ) + if ( + self.dominance_flag != -1 + ): # means, that new parameter set is a new non-dominated solution self.metric = self.calc_metric(metric) self.parameter_current = x_curr # update the new status structure - self.status.params_max, self.status.params_min = self.parameter_current, self.parameter_current - - print('Best solution found has obj function value of ' + str(self.best_value.best_obj_val) + ' at ' - + str(repitionno_best) + '\n\n') - debug_results.append({"sbest": self.best_value.parameters , "objfunc_val": self.best_value.best_obj_val}) + self.status.params_max, self.status.params_min = ( + self.parameter_current, + self.parameter_current, + ) + + print( + "Best solution found has obj function value of " + + str(self.best_value.best_obj_val) + + " at " + + str(repitionno_best) + + "\n\n" + ) + debug_results.append( + { + "sbest": self.best_value.parameters, + "objfunc_val": self.best_value.best_obj_val, + } + ) self.final_call() return debug_results @@ -240,11 +317,11 @@ def calc_metric(self, metric): if metric == "ones": return np.array([1] * self.pareto_front.shape[0]) elif metric == "crowd_distance": - return crowd_dist(np.array([w for w in self.pareto_front[:,0]])) + return crowd_dist(np.array([w for w in self.pareto_front[:, 0]])) elif metric == "chc": - return chc(np.array([w for w in self.pareto_front[:,0]])) + return chc(np.array([w for w in self.pareto_front[:, 0]])) elif metric == "hvc": - return self.hvc(np.array([w for w in self.pareto_front[:,0]])) + return self.hvc(np.array([w for w in self.pareto_front[:, 0]])) else: raise AttributeError("metric argument is invalid") @@ -254,30 +331,48 @@ def calc_initial_pareto_front(self, its): :param its: amount of initial parameters """ - dominance_flag = -1 for i in range(its): for j in range(self.number_of_parameters): if self.best_value.parameters.as_int[j]: - self.parameter_current[j] = self.np_random.randint(self.best_value.parameters.minbound[j], - self.best_value.parameters.maxbound[j]) + self.parameter_current[j] = self.np_random.randint( + self.best_value.parameters.minbound[j], + self.best_value.parameters.maxbound[j], + ) else: - self.parameter_current[j] = self.best_value.parameters.minbound[j] + self.parameter_range[ - j] * self.np_random.rand() # uniform random - - id, params, model_simulations = self.simulate((range(len(self.parameter_current)), self.parameter_current)) - self.obj_func_current = self.getfitness(simulation=model_simulations, params=self.parameter_current) + self.parameter_current[j] = ( + self.best_value.parameters.minbound[j] + + self.parameter_range[j] * self.np_random.rand() + ) # uniform random + + id, params, model_simulations = self.simulate( + (range(len(self.parameter_current)), self.parameter_current) + ) + self.obj_func_current = self.getfitness( + simulation=model_simulations, params=self.parameter_current + ) # First value will be used to initialize the values if i == 0: self.pareto_front = np.vstack( - [self.pareto_front[0], np.array([self.obj_func_current.copy(), self.parameter_current.copy() + 0])]) + [ + self.pareto_front[0], + np.array( + [ + self.obj_func_current.copy(), + self.parameter_current.copy() + 0, + ] + ), + ] + ) else: - (self.pareto_front, dominance_flag) = nd_check(self.pareto_front, self.obj_func_current, - self.parameter_current.copy()) + (self.pareto_front, dominance_flag) = nd_check( + self.pareto_front, + self.obj_func_current, + self.parameter_current.copy(), + ) self.dominance_flag = dominance_flag - def calculate_next_s_test(self, previous_x_curr, rep, rep_limit, r): """ Needs to run inside `sample` method. Calculate the next set of parameters based on a given set. @@ -297,7 +392,9 @@ def calculate_next_s_test(self, previous_x_curr, rep, rep_limit, r): :return: next parameter set """ amount_params = len(previous_x_curr) - new_x_curr = previous_x_curr.copy() # define new_x_curr initially as current (previous_x_curr for greedy) + new_x_curr = ( + previous_x_curr.copy() + ) # define new_x_curr initially as current (previous_x_curr for greedy) randompar = self.np_random.rand(amount_params) @@ -305,17 +402,29 @@ def calculate_next_s_test(self, previous_x_curr, rep, rep_limit, r): dvn_count = 0 # counter for how many decision variables vary in neighbour for j in range(amount_params): - if randompar[j] < probability_neighborhood: # then j th DV selected to vary in neighbour + if ( + randompar[j] < probability_neighborhood + ): # then j th DV selected to vary in neighbour dvn_count = dvn_count + 1 - new_value = self.dds_generator.neigh_value_mixed(previous_x_curr, r, j, self.min_bound[j],self.max_bound[j]) + new_value = self.dds_generator.neigh_value_mixed( + previous_x_curr, r, j, self.min_bound[j], self.max_bound[j] + ) new_x_curr[j] = new_value # change relevant dec var value in x_curr if dvn_count == 0: # no DVs selected at random, so select ONE - dec_var = np.int(np.ceil(amount_params * self.np_random.rand())) - new_value = self.dds_generator.neigh_value_mixed(previous_x_curr, r, dec_var - 1, self.min_bound[dec_var - 1],self.max_bound[dec_var - 1]) - - new_x_curr[dec_var - 1] = new_value # change relevant decision variable value in s_test + dec_var = int(np.ceil(amount_params * self.np_random.rand())) + new_value = self.dds_generator.neigh_value_mixed( + previous_x_curr, + r, + dec_var - 1, + self.min_bound[dec_var - 1], + self.max_bound[dec_var - 1], + ) + + new_x_curr[ + dec_var - 1 + ] = new_value # change relevant decision variable value in s_test return new_x_curr @@ -360,7 +469,7 @@ def nd_check(nd_set_input, objective_values, parameter_set): try: _ = objective_values < nd_set[i][0] except ValueError: - nd_set[i][0] = np.array([np.inf]*objective_values.shape[0]) + nd_set[i][0] = np.array([np.inf] * objective_values.shape[0]) num_eql = np.sum(objective_values == nd_set[i][0]) num_imp = np.sum(objective_values < nd_set[i][0]) @@ -372,7 +481,10 @@ def nd_check(nd_set_input, objective_values, parameter_set): elif num_eql == like_struct_len: # Objective functions are the same for parameter_set and archived solution i # TODO check if this line still works - nd_set[i][0], nd_set[i][1] = objective_values, parameter_set # Replace solution i in ND_set with X + nd_set[i][0], nd_set[i][1] = ( + objective_values, + parameter_set, + ) # Replace solution i in ND_set with X dominance_flag = 0 # X is non - dominated return nd_set, dominance_flag elif num_imp > 0 and num_deg == 0: # X dominates ith solution in the ND_set @@ -381,10 +493,13 @@ def nd_check(nd_set_input, objective_values, parameter_set): dominance_flag = 1 if nd_set.size == 0: # that means the array is completely empty - nd_set = np.array([objective_values, parameter_set]) # Set solution i in ND_set with X + nd_set = np.array( + [objective_values, parameter_set] + ) # Set solution i in ND_set with X else: # If X dominated a portion of solutions in ND_set nd_set = np.vstack( - [nd_set, np.array([objective_values, parameter_set])]) # Add the new solution to the end of ND_set (for later use and comparing! + [nd_set, np.array([objective_values, parameter_set])] + ) # Add the new solution to the end of ND_set (for later use and comparing! return nd_set, dominance_flag @@ -412,7 +527,7 @@ def crowd_dist(points): max_f = np.nanmax(points, 0) min_f = np.nanmin(points, 0) - levels = (max_f == min_f) + levels = max_f == min_f length_y = points.shape[0] indicies = np.array(range(length_x))[levels] @@ -436,7 +551,9 @@ def crowd_dist(points): endpointIndx = np.array([0] * 2 * length_x) # Main Calculation - if length_y <= length_x + 1: # Less than or equal # obj + 1 solutions are non-dominated + if ( + length_y <= length_x + 1 + ): # Less than or equal # obj + 1 solutions are non-dominated temp[:, ij] = 1 # The crowding distance is 1 for all archived solutions return temp[:, ij] else: # More than 2 solutions are non - dominated @@ -444,7 +561,9 @@ def crowd_dist(points): # https://stackoverflow.com/a/22699957/5885054 temp = temp[temp[:, i].argsort()] temp[0, ij] = temp[0, ij] + 2 * (temp[1, i] - temp[0, i]) - temp[length_y - 1, ij] = temp[length_y - 1, ij] + 2 * (temp[length_y - 1, i] - temp[length_y - 2, i]) + temp[length_y - 1, ij] = temp[length_y - 1, ij] + 2 * ( + temp[length_y - 1, i] - temp[length_y - 2, i] + ) for j in range(1, length_y - 1): temp[j, ij] = temp[j, ij] + (temp[j + 1, i] - temp[j - 1, i]) @@ -453,8 +572,9 @@ def crowd_dist(points): endpointIndx[2 * (i - 1) + 1] = temp[-1, -1] # Endpoints of Pareto Front - temp = temp[temp[:, temp.shape[ - 1] - 1].argsort()] # Sort points based on the last column to restore the original order of points in the archive + temp = temp[ + temp[:, temp.shape[1] - 1].argsort() + ] # Sort points based on the last column to restore the original order of points in the archive endpointIndx = np.unique(endpointIndx) non_endpointIndx = np.array(range(length_y)).reshape((length_y, 1)) @@ -464,11 +584,14 @@ def crowd_dist(points): Y = points[endpointIndx, :] X = points[non_endpointIndx, :] - IDX = dsearchn(X, Y) # Identify the closest point in the objective space to each endpoint (dsearchn in Matlab) + IDX = dsearchn( + X, Y + ) # Identify the closest point in the objective space to each endpoint (dsearchn in Matlab) if IDX.size > 0: for i in range(endpointIndx.shape[0]): - temp[endpointIndx[i], ij] = np.nanmax([temp[endpointIndx[i], ij], temp[non_endpointIndx[IDX[ - i]], ij]]) # IF the closest point to the endpoint has a higher CD value, assign that to the endpoint; otherwise leave the CD value of the endpoint unchanged + temp[endpointIndx[i], ij] = np.nanmax( + [temp[endpointIndx[i], ij], temp[non_endpointIndx[IDX[i]], ij]] + ) # IF the closest point to the endpoint has a higher CD value, assign that to the endpoint; otherwise leave the CD value of the endpoint unchanged return temp[:, ij] @@ -482,18 +605,21 @@ def dsearchn(x, y): IDX = [] for line in range(y.shape[0]): distances = np.sqrt(np.nansum(np.power(x - y[line, :], 2), axis=1)) - found_min_dist_ind = (np.nanmin(distances, axis=0) == distances) + found_min_dist_ind = np.nanmin(distances, axis=0) == distances length = found_min_dist_ind.shape[0] IDX.append(np.array(range(length))[found_min_dist_ind][0]) return np.array(IDX) -class HVC(): +class HVC: def __init__(self, *args, **kwargs): - self.fakerandom = ('fakerandom' in kwargs and kwargs['fakerandom']) or ('np_random' in kwargs) - self.has_random_class = ('np_random' in kwargs) - self.random_class = (self.has_random_class and kwargs['np_random']) + self.fakerandom = ("fakerandom" in kwargs and kwargs["fakerandom"]) or ( + "np_random" in kwargs + ) + self.has_random_class = "np_random" in kwargs + self.random_class = self.has_random_class and kwargs["np_random"] from deap.tools._hypervolume import hv + self.hv = hv.hypervolume self.h_vol_c = self.hv_wrapper @@ -501,11 +627,11 @@ def hv_wrapper(self, points): ref = np.max(points, axis=0) return self.hv(points, ref) - def _set_np_random(self,f_rand): + def _set_np_random(self, f_rand): self.random_class = f_rand self.has_random_class = True - def hype_indicator_sampled(self,points, bounds, nrOfSamples): + def hype_indicator_sampled(self, points, bounds, nrOfSamples): try: nrP, dim = points.shape except ValueError: @@ -514,7 +640,9 @@ def hype_indicator_sampled(self,points, bounds, nrOfSamples): F = np.array([0] * nrP) BoxL = np.min(points, 0) - S = np.dot(self.__rand(nrOfSamples, dim), np.diag(bounds - BoxL)) + np.dot(np.ones([nrOfSamples, dim]), np.diag(BoxL)) + S = np.dot(self.__rand(nrOfSamples, dim), np.diag(bounds - BoxL)) + np.dot( + np.ones([nrOfSamples, dim]), np.diag(BoxL) + ) dominated = np.array([0] * nrOfSamples) dominated_ind = np.array([0] * nrOfSamples) @@ -544,8 +672,7 @@ def hype_indicator_sampled(self,points, bounds, nrOfSamples): F = np.transpose(F) * np.prod(bounds - BoxL) / nrOfSamples return F # transpose?? - - def hv_apprx(self,points): + def hv_apprx(self, points): p_xlen = np.shape(points)[0] p_ylen = np.shape(points)[1] indexis = np.array(range(p_xlen)) + 1 @@ -559,7 +686,9 @@ def hv_apprx(self,points): endpointIndx[2 * (i) + 1] = temp[-1:, -1:] endpointIndx = np.int32(np.unique(endpointIndx) - 1) - nrOfSamples = np.max([10000, 2 * p_xlen]) # Dictates the accuracy of approximation + nrOfSamples = np.max( + [10000, 2 * p_xlen] + ) # Dictates the accuracy of approximation HVInf = self.hype_indicator_sampled(points, np.array([1] * p_ylen), nrOfSamples) @@ -577,24 +706,21 @@ def hv_apprx(self,points): return HVInf - def __rand(self, x, y): if self.fakerandom: if self.has_random_class: - return self.random_class.rand(x,y) + return self.random_class.rand(x, y) else: dim = x * y + 1 step = 1.0 / dim - data = np.arange(step, 1, step)[0:x * y] + data = np.arange(step, 1, step)[0 : x * y] if y <= 1: reshaper = [x] else: reshaper = [x, y] return data.reshape(reshaper) else: - return np.random.rand(x,y) - - + return np.random.rand(x, y) def sortrows(self, arr, index): """ @@ -605,7 +731,7 @@ def sortrows(self, arr, index): """ return arr[arr[:, index].argsort()] - def hv_exact(self,points): + def hv_exact(self, points): p_xlen = np.shape(points)[0] p_ylen = np.shape(points)[1] @@ -639,7 +765,9 @@ def hv_exact(self,points): HVInf[indexis[i] - 1] = totalHV - subhv non_endpointIndx = np.array(range(p_xlen)) - non_endpointIndx = non_endpointIndx[[i for i in range(p_xlen) if i not in endpointIndx]] + non_endpointIndx = non_endpointIndx[ + [i for i in range(p_xlen) if i not in endpointIndx] + ] Y = points[endpointIndx, :] X = points[non_endpointIndx, :] @@ -653,7 +781,6 @@ def hv_exact(self,points): return HVInf - def __call__(self, points): p_xlen = np.shape(points)[0] p_ylen = np.shape(points)[1] @@ -683,7 +810,6 @@ def __call__(self, points): return self.hv_apprx(points) - def chc(points): """ function CHC_metric = CHC(points) @@ -731,7 +857,7 @@ def chc(points): hull = ConvexHull(points) except ValueError: hull = None - return np.array([1.] * np.max(points.shape)) + return np.array([1.0] * np.max(points.shape)) Totalv = hull.volume # hull.vertices @@ -748,16 +874,22 @@ def chc(points): all_CHpts_ind[ZEROind] = [] # Identify points in groups ii, iii, iv as defined above - top_facets_ind = np.min(norm, 1) >= 0 # facets with outward norm that has only non-negative components + top_facets_ind = ( + np.min(norm, 1) >= 0 + ) # facets with outward norm that has only non-negative components - ii_iv_CHpts_ind = np.unique(vertices[top_facets_ind]) # points on top of CH, i.e. groups ii and iv + ii_iv_CHpts_ind = np.unique( + vertices[top_facets_ind] + ) # points on top of CH, i.e. groups ii and iv ZEROind = ii_iv_CHpts_ind == 0 ii_iv_CHpts_ind[ZEROind] = [] other_facets_ind = np.array(range(norm.shape[0])) # All facets other_facets_ind = np.delete(other_facets_ind, other_facets_ind[top_facets_ind]) - iii_iv_CHpts_ind = np.unique(vertices[other_facets_ind, :]) # points on bottom of CH, i.e. groups iii and iv + iii_iv_CHpts_ind = np.unique( + vertices[other_facets_ind, :] + ) # points on bottom of CH, i.e. groups iii and iv ZEROind = iii_iv_CHpts_ind == 0 @@ -767,7 +899,9 @@ def chc(points): bor_CHpts_ind = iii_iv_CHpts_ind[bor_ind] bot_CHpts_ind = iii_iv_CHpts_ind - bot_CHpts_ind = bot_CHpts_ind[bor_ind == False] # Remove border points from bottom points + bot_CHpts_ind = bot_CHpts_ind[ + bor_ind == False + ] # Remove border points from bottom points # When number of bottom points and border points are not enough to form CH if bot_CHpts_ind.shape[0] == 0: @@ -776,7 +910,9 @@ def chc(points): return CHC_metric for i in range(bot_CHpts_ind.shape[0]): - y = points[all_CHpts_ind - 1] # Only consider points that are on the vertices of convex hull + y = points[ + all_CHpts_ind - 1 + ] # Only consider points that are on the vertices of convex hull # Meaning that forget the points inside the convex hull ind = np.array([j == bot_CHpts_ind[i] for j in all_CHpts_ind]) y = y[ind == False] diff --git a/spotpy/algorithms/rope.py b/src/spotpy/algorithms/rope.py similarity index 74% rename from spotpy/algorithms/rope.py rename to src/spotpy/algorithms/rope.py index caf1ae2a..88f60224 100644 --- a/spotpy/algorithms/rope.py +++ b/src/spotpy/algorithms/rope.py @@ -1,29 +1,30 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska and Alejandro Chamorro-Chavez -''' -from __future__ import unicode_literals, division, absolute_import -from . import _algorithm +""" +import random import time + import numpy as np -import random + +from . import _algorithm class rope(_algorithm): - ''' + """ This class holds the Robust Parameter Estimation (ROPE) algorithm based on Bárdossy and Singh (2008). Bárdossy, A. and Singh, S. K.: Robust estimation of hydrological model parameters, Hydrol. Earth Syst. Sci. Discuss., 5(3), 1641–1675, 2008. - ''' + """ + + def __init__(self, *args, **kwargs): - def __init__(self, *args, **kwargs): - - ''' + """ Input ---------- :param spot_setup: class @@ -39,16 +40,16 @@ def __init__(self, *args, **kwargs): simulation and observation. evaluation: function Should return the true values as return by the model. - + :param dbname: str * Name of the database where parameter, objectivefunction value and simulation results will be saved. - + :param dbformat: str * ram: fast suited for short sampling time. no file will be created and results are saved in an array. * csv: A csv file will be created, which you can import afterwards. - + :param parallel: str * seq: Sequentiel sampling (default): Normal iterations on one core of your cpu. @@ -56,23 +57,34 @@ def __init__(self, *args, **kwargs): (recommended for windows os). * mpi: Message Passing Interface: Parallel computing on cluster pcs (recommended for unix os). - + :param save_sim: boolean *True: Simulation results will be saved *False: Simulation results will not be saved - ''' - kwargs['optimization_direction'] = 'maximize' - kwargs['algorithm_name'] = 'RObust Parameter Estimation (ROPE) algorithm' + """ + kwargs["optimization_direction"] = "maximize" + kwargs["algorithm_name"] = "RObust Parameter Estimation (ROPE) algorithm" super(rope, self).__init__(*args, **kwargs) def get_best_runs(self, likes, pars, runs, percentage): - ''' - Returns the best xx% of the runs''' - return [new_pars for (new_likes, new_pars) in sorted(zip(likes, pars))[int(len(likes) * (1 - percentage)):]] - - def sample(self, repetitions=None, repetitions_first_run=None, - subsets=5, percentage_first_run=0.10, - percentage_following_runs=0.10, NDIR=None): + """ + Returns the best xx% of the runs""" + return [ + new_pars + for (new_likes, new_pars) in sorted(zip(likes, pars))[ + int(len(likes) * (1 - percentage)) : + ] + ] + + def sample( + self, + repetitions=None, + repetitions_first_run=None, + subsets=5, + percentage_first_run=0.10, + percentage_following_runs=0.10, + NDIR=None, + ): """ Samples from the ROPE algorithm. @@ -90,39 +102,41 @@ def sample(self, repetitions=None, repetitions_first_run=None, next step after in all following subsets NDIR = The number of samples to draw """ - #Reported behaviour: + # Reported behaviour: # Takes ways´to long for npar >8 # wenn mehr parameter produziert werden sollen als reingehen, rechnet er sich tot (ngen>n) - #Subsets < 5 führt manchmal zu Absturz - print('Starting the ROPE algotrithm with '+str(repetitions)+ ' repetitions...') + # Subsets < 5 führt manchmal zu Absturz + print( + "Starting the ROPE algotrithm with " + str(repetitions) + " repetitions..." + ) self.set_repetiton(repetitions) if repetitions_first_run is None: - #Take the first have of the repetitions as burn-in + # Take the first have of the repetitions as burn-in first_run = int(repetitions / 2.0) else: - #Make user defined number of burn-in repetitions + # Make user defined number of burn-in repetitions first_run = repetitions_first_run - repetitions_following_runs = int((repetitions-first_run) - / (subsets-1)) + repetitions_following_runs = int((repetitions - first_run) / (subsets - 1)) # Needed to avoid an error in integer division somewhere in depth function if repetitions_following_runs % 2 != 0: - print('Warning: Burn-in samples and total number of repetions are not compatible.\n' - 'SPOTPY will automatically adjust the number of total repetitions.') - repetitions_following_runs+=1 + print( + "Warning: Burn-in samples and total number of repetions are not compatible.\n" + "SPOTPY will automatically adjust the number of total repetitions." + ) + repetitions_following_runs += 1 - if NDIR is None: NDIR = int(repetitions_following_runs / 100.0) self.NDIR = NDIR starttime = time.time() intervaltime = starttime - parset =self.parameter() - self.min_bound, self.max_bound = parset['minbound'], parset['maxbound'] - + parset = self.parameter() + self.min_bound, self.max_bound = parset["minbound"], parset["maxbound"] + # Init ROPE with one subset likes = [] pars = [] @@ -141,10 +155,9 @@ def sample(self, repetitions=None, repetitions_first_run=None, random.shuffle(matrix[:, i]) # A generator that produces the parameters - param_generator = ((rep, matrix[rep]) - for rep in range(int(first_run) - 1)) + param_generator = ((rep, matrix[rep]) for rep in range(int(first_run) - 1)) for rep, randompar, simulations in self.repeat(param_generator): - # A function that calculates the fitness of the run and the manages the database + # A function that calculates the fitness of the run and the manages the database like = self.postprocessing(rep, randompar, simulations) likes.append(like) pars.append(randompar) @@ -154,21 +167,26 @@ def sample(self, repetitions=None, repetitions_first_run=None, if self.status.stop: break if acttime - intervaltime >= 2: - text = '1 Subset: Run %i of %i (best like=%g)' % ( - rep, first_run, self.status.objectivefunction_max) + text = "1 Subset: Run %i of %i (best like=%g)" % ( + rep, + first_run, + self.status.objectivefunction_max, + ) print(text) intervaltime = time.time() for subset in range(subsets - 1): if subset == 0: - best_pars = self.get_best_runs(likes, pars, repetitions_following_runs, - percentage_first_run) + best_pars = self.get_best_runs( + likes, pars, repetitions_following_runs, percentage_first_run + ) else: - best_pars = self.get_best_runs(likes, pars, repetitions_following_runs, - percentage_following_runs) + best_pars = self.get_best_runs( + likes, pars, repetitions_following_runs, percentage_following_runs + ) valid = False trials = 0 - while valid is False and trials < 10 and repetitions_following_runs>1: + while valid is False and trials < 10 and repetitions_following_runs > 1: new_pars = self.programm_depth(best_pars, repetitions_following_runs) if len(new_pars) == repetitions_following_runs: valid = True @@ -176,17 +194,22 @@ def sample(self, repetitions=None, repetitions_first_run=None, trials += 1 pars = [] likes = [] - if(int(repetitions_following_runs) > len(new_pars)): + if int(repetitions_following_runs) > len(new_pars): repetitions_following_runs = len(new_pars) param_generator = ( - (rep, new_pars[rep]) for rep in range(int(repetitions_following_runs))) + (rep, new_pars[rep]) for rep in range(int(repetitions_following_runs)) + ) for rep, ropepar, simulations in self.repeat(param_generator): # Calculate the objective function - like = self.postprocessing(first_run + rep + repetitions_following_runs * subset, ropepar, simulations) + like = self.postprocessing( + first_run + rep + repetitions_following_runs * subset, + ropepar, + simulations, + ) likes.append(like) pars.append(ropepar) if self.status.stop: - print('Stopping samplig') + print("Stopping samplig") break # Progress bar @@ -194,30 +217,30 @@ def sample(self, repetitions=None, repetitions_first_run=None, if repetitions_following_runs is not None: # Refresh progressbar every second if acttime - intervaltime >= 2: - text = '%i Subset: Run %i of %i (best like=%g)' % ( + text = "%i Subset: Run %i of %i (best like=%g)" % ( subset + 2, rep, repetitions_following_runs, - self.status.objectivefunction_max) + self.status.objectivefunction_max, + ) print(text) intervaltime = time.time() if self.status.stop: break self.final_call() - def programm_depth(self, pars, runs): X = np.array(pars) N, NP = X.shape - text = str(N) + ' input vectors with ' + str(NP) + ' parameters' + text = str(N) + " input vectors with " + str(NP) + " parameters" print(text) Ngen = int(runs) # int(N*(1/self.percentage)) - print(('Generating ' + str(Ngen) + ' parameters:')) + print(("Generating " + str(Ngen) + " parameters:")) - NPOSI = Ngen # Number of points to generate + NPOSI = Ngen # Number of points to generate EPS = 0.00001 @@ -236,8 +259,8 @@ def programm_depth(self, pars, runs): CL = np.zeros(NP) TL = np.zeros(shape=(LLEN, NP)) - while (IPOS < NPOSI): - for IM in range(LLEN): # LLEN=1000 Random Vectors of dim NP + while IPOS < NPOSI: + for IM in range(LLEN): # LLEN=1000 Random Vectors of dim NP for j in range(NP): DRAND = np.random.rand() TL[IM, j] = XMIN[j] + DRAND * (XMAX[j] - XMIN[j]) @@ -282,7 +305,7 @@ def fDEP(self, N, NP, X, TL, EPS, LLEN): NSAMP = NSAMP + 1 JSAMP[index + 1] = L - # Covariance matrix of the sample + # Covariance matrix of the sample S = np.zeros(shape=(NP, NP)) for i in range(NP): row = JSAMP[i] @@ -307,10 +330,10 @@ def fDEP(self, N, NP, X, TL, EPS, LLEN): # Eigenvector in the direction of min eigenvalue EVECT = EVE[:, arg[0]] - # Project all points on the line through theta with direction + # Project all points on the line through theta with direction # given by the eigenvector of the smallest eigenvalue, i.e. # the direction orthogonal on the hyperplane given by the np-subset - # Compute the one-dimensional halfspace depth of theta on this line + # Compute the one-dimensional halfspace depth of theta on this line HELP = [] for L in range(N): k = np.dot(EVECT, X[L, :]) @@ -320,12 +343,11 @@ def fDEP(self, N, NP, X, TL, EPS, LLEN): ICROSS = 0 for LU in range(LLEN): - if (LNDEP[LU] > ICROSS): - EKT = 0. + if LNDEP[LU] > ICROSS: + EKT = 0.0 NT = 0 EKT = EKT + np.dot(TL[LU, :], EVECT) - if ((EKT < (HELP[0] - EPS)) or ( - EKT > (HELP[N - 1] + EPS))): + if (EKT < (HELP[0] - EPS)) or (EKT > (HELP[N - 1] + EPS)): N1 = 0 else: N1 = 1 @@ -333,13 +355,13 @@ def fDEP(self, N, NP, X, TL, EPS, LLEN): dirac = 0 while dirac == 0: dirac = 1 - N3 = (N1 + N2) / 2. - if (HELP[int(N3)] < EKT): + N3 = (N1 + N2) / 2.0 + if HELP[int(N3)] < EKT: N1 = N3 else: N2 = N3 - if(N2 - N1) > 1: + if (N2 - N1) > 1: dirac = 0 NUMH = N1 LNDEP[LU] = min(LNDEP[LU], min(NUMH + NT, N - NUMH)) - return LNDEP \ No newline at end of file + return LNDEP diff --git a/spotpy/algorithms/sa.py b/src/spotpy/algorithms/sa.py similarity index 74% rename from spotpy/algorithms/sa.py rename to src/spotpy/algorithms/sa.py index 15bc3f31..ba787fae 100644 --- a/spotpy/algorithms/sa.py +++ b/src/spotpy/algorithms/sa.py @@ -1,23 +1,20 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska and Alejandro Chamorro-Chavez -''' +""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from . import _algorithm import numpy as np +from . import _algorithm + class sa(_algorithm): """ This class holds the Simulated Annealing (SA) algorithm based on: - - Kirkpatrick, S., Gelatt, C. D., Vecchi, M. P. and others (2013). + + Kirkpatrick, S., Gelatt, C. D., Vecchi, M. P. and others (2013). Optimization by simmulated annealing, science. """ @@ -53,10 +50,10 @@ def __init__(self, *args, **kwargs): * True: Simulation results will be saved * False: Simulation results will not be saved """ - kwargs['optimization_direction'] = 'maximize' - kwargs['algorithm_name'] = 'Simulated Annealing (SA) algorithm' + kwargs["optimization_direction"] = "maximize" + kwargs["algorithm_name"] = "Simulated Annealing (SA) algorithm" super(sa, self).__init__(*args, **kwargs) - + def check_par_validity(self, par): if len(par) == len(self.min_bound) and len(par) == len(self.max_bound): for i in range(len(par)): @@ -65,7 +62,7 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - print('ERROR: Bounds have not the same lenghts as Parameterarray') + print("ERROR: Bounds have not the same lenghts as Parameterarray") return par def sample(self, repetitions, Tini=80, Ntemp=50, alpha=0.99): @@ -74,26 +71,28 @@ def sample(self, repetitions, Tini=80, Ntemp=50, alpha=0.99): Input ---------- - repetitions: int - Maximum number of runs. + repetitions: int + Maximum number of runs. """ self.set_repetiton(repetitions) - print('Starting the SA algotrithm with '+str(repetitions)+ ' repetitions...') - self.min_bound, self.max_bound = self.parameter( - )['minbound'], self.parameter()['maxbound'] - stepsizes = self.parameter()['step'] + print("Starting the SA algotrithm with " + str(repetitions) + " repetitions...") + self.min_bound, self.max_bound = ( + self.parameter()["minbound"], + self.parameter()["maxbound"], + ) + stepsizes = self.parameter()["step"] Eopt = 999999 Titer = Tini - x = self.parameter()['optguess'] + x = self.parameter()["optguess"] Xopt = x _, _, simulations = self.simulate((1, x)) Enew = self.postprocessing(1, x, simulations) Eopt = Enew - rep = 1 # Because the model has been started once already - while (Titer > 0.001 * Tini and rep < repetitions): + rep = 1 # Because the model has been started once already + while Titer > 0.001 * Tini and rep < repetitions: for counter in range(Ntemp): - if (Enew > Eopt): # Run was better + if Enew > Eopt: # Run was better Eopt = Enew Xopt = x Eopt = Enew @@ -103,33 +102,34 @@ def sample(self, repetitions, Tini=80, Ntemp=50, alpha=0.99): accepted = frandom(Enew, Eopt, Titer) if accepted == True: Xopt = x - x = np.random.uniform(low=Xopt - stepsizes, high=Xopt + stepsizes) + x = np.random.uniform( + low=Xopt - stepsizes, high=Xopt + stepsizes + ) else: x = np.random.normal(loc=Xopt, scale=stepsizes) x = self.check_par_validity(x) - _, _, simulations = self.simulate((rep+1, x)) - Enew = self.postprocessing(rep+1, x, simulations) + _, _, simulations = self.simulate((rep + 1, x)) + Enew = self.postprocessing(rep + 1, x, simulations) rep += 1 if self.status.stop: break - Titer = alpha * Titer - self.final_call() + self.final_call() def frandom(Enew, Eold, Titer): dE = Eold - Enew accepted = False - if (dE > 0): + if dE > 0: P = np.exp(-(dE) / Titer) # Boltzmann distr. rn = np.random.rand() - if (rn <= P): # New configuration accepted + if rn <= P: # New configuration accepted accepted = True else: accepted = True - return accepted \ No newline at end of file + return accepted diff --git a/spotpy/algorithms/sceua.py b/src/spotpy/algorithms/sceua.py similarity index 61% rename from spotpy/algorithms/sceua.py rename to src/spotpy/algorithms/sceua.py index fe5e59bf..a0841b5c 100644 --- a/spotpy/algorithms/sceua.py +++ b/src/spotpy/algorithms/sceua.py @@ -1,29 +1,26 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska and Stijn Van Hoey -''' +""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from . import _algorithm import numpy as np +from . import _algorithm + class sceua(_algorithm): """ - This class holds the Shuffled Complex Evolution Algorithm (SCE-UA) algorithm, + This class holds the Shuffled Complex Evolution Algorithm (SCE-UA) algorithm, based on: - Duan, Q., Sorooshian, S. and Gupta, V. K. (1994) + Duan, Q., Sorooshian, S. and Gupta, V. K. (1994) Optimal use of the SCE-UA global optimization method for calibrating watershed models, J. Hydrol. Based on the PYthon package Optimization_SCE Copyright (c) 2011 Stijn Van Hoey. Restructured and parallelized by Houska et al (2015): - Houska, T., Kraft, P., Chamorro-Chavez, A. and Breuer, L. (2015) + Houska, T., Kraft, P., Chamorro-Chavez, A. and Breuer, L. (2015) SPOTting Model Parameters Using a Ready-Made Python Package, PLoS ONE. """ @@ -60,8 +57,8 @@ def __init__(self, *args, **kwargs): * True: Simulation results will be saved * False: Simulation results will not be saved """ - kwargs['optimization_direction'] = 'minimize' - kwargs['algorithm_name'] = 'Shuffled Complex Evolution (SCE-UA) algorithm' + kwargs["optimization_direction"] = "minimize" + kwargs["algorithm_name"] = "Shuffled Complex Evolution (SCE-UA) algorithm" super(sceua, self).__init__(*args, **kwargs) def simulate(self, id_params_tuple): @@ -70,13 +67,23 @@ def simulate(self, id_params_tuple): 1) burn-in 2) complex evolution """ - + if not self.repeat.phase: # burn-in return _algorithm.simulate(self, id_params_tuple) else: # complex-evolution igs, x, xf, cx, cf, sce_vars = id_params_tuple - self.npg, self.nopt, self.ngs, self.nspl, self.nps, self.bl, self.bu, self.stochastic_parameters, discarded_runs = sce_vars + ( + self.npg, + self.nopt, + self.ngs, + self.nspl, + self.nps, + self.bl, + self.bu, + self.stochastic_parameters, + discarded_runs, + ) = sce_vars # Partition the population into complexes (sub-populations); k1 = np.arange(self.npg, dtype=int) k2 = k1 * self.ngs + igs @@ -93,8 +100,16 @@ def simulate(self, id_params_tuple): lcs[0] = 1 for k3 in range(1, self.nps): for i in range(1000): - lpos = int(np.floor( - self.npg + 0.5 - np.sqrt((self.npg + 0.5)**2 - self.npg * (self.npg + 1) * np.random.random()))) + lpos = int( + np.floor( + self.npg + + 0.5 + - np.sqrt( + (self.npg + 0.5) ** 2 + - self.npg * (self.npg + 1) * np.random.random() + ) + ) + ) # check if the element has already been chosen idx = (lcs[0:k3] == lpos).nonzero() if idx[0].size == 0: @@ -106,11 +121,13 @@ def simulate(self, id_params_tuple): s = cx[lcs, :] sf = cf[lcs] - snew, fnew, simulation, discarded_runs = self._cceua(s, sf, discarded_runs) + snew, fnew, simulation, discarded_runs = self._cceua( + s, sf, discarded_runs + ) likes.append(fnew) pars.append(snew) sims.append(simulation) - + # Replace the worst point in Simplex with the new point: s[-1, :] = snew sf[-1] = fnew @@ -123,13 +140,21 @@ def simulate(self, id_params_tuple): idx = np.argsort(cf) cf = np.sort(cf) cx = cx[idx, :] - + # Replace the complex back into the population; return igs, likes, pars, sims, cx, cf, k1, k2, discarded_runs - def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.0000001, max_loop_inc=None): + def sample( + self, + repetitions, + ngs=20, + kstop=100, + pcento=0.0000001, + peps=0.0000001, + max_loop_inc=None, + ): """ - Samples from parameter distributions using SCE-UA (Duan, 2004), + Samples from parameter distributions using SCE-UA (Duan, 2004), converted to python by Van Hoey (2011), restructured and parallelized by Houska et al (2015). Parameters @@ -151,7 +176,7 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 self.set_repetiton(repetitions) # Initialize SCE parameters: self.ngs = ngs - randompar = self.parameter()['random'] + randompar = self.parameter()["random"] self.nopt = randompar.size self.npg = 2 * self.nopt + 1 self.nps = self.nopt + 1 @@ -159,8 +184,7 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 npt = self.npg * self.ngs self.iseed = 1 self.discarded_runs = 0 - self.bl, self.bu = self.parameter()['minbound'], self.parameter()[ - 'maxbound'] + self.bl, self.bu = self.parameter()["minbound"], self.parameter()["maxbound"] bound = self.bu - self.bl # np.array self.stochastic_parameters = bound != 0 proceed = True @@ -168,31 +192,33 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 # burnin_only, needed to indictat if only the burnin phase should be run burnin_only = False - if self.breakpoint == 'read' or self.breakpoint == 'readandwrite': + if self.breakpoint == "read" or self.breakpoint == "readandwrite": data_frombreak = self.read_breakdata(self.dbname) icall = data_frombreak[0] x = data_frombreak[1][0] xf = data_frombreak[1][1] gnrng = data_frombreak[2] - elif self.breakpoint is None or self.breakpoint == 'write': + elif self.breakpoint is None or self.breakpoint == "write": # Create an initial population to fill array x(npt,self.self.nopt): x = self._sampleinputmatrix(npt, self.nopt) nloop = 0 icall = 0 xf = np.zeros(npt) - print('Starting burn-in sampling...') + print("Starting burn-in sampling...") # Burn in param_generator = ((rep, x[rep]) for rep in range(int(npt))) for rep, randompar, simulations in self.repeat(param_generator): # Calculate the objective function - like = self.postprocessing(icall, randompar, simulations,chains=0) + like = self.postprocessing(icall, randompar, simulations, chains=0) xf[rep] = like - icall+=1 + icall += 1 if self.status.stop: - print('Stopping samplig. Maximum number of repetitions reached already during burn-in') + print( + "Stopping samplig. Maximum number of repetitions reached already during burn-in" + ) proceed = False break # Sort the population in order of increasing function values; @@ -203,11 +229,13 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 if max_loop_inc == 1: burnin_only = True - print('Burn-in sampling completed...') + print("Burn-in sampling completed...") else: - raise ValueError("Don't know the breakpoint keyword {}".format(self.breakpoint)) - + raise ValueError( + "Don't know the breakpoint keyword {}".format(self.breakpoint) + ) + # Record the best points; bestx = x[0, :] bestf = xf[0] @@ -217,75 +245,104 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 # Computes the normalized geometric range of the parameters gnrng = np.exp( - np.mean(np.log((np.max(x[:, self.stochastic_parameters], axis=0) - np.min(x[:, self.stochastic_parameters], axis=0)) / bound[self.stochastic_parameters]))) + np.mean( + np.log( + ( + np.max(x[:, self.stochastic_parameters], axis=0) + - np.min(x[:, self.stochastic_parameters], axis=0) + ) + / bound[self.stochastic_parameters] + ) + ) + ) # Check for convergency; if self.status.rep >= repetitions: - print('*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT') - print('ON THE MAXIMUM NUMBER OF TRIALS ') + print("*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT") + print("ON THE MAXIMUM NUMBER OF TRIALS ") print(repetitions) - print('HAS BEEN EXCEEDED. SEARCH WAS STOPPED AT TRIAL NUMBER:') + print("HAS BEEN EXCEEDED. SEARCH WAS STOPPED AT TRIAL NUMBER:") print(self.status.rep) - print('OF THE INITIAL LOOP!') + print("OF THE INITIAL LOOP!") if gnrng < peps: print( - 'THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE') - - - + "THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE" + ) # Begin evolution loops: nloop = 0 criter = [] - criter_change_pcent = 1e+5 + criter_change_pcent = 1e5 proceed = True # if only burnin, stop the following while loop to be started # write brakpoint if only a single generation shall be computed and # the main loop will not be executed if burnin_only: - if self.breakpoint == 'write' or self.breakpoint == 'readandwrite': + if self.breakpoint == "write" or self.breakpoint == "readandwrite": work = (self.status.rep, (x, xf), gnrng) self.write_breakdata(self.dbname, work) proceed = False - print('ONLY THE BURNIN PHASE WAS COMPUTED') + print("ONLY THE BURNIN PHASE WAS COMPUTED") else: - self.repeat.setphase('ComplexEvo') - print('Starting Complex Evolution...') - - while icall < repetitions and gnrng > peps and criter_change_pcent > pcento and proceed == True: + self.repeat.setphase("ComplexEvo") + print("Starting Complex Evolution...") + + while ( + icall < repetitions + and gnrng > peps + and criter_change_pcent > pcento + and proceed == True + ): nloop += 1 - print ('ComplexEvo loop #%d in progress...' % nloop) + print("ComplexEvo loop #%d in progress..." % nloop) # Loop on complexes (sub-populations); cx = np.zeros((self.npg, self.nopt)) cf = np.zeros((self.npg)) remaining_runs = repetitions - self.status.rep if remaining_runs <= self.ngs: - self.ngs = remaining_runs-1 + self.ngs = remaining_runs - 1 proceed = False - - sce_vars = [self.npg, self.nopt, self.ngs, self.nspl, - self.nps, self.bl, self.bu, self.stochastic_parameters, self.discarded_runs] - param_generator = ((rep, x, xf, cx, cf, sce_vars) - for rep in range(int(self.ngs))) - for igs, likes, pars, sims, cx, cf, k1, k2, discarded_runs in self.repeat(param_generator): + + sce_vars = [ + self.npg, + self.nopt, + self.ngs, + self.nspl, + self.nps, + self.bl, + self.bu, + self.stochastic_parameters, + self.discarded_runs, + ] + param_generator = ( + (rep, x, xf, cx, cf, sce_vars) for rep in range(int(self.ngs)) + ) + for igs, likes, pars, sims, cx, cf, k1, k2, discarded_runs in self.repeat( + param_generator + ): x[k2, :] = cx[k1, :] xf[k2] = cf[k1] self.discard_runs = discarded_runs for i in range(len(likes)): - if not self.status.stop: - like = self.postprocessing(i, pars[i], sims[i], chains=i+1) + if not self.status.stop: + like = self.postprocessing(i, pars[i], sims[i], chains=i + 1) else: - #Collect data from all slaves but do not save - proceed=False - like = self.postprocessing(i, pars[i], sims[i], chains=i+1, save_run=False) - self.discarded_runs+=1 - print('Skipping saving') - - if self.breakpoint == 'write' or self.breakpoint == 'readandwrite'\ - and self.status.rep >= self.backup_every_rep: + # Collect data from all slaves but do not save + proceed = False + like = self.postprocessing( + i, pars[i], sims[i], chains=i + 1, save_run=False + ) + self.discarded_runs += 1 + print("Skipping saving") + + if ( + self.breakpoint == "write" + or self.breakpoint == "readandwrite" + and self.status.rep >= self.backup_every_rep + ): work = (self.status.rep, (x, xf), gnrng) self.write_breakdata(self.dbname, work) @@ -306,37 +363,54 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 # Computes the normalized geometric range of the parameters gnrng = np.exp( - np.mean(np.log((np.max(x[:, self.stochastic_parameters], axis=0) - np.min(x[:, self.stochastic_parameters], axis=0)) / bound[self.stochastic_parameters]))) + np.mean( + np.log( + ( + np.max(x[:, self.stochastic_parameters], axis=0) + - np.min(x[:, self.stochastic_parameters], axis=0) + ) + / bound[self.stochastic_parameters] + ) + ) + ) criter = np.append(criter, bestf) - + # Check for convergency; if self.status.rep >= repetitions: - print('*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT') - print('ON THE MAXIMUM NUMBER OF TRIALS ') + print("*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT") + print("ON THE MAXIMUM NUMBER OF TRIALS ") print(repetitions) - print('HAS BEEN EXCEEDED.') + print("HAS BEEN EXCEEDED.") elif gnrng < peps: print( - 'THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE') + "THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE" + ) - - elif nloop >= kstop: # necessary so that the area of high posterior density is visited as much as possible - print ('Objective function convergence criteria is now being updated and assessed...') - absolute_change = np.abs( - criter[nloop - 1] - criter[nloop - kstop])*100 - denominator = np.mean(np.abs(criter[(nloop - kstop):nloop])) + elif ( + nloop >= kstop + ): # necessary so that the area of high posterior density is visited as much as possible + print( + "Objective function convergence criteria is now being updated and assessed..." + ) + absolute_change = ( + np.abs(criter[nloop - 1] - criter[nloop - kstop]) * 100 + ) + denominator = np.mean(np.abs(criter[(nloop - kstop) : nloop])) if denominator == 0.0: criter_change_pcent = 0.0 else: criter_change_pcent = absolute_change / denominator - print ('Updated convergence criteria: %f' % criter_change_pcent) + print("Updated convergence criteria: %f" % criter_change_pcent) if criter_change_pcent <= pcento: - print('THE BEST POINT HAS IMPROVED IN LAST %d LOOPS BY LESS THAN THE USER-SPECIFIED THRESHOLD %f' % ( - kstop, pcento)) print( - 'CONVERGENCY HAS ACHIEVED BASED ON OBJECTIVE FUNCTION CRITERIA!!!') + "THE BEST POINT HAS IMPROVED IN LAST %d LOOPS BY LESS THAN THE USER-SPECIFIED THRESHOLD %f" + % (kstop, pcento) + ) + print( + "CONVERGENCY HAS ACHIEVED BASED ON OBJECTIVE FUNCTION CRITERIA!!!" + ) elif self.status.stop: proceed = False break @@ -344,37 +418,38 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 # stop, if max number of loop iteration was reached elif max_loop_inc and nloop >= max_loop_inc: proceed = False - print('THE MAXIMAL NUMBER OF LOOPS PER EXECUTION WAS REACHED') + print("THE MAXIMAL NUMBER OF LOOPS PER EXECUTION WAS REACHED") break - + # End of the Outer Loops - print('SEARCH WAS STOPPED AT TRIAL NUMBER: %d' % self.status.rep) - print('NUMBER OF DISCARDED TRIALS: %d' % self.discarded_runs) - print('NORMALIZED GEOMETRIC RANGE = %f' % gnrng) - print('THE BEST POINT HAS IMPROVED IN LAST %d LOOPS BY %f PERCENT' % ( - kstop, criter_change_pcent)) + print("SEARCH WAS STOPPED AT TRIAL NUMBER: %d" % self.status.rep) + print("NUMBER OF DISCARDED TRIALS: %d" % self.discarded_runs) + print("NORMALIZED GEOMETRIC RANGE = %f" % gnrng) + print( + "THE BEST POINT HAS IMPROVED IN LAST %d LOOPS BY %f PERCENT" + % (kstop, criter_change_pcent) + ) # reshape BESTX - #BESTX = BESTX.reshape(BESTX.size // self.nopt, self.nopt) + # BESTX = BESTX.reshape(BESTX.size // self.nopt, self.nopt) self.final_call() - def _cceua(self, s, sf, discarded_runs): - # This is the subroutine for generating a new point in a simplex - # - # s(.,.) = the sorted simplex in order of increasing function values - # s(.) = function values in increasing order - # - # LIST OF LOCAL VARIABLES - # sb(.) = the best point of the simplex - # sw(.) = the worst point of the simplex - # w2(.) = the second worst point of the simplex - # fw = function value of the worst point - # ce(.) = the centroid of the simplex excluding wo - # snew(.) = new point generated from the simplex - # iviol = flag indicating if constraints are violated - # = 1 , yes - # = 0 , no + # This is the subroutine for generating a new point in a simplex + # + # s(.,.) = the sorted simplex in order of increasing function values + # s(.) = function values in increasing order + # + # LIST OF LOCAL VARIABLES + # sb(.) = the best point of the simplex + # sw(.) = the worst point of the simplex + # w2(.) = the second worst point of the simplex + # fw = function value of the worst point + # ce(.) = the centroid of the simplex excluding wo + # snew(.) = new point generated from the simplex + # iviol = flag indicating if constraints are violated + # = 1 , yes + # = 0 , no constant_parameters = np.invert(self.stochastic_parameters) self.nps, self.nopt = s.shape alpha = 1.0 @@ -407,9 +482,11 @@ def _cceua(self, s, sf, discarded_runs): ## fnew = functn(self.nopt,snew); _, _, simulations = _algorithm.simulate(self, (1, snew)) - like = self.postprocessing(1, snew, simulations, save_run=False, block_print=True) - discarded_runs+=1 - + like = self.postprocessing( + 1, snew, simulations, save_run=False, block_print=True + ) + discarded_runs += 1 + fnew = like # Reflection failed; now attempt a contraction point: @@ -418,29 +495,33 @@ def _cceua(self, s, sf, discarded_runs): snew[constant_parameters] = sw[constant_parameters] _, _, simulations = _algorithm.simulate(self, (2, snew)) - like = self.postprocessing(2, snew, simulations, save_run=False, block_print=True) - discarded_runs+=1 + like = self.postprocessing( + 2, snew, simulations, save_run=False, block_print=True + ) + discarded_runs += 1 fnew = like - # Both reflection and contraction have failed, attempt a random point; + # Both reflection and contraction have failed, attempt a random point; if fnew > fw: snew = self._sampleinputmatrix(1, self.nopt)[0] _, _, simulations = _algorithm.simulate(self, (3, snew)) - like = self.postprocessing(3, snew, simulations, save_run=False, block_print=True) - discarded_runs+=1 + like = self.postprocessing( + 3, snew, simulations, save_run=False, block_print=True + ) + discarded_runs += 1 fnew = like # END OF CCE return snew, fnew, simulations, discarded_runs def _sampleinputmatrix(self, nrows, npars): - ''' + """ Create inputparameter matrix for nrows simualtions, for npars with bounds ub and lb (np.array from same size) distname gives the initial sampling ditribution (currently one for all parameters) returns np.array - ''' + """ x = np.zeros((nrows, npars)) for i in range(nrows): - x[i, :] = self.parameter()['random'] + x[i, :] = self.parameter()["random"] return x diff --git a/spotpy/analyser.py b/src/spotpy/analyser.py similarity index 50% rename from spotpy/analyser.py rename to src/spotpy/analyser.py index e6c83687..7481bcfd 100644 --- a/spotpy/analyser.py +++ b/src/spotpy/analyser.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). @@ -8,14 +8,14 @@ Holds functions to analyse results out of the database. Note: This part of SPOTPY is in alpha status and not yet ready for production use. -''' +""" import numpy as np + import spotpy -font = {'family' : 'calibri', - 'weight' : 'normal', - 'size' : 18} +font = {"family": "calibri", "weight": "normal", "size": 18} + def load_csv_results(filename, usecols=None): """ @@ -28,26 +28,38 @@ def load_csv_results(filename, usecols=None): :rtype: array """ if usecols == None: - return np.genfromtxt(filename+'.csv',delimiter=',',names=True,invalid_raise=False) + return np.genfromtxt( + filename + ".csv", delimiter=",", names=True, invalid_raise=False + ) else: - return np.genfromtxt(filename+'.csv',delimiter=',',names=True,skip_footer=1,invalid_raise=False,usecols=usecols)[1:] + return np.genfromtxt( + filename + ".csv", + delimiter=",", + names=True, + skip_footer=1, + invalid_raise=False, + usecols=usecols, + )[1:] + def load_hdf5_results(filename): """ - Get an array of your results in the given file. - - :filename: Expects an available filename, without the .h5 ending, + Get an array of your results in the given file. + + :filename: Expects an available filename, without the .h5 ending, in your working directory :type: str - :return: Result array, simulation is an ndarray, + :return: Result array, simulation is an ndarray, which is different to structured arrays return by the csv/sql/ram databases :rtype: array """ import h5py - with h5py.File(filename+'.h5', 'r') as f: + + with h5py.File(filename + ".h5", "r") as f: return f[filename][()] + def load_csv_parameter_results(filename, usecols=None): """ Get an array of your results in the given file, without the first and the @@ -60,38 +72,49 @@ def load_csv_parameter_results(filename, usecols=None): :return: Result array :rtype: array """ - ofile=open(filename+'.csv') + ofile = open(filename + ".csv") line = ofile.readline() - header=line.split(',') + header = line.split(",") ofile.close() - - words=[] - index =[] - for i,word in enumerate(header): - if word.startswith('par'): - words.append(word) - index.append(i) - return np.genfromtxt(filename+'.csv', delimiter=',', names=words, - usecols=index, invalid_raise=False, skip_header=1) + + words = [] + index = [] + for i, word in enumerate(header): + if word.startswith("par"): + words.append(word) + index.append(i) + return np.genfromtxt( + filename + ".csv", + delimiter=",", + names=words, + usecols=index, + invalid_raise=False, + skip_header=1, + ) + def get_header(results): return results.dtype.names + def get_like_fields(results): header = get_header(results) - fields=[word for word in header if word.startswith('like')] + fields = [word for word in header if word.startswith("like")] return fields + def get_parameter_fields(results): header = get_header(results) - fields=[word for word in header if word.startswith('par')] + fields = [word for word in header if word.startswith("par")] return fields + def get_simulation_fields(results): header = get_header(results) - fields=[word for word in header if word.startswith('sim')] + fields = [word for word in header if word.startswith("sim")] return fields + def get_modelruns(results): """ Get an shorter array out of your result array, containing just the @@ -103,9 +126,10 @@ def get_modelruns(results): :return: Array containing just the columns beginnning with the indice "sim" :rtype: array """ - fields=[word for word in results.dtype.names if word.startswith('sim')] + fields = [word for word in results.dtype.names if word.startswith("sim")] return results[fields] + def get_parameters(results): """ Get an shorter array out of your result array, containing just the @@ -117,10 +141,11 @@ def get_parameters(results): :return: Array containing just the columns beginnning with the indice "par" :rtype: array """ - fields=[word for word in results.dtype.names if word.startswith('par')] + fields = [word for word in results.dtype.names if word.startswith("par")] results = results[fields] return results + def get_parameternames(results): """ Get list of strings with the names of the parameters of your model. @@ -132,14 +157,15 @@ def get_parameternames(results): :rtype: list """ - fields=[word for word in results.dtype.names if word.startswith('par')] + fields = [word for word in results.dtype.names if word.startswith("par")] - parnames=[] + parnames = [] for field in fields: parnames.append(field[3:]) return parnames -def get_maxlikeindex(results,verbose=True): + +def get_maxlikeindex(results, verbose=True): """ Get the maximum objectivefunction of your result array @@ -151,19 +177,20 @@ def get_maxlikeindex(results,verbose=True): :rtype: int and float """ try: - likes=results['like'] + likes = results["like"] except ValueError: - likes=results['like1'] - maximum=np.nanmax(likes) - value=str(round(maximum,4)) - text=str('Run number ' ) - index=np.where(likes==maximum) - text2=str(' has the highest objectivefunction with: ') - textv=text+str(index[0][0])+text2+value + likes = results["like1"] + maximum = np.nanmax(likes) + value = str(round(maximum, 4)) + text = str("Run number ") + index = np.where(likes == maximum) + text2 = str(" has the highest objectivefunction with: ") + textv = text + str(index[0][0]) + text2 + value if verbose: print(textv) return index, maximum + def get_minlikeindex(results): """ Get the minimum objectivefunction of your result array @@ -176,21 +203,21 @@ def get_minlikeindex(results): :rtype: int and float """ try: - likes=results['like'] + likes = results["like"] except ValueError: - likes=results['like1'] - minimum=np.nanmin(likes) - value=str(round(minimum,4)) - text=str('Run number ' ) - index=np.where(likes==minimum) - text2=str(' has the lowest objectivefunction with: ') - textv=text+str(index[0][0])+text2+value + likes = results["like1"] + minimum = np.nanmin(likes) + value = str(round(minimum, 4)) + text = str("Run number ") + index = np.where(likes == minimum) + text2 = str(" has the lowest objectivefunction with: ") + textv = text + str(index[0][0]) + text2 + value print(textv) return index[0][0], minimum -def get_percentiles(results,sim_number=''): +def get_percentiles(results, sim_number=""): """ Get 5,25,50,75 and 95 percentiles of your simulations @@ -203,17 +230,22 @@ def get_percentiles(results,sim_number=''): :return: Percentiles of simulations :rtype: int and float """ - p5,p25,p50,p75,p95=[],[],[],[],[] - fields=[word for word in results.dtype.names if word.startswith('simulation'+str(sim_number))] + p5, p25, p50, p75, p95 = [], [], [], [], [] + fields = [ + word + for word in results.dtype.names + if word.startswith("simulation" + str(sim_number)) + ] for i in range(len(fields)): - p5.append(np.percentile(list(results[fields[i]]),5)) - p25.append(np.percentile(list(results[fields[i]]),25)) - p50.append(np.percentile(list(results[fields[i]]),50)) - p75.append(np.percentile(list(results[fields[i]]),75)) - p95.append(np.percentile(list(results[fields[i]]),95)) - return p5,p25,p50,p75,p95 - -def calc_like(results,evaluation,objectivefunction): + p5.append(np.percentile(list(results[fields[i]]), 5)) + p25.append(np.percentile(list(results[fields[i]]), 25)) + p50.append(np.percentile(list(results[fields[i]]), 50)) + p75.append(np.percentile(list(results[fields[i]]), 75)) + p95.append(np.percentile(list(results[fields[i]]), 95)) + return p5, p25, p50, p75, p95 + + +def calc_like(results, evaluation, objectivefunction): """ Calculate another objectivefunction of your results @@ -222,20 +254,21 @@ def calc_like(results,evaluation,objectivefunction): :evaluation: Expects values, which correspond to your simulations :type: list - + :objectivefunction: Takes evaluation and simulation data and returns a objectivefunction, e.g. spotpy.objectvefunction.rmse :type: function :return: New objectivefunction list :rtype: list """ - likes=[] - sim=get_modelruns(results) + likes = [] + sim = get_modelruns(results) for s in sim: - likes.append(objectivefunction(evaluation,list(s))) + likes.append(objectivefunction(evaluation, list(s))) return likes -def compare_different_objectivefunctions(like1,like2): + +def compare_different_objectivefunctions(like1, like2): """ Performs the Welch’s t-test (aka unequal variances t-test) @@ -249,15 +282,17 @@ def compare_different_objectivefunctions(like1,like2): :rtype: list """ from scipy import stats + out = stats.ttest_ind(like1, like2, equal_var=False) print(out) - if out[1]>0.05: - print('like1 is NOT signifikant different to like2: p>0.05') + if out[1] > 0.05: + print("like1 is NOT signifikant different to like2: p>0.05") else: - print('like1 is signifikant different to like2: p<0.05' ) + print("like1 is signifikant different to like2: p<0.05") return out -def get_posterior(results,percentage=10, maximize=True): + +def get_posterior(results, percentage=10, maximize=True): """ Get the best XX% of your result array (e.g. best 10% model runs would be a threshold setting of 0.9) @@ -266,7 +301,7 @@ def get_posterior(results,percentage=10, maximize=True): :percentag: Optional, ratio of values that will be deleted. :type: float - + :maximize: If True (default), higher "like1" column values are assumed to be better. If False, lower "like1" column values are assumed to be better. @@ -274,60 +309,82 @@ def get_posterior(results,percentage=10, maximize=True): :rtype: array """ if maximize: - index = np.where(results['like1']>=np.percentile(results['like1'],100.0-percentage)) + index = np.where( + results["like1"] >= np.percentile(results["like1"], 100.0 - percentage) + ) else: - index = np.where(results['like1']>=np.percentile(results['like1'],100.0-percentage)) + index = np.where( + results["like1"] >= np.percentile(results["like1"], 100.0 - percentage) + ) return results[index] + def plot_parameter_trace(ax, results, parameter): - #THis function plots the parameter setting for each run - for i in range(int(max(results['chain']))): - index=np.where(results['chain']==i) - ax.plot(results['par'+parameter['name']][index],'.', markersize=2) - ax.set_ylabel(parameter['name']) - ax.set_ylim(parameter['minbound'], parameter['maxbound']) - + # THis function plots the parameter setting for each run + for i in range(int(max(results["chain"]))): + index = np.where(results["chain"] == i) + ax.plot(results["par" + parameter["name"]][index], ".", markersize=2) + ax.set_ylabel(parameter["name"]) + ax.set_ylim(parameter["minbound"], parameter["maxbound"]) + + def plot_posterior_parameter_histogram(ax, results, parameter): - #This functing is the last 100 runs - ax.hist(results['par'+parameter['name']][-100:], - bins =np.linspace(parameter['minbound'],parameter['maxbound'],20)) - ax.set_ylabel('Density') - ax.set_xlim(parameter['minbound'], parameter['maxbound']) - -def plot_parameter_uncertainty(posterior_results,evaluation, fig_name='Posterior_parameter_uncertainty.png'): + # This functing is the last 100 runs + ax.hist( + results["par" + parameter["name"]][-100:], + bins=np.linspace(parameter["minbound"], parameter["maxbound"], 20), + ) + ax.set_ylabel("Density") + ax.set_xlim(parameter["minbound"], parameter["maxbound"]) + + +def plot_parameter_uncertainty( + posterior_results, evaluation, fig_name="Posterior_parameter_uncertainty.png" +): import matplotlib.pyplot as plt simulation_fields = get_simulation_fields(posterior_results) - fig= plt.figure(figsize=(16,9)) + fig = plt.figure(figsize=(16, 9)) for i in range(len(evaluation)): if evaluation[i] == -9999: evaluation[i] = np.nan - ax = plt.subplot(1,1,1) - q5,q95=[],[] + ax = plt.subplot(1, 1, 1) + q5, q95 = [], [] for field in simulation_fields: - q5.append(np.percentile(list(posterior_results[field]),2.5)) - q95.append(np.percentile(list(posterior_results[field]),97.5)) - ax.plot(q5,color='dimgrey',linestyle='solid') - ax.plot(q95,color='dimgrey',linestyle='solid') - ax.fill_between(np.arange(0,len(q5),1),list(q5),list(q95),facecolor='dimgrey',zorder=0, - linewidth=0,label='parameter uncertainty') - ax.plot(evaluation,'r.',markersize=1, label='Observation data') - - bestindex,bestobjf = get_maxlikeindex(posterior_results,verbose=False) - plt.plot(list(posterior_results[simulation_fields][bestindex][0]),'b-',label='Obj='+str(round(bestobjf,2))) - plt.xlabel('Number of Observation Points') - plt.ylabel ('Simulated value') - plt.legend(loc='upper right') - fig.savefig(fig_name,dpi=300) - text='A plot of the parameter uncertainty has been saved as '+fig_name + q5.append(np.percentile(list(posterior_results[field]), 2.5)) + q95.append(np.percentile(list(posterior_results[field]), 97.5)) + ax.plot(q5, color="dimgrey", linestyle="solid") + ax.plot(q95, color="dimgrey", linestyle="solid") + ax.fill_between( + np.arange(0, len(q5), 1), + list(q5), + list(q95), + facecolor="dimgrey", + zorder=0, + linewidth=0, + label="parameter uncertainty", + ) + ax.plot(evaluation, "r.", markersize=1, label="Observation data") + + bestindex, bestobjf = get_maxlikeindex(posterior_results, verbose=False) + plt.plot( + list(posterior_results[simulation_fields][bestindex][0]), + "b-", + label="Obj=" + str(round(bestobjf, 2)), + ) + plt.xlabel("Number of Observation Points") + plt.ylabel("Simulated value") + plt.legend(loc="upper right") + fig.savefig(fig_name, dpi=300) + text = "A plot of the parameter uncertainty has been saved as " + fig_name print(text) - def sort_like(results): - return np.sort(results,axis=0) + return np.sort(results, axis=0) + -def get_best_parameterset(results,maximize=True): +def get_best_parameterset(results, maximize=True): """ Get the best parameter set of your result array, depending on your first objectivefunction @@ -341,24 +398,25 @@ def get_best_parameterset(results,maximize=True): :rtype: array """ try: - likes=results['like'] + likes = results["like"] except ValueError: - likes=results['like1'] + likes = results["like1"] if maximize: - best=np.nanmax(likes) + best = np.nanmax(likes) else: - best=np.nanmin(likes) - index=np.where(likes==best) + best = np.nanmin(likes) + index = np.where(likes == best) best_parameter_set = get_parameters(results[index])[0] parameter_names = get_parameternames(results) - text='' + text = "" for i in range(len(parameter_names)): - text+=parameter_names[i]+'='+str(best_parameter_set[i])+', ' - print('Best parameter set:\n'+text[:-2]) + text += parameter_names[i] + "=" + str(best_parameter_set[i]) + ", " + print("Best parameter set:\n" + text[:-2]) return get_parameters(results[index]) + def get_min_max(spotpy_setup): """ Get the minimum and maximum values of your parameters function of the spotpy setup @@ -369,12 +427,15 @@ def get_min_max(spotpy_setup): :return: Possible minimal and maximal values of all parameters in the parameters function of the spotpy_setup class :rtype: Two arrays """ - parameter_obj = spotpy.parameter.generate(spotpy.parameter.get_parameters_from_setup(spotpy_setup)) - randompar = parameter_obj['random'] + parameter_obj = spotpy.parameter.generate( + spotpy.parameter.get_parameters_from_setup(spotpy_setup) + ) + randompar = parameter_obj["random"] for i in range(1000): - randompar = np.column_stack((randompar, parameter_obj['random'])) + randompar = np.column_stack((randompar, parameter_obj["random"])) return np.amin(randompar, axis=1), np.amax(randompar, axis=1) + def get_parbounds(spotpy_setup): """ Get the minimum and maximum parameter bounds of your parameters function of the spotpy setup @@ -385,13 +446,14 @@ def get_parbounds(spotpy_setup): :return: Possible minimal and maximal values of all parameters in the parameters function of the spotpy_setup class :rtype: list """ - parmin,parmax=get_min_max(spotpy_setup) - bounds=[] + parmin, parmax = get_min_max(spotpy_setup) + bounds = [] for i in range(len(parmin)): - bounds.append([parmin[i],parmax[i]]) + bounds.append([parmin[i], parmax[i]]) return bounds -def get_sensitivity_of_fast(results,like_index=1,M=4, print_to_console=True): + +def get_sensitivity_of_fast(results, like_index=1, M=4, print_to_console=True): """ Get the sensitivity for every parameter of your result array, created with the FAST algorithm @@ -405,19 +467,24 @@ def get_sensitivity_of_fast(results,like_index=1,M=4, print_to_console=True): :rtype: list """ import math - likes=results['like'+str(like_index)] - print('Number of model runs:', likes.size) + + likes = results["like" + str(like_index)] + print("Number of model runs:", likes.size) parnames = get_parameternames(results) - parnumber=len(parnames) - print('Number of parameters:', parnumber) - + parnumber = len(parnames) + print("Number of parameters:", parnumber) + rest = likes.size % (parnumber) if rest != 0: - print("""" + print( + """" Number of samples in model output file must be a multiple of D, where D is the number of parameters in your parameter file. - We handle this by ignoring the last """, rest, """runs.""") - likes = likes[:-rest ] + We handle this by ignoring the last """, + rest, + """runs.""", + ) + likes = likes[:-rest] N = int(likes.size / parnumber) # Recreate the vector omega used in the sampling @@ -425,30 +492,32 @@ def get_sensitivity_of_fast(results,like_index=1,M=4, print_to_console=True): omega[0] = math.floor((N - 1) / (2 * M)) m = math.floor(omega[0] / (2 * M)) - print('m =', m) + print("m =", m) if m >= (parnumber - 1): omega[1:] = np.floor(np.linspace(1, m, parnumber - 1)) else: omega[1:] = np.arange(parnumber - 1) % m + 1 - print('Omega =', omega) + print("Omega =", omega) # Calculate and Output the First and Total Order Values if print_to_console: print("Parameter First Total") - Si = dict((k, [None] * parnumber) for k in ['S1', 'ST']) + Si = dict((k, [None] * parnumber) for k in ["S1", "ST"]) print(Si) for i in range(parnumber): l = np.arange(i * N, (i + 1) * N) print(l) - Si['S1'][i] = _compute_first_order(likes[l], N, M, omega[0]) - Si['ST'][i] = _compute_total_order(likes[l], N, omega[0]) + Si["S1"][i] = _compute_first_order(likes[l], N, M, omega[0]) + Si["ST"][i] = _compute_total_order(likes[l], N, omega[0]) print(Si) if print_to_console: - print("%s %f %f" % - (parnames[i], Si['S1'][i], Si['ST'][i])) + print("%s %f %f" % (parnames[i], Si["S1"][i], Si["ST"][i])) return Si -def plot_fast_sensitivity(results,like_index=1,number_of_sensitiv_pars=10,fig_name='FAST_sensitivity.png'): + +def plot_fast_sensitivity( + results, like_index=1, number_of_sensitiv_pars=10, fig_name="FAST_sensitivity.png" +): """ Example, how to plot the sensitivity for every parameter of your result array, created with the FAST algorithm @@ -467,195 +536,207 @@ def plot_fast_sensitivity(results,like_index=1,number_of_sensitiv_pars=10,fig_na import matplotlib.pyplot as plt - parnames=get_parameternames(results) - fig=plt.figure(figsize=(9,6)) + parnames = get_parameternames(results) + fig = plt.figure(figsize=(9, 6)) - ax = plt.subplot(1,1,1) - Si = get_sensitivity_of_fast(results, like_index=like_index) + ax = plt.subplot(1, 1, 1) + Si = get_sensitivity_of_fast(results, like_index=like_index) names = [] values = [] no_names = [] no_values = [] - index=[] - no_index=[] - + index = [] + no_index = [] + try: - threshold = np.sort(list(Si.values())[1])[-number_of_sensitiv_pars] + threshold = np.sort(list(Si.values())[1])[-number_of_sensitiv_pars] except IndexError: threshold = 0 - - first_sens_call=True - first_insens_call=True + + first_sens_call = True + first_insens_call = True try: Si.values() except AttributeError: - exit("Our SI is wrong: " +str(Si)) + exit("Our SI is wrong: " + str(Si)) for j in range(len(list(Si.values())[1])): - if list(Si.values())[1][j]>=threshold: + if list(Si.values())[1][j] >= threshold: names.append(j) values.append(list(Si.values())[1][j]) index.append(j) if first_sens_call: - ax.bar(j, list(Si.values())[1][j], color='blue', label='Sensitive Parameters') + ax.bar( + j, + list(Si.values())[1][j], + color="blue", + label="Sensitive Parameters", + ) else: - ax.bar(j, list(Si.values())[1][j], color='blue') - first_sens_call=False - + ax.bar(j, list(Si.values())[1][j], color="blue") + first_sens_call = False else: - #names.append('') + # names.append('') no_values.append(list(Si.values())[1][j]) no_index.append(j) if first_insens_call: - ax.bar(j,list(Si.values())[1][j],color='orange', label = 'Insensitive parameter') + ax.bar( + j, + list(Si.values())[1][j], + color="orange", + label="Insensitive parameter", + ) else: - ax.bar(j,list(Si.values())[1][j],color='orange') - first_insens_call=False + ax.bar(j, list(Si.values())[1][j], color="orange") + first_insens_call = False - ax.set_ylim([0,1]) + ax.set_ylim([0, 1]) - ax.set_xlabel('Model Paramters') - ax.set_ylabel('Total Sensititivity Index') + ax.set_xlabel("Model Paramters") + ax.set_ylabel("Total Sensititivity Index") ax.legend() - ax.set_xticks(np.arange(0,len(parnames))) - xtickNames = ax.set_xticklabels(parnames, color='grey') - + ax.set_xticks(np.arange(0, len(parnames))) + xtickNames = ax.set_xticklabels(parnames, color="grey") + plt.setp(xtickNames, rotation=90) for name_id in names: ax.get_xticklabels()[name_id].set_color("black") - - #ax.set_xticklabels(['0']+parnames) - ax.plot(np.arange(-1,len(parnames)+1,1),[threshold]*(len(parnames)+2),'r--') - ax.set_xlim(-0.5,len(parnames)-0.5) + + # ax.set_xticklabels(['0']+parnames) + ax.plot( + np.arange(-1, len(parnames) + 1, 1), [threshold] * (len(parnames) + 2), "r--" + ) + ax.set_xlim(-0.5, len(parnames) - 0.5) plt.tight_layout() - fig.savefig(fig_name,dpi=150) - + fig.savefig(fig_name, dpi=150) -def plot_heatmap_griewank(results,algorithms, fig_name='heatmap_griewank.png'): + +def plot_heatmap_griewank(results, algorithms, fig_name="heatmap_griewank.png"): """Example Plot as seen in the SPOTPY Documentation""" import matplotlib.pyplot as plt - - from matplotlib import ticker - from matplotlib import cm - font = {'family' : 'calibri', - 'weight' : 'normal', - 'size' : 20} - plt.rc('font', **font) - subplots=len(results) - xticks=[-40,0,40] - yticks=[-40,0,40] - fig=plt.figure(figsize=(16,6)) + from matplotlib import cm, ticker + + font = {"family": "calibri", "weight": "normal", "size": 20} + plt.rc("font", **font) + subplots = len(results) + xticks = [-40, 0, 40] + yticks = [-40, 0, 40] + fig = plt.figure(figsize=(16, 6)) N = 2000 x = np.linspace(-50.0, 50.0, N) y = np.linspace(-50.0, 50.0, N) x, y = np.meshgrid(x, y) - z=1+ (x**2+y**2)/4000 - np.cos(x/np.sqrt(2))*np.cos(y/np.sqrt(3)) + z = 1 + (x**2 + y**2) / 4000 - np.cos(x / np.sqrt(2)) * np.cos(y / np.sqrt(3)) - cmap = plt.get_cmap('autumn') + cmap = plt.get_cmap("autumn") - rows=2 + rows = 2 for i in range(subplots): - amount_row = int(np.ceil(subplots/rows)) - ax = plt.subplot(rows, amount_row, i+1) - CS = ax.contourf(x, y, z,locator=ticker.LogLocator(),cmap=cm.rainbow) + amount_row = int(np.ceil(subplots / rows)) + ax = plt.subplot(rows, amount_row, i + 1) + CS = ax.contourf(x, y, z, locator=ticker.LogLocator(), cmap=cm.rainbow) - ax.plot(results[i]['par0'],results[i]['par1'],'ko',alpha=0.2,markersize=1.9) + ax.plot(results[i]["par0"], results[i]["par1"], "ko", alpha=0.2, markersize=1.9) ax.xaxis.set_ticks([]) - if i==0: - ax.set_ylabel('y') - if i==subplots/rows: - ax.set_ylabel('y') - if i>=subplots/rows: - ax.set_xlabel('x') + if i == 0: + ax.set_ylabel("y") + if i == subplots / rows: + ax.set_ylabel("y") + if i >= subplots / rows: + ax.set_xlabel("x") ax.xaxis.set_ticks(xticks) - if i!=0 and i!=subplots/rows: + if i != 0 and i != subplots / rows: ax.yaxis.set_ticks([]) - ax.set_title(algorithms[i]) - fig.savefig(fig_name, bbox_inches='tight') + fig.savefig(fig_name, bbox_inches="tight") -def plot_objectivefunction(results,evaluation,limit=None,sort=True, fig_name = 'objective_function.png'): +def plot_objectivefunction( + results, evaluation, limit=None, sort=True, fig_name="objective_function.png" +): """Example Plot as seen in the SPOTPY Documentation""" import matplotlib.pyplot as plt - likes=calc_like(results,evaluation,spotpy.objectivefunctions.rmse) - data=likes - #Calc confidence Interval + + likes = calc_like(results, evaluation, spotpy.objectivefunctions.rmse) + data = likes + # Calc confidence Interval mean = np.average(data) # evaluate sample variance by setting delta degrees of freedom (ddof) to # 1. The degree used in calculations is N - ddof stddev = np.std(data, ddof=1) from scipy.stats import t + # Get the endpoints of the range that contains 95% of the distribution t_bounds = t.interval(0.999, len(data) - 1) # sum mean to the confidence interval ci = [mean + critval * stddev / np.sqrt(len(data)) for critval in t_bounds] - value="Mean: %f" % mean + value = "Mean: %f" % mean print(value) - value="Confidence Interval 95%%: %f, %f" % (ci[0], ci[1]) + value = "Confidence Interval 95%%: %f, %f" % (ci[0], ci[1]) print(value) - threshold=ci[1] - happend=None - bestlike=[data[0]] + threshold = ci[1] + happend = None + bestlike = [data[0]] for like in data: - if like0: + if i > 0: ax.yaxis.set_ticks([]) - if j==rows-1: - ax.set_xlabel(algorithmnames[i-subplots]) + if j == rows - 1: + ax.set_xlabel(algorithmnames[i - subplots]) else: ax.xaxis.set_ticks([]) - ax.plot([1]*rep,'r--') - ax.set_xlim(0,rep) - ax.set_ylim(parameter['minbound'][j],parameter['maxbound'][j]) - - #plt.tight_layout() - fig.savefig(fig_name, bbox_inches='tight') + ax.plot([1] * rep, "r--") + ax.set_xlim(0, rep) + ax.set_ylim(parameter["minbound"][j], parameter["maxbound"][j]) + # plt.tight_layout() + fig.savefig(fig_name, bbox_inches="tight") -def plot_parametertrace(results,parameternames=None,fig_name='Parameter_trace.png'): + +def plot_parametertrace(results, parameternames=None, fig_name="Parameter_trace.png"): """ Get a plot with all values of a given parameter in your result array. The plot will be saved as a .png file. @@ -670,27 +751,31 @@ def plot_parametertrace(results,parameternames=None,fig_name='Parameter_trace.pn :rtype: figure """ import matplotlib.pyplot as plt - fig=plt.figure(figsize=(16,9)) + + fig = plt.figure(figsize=(16, 9)) if not parameternames: - parameternames=get_parameternames(results) - names='' - i=1 + parameternames = get_parameternames(results) + names = "" + i = 1 for name in parameternames: - ax = plt.subplot(len(parameternames),1,i) - ax.plot(results['par'+name],label=name) - names+=name+'_' + ax = plt.subplot(len(parameternames), 1, i) + ax.plot(results["par" + name], label=name) + names += name + "_" ax.set_ylabel(name) - if i==len(parameternames): - ax.set_xlabel('Repetitions') - if i==1: - ax.set_title('Parametertrace') + if i == len(parameternames): + ax.set_xlabel("Repetitions") + if i == 1: + ax.set_title("Parametertrace") ax.legend() - i+=1 + i += 1 fig.savefig(fig_name) - text='The figure as been saved as "'+fig_name + text = 'The figure as been saved as "' + fig_name print(text) -def plot_posterior_parametertrace(results,parameternames=None,threshold=0.1, fig_name='Posterior_parametertrace.png'): + +def plot_posterior_parametertrace( + results, parameternames=None, threshold=0.1, fig_name="Posterior_parametertrace.png" +): """ Get a plot with all values of a given parameter in your result array. The plot will be saved as a .png file. @@ -705,29 +790,39 @@ def plot_posterior_parametertrace(results,parameternames=None,threshold=0.1, fig :rtype: figure """ import matplotlib.pyplot as plt - fig=plt.figure(figsize=(16,9)) - results=sort_like(results) + fig = plt.figure(figsize=(16, 9)) + + results = sort_like(results) if not parameternames: - parameternames=get_parameternames(results) - names='' - i=1 + parameternames = get_parameternames(results) + names = "" + i = 1 for name in parameternames: - ax = plt.subplot(len(parameternames),1,i) - ax.plot(results['par'+name][int(len(results)*threshold):],label=name) - names+=name+'_' + ax = plt.subplot(len(parameternames), 1, i) + ax.plot(results["par" + name][int(len(results) * threshold) :], label=name) + names += name + "_" ax.set_ylabel(name) - if i==len(parameternames): - ax.set_xlabel('Repetitions') - if i==1: - ax.set_title('Parametertrace') + if i == len(parameternames): + ax.set_xlabel("Repetitions") + if i == 1: + ax.set_title("Parametertrace") ax.legend() - i+=1 + i += 1 fig.savefig(fig_name) - text='The figure as been saved as '+fig_name + text = "The figure as been saved as " + fig_name print(text) -def plot_posterior(results,evaluation,dates=None,ylabel='Posterior model simulation',xlabel='Time',bestperc=0.1, fig_name='bestmodelrun.png'): + +def plot_posterior( + results, + evaluation, + dates=None, + ylabel="Posterior model simulation", + xlabel="Time", + bestperc=0.1, + fig_name="bestmodelrun.png", +): """ Get a plot with the maximum objectivefunction of your simulations in your result array. @@ -756,31 +851,36 @@ def plot_posterior(results,evaluation,dates=None,ylabel='Posterior model simulat """ import matplotlib.pyplot as plt - index,maximum=get_maxlikeindex(results) - sim=get_modelruns(results) - bestmodelrun=list(sim[index][0])#Transform values into list to ensure plotting - bestparameterset=list(get_parameters(results)[index][0]) - - parameternames=list(get_parameternames(results) ) - bestparameterstring='' - maxNSE=spotpy.objectivefunctions.nashsutcliffe(bestmodelrun,evaluation) + + index, maximum = get_maxlikeindex(results) + sim = get_modelruns(results) + bestmodelrun = list(sim[index][0]) # Transform values into list to ensure plotting + bestparameterset = list(get_parameters(results)[index][0]) + + parameternames = list(get_parameternames(results)) + bestparameterstring = "" + maxNSE = spotpy.objectivefunctions.nashsutcliffe(bestmodelrun, evaluation) for i in range(len(parameternames)): - if i%8==0: - bestparameterstring+='\n' - bestparameterstring+=parameternames[i]+'='+str(round(bestparameterset[i],4))+',' - fig=plt.figure(figsize=(16,8)) - plt.plot(bestmodelrun,'b-',label='Simulation='+str(round(maxNSE,4))) - plt.plot(evaluation,'ro',label='Evaluation') + if i % 8 == 0: + bestparameterstring += "\n" + bestparameterstring += ( + parameternames[i] + "=" + str(round(bestparameterset[i], 4)) + "," + ) + fig = plt.figure(figsize=(16, 8)) + plt.plot(bestmodelrun, "b-", label="Simulation=" + str(round(maxNSE, 4))) + plt.plot(evaluation, "ro", label="Evaluation") plt.legend() plt.ylabel(ylabel) plt.xlabel(xlabel) - plt.title('Maximum objectivefunction of Simulations with '+bestparameterstring[0:-2]) + plt.title( + "Maximum objectivefunction of Simulations with " + bestparameterstring[0:-2] + ) fig.savefig(fig_name) - text='The figure as been saved as '+fig_name + text = "The figure as been saved as " + fig_name print(text) -def plot_bestmodelrun(results,evaluation,fig_name ='Best_model_run.png'): +def plot_bestmodelrun(results, evaluation, fig_name="Best_model_run.png"): """ Get a plot with the maximum objectivefunction of your simulations in your result array. @@ -797,24 +897,38 @@ def plot_bestmodelrun(results,evaluation,fig_name ='Best_model_run.png'): figure. Plot of the simulation with the maximum objectivefunction value in the result array as a blue line and dots for the evaluation data. """ import matplotlib.pyplot as plt - fig= plt.figure(figsize=(16,9)) + + fig = plt.figure(figsize=(16, 9)) for i in range(len(evaluation)): if evaluation[i] == -9999: evaluation[i] = np.nan - plt.plot(evaluation,'ro',markersize=1, label='Observation data') + plt.plot(evaluation, "ro", markersize=1, label="Observation data") simulation_fields = get_simulation_fields(results) - bestindex,bestobjf = get_maxlikeindex(results,verbose=False) - plt.plot(list(results[simulation_fields][bestindex][0]),'b-',label='Obj='+str(round(bestobjf,2))) - plt.xlabel('Number of Observation Points') - plt.ylabel ('Simulated value') - plt.legend(loc='upper right') - fig.savefig(fig_name,dpi=300) - text='A plot of the best model run has been saved as '+fig_name + bestindex, bestobjf = get_maxlikeindex(results, verbose=False) + plt.plot( + list(results[simulation_fields][bestindex][0]), + "b-", + label="Obj=" + str(round(bestobjf, 2)), + ) + plt.xlabel("Number of Observation Points") + plt.ylabel("Simulated value") + plt.legend(loc="upper right") + fig.savefig(fig_name, dpi=300) + text = "A plot of the best model run has been saved as " + fig_name print(text) - -def plot_bestmodelruns(results,evaluation,algorithms=None,dates=None,ylabel='Best model simulation',xlabel='Date',objectivefunctionmax=True,calculatelike=True,fig_name='bestmodelrun.png'): +def plot_bestmodelruns( + results, + evaluation, + algorithms=None, + dates=None, + ylabel="Best model simulation", + xlabel="Date", + objectivefunctionmax=True, + calculatelike=True, + fig_name="bestmodelrun.png", +): """ Get a plot with the maximum objectivefunction of your simulations in your result array. @@ -847,70 +961,96 @@ def plot_bestmodelruns(results,evaluation,algorithms=None,dates=None,ylabel='Bes """ import matplotlib.pyplot as plt - plt.rc('font', **font) - fig=plt.figure(figsize=(17,8)) - colors=['grey', 'black', 'brown','red','orange', 'yellow','green','blue',] - plt.plot(dates,evaluation,'ro',label='Evaluation data') + plt.rc("font", **font) + fig = plt.figure(figsize=(17, 8)) + colors = [ + "grey", + "black", + "brown", + "red", + "orange", + "yellow", + "green", + "blue", + ] + plt.plot(dates, evaluation, "ro", label="Evaluation data") for i in range(len(results)): if calculatelike: - likes=[] - sim=get_modelruns(results[i]) - par=get_parameters(results[i]) + likes = [] + sim = get_modelruns(results[i]) + par = get_parameters(results[i]) for s in sim: - likes.append(spotpy.objectivefunctions.lognashsutcliffe(evaluation,list(s))) - - maximum=max(likes) - index=likes.index(maximum) - bestmodelrun=list(sim[index]) - bestparameterset=list(par[index]) + likes.append( + spotpy.objectivefunctions.lognashsutcliffe(evaluation, list(s)) + ) + + maximum = max(likes) + index = likes.index(maximum) + bestmodelrun = list(sim[index]) + bestparameterset = list(par[index]) print(bestparameterset) else: - if objectivefunctionmax==True: - index,maximum=get_maxlikeindex(results[i]) + if objectivefunctionmax == True: + index, maximum = get_maxlikeindex(results[i]) else: - index,maximum=get_minlikeindex(results[i]) - bestmodelrun=list(get_modelruns(results[i])[index][0])#Transform values into list to ensure plotting + index, maximum = get_minlikeindex(results[i]) + bestmodelrun = list( + get_modelruns(results[i])[index][0] + ) # Transform values into list to ensure plotting - maxLike=spotpy.objectivefunctions.lognashsutcliffe(evaluation,bestmodelrun) + maxLike = spotpy.objectivefunctions.lognashsutcliffe(evaluation, bestmodelrun) if dates is not None: - plt.plot(dates,bestmodelrun,'-',color=colors[i],label=algorithms[i]+': LogNSE='+str(round(maxLike,4))) + plt.plot( + dates, + bestmodelrun, + "-", + color=colors[i], + label=algorithms[i] + ": LogNSE=" + str(round(maxLike, 4)), + ) else: - plt.plot(bestmodelrun,'-',color=colors[i],label=algorithms[i]+': AI='+str(round(maxLike,4))) - #plt.plot(evaluation,'ro',label='Evaluation data') - plt.legend(bbox_to_anchor=(.0, 0), loc=3) + plt.plot( + bestmodelrun, + "-", + color=colors[i], + label=algorithms[i] + ": AI=" + str(round(maxLike, 4)), + ) + # plt.plot(evaluation,'ro',label='Evaluation data') + plt.legend(bbox_to_anchor=(0.0, 0), loc=3) plt.ylabel(ylabel) plt.xlabel(xlabel) - plt.ylim(15,50) #DELETE WHEN NOT USED WITH SOIL MOISTUR RESULTS + plt.ylim(15, 50) # DELETE WHEN NOT USED WITH SOIL MOISTUR RESULTS fig.savefig(fig_name) - text='The figure as been saved as '+fig_name + text = "The figure as been saved as " + fig_name print(text) -def plot_objectivefunctiontraces(results,evaluation,algorithms,fig_name='Like_trace.png'): + +def plot_objectivefunctiontraces( + results, evaluation, algorithms, fig_name="Like_trace.png" +): import matplotlib.pyplot as plt from matplotlib import colors - cnames=list(colors.cnames) - font = {'family' : 'calibri', - 'weight' : 'normal', - 'size' : 20} - plt.rc('font', **font) - fig=plt.figure(figsize=(16,3)) - xticks=[5000,15000] + + cnames = list(colors.cnames) + font = {"family": "calibri", "weight": "normal", "size": 20} + plt.rc("font", **font) + fig = plt.figure(figsize=(16, 3)) + xticks = [5000, 15000] for i in range(len(results)): - ax = plt.subplot(1,len(results),i+1) - likes=calc_like(results[i],evaluation,spotpy.objectivefunctions.rmse) - ax.plot(likes,'b-') - ax.set_ylim(0,25) - ax.set_xlim(0,len(results[0])) + ax = plt.subplot(1, len(results), i + 1) + likes = calc_like(results[i], evaluation, spotpy.objectivefunctions.rmse) + ax.plot(likes, "b-") + ax.set_ylim(0, 25) + ax.set_xlim(0, len(results[0])) ax.set_xlabel(algorithms[i]) ax.xaxis.set_ticks(xticks) - if i==0: - ax.set_ylabel('RMSE') - ax.yaxis.set_ticks([0,10,20]) + if i == 0: + ax.set_ylabel("RMSE") + ax.yaxis.set_ticks([0, 10, 20]) else: ax.yaxis.set_ticks([]) @@ -918,94 +1058,101 @@ def plot_objectivefunctiontraces(results,evaluation,algorithms,fig_name='Like_tr fig.savefig(fig_name) -def plot_regression(results,evaluation,fig_name='regressionanalysis.png'): +def plot_regression(results, evaluation, fig_name="regressionanalysis.png"): import matplotlib.pyplot as plt - fig=plt.figure(figsize=(16,9)) - simulations=get_modelruns(results) + + fig = plt.figure(figsize=(16, 9)) + simulations = get_modelruns(results) for sim in simulations: - plt.plot(evaluation,list(sim),'bo',alpha=.05) - plt.ylabel('simulation') - plt.xlabel('evaluation') - plt.title('Regression between simulations and evaluation data') + plt.plot(evaluation, list(sim), "bo", alpha=0.05) + plt.ylabel("simulation") + plt.xlabel("evaluation") + plt.title("Regression between simulations and evaluation data") fig.savefig(fig_name) - text='The figure as been saved as '+fig_name + text = "The figure as been saved as " + fig_name print(text) -def plot_parameterInteraction(results, fig_name ='ParameterInteraction.png'): - '''Input: List with values of parameters and list of strings with parameter names - Output: Dotty plot of parameter distribution and gaussian kde distribution''' +def plot_parameterInteraction(results, fig_name="ParameterInteraction.png"): + """Input: List with values of parameters and list of strings with parameter names + Output: Dotty plot of parameter distribution and gaussian kde distribution""" import matplotlib.pyplot as plt import pandas as pd - parameterdistribtion=get_parameters(results) - parameternames=get_parameternames(results) - df = pd.DataFrame(np.asarray(parameterdistribtion).T.tolist(), columns=parameternames) - pd.plotting.scatter_matrix(df, alpha=0.2, figsize=(12, 12), diagonal='kde') - plt.savefig(fig_name,dpi=300) + parameterdistribtion = get_parameters(results) + parameternames = get_parameternames(results) + df = pd.DataFrame( + np.asarray(parameterdistribtion).T.tolist(), columns=parameternames + ) + + pd.plotting.scatter_matrix(df, alpha=0.2, figsize=(12, 12), diagonal="kde") + plt.savefig(fig_name, dpi=300) -def plot_allmodelruns(modelruns,observations,dates=None, fig_name='bestmodel.png'): - '''Input: Array of modelruns and list of Observations - Output: Plot with all modelruns as a line and dots with the Observations - ''' +def plot_allmodelruns(modelruns, observations, dates=None, fig_name="bestmodel.png"): + """Input: Array of modelruns and list of Observations + Output: Plot with all modelruns as a line and dots with the Observations + """ import matplotlib.pyplot as plt - fig=plt.figure(figsize=(16,9)) - ax = plt.subplot(1,1,1) + + fig = plt.figure(figsize=(16, 9)) + ax = plt.subplot(1, 1, 1) if dates is not None: for i in range(len(modelruns)): - if i==0: - ax.plot(dates, modelruns[i],'b',alpha=.05,label='Simulations') + if i == 0: + ax.plot(dates, modelruns[i], "b", alpha=0.05, label="Simulations") else: - ax.plot(dates, modelruns[i],'b',alpha=.05) + ax.plot(dates, modelruns[i], "b", alpha=0.05) else: for i in range(len(modelruns)): - if i==0: - ax.plot(modelruns[i],'b',alpha=.05,label='Simulations') + if i == 0: + ax.plot(modelruns[i], "b", alpha=0.05, label="Simulations") else: - ax.plot(modelruns[i],'b',alpha=.05) - ax.plot(observations,'ro',label='Evaluation') + ax.plot(modelruns[i], "b", alpha=0.05) + ax.plot(observations, "ro", label="Evaluation") ax.legend() - ax.set_xlabel = 'Best model simulation' - ax.set_ylabel = 'Evaluation points' - ax.set_title = 'Maximum objectivefunction of Simulations' + ax.set_xlabel = "Best model simulation" + ax.set_ylabel = "Evaluation points" + ax.set_title = "Maximum objectivefunction of Simulations" fig.savefig(fig_name) - text='The figure as been saved as '+fig_name + text = "The figure as been saved as " + fig_name print(text) -def plot_gelman_rubin(results, r_hat_values,fig_name='gelman_rub.png'): - '''Input: List of R_hat values of chains (see Gelman & Rubin 1992) - Output: Plot as seen for e.g. in (Sadegh and Vrugt 2014)''' +def plot_gelman_rubin(results, r_hat_values, fig_name="gelman_rub.png"): + """Input: List of R_hat values of chains (see Gelman & Rubin 1992) + Output: Plot as seen for e.g. in (Sadegh and Vrugt 2014)""" import matplotlib.pyplot as plt - - fig= plt.figure(figsize=(9,6)) - ax1 = plt.subplot(2,1,1) - for i in range(int(max(results['chain']))+1): - index=np.where(results['chain']==i) - ax1.plot(results['like1'][index], label='Chain '+str(i+1)) - - ax1.set_ylabel('Likelihood value') + + fig = plt.figure(figsize=(9, 6)) + ax1 = plt.subplot(2, 1, 1) + for i in range(int(max(results["chain"])) + 1): + index = np.where(results["chain"] == i) + ax1.plot(results["like1"][index], label="Chain " + str(i + 1)) + + ax1.set_ylabel("Likelihood value") ax1.legend() - - ax2 =plt.subplot(2,1,2) - r_hat=np.array(r_hat_values) - ax2.plot([1.2]*len(r_hat),'k--') + + ax2 = plt.subplot(2, 1, 2) + r_hat = np.array(r_hat_values) + ax2.plot([1.2] * len(r_hat), "k--") for i in range(len(r_hat[0])): - ax2.plot(r_hat[:,i],label='x'+str(i+1)) - - ax2.set_yscale("log", nonpositive='clip') - ax2.set_ylabel('R$^d$ - convergence diagnostic') - ax2.set_xlabel('Number of chainruns') + ax2.plot(r_hat[:, i], label="x" + str(i + 1)) + + ax2.set_yscale("log", nonpositive="clip") + ax2.set_ylabel("R$^d$ - convergence diagnostic") + ax2.set_xlabel("Number of chainruns") ax2.legend() - fig.savefig(fig_name,dpi=150) + fig.savefig(fig_name, dpi=150) + def gelman_rubin(x): - '''NOT USED YET''' + """NOT USED YET""" if np.shape(x) < (2,): raise ValueError( - 'Gelman-Rubin diagnostic requires multiple chains of the same length.') + "Gelman-Rubin diagnostic requires multiple chains of the same length." + ) try: m, n = np.shape(x) except ValueError: @@ -1013,10 +1160,9 @@ def gelman_rubin(x): # Calculate between-chain variance B_over_n = np.sum((np.mean(x, 1) - np.mean(x)) ** 2) / (m - 1) # Calculate within-chain variances - W = np.sum( - [(x[i] - xbar) ** 2 for i, - xbar in enumerate(np.mean(x, - 1))]) / (m * (n - 1)) + W = np.sum([(x[i] - xbar) ** 2 for i, xbar in enumerate(np.mean(x, 1))]) / ( + m * (n - 1) + ) # (over) estimate of variance s2 = W * (n - 1) / n + B_over_n # Pooled posterior variance estimate @@ -1026,9 +1172,9 @@ def gelman_rubin(x): return R -def plot_Geweke(parameterdistribution,parametername): - '''Input: Takes a list of sampled values for a parameter and his name as a string - Output: Plot as seen for e.g. in BUGS or PyMC''' +def plot_Geweke(parameterdistribution, parametername): + """Input: Takes a list of sampled values for a parameter and his name as a string + Output: Plot as seen for e.g. in BUGS or PyMC""" import matplotlib.pyplot as plt # perform the Geweke test @@ -1036,16 +1182,17 @@ def plot_Geweke(parameterdistribution,parametername): # plot the results fig = plt.figure() - plt.plot(Geweke_values,label=parametername) + plt.plot(Geweke_values, label=parametername) plt.legend() - plt.title(parametername + '- Geweke_Test') - plt.xlabel('Subinterval') - plt.ylabel('Geweke Test') - plt.ylim([-3,3]) + plt.title(parametername + "- Geweke_Test") + plt.xlabel("Subinterval") + plt.ylabel("Geweke Test") + plt.ylim([-3, 3]) # plot the delimiting line - plt.plot( [2]*len(Geweke_values), 'r-.') - plt.plot( [-2]*len(Geweke_values), 'r-.') + plt.plot([2] * len(Geweke_values), "r-.") + plt.plot([-2] * len(Geweke_values), "r-.") + def _compute_first_order(outputs, N, M, omega): f = np.fft.fft(outputs) @@ -1054,37 +1201,39 @@ def _compute_first_order(outputs, N, M, omega): D1 = 2 * np.sum(Sp[np.arange(1, M + 1) * int(omega) - 1]) return D1 / V + def _compute_total_order(outputs, N, omega): f = np.fft.fft(outputs) Sp = np.power(np.absolute(f[np.arange(1, int((N + 1) / 2))]) / N, 2) V = 2 * np.sum(Sp) Dt = 2 * sum(Sp[np.arange(int(omega / 2))]) - return (1 - Dt / V) + return 1 - Dt / V + def _Geweke(samples, intervals=20): - '''Calculates Geweke Z-Scores''' - length=int(len(samples)/intervals/2) + """Calculates Geweke Z-Scores""" + length = int(len(samples) / intervals / 2) # discard the first 10 per cent - first = 0.1*len(samples) + first = 0.1 * len(samples) # create empty array to store the results z = np.empty(intervals) for k in np.arange(0, intervals): # starting points of the two different subsamples - start1 = int(first + k*length) - start2 = int(len(samples)/2 + k*length) + start1 = int(first + k * length) + start2 = int(len(samples) / 2 + k * length) # extract the sub samples - subsamples1 = samples[start1:start1+length] - subsamples2 = samples[start2:start2+length] + subsamples1 = samples[start1 : start1 + length] + subsamples2 = samples[start2 : start2 + length] # calculate the mean and the variance mean1 = np.mean(subsamples1) mean2 = np.mean(subsamples2) - var1 = np.var(subsamples1) - var2 = np.var(subsamples2) + var1 = np.var(subsamples1) + var2 = np.var(subsamples2) # calculate the Geweke test - z[k] = (mean1-mean2)/np.sqrt(var1+var2) + z[k] = (mean1 - mean2) / np.sqrt(var1 + var2) return z diff --git a/src/spotpy/cli.py b/src/spotpy/cli.py new file mode 100644 index 00000000..9c0e4385 --- /dev/null +++ b/src/spotpy/cli.py @@ -0,0 +1,119 @@ +import inspect +import io +import os + +import click + +from . import algorithms, database, describe + + +def get_config_from_file(): + """ + Gets the spotpy configuration from a config file 'spotpy.conf'. + + Example: + + sampler = mc + dbtype = csv + parallel = seq + # This is a comment + runs = 10 + """ + config = {} + if os.path.exists("spotpy.conf"): + with io.open("spotpy.conf") as f: + for line in f: + if not line.strip().startswith("#"): + try: + k, v = line.split("=", 1) + config[k.strip()] = v.strip() + except ValueError: + pass + return config + + +def get_sampler_from_string(sampler_name): + return getattr(algorithms, sampler_name) + + +def make_type_from_module(module, *exclude): + def use(cl): + # Check if class name starts with an exclusion term + return inspect.isclass(cl) and not any( + [cl.__name__.startswith(ex) for ex in ("_",) + exclude] + ) + + members = inspect.getmembers(module, use) + return click.Choice([n for n, m in members if not n.startswith("_")]) + + +@click.group(context_settings=dict(help_option_names=["-h", "--help"])) +def cli(): + pass + + +@cli.command() +@click.pass_context +@click.option( + "--sampler", + "-s", + type=make_type_from_module(algorithms), + default="mc", + help="Select the spotpy sampler", +) +@click.option( + "--dbformat", + type=click.Choice(database.__dir__()), + default="ram", + help="The type of the database", +) +@click.option( + "--dbname", type=click.STRING, help="The name of the database, leave open for ram" +) +@click.option( + "--parallel", + "-p", + type=click.Choice(["seq", "mpc", "mpi"]), + default="seq", + help="Parallelization: seq = no parallelization, mpi = MPI (for clusters), mpc = multiprocessing", +) +@click.option("--runs", "-n", type=click.INT, default=1, help="Number of runs") +@click.option( + "--config", + "-c", + is_flag=True, + help="Print only the configuration, can be used to create a config file with your_model.py > spotpy.conf", +) +def run(ctx, **kwargs): + """ + Runs a sampler for automatic calibration + """ + setup = ctx.obj + if kwargs.pop("config", None): + click.echo("\n".join("{} = {}".format(k, v) for k, v in kwargs.items())) + else: + sampler_name = kwargs.pop("sampler") + sampler_class = get_sampler_from_string(sampler_name) + runs = kwargs.pop("runs") + sampler = sampler_class(setup, **kwargs) + sampler.sample(runs) + + +@cli.command() +@click.pass_context +def gui(ctx): + """ + Shows a GUI for manual calibration + """ + from spotpy.gui.mpl import GUI + + setup = ctx.obj + gui = GUI(setup) + gui.show() + + +def main(setup): + # Prevent help text from wrapping + cli.help = "\b\n" + describe.setup(setup).replace("\n\n", "\n\b\n") + config = get_config_from_file() + cli(obj=setup, auto_envvar_prefix="SPOTPY", default_map=config) diff --git a/spotpy/database/__init__.py b/src/spotpy/database/__init__.py similarity index 62% rename from spotpy/database/__init__.py rename to src/spotpy/database/__init__.py index 8c539ba2..89ed286f 100644 --- a/spotpy/database/__init__.py +++ b/src/spotpy/database/__init__.py @@ -8,12 +8,13 @@ def __dir__(): :return: """ import pkgutil + names = [ - name for importer, name, ispkg - in pkgutil.iter_modules(__path__) - if not ispkg and name != 'base' + name + for importer, name, ispkg in pkgutil.iter_modules(__path__) + if not ispkg and name != "base" ] - return names + ['custom', 'noData'] + return names + ["custom", "noData"] def __getattr__(name): @@ -21,17 +22,17 @@ def __getattr__(name): print(names) if name in names: try: - db_module = import_module('.' + name, __name__) + db_module = import_module("." + name, __name__) except ImportError: - db_module = import_module('.base', __name__) + db_module = import_module(".base", __name__) return getattr(db_module, name) else: - raise AttributeError('{} is not a member of spotpy.database') + raise AttributeError("{} is not a member of spotpy.database") def get_datawriter(dbformat, *args, **kwargs): """Given a dbformat (ram, csv, sql, noData, etc), return the constructor - of the appropriate class from this file. + of the appropriate class from this file. """ db_class = __getattr__(dbformat) datawriter = db_class(*args, **kwargs) diff --git a/spotpy/database/base.py b/src/spotpy/database/base.py similarity index 72% rename from spotpy/database/base.py rename to src/spotpy/database/base.py index 6ade100c..5e64b149 100644 --- a/spotpy/database/base.py +++ b/src/spotpy/database/base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Optimization Tool (SPOTPY). @@ -8,19 +8,14 @@ This is the parent class of all algorithms, which can handle the database structure during the sample. -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -import numpy as np +""" +import sys import time +from importlib import import_module from itertools import product -import sys -if sys.version_info[0] >= 3: - unicode = str -from importlib import import_module +import numpy as np + class database(object): """ @@ -28,8 +23,18 @@ class database(object): databases. """ - def __init__(self, dbname, parnames, like, randompar, simulations=None, - chains=1, save_sim=True, db_precision=np.float32, **kwargs): + def __init__( + self, + dbname, + parnames, + like, + randompar, + simulations=None, + chains=1, + save_sim=True, + db_precision=np.float32, + **kwargs, + ): # Just needed for the first line in the database self.chains = chains self.dbname = dbname @@ -41,24 +46,30 @@ def __init__(self, dbname, parnames, like, randompar, simulations=None, if not save_sim: simulations = None self.dim_dict = {} - self.singular_data_lens = [self._check_dims(name, obj) for name, obj in [( - 'like', like), ('par', randompar), ('simulation', simulations)]] + self.singular_data_lens = [ + self._check_dims(name, obj) + for name, obj in [ + ("like", like), + ("par", randompar), + ("simulation", simulations), + ] + ] self._make_header(simulations, parnames) self.last_flush = time.time() def _check_dims(self, name, obj): - '''checks dimensionality of obj and stores function in dict''' + """checks dimensionality of obj and stores function in dict""" if obj is None: # None object self.dim_dict[name] = self._empty_list return (0,) - elif hasattr(obj, '__len__'): - if hasattr(obj, 'shape'): + elif hasattr(obj, "__len__"): + if hasattr(obj, "shape"): # np.array style obj self.dim_dict[name] = self._array_to_list return obj.shape - elif all([hasattr(x, '__len__') for x in obj]): + elif all([hasattr(x, "__len__") for x in obj]): # nested list, checked only for singular nesting # assumes all lists have same length self.dim_dict[name] = self._nestediterable_to_list @@ -104,21 +115,29 @@ def _nestediterable_to_list(self, obj): def _make_header(self, simulations, parnames): self.header = [] - self.header.extend(['like' + '_'.join(map(str, x)) - for x in product(*self._tuple_2_xrange(self.singular_data_lens[0]))]) - self.header.extend(['par{0}'.format(x) for x in parnames]) + self.header.extend( + [ + "like" + "_".join(map(str, x)) + for x in product(*self._tuple_2_xrange(self.singular_data_lens[0])) + ] + ) + self.header.extend(["par{0}".format(x) for x in parnames]) # print(self.singular_data_lens[2]) # print(type(self.singular_data_lens[2])) if self.save_sim: for i in range(len(simulations)): - if isinstance(simulations[0], list) or type(simulations[0]) == type(np.array([])): + if isinstance(simulations[0], list) or type(simulations[0]) == type( + np.array([]) + ): for j in range(len(simulations[i])): - self.header.extend(['simulation' + str(i + 1) + '_' + str(j + 1)]) + self.header.extend( + ["simulation" + str(i + 1) + "_" + str(j + 1)] + ) else: - self.header.extend(['simulation' + '_' + str(i)]) + self.header.extend(["simulation" + "_" + str(i)]) # for x in product(*self._tuple_2_xrange(self.singular_data_lens[2]))]) - self.header.append('chain') + self.header.append("chain") def _tuple_2_xrange(self, t): return (range(1, x + 1) for x in t) @@ -142,6 +161,7 @@ def finalize(self): def getdata(self): pass + class custom(database): """ This class is a simple wrapper over the database API, and can be used @@ -149,14 +169,18 @@ class custom(database): """ def __init__(self, *args, **kwargs): - if 'setup' not in kwargs: - raise ValueError(""" + if "setup" not in kwargs: + raise ValueError( + """ You are using the 'custom' Datawriter. To use it, the setup must be specified on creation, but it is missing - """) - self.setup = kwargs['setup'] - if not hasattr(self.setup, 'save'): - raise AttributeError('Your setup needs a "save" method in order to use the "custom" dbformat') + """ + ) + self.setup = kwargs["setup"] + if not hasattr(self.setup, "save"): + raise AttributeError( + 'Your setup needs a "save" method in order to use the "custom" dbformat' + ) super(custom, self).__init__(*args, **kwargs) @@ -168,6 +192,3 @@ def finalize(self): def getdata(self): pass - - - diff --git a/spotpy/database/csv.py b/src/spotpy/database/csv.py similarity index 56% rename from spotpy/database/csv.py rename to src/spotpy/database/csv.py index 9fc7c596..008f1f34 100644 --- a/spotpy/database/csv.py +++ b/src/spotpy/database/csv.py @@ -1,14 +1,11 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from .base import database -import numpy as np import io -import time import sys -if sys.version_info[0] >= 3: - unicode = str +import time + +import numpy as np + +from .base import database + class csv(database): """ @@ -20,32 +17,29 @@ def __init__(self, *args, **kwargs): # init base class super(csv, self).__init__(*args, **kwargs) # store init item only if dbinit - if kwargs.get('dbappend', False) is False: + if kwargs.get("dbappend", False) is False: print("* Database file '{}.csv' created.".format(self.dbname)) # Create a open file, which needs to be closed after the sampling - mode = 'w' - if sys.version_info.major < 3: - mode += 'b' - self.db = io.open(self.dbname + '.csv', mode) + mode = "w" + self.db = io.open(self.dbname + ".csv", mode) # write header line - self.db.write(unicode(','.join(self.header) + '\n')) + self.db.write(str(",".join(self.header) + "\n")) else: print("* Appending to database file '{}.csv'.".format(self.dbname)) # Continues writing file - mode = 'a' - if sys.version_info.major < 3: - mode += 'b' - self.db = io.open(self.dbname + '.csv', mode) + mode = "a" + self.db = io.open(self.dbname + ".csv", mode) def save(self, objectivefunction, parameterlist, simulations=None, chains=1): - coll = (self.dim_dict['like'](objectivefunction) + - self.dim_dict['par'](parameterlist) + - self.dim_dict['simulation'](simulations) + - [chains]) + coll = ( + self.dim_dict["like"](objectivefunction) + + self.dim_dict["par"](parameterlist) + + self.dim_dict["simulation"](simulations) + + [chains] + ) # Apply rounding of floats coll = map(self.db_precision, coll) - self.db.write( - ','.join(map(str, coll)) + '\n') + self.db.write(",".join(map(str, coll)) + "\n") acttime = time.time() # Force writing to disc at least every two seconds @@ -58,5 +52,5 @@ def finalize(self): self.db.close() def getdata(self): - data = np.genfromtxt(self.dbname + '.csv', delimiter=',', names=True) + data = np.genfromtxt(self.dbname + ".csv", delimiter=",", names=True) return data diff --git a/spotpy/database/hdf5.py b/src/spotpy/database/hdf5.py similarity index 72% rename from spotpy/database/hdf5.py rename to src/spotpy/database/hdf5.py index ea5de0a8..a4a4153a 100644 --- a/spotpy/database/hdf5.py +++ b/src/spotpy/database/hdf5.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Optimization Tool (SPOTPY). @@ -8,23 +8,21 @@ This is the parent class of all algorithms, which can handle the database structure during the sample. -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +""" import numpy as np + from .base import database + try: import tables except ImportError: - print('ImportError: Pytables is not correctly installed. Please also make sure you', - 'installed the hdf5 extension (https://www.hdfgroup.org/downloads/hdf5/)') + print( + "ImportError: Pytables is not correctly installed. Please also make sure you", + "installed the hdf5 extension (https://www.hdfgroup.org/downloads/hdf5/)", + ) raise import sys -if sys.version_info[0] >= 3: - unicode = str class hdf5(database): @@ -33,6 +31,7 @@ class hdf5(database): This is only available if PyTables is installed """ + def get_table_def(self): """ Returns a dict of column definitions using multidimensional @@ -63,12 +62,10 @@ def get_table_def(self): # Get the appropriate dtype for the n-d cell # (tables.Col.from_dtype does not take a shape parameter) sim_dtype = np.dtype((self.db_precision, sim_shape)) - columns['simulation'] = tables.Col.from_dtype( - sim_dtype, pos=sim_pos - ) + columns["simulation"] = tables.Col.from_dtype(sim_dtype, pos=sim_pos) chain_pos += 1 # Add a column chains - columns['chains'] = tables.UInt16Col(pos=chain_pos) + columns["chains"] = tables.UInt16Col(pos=chain_pos) return columns @@ -81,29 +78,33 @@ def __init__(self, *args, **kwargs): # init base class super(hdf5, self).__init__(*args, **kwargs) # store init item only if dbinit - if not kwargs.get('dbappend', False): + if not kwargs.get("dbappend", False): # Create an open file, which needs to be closed after the sampling - self.db = tables.open_file(self.dbname + '.h5', 'w', self.dbname) - self.table = self.db.create_table('/', self.dbname, description=self.get_table_def()) + self.db = tables.open_file(self.dbname + ".h5", "w", self.dbname) + self.table = self.db.create_table( + "/", self.dbname, description=self.get_table_def() + ) else: # Continues writing file - self.db = tables.open_file(self.dbname + '.h5', 'a') + self.db = tables.open_file(self.dbname + ".h5", "a") self.table = self.db.root[self.dbname] def save(self, objectivefunction, parameterlist, simulations=None, chains=1): new_row = self.table.row - coll = self.dim_dict['like'](objectivefunction) + self.dim_dict['par'](parameterlist) + coll = self.dim_dict["like"](objectivefunction) + self.dim_dict["par"]( + parameterlist + ) for header, value in zip(self.header, coll): new_row[header] = value if self.save_sim: - new_row['simulation'] = simulations - new_row['chains'] = chains + new_row["simulation"] = simulations + new_row["chains"] = chains new_row.append() def finalize(self): self.db.close() def getdata(self): - with tables.open_file(self.dbname + '.h5', 'a') as db: + with tables.open_file(self.dbname + ".h5", "a") as db: return db.root[self.dbname][:] diff --git a/spotpy/database/ram.py b/src/spotpy/database/ram.py similarity index 66% rename from spotpy/database/ram.py rename to src/spotpy/database/ram.py index e5652c70..6523641d 100644 --- a/spotpy/database/ram.py +++ b/src/spotpy/database/ram.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Optimization Tool (SPOTPY). @@ -8,17 +8,13 @@ This is the parent class of all algorithms, which can handle the database structure during the sample. -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +""" + +import sys import numpy as np + from .base import database -import sys -if sys.version_info[0] >= 3: - unicode = str class ram(database): @@ -33,12 +29,15 @@ def __init__(self, *args, **kwargs): # init the status vars self.ram = [] - def save(self, objectivefunction, parameterlist, simulations=None, - chains=1): - self.ram.append(tuple(self.dim_dict['like'](objectivefunction) + - self.dim_dict['par'](parameterlist) + - self.dim_dict['simulation'](simulations) + - [chains])) + def save(self, objectivefunction, parameterlist, simulations=None, chains=1): + self.ram.append( + tuple( + self.dim_dict["like"](objectivefunction) + + self.dim_dict["par"](parameterlist) + + self.dim_dict["simulation"](simulations) + + [chains] + ) + ) def finalize(self): """ @@ -46,8 +45,7 @@ def finalize(self): Forms the List of values into a strutured numpy array in order to have the same structure as a csv database. """ - dt = {'names': self.header, - 'formats': [np.float] * len(self.header)} + dt = {"names": self.header, "formats": [float] * len(self.header)} i = 0 Y = np.zeros(len(self.ram), dtype=dt) diff --git a/src/spotpy/database/sql.py b/src/spotpy/database/sql.py new file mode 100644 index 00000000..2231fc1c --- /dev/null +++ b/src/spotpy/database/sql.py @@ -0,0 +1,99 @@ +import sqlite3 +import sys + +import numpy as np + +from .base import database + + +class PickalableSWIG: + def __setstate__(self, state): + self.__init__(*state["args"]) + + def __getstate__(self): + return {"args": self.args} + + +class PickalableSQL3Connect(sqlite3.Connection, PickalableSWIG): + def __init__(self, *args, **kwargs): + self.args = args + sqlite3.Connection.__init__(self, *args, **kwargs) + + +class PickalableSQL3Cursor(sqlite3.Cursor, PickalableSWIG): + def __init__(self, *args, **kwargs): + self.args = args + sqlite3.Cursor.__init__(self, *args, **kwargs) + + +class sql(database): + """ + This class saves the process in the working storage. It can be used if + safety matters. + """ + + def __init__(self, *args, **kwargs): + import os + + # init base class + super(sql, self).__init__(*args, **kwargs) + # Create a open file, which needs to be closed after the sampling + try: + os.remove(self.dbname + ".db") + except: + pass + + self.db = PickalableSQL3Connect(self.dbname + ".db") + self.db_cursor = PickalableSQL3Cursor(self.db) + # Create Table + # self.db_cursor.execute('''CREATE TABLE IF NOT EXISTS '''+self.dbname+''' + # (like1 real, parx real, pary real, simulation1 real, chain int)''') + self.db_cursor.execute( + """CREATE TABLE IF NOT EXISTS """ + + self.dbname + + """ + (""" + + " real ,".join(self.header) + + """)""" + ) + + def save(self, objectivefunction, parameterlist, simulations=None, chains=1): + coll = ( + self.dim_dict["like"](objectivefunction) + + self.dim_dict["par"](parameterlist) + + self.dim_dict["simulation"](simulations) + + [chains] + ) + # Apply rounding of floats + coll = map(self.db_precision, coll) + self.db_cursor.execute( + "INSERT INTO " + + self.dbname + + " VALUES (" + + '"' + + str('","'.join(map(str, coll))) + + '"' + + ")" + ) + + self.db.commit() + + def finalize(self): + self.db.close() + + def getdata(self): + self.db = PickalableSQL3Connect(self.dbname + ".db") + self.db_cursor = PickalableSQL3Cursor(self.db) + + headers = [ + (row[1], ">> spotpy.describe.sampler(sampler) +>>> spotpy.describe.setup(model) +""" +import sys +from inspect import getdoc as _getdoc + +from .algorithms._algorithm import _algorithm +from .parameter import get_parameters_from_setup + +try: + from docutils.core import publish_string +except ImportError: + publish_string = None + + +def describe(obj): + """ + Returns a long string description of a sampler with its model + :param obj: A sampler + :return: str + """ + return "Sampler:\n--------\n{}\n\nModel:\n------\n{}".format( + sampler(obj), setup(obj.setup) + ) + + +def sampler(obj): + """ + Returns a string representation of the sampler. + By design, it is rather verbose and returns a + large multiline description + :return: + """ + cname = str(type(obj).__name__) + s = [ + cname, + "=" * len(cname), + _getdoc(obj), + " db format: " + obj.dbformat, + " db name: " + obj.dbname, + " save simulation: " + str(obj.save_sim), + " parallel: " + type(obj.repeat).__module__.split(".")[-1], + ] + return "\n".join(s) + + +def setup(obj): + """ + Describes a spotpy setup using its class name, docstring and parameters + :param obj: A spotpy compatible model setup + :return: A describing string + """ + # Get class name + cname = str(type(obj).__name__) + # Add doc string + mdoc = _getdoc(obj).strip("\n").replace("\r", "\n") + # Get parameters from class + params = "\n".join( + " - {p}".format(p=str(p)) for p in get_parameters_from_setup(obj) + ) + parts = [cname, "=" * len(cname), mdoc, "Parameters:", "-" * 11, params] + return "\n".join(parts) + + +import webbrowser +from pathlib import Path + + +class rst: + """ + Creates a reStructuredText description of a sampler or a setup + + Usage: + >>>description = spotpy.describe.rst(sampler) + >>>print(description) # Prints the rst source text + >>># Add additional text section + >>>description.append('#. One idea' + '\n' + '#. Another one.', title='Ideas', titlelevel=2) + >>>description.append_image('media/image.png') + >>>print(description.as_html()) # Create html + >>>description.show_in_browser() + """ + + caption_characters = "=-#~*+^" + + def __init__(self, setup_or_sampler): + """ + Creates a reStructuredText description of a sampler or a setup + :param setup_or_sampler: Either a spotpy.algorithm sampler or a spotpy setup + """ + if isinstance(setup_or_sampler, _algorithm): + self.setup = setup_or_sampler.setup + self.sampler = setup_or_sampler + self.rst_text = [self._sampler_text()] + else: + self.setup = setup_or_sampler + self.sampler = None + self.rst_text = [] + + if self.setup: + self.rst_text.append(self._setup_text()) + + def append(self, text="", title=None, titlelevel=1): + """ + Appends additional descriptions in reStructured text to the generated text + :param text: The rst text to add + :param title: A title for the text + :param titlelevel: The level of the section (0->h1.title, 1->h1, 2->h2, etc.) + :return: + """ + res = "\n" + if title: + res += rst._as_rst_caption(title, titlelevel) + self.rst_text.append(res + text) + + def append_image(self, imgpath, **kwargs): + """ + Links an image to the output + :param imgpath: Path to the image (must be found from the http server) + :param kwargs: Any keyword with value is translated in rst as `:keyword: value` + and added to the image description + + >>>description.append_image('https://img.shields.io/travis/thouska/spotpy/master.svg', + ... target='https://github.com/thouska', + ... width='200px') + """ + rst = ".. image:: {}".format(imgpath) + for k, v in kwargs.items(): + rst += "\n :{}: {}".format(k, v) + rst += "\n" + self.append(rst) + + def append_math(self, latex): + """ + Appends a block equation to the output + :param latex: Latex formula + """ + rst = ".. math::\n" + rst += " " + latex + "\n" + self.append(rst) + + def __str__(self): + return "\n".join(self.rst_text) + + @classmethod + def _as_rst_caption(cls, s, level=1): + """ + Marks text as a section caption + :param s: String to be marked as caption + :param level: Caption level 0-6, translates to 0=h1.title, 1=h1, 2=h2, etc. + :return: The string as rst caption + """ + return s + "\n" + cls.caption_characters[level] * len(s) + "\n\n" + + css = """ + body, table, div, p, dl { + font-family: Lucida Grande, Verdana, Geneva, Arial, sans-serif; + font-size: 16px; + } + li>p { + margin: 0px; + } + /* @group Heading Levels */ + h1.title { + background-color: #fff; + color: #0040A0; + text-align: left; + font-size: 200%; + border: solid 2px #1f6992; + } + h1 { + background-color: #1f6992; + color: #fff; + padding: .2em .5em; + font-size: 150%; + } + h2 { + background-color: #cde; + color: #000; + padding: .2em .5em; + border-bottom: solid 2px #1f6992; + font-size: 120%; + } + + h3 { + font-size: 100%; + border-bottom: solid 2px #0040A0; + } + div.line { + font-family: "Lucida Console", "Lucida Sans Typewriter","DejaVu Sans Mono",monospace; + font-size: 100%; + } + + img { + max-width: 720px; + } + + """ + + def as_html(self, css=None): + """ + Converts the generated reStructuredText as html5 + + :css: A string containing a cascading style sheet. If None, the default css is used + :return: The html document as string + """ + if publish_string is None: + raise NotImplementedError("The docutils package needs to be installed") + args = {"input_encoding": "unicode", "output_encoding": "unicode"} + res = publish_string( + source=str(self), writer_name="html5", settings_overrides=args + ) + style_idx = res.index("") + css = css or self.css + # Include css + res = res[:style_idx] + css + res[style_idx:] + return res + + def show_in_browser(self, filename=None, css=None): + """ + Writes the content as html to disk and opens a browser showing the result + + :param filename: The html filename, if None use .html + :param css: A style string, if None the default style is used + """ + html = self.as_html(css).replace("unicode", "utf-8") + fn = filename or type(self.setup).__name__ + ".html" + path = Path(fn).absolute() + path.write_text(html, encoding="utf-8") + webbrowser.open_new_tab(path.as_uri()) + + def _sampler_text(self): + """ + Generates the rst for the sampler + :return: + """ + obj = self.sampler + cname = rst._as_rst_caption(type(obj).__name__, 0) + s = [ + "- **db format:** " + obj.dbformat, + "- **db name:** " + obj.dbname, + "- **save simulation:** " + str(obj.save_sim), + "- **parallel:** " + type(obj.repeat).__module__.split(".")[-1], + "", + "", + ] + return cname + _getdoc(obj).strip("\n") + "\n\n" + "\n".join(s) + + def _setup_text(self): + """ + Generates the rst for the setup + :return: + """ + # Get class name + obj = self.setup + cname = rst._as_rst_caption(type(obj).__name__, 0) + # Add doc string + mdoc = _getdoc(obj).strip("\n").replace("\r", "\n") + "\n\n" + # Get parameters from class + param_caption = rst._as_rst_caption("Parameters", 1) + params = "\n".join( + "#. **{p.name}:** {p}".format(p=p) for p in get_parameters_from_setup(obj) + ) + return cname + mdoc + param_caption + params diff --git a/spotpy/examples/__init__.py b/src/spotpy/examples/__init__.py similarity index 94% rename from spotpy/examples/__init__.py rename to src/spotpy/examples/__init__.py index 473323e1..ff4ed0d2 100644 --- a/spotpy/examples/__init__.py +++ b/src/spotpy/examples/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -''' +""" Copyright 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). :author: Tobias Houska -''' +""" diff --git a/spotpy/examples/hymod_python/__init__.py b/src/spotpy/examples/cmf_data/__init__.py similarity index 99% rename from spotpy/examples/hymod_python/__init__.py rename to src/spotpy/examples/cmf_data/__init__.py index c91096bc..f107b27a 100644 --- a/spotpy/examples/hymod_python/__init__.py +++ b/src/spotpy/examples/cmf_data/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). @@ -28,4 +28,4 @@ https://github.com/CamDavidsonPilon/Probabilistic-Programming-and-Bayesian-Methods-for-Hackers/tree/master/ Please cite our paper, if you are using SPOTPY. -''' +""" diff --git a/spotpy/examples/cmf_data/driver_data_site24.csv b/src/spotpy/examples/cmf_data/driver_data_site24.csv similarity index 100% rename from spotpy/examples/cmf_data/driver_data_site24.csv rename to src/spotpy/examples/cmf_data/driver_data_site24.csv diff --git a/spotpy/examples/cmf_data/fulda_climate.csv b/src/spotpy/examples/cmf_data/fulda_climate.csv similarity index 100% rename from spotpy/examples/cmf_data/fulda_climate.csv rename to src/spotpy/examples/cmf_data/fulda_climate.csv diff --git a/spotpy/examples/cmf_data/soilmoisture_site24.csv b/src/spotpy/examples/cmf_data/soilmoisture_site24.csv similarity index 100% rename from spotpy/examples/cmf_data/soilmoisture_site24.csv rename to src/spotpy/examples/cmf_data/soilmoisture_site24.csv diff --git a/spotpy/examples/hymod_exe/HYMODsilent.exe b/src/spotpy/examples/hymod_exe/HYMODsilent.exe similarity index 100% rename from spotpy/examples/hymod_exe/HYMODsilent.exe rename to src/spotpy/examples/hymod_exe/HYMODsilent.exe diff --git a/spotpy/examples/hymod_exe/Param.in b/src/spotpy/examples/hymod_exe/Param.in similarity index 100% rename from spotpy/examples/hymod_exe/Param.in rename to src/spotpy/examples/hymod_exe/Param.in diff --git a/spotpy/examples/hymod_exe/__init__.py b/src/spotpy/examples/hymod_exe/__init__.py similarity index 99% rename from spotpy/examples/hymod_exe/__init__.py rename to src/spotpy/examples/hymod_exe/__init__.py index c91096bc..f107b27a 100644 --- a/spotpy/examples/hymod_exe/__init__.py +++ b/src/spotpy/examples/hymod_exe/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). @@ -28,4 +28,4 @@ https://github.com/CamDavidsonPilon/Probabilistic-Programming-and-Bayesian-Methods-for-Hackers/tree/master/ Please cite our paper, if you are using SPOTPY. -''' +""" diff --git a/spotpy/examples/hymod_exe/bound.txt b/src/spotpy/examples/hymod_exe/bound.txt similarity index 100% rename from spotpy/examples/hymod_exe/bound.txt rename to src/spotpy/examples/hymod_exe/bound.txt diff --git a/spotpy/examples/hymod_exe/bound_units.xlsx b/src/spotpy/examples/hymod_exe/bound_units.xlsx similarity index 100% rename from spotpy/examples/hymod_exe/bound_units.xlsx rename to src/spotpy/examples/hymod_exe/bound_units.xlsx diff --git a/spotpy/examples/hymod_exe/hymod_input.csv b/src/spotpy/examples/hymod_exe/hymod_input.csv similarity index 100% rename from spotpy/examples/hymod_exe/hymod_input.csv rename to src/spotpy/examples/hymod_exe/hymod_input.csv diff --git a/spotpy/examples/hymod_exe/license.txt b/src/spotpy/examples/hymod_exe/license.txt similarity index 100% rename from spotpy/examples/hymod_exe/license.txt rename to src/spotpy/examples/hymod_exe/license.txt diff --git a/spotpy/examples/dds/__init__.py b/src/spotpy/examples/hymod_python/__init__.py similarity index 99% rename from spotpy/examples/dds/__init__.py rename to src/spotpy/examples/hymod_python/__init__.py index c91096bc..f107b27a 100644 --- a/spotpy/examples/dds/__init__.py +++ b/src/spotpy/examples/hymod_python/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). @@ -28,4 +28,4 @@ https://github.com/CamDavidsonPilon/Probabilistic-Programming-and-Bayesian-Methods-for-Hackers/tree/master/ Please cite our paper, if you are using SPOTPY. -''' +""" diff --git a/spotpy/examples/hymod_python/hymod.py b/src/spotpy/examples/hymod_python/hymod.py similarity index 79% rename from spotpy/examples/hymod_python/hymod.py rename to src/spotpy/examples/hymod_python/hymod.py index f74bebdf..dc4d0af1 100644 --- a/spotpy/examples/hymod_python/hymod.py +++ b/src/spotpy/examples/hymod_python/hymod.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). @@ -9,17 +9,17 @@ :paper: Houska, T., Kraft, P., Chamorro-Chavez, A. and Breuer, L.: SPOTting Model Parameters Using a Ready-Made Python Package, PLoS ONE, 10(12), e0145180, doi:10.1371/journal.pone.0145180, 2015. -''' +""" from numba import jit -def hymod(Precip, PET, cmax,bexp,alpha,Rs,Rq): +def hymod(Precip, PET, cmax, bexp, alpha, Rs, Rq): """ See https://www.proc-iahs.net/368/180/2015/piahs-368-180-2015.pdf for a scientific paper: - - Quan, Z.; Teng, J.; Sun, W.; Cheng, T. & Zhang, J. (2015): Evaluation of the HYMOD model - for rainfall–runoff simulation using the GLUE method. Remote Sensing and GIS for Hydrology + + Quan, Z.; Teng, J.; Sun, W.; Cheng, T. & Zhang, J. (2015): Evaluation of the HYMOD model + for rainfall–runoff simulation using the GLUE method. Remote Sensing and GIS for Hydrology and Water Resources, 180 - 185, IAHS Publ. 368. DOI: 10.5194/piahs-368-180-2015. :param cmax: @@ -37,12 +37,12 @@ def hymod(Precip, PET, cmax,bexp,alpha,Rs,Rq): x_slow = 2.3503 / (Rs * 22.5) x_slow = 0 # --> works ok if calibration data starts with low discharge # Initialize state(s) of quick tank(s) - x_quick = [0,0,0] + x_quick = [0, 0, 0] t = 0 output = [] # START PROGRAMMING LOOP WITH DETERMINING RAINFALL - RUNOFF AMOUNTS - while t <= len(Precip)-1: + while t <= len(Precip) - 1: Pval = Precip[t] PETval = PET[t] # Compute excess precipitation and evaporation @@ -64,27 +64,32 @@ def hymod(Precip, PET, cmax,bexp,alpha,Rs,Rq): # Compute total flow for timestep output.append(QS + outflow) - t = t+1 + t = t + 1 return output + @jit -def power(X,Y): - X=abs(X) # Needed to capture invalid overflow with netgative values +def power(X, Y): + X = abs(X) # Needed to capture invalid overflow with netgative values return X**Y + @jit -def linres(x_slow,inflow,Rs): +def linres(x_slow, inflow, Rs): # Linear reservoir x_slow = (1 - Rs) * x_slow + (1 - Rs) * inflow outflow = (Rs / (1 - Rs)) * x_slow - return x_slow,outflow + return x_slow, outflow + @jit -def excess(x_loss,cmax,bexp,Pval,PETval): +def excess(x_loss, cmax, bexp, Pval, PETval): # this function calculates excess precipitation and evaporation xn_prev = x_loss - ct_prev = cmax * (1 - power((1 - ((bexp + 1) * (xn_prev) / cmax)), (1 / (bexp + 1)))) + ct_prev = cmax * ( + 1 - power((1 - ((bexp + 1) * (xn_prev) / cmax)), (1 / (bexp + 1))) + ) # Calculate Effective rainfall 1 ER1 = max((Pval - cmax + ct_prev), 0.0) Pval = Pval - ER1 @@ -95,7 +100,9 @@ def excess(x_loss,cmax,bexp,Pval,PETval): ER2 = max(Pval - (xn - xn_prev), 0) # Alternative approach - evap = (1 - (((cmax / (bexp + 1)) - xn) / (cmax / (bexp + 1)))) * PETval # actual ET is linearly related to the soil moisture state + evap = ( + 1 - (((cmax / (bexp + 1)) - xn) / (cmax / (bexp + 1))) + ) * PETval # actual ET is linearly related to the soil moisture state xn = max(xn - evap, 0) # update state - return ER1,ER2,xn + return ER1, ER2, xn diff --git a/spotpy/examples/hymod_python/hymod_input.csv b/src/spotpy/examples/hymod_python/hymod_input.csv similarity index 100% rename from spotpy/examples/hymod_python/hymod_input.csv rename to src/spotpy/examples/hymod_python/hymod_input.csv diff --git a/spotpy/examples/cmf_data/__init__.py b/src/spotpy/examples/hymod_unix/__init__.py similarity index 99% rename from spotpy/examples/cmf_data/__init__.py rename to src/spotpy/examples/hymod_unix/__init__.py index c91096bc..f107b27a 100644 --- a/spotpy/examples/cmf_data/__init__.py +++ b/src/spotpy/examples/hymod_unix/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). @@ -28,4 +28,4 @@ https://github.com/CamDavidsonPilon/Probabilistic-Programming-and-Bayesian-Methods-for-Hackers/tree/master/ Please cite our paper, if you are using SPOTPY. -''' +""" diff --git a/spotpy/examples/hymod_unix/bound.txt b/src/spotpy/examples/hymod_unix/bound.txt similarity index 100% rename from spotpy/examples/hymod_unix/bound.txt rename to src/spotpy/examples/hymod_unix/bound.txt diff --git a/spotpy/examples/hymod_unix/bound_units.xlsx b/src/spotpy/examples/hymod_unix/bound_units.xlsx similarity index 100% rename from spotpy/examples/hymod_unix/bound_units.xlsx rename to src/spotpy/examples/hymod_unix/bound_units.xlsx diff --git a/spotpy/examples/hymod_unix/hymod_3.6 b/src/spotpy/examples/hymod_unix/hymod_3.6 similarity index 100% rename from spotpy/examples/hymod_unix/hymod_3.6 rename to src/spotpy/examples/hymod_unix/hymod_3.6 diff --git a/spotpy/examples/hymod_unix/hymod_3.7 b/src/spotpy/examples/hymod_unix/hymod_3.7 similarity index 100% rename from spotpy/examples/hymod_unix/hymod_3.7 rename to src/spotpy/examples/hymod_unix/hymod_3.7 diff --git a/spotpy/examples/hymod_unix/hymod_cython/Makefile b/src/spotpy/examples/hymod_unix/hymod_cython/Makefile similarity index 100% rename from spotpy/examples/hymod_unix/hymod_cython/Makefile rename to src/spotpy/examples/hymod_unix/hymod_cython/Makefile diff --git a/spotpy/examples/hymod_unix/hymod_cython/compile_hymod.sh b/src/spotpy/examples/hymod_unix/hymod_cython/compile_hymod.sh similarity index 100% rename from spotpy/examples/hymod_unix/hymod_cython/compile_hymod.sh rename to src/spotpy/examples/hymod_unix/hymod_cython/compile_hymod.sh diff --git a/spotpy/examples/hymod_unix/hymod_cython/hymod.pyx b/src/spotpy/examples/hymod_unix/hymod_cython/hymod.pyx similarity index 99% rename from spotpy/examples/hymod_unix/hymod_cython/hymod.pyx rename to src/spotpy/examples/hymod_unix/hymod_cython/hymod.pyx index db53be61..8641c8fd 100644 --- a/spotpy/examples/hymod_unix/hymod_cython/hymod.pyx +++ b/src/spotpy/examples/hymod_unix/hymod_cython/hymod.pyx @@ -1,5 +1,6 @@ -import sys, re, os - +import os +import re +import sys #cdef public list hymod(list Precip, list PET, float cmax, float bexp, float alpha, float Rs, float Rq): # public function declaration diff --git a/spotpy/examples/hymod_unix/hymod_cython/main_hymod.cpp b/src/spotpy/examples/hymod_unix/hymod_cython/main_hymod.cpp similarity index 100% rename from spotpy/examples/hymod_unix/hymod_cython/main_hymod.cpp rename to src/spotpy/examples/hymod_unix/hymod_cython/main_hymod.cpp diff --git a/src/spotpy/examples/spot_setup_ackley.py b/src/spotpy/examples/spot_setup_ackley.py new file mode 100644 index 00000000..a38c031d --- /dev/null +++ b/src/spotpy/examples/spot_setup_ackley.py @@ -0,0 +1,48 @@ +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +This example implements the Ackley function into SPOT. +""" +import numpy as np + +import spotpy + + +class spot_setup(object): + def __init__(self, dim=30): + self.dim = dim + self.params = [] + for i in range(self.dim): + self.params.append( + spotpy.parameter.Uniform(str(i), -32.768, 32.768, 2.5, -20.0) + ) + + def parameters(self): + return spotpy.parameter.generate(self.params) + + def simulation(self, vector): + firstSum = 0.0 + secondSum = 0.0 + for c in range(len(vector)): + firstSum += c**2.0 + secondSum += np.cos(2.0 * np.pi * vector[c]) + n = float(len(vector)) + return [ + -20.0 * np.exp(-0.2 * np.sqrt(firstSum / n)) + - np.exp(secondSum / n) + + 20 + + np.e + ] + + def evaluation(self): + observations = [0] + return observations + + def objectivefunction(self, simulation, evaluation): + objectivefunction = -spotpy.objectivefunctions.rmse( + evaluation=evaluation, simulation=simulation + ) + return objectivefunction diff --git a/spotpy/examples/spot_setup_cmf1d.py b/src/spotpy/examples/spot_setup_cmf1d.py similarity index 63% rename from spotpy/examples/spot_setup_cmf1d.py rename to src/spotpy/examples/spot_setup_cmf1d.py index 566bd3bf..e7515481 100644 --- a/spotpy/examples/spot_setup_cmf1d.py +++ b/src/spotpy/examples/spot_setup_cmf1d.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" Copyright 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). @@ -7,13 +7,16 @@ A one dimensional cmf model analysing data from the Schwingbach hydrological observatory. You need to have cmf and pandas installed on your system: svn checkout svn://fb09-pasig.umwelt.uni-giessen.de/cmf/trunk cmf -''' +""" -import cmf +import os +import sys from datetime import datetime, timedelta + +import cmf import numpy as np + import spotpy -import os, sys class _CmfProject: @@ -26,12 +29,16 @@ def __init__(self, par): # run the model self.project = cmf.project() - self.cell = self.project.NewCell(x=0, y=0, z=238.628, area=1000, with_surfacewater=True) + self.cell = self.project.NewCell( + x=0, y=0, z=238.628, area=1000, with_surfacewater=True + ) c = self.cell - r_curve = cmf.VanGenuchtenMualem(Ksat=10**par.pKsat, phi=par.porosity, alpha=par.alpha, n=par.n) + r_curve = cmf.VanGenuchtenMualem( + Ksat=10**par.pKsat, phi=par.porosity, alpha=par.alpha, n=par.n + ) # Make the layer boundaries and create soil layers - lthickness = [.01] * 5 + [0.025] * 6 + [0.05] * 6 + [0.1] * 5 + lthickness = [0.01] * 5 + [0.025] * 6 + [0.05] * 6 + [0.1] * 5 ldepth = np.cumsum(lthickness) # Create soil layers and pick the new soil layers if they are inside of the evaluation depth's @@ -47,9 +54,9 @@ def __init__(self, par): c.saturated_depth = 0.5 - self.gw = self.project.NewOutlet('groundwater', x=0, y=0, z=.9) + self.gw = self.project.NewOutlet("groundwater", x=0, y=0, z=0.9) cmf.Richards(c.layers[-1], self.gw) - self.gw.potential = c.z - 0.5 #IMPORTANT + self.gw.potential = c.z - 0.5 # IMPORTANT self.gw.is_source = True self.gwhead = cmf.timeseries.from_scalar(c.z - 0.5) @@ -63,13 +70,15 @@ def set_parameters(self, par): """ try: for l in self.cell.layers: - r_curve = cmf.VanGenuchtenMualem(Ksat=10**par.pKsat, phi=par.porosity, alpha=par.alpha, n=par.n) + r_curve = cmf.VanGenuchtenMualem( + Ksat=10**par.pKsat, phi=par.porosity, alpha=par.alpha, n=par.n + ) r_curve.w0 = r_curve.fit_w0() l.soil = r_curve self.cell.saturated_depth = 0.5 self.gw.potential = self.cell.z - 0.5 except RuntimeError as e: - sys.stderr.write('Set parameters failed with:\n' + str(par) + '\n' + str(e)) + sys.stderr.write("Set parameters failed with:\n" + str(par) + "\n" + str(e)) raise def load_meteo(self, driver_data): @@ -77,19 +86,26 @@ def load_meteo(self, driver_data): datastart = driver_data.time[0] # Create meteo station for project - meteo = self.project.meteo_stations.add_station('Schwingbach', position=(0, 0, 0), tz=1, timestep=cmf.h) + meteo = self.project.meteo_stations.add_station( + "Schwingbach", position=(0, 0, 0), tz=1, timestep=cmf.h + ) rain = cmf.timeseries.from_array(datastart, cmf.h, driver_data.rain_mmday) - meteo.rHmean = cmf.timeseries.from_array(datastart, cmf.h, driver_data.relhum_perc) - meteo.Windspeed = cmf.timeseries.from_array(datastart, cmf.h, driver_data.windspeed_ms) - meteo.Rs = cmf.timeseries.from_array(datastart, cmf.h, driver_data.solarrad_wm2 * 86400e-6) + meteo.rHmean = cmf.timeseries.from_array( + datastart, cmf.h, driver_data.relhum_perc + ) + meteo.Windspeed = cmf.timeseries.from_array( + datastart, cmf.h, driver_data.windspeed_ms + ) + meteo.Rs = cmf.timeseries.from_array( + datastart, cmf.h, driver_data.solarrad_wm2 * 86400e-6 + ) meteo.T = cmf.timeseries.from_array(datastart, cmf.h, driver_data.airtemp_degc) meteo.Tmax = meteo.T.floating_max(cmf.day) meteo.Tmin = meteo.T.floating_min(cmf.day) - self.project.rainfall_stations.add('Schwingbach', rain, (0, 0, 0)) + self.project.rainfall_stations.add("Schwingbach", rain, (0, 0, 0)) self.project.use_nearest_rainfall() - # Use the meteorological station for each cell of the project self.project.use_nearest_meteo() self.meteo = meteo @@ -111,18 +127,23 @@ def _load_data(filename): Loads data from csv, where the first column has the measurement date and all the other columns additional data :return: Rec-Array with date as a first column holding datetimes """ + def str2date(s): """Converts a string to a datetime""" - return datetime.strptime(s.decode(), '%Y-%m-%d %H:%M:%S') + return datetime.strptime(s.decode(), "%Y-%m-%d %H:%M:%S") + # Load the data - return np.recfromcsv(filename, converters={0: str2date}, comments='#') + return np.recfromcsv(filename, converters={0: str2date}, comments="#") class _ProgressReporter: """ Simple helper class to report progress and check for too long runtime """ - def __init__(self, start, end, frequency=cmf.week, max_runtime=15 * 60, verbose=False): + + def __init__( + self, start, end, frequency=cmf.week, max_runtime=15 * 60, verbose=False + ): self.verbose = verbose self.frequency = frequency self.max_runtime = max_runtime @@ -131,13 +152,19 @@ def __init__(self, start, end, frequency=cmf.week, max_runtime=15 * 60, verbose= def __call__(self, t): elapsed, total, remaining = self.stopwatch(t) if self.verbose and not t % cmf.week: - print('{modeltime:%Y-%m-%d}: {elapsed}/{total}' - .format(modeltime=t.AsPython(), - elapsed=elapsed * cmf.sec, - total=total * cmf.sec)) + print( + "{modeltime:%Y-%m-%d}: {elapsed}/{total}".format( + modeltime=t.AsPython(), + elapsed=elapsed * cmf.sec, + total=total * cmf.sec, + ) + ) if elapsed > self.max_runtime: - raise RuntimeError('{:%Y-%m-%d %H:%S} model took {:0.0f}min until now, stopping' - .format(t.AsPython(), elapsed / 60)) + raise RuntimeError( + "{:%Y-%m-%d %H:%S} model took {:0.0f}min until now, stopping".format( + t.AsPython(), elapsed / 60 + ) + ) class Cmf1d_Model(object): @@ -145,16 +172,26 @@ class Cmf1d_Model(object): A 1d Richards based soilmoisture model for Schwingbach site #24 """ - alpha = spotpy.parameter.Uniform(0.0001, 0.2, optguess=0.1156, doc=u'α in 1/cm for van Genuchten Mualem model') - pKsat = spotpy.parameter.Uniform(-2, 2, optguess=0, doc=u'log10 of saturated conductivity of the soil in m/day') - n = spotpy.parameter.Uniform(1.08, 1.8, optguess=1.1787, doc=u'van Genuchten-Mualem n') - porosity = spotpy.parameter.Uniform(0.3, 0.65, optguess=0.43359, doc=u'φ in m³/m³') + alpha = spotpy.parameter.Uniform( + 0.0001, 0.2, optguess=0.1156, doc="α in 1/cm for van Genuchten Mualem model" + ) + pKsat = spotpy.parameter.Uniform( + -2, 2, optguess=0, doc="log10 of saturated conductivity of the soil in m/day" + ) + n = spotpy.parameter.Uniform( + 1.08, 1.8, optguess=1.1787, doc="van Genuchten-Mualem n" + ) + porosity = spotpy.parameter.Uniform(0.3, 0.65, optguess=0.43359, doc="φ in m³/m³") def __init__(self, days=None): - self.driver_data = _load_data('cmf_data/driver_data_site24.csv') - self.evaluation_data = np.loadtxt('cmf_data/soilmoisture_site24.csv', delimiter=',', - comments='#', usecols=[1,2,3]) + self.driver_data = _load_data("cmf_data/driver_data_site24.csv") + self.evaluation_data = np.loadtxt( + "cmf_data/soilmoisture_site24.csv", + delimiter=",", + comments="#", + usecols=[1, 2, 3], + ) self.datastart = self.driver_data.time[0] @@ -168,19 +205,18 @@ def __init__(self, days=None): # The depth below ground in m where the evaluation data belongs to self.eval_depth = [0.1, 0.25, 0.4] - # Make the model self.model = _CmfProject(self.make_parameters()) # Load meteo data self.model.load_meteo(driver_data=self.driver_data) - self.__doc__ += '\n\n' + cmf.describe(self.model.project) + self.__doc__ += "\n\n" + cmf.describe(self.model.project) def make_parameters(self, random=False, **kwargs): return spotpy.parameter.create_set(self) def evaluation(self): """ - :return: The evaluation soilmoisture as a 2d array + :return: The evaluation soilmoisture as a 2d array """ return self.evaluation_data @@ -188,14 +224,16 @@ def evaluation(self): def objectivefunction(self, simulation, evaluation): """ Returns the negative RMSE for all values - :param simulation: - :param evaluation: - :return: + :param simulation: + :param evaluation: + :return: """ # Find all data positions where simulation and evaluation data is present take = np.isfinite(simulation) & np.isfinite(evaluation) - rmse = -spotpy.objectivefunctions.rmse(evaluation=evaluation[take], simulation=simulation[take]) + rmse = -spotpy.objectivefunctions.rmse( + evaluation=evaluation[take], simulation=simulation[take] + ) return rmse def get_eval_layers(self, eval_depth=None): @@ -207,10 +245,13 @@ def get_eval_layers(self, eval_depth=None): """ c = self.model.cell if eval_depth: - edi = 0 # current index of the evaluation depth - eval_layers = [] # the layers to do the evaluation are stored here + edi = 0 # current index of the evaluation depth + eval_layers = [] # the layers to do the evaluation are stored here for l in c.layers: - if edi < len(self.eval_depth) and l.upper_boundary <= self.eval_depth[edi] < l.lower_boundary: + if ( + edi < len(self.eval_depth) + and l.upper_boundary <= self.eval_depth[edi] < l.lower_boundary + ): eval_layers.append(l) edi += 1 else: @@ -218,34 +259,38 @@ def get_eval_layers(self, eval_depth=None): return eval_layers def simulation(self, par=None, verbose=False): - ''' + """ Runs the model instance - + :param par: A namedtuple of model parameters :param verbose: Write the progress for the single run :return: A 2d array of simulated soil moisture values per depth - ''' + """ # Set the parameters par = par or self.make_parameters() self.model.set_parameters(par) if verbose: - print('Parameters:') + print("Parameters:") for k, v in par._asdict().items(): - print(' {} = {:0.4g}'.format(k,v)) + print(" {} = {:0.4g}".format(k, v)) eval_layers = self.get_eval_layers(self.eval_depth) # Prepare result array with nan's result = np.nan * np.ones(shape=(len(self.evaluation_data), len(eval_layers))) # Save the starting conditions result[0] = [l.theta for l in eval_layers] - reporter = _ProgressReporter(self.datastart, self.dataend, max_runtime=15 * 60, verbose=verbose) + reporter = _ProgressReporter( + self.datastart, self.dataend, max_runtime=15 * 60, verbose=verbose + ) try: - for i, t in enumerate(self.model.solver.run(self.datastart, self.dataend, timedelta(hours=1))): + for i, t in enumerate( + self.model.solver.run(self.datastart, self.dataend, timedelta(hours=1)) + ): self.model.set_boundary(t) # Get result - result[i+1] = [l.theta for l in eval_layers] + result[i + 1] = [l.theta for l in eval_layers] # Raises if the runtime is too long reporter(t) except RuntimeError as e: @@ -255,7 +300,7 @@ def simulation(self, par=None, verbose=False): return result -if __name__ == '__main__': +if __name__ == "__main__": print(spotpy.__file__, spotpy.__version__) if len(sys.argv) > 1: runs = int(sys.argv[1]) @@ -265,14 +310,13 @@ def simulation(self, par=None, verbose=False): model = Cmf1d_Model() print(spotpy.describe.setup(model)) if runs > 1: - parallel = 'mpi' if 'OMPI_COMM_WORLD_SIZE' in os.environ else 'seq' - sampler = spotpy.algorithms.mc(model, - dbformat='csv', - dbname='cmf1d_mc', - parallel=parallel, - save_sim=False) + parallel = "mpi" if "OMPI_COMM_WORLD_SIZE" in os.environ else "seq" + sampler = spotpy.algorithms.mc( + model, dbformat="csv", dbname="cmf1d_mc", parallel=parallel, save_sim=False + ) sampler.sample(runs) else: from spotpy.gui.mpl import GUI + gui = GUI(model) gui.show() diff --git a/spotpy/examples/spot_setup_cmf_lumped.py b/src/spotpy/examples/spot_setup_cmf_lumped.py similarity index 76% rename from spotpy/examples/spot_setup_cmf_lumped.py rename to src/spotpy/examples/spot_setup_cmf_lumped.py index 411d3d65..b11bb24d 100644 --- a/spotpy/examples/spot_setup_cmf_lumped.py +++ b/src/spotpy/examples/spot_setup_cmf_lumped.py @@ -7,17 +7,16 @@ This example can be easily extended with more storages """ -from __future__ import division, print_function import datetime + import cmf +import numpy as np import spotpy - from spotpy.parameter import Uniform -import numpy as np # Make sure we do not get pestered with divIde by zero errors -np.seterr(all='ignore') +np.seterr(all="ignore") class DataProvider(object): @@ -27,11 +26,11 @@ class DataProvider(object): def __init__(self): # Load data from file using numpy magic - data = np.recfromcsv('cmf_data/fulda_climate.csv', encoding='utf-8') + data = np.recfromcsv("cmf_data/fulda_climate.csv", encoding="utf-8") def bstr2date(bs): """Helper function to convert date byte string to datetime object""" - return datetime.datetime.strptime(bs, '%d.%m.%Y') + return datetime.datetime.strptime(bs, "%d.%m.%Y") # Get begin, step and end from the date column self.begin = bstr2date(data.date[0]) @@ -60,13 +59,13 @@ def add_stations(self, project): :param project: A cmf.project :return: rainstation, meteo """ - rainstation = project.rainfall_stations.add('Grebenau avg', self.P, (0, 0, 0)) + rainstation = project.rainfall_stations.add("Grebenau avg", self.P, (0, 0, 0)) # Tell the project to use the meteo station just created project.use_nearest_rainfall() # Temperature data - meteo = project.meteo_stations.add_station('Grebenau avg', (0, 0, 0)) + meteo = project.meteo_stations.add_station("Grebenau avg", (0, 0, 0)) meteo.T = self.T meteo.Tmin = self.Tmin meteo.Tmax = self.Tmax @@ -82,19 +81,29 @@ class SingleStorage(object): A simple hydrological single storage model. No snow, interception or routing. """ + # Catchment area area = 2976.41e6 # sq m # General storage parameter V0 = Uniform(10, 10000, optguess=1000) # ET parameters - fETV1 = Uniform(0.01, 1, optguess=0.2, doc='if V datetime.timedelta(minutes=self.max_run_minutes): - print('Cancelled, since it took more than {} minutes'.format(self.max_run_minutes)) + print(t, "P={:5.3f}".format(c.get_rainfall(t))) + if datetime.datetime.now() - tstart > datetime.timedelta( + minutes=self.max_run_minutes + ): + print( + "Cancelled, since it took more than {} minutes".format( + self.max_run_minutes + ) + ) for t in cmf.timerange(solver.t, self.end, cmf.day): res_q.add(np.nan) @@ -196,7 +216,7 @@ def simulation(self, vector=None, verbose=False): self.setparameters(vector) result_q = self.runmodel(verbose) - return np.array(result_q[self.begin:self.end]) + return np.array(result_q[self.begin : self.end]) @staticmethod def objectivefunction(simulation, evaluation): @@ -205,7 +225,7 @@ def objectivefunction(simulation, evaluation): """ return [ spotpy.objectivefunctions.nashsutcliffe(evaluation, simulation), - spotpy.objectivefunctions.pbias(evaluation, simulation) + spotpy.objectivefunctions.pbias(evaluation, simulation), ] def evaluation(self): @@ -213,4 +233,4 @@ def evaluation(self): Returns the evaluation data """ runoff_mm = self.data.runoff_mm(self.area) - return np.array(runoff_mm[self.begin:self.end]) + return np.array(runoff_mm[self.begin : self.end]) diff --git a/src/spotpy/examples/spot_setup_dds.py b/src/spotpy/examples/spot_setup_dds.py new file mode 100644 index 00000000..05e7f3f0 --- /dev/null +++ b/src/spotpy/examples/spot_setup_dds.py @@ -0,0 +1,186 @@ +import numpy as np + +import spotpy +from spotpy.objectivefunctions import rmse +from spotpy.parameter import Uniform + + +def ackley10(vector): + length = len(vector) + sum1 = 0 + sum2 = 0 + for i in range(length): + sum1 = sum1 + vector[i] ** 2 + sum2 = sum2 + np.cos(2 * np.pi * vector[i]) + return -1 * (-20 * np.exp(-0.2 * (sum1 / length) ** 0.5) - np.exp(sum2 / length)) + + +def griewank10(vector): + sum1 = 0 + term2 = 1 + term3 = 1 + + for i in range(len(vector)): + sum1 = sum1 + (vector[i] ** 2) / 4000 + term2 = term2 * np.cos(vector[i] / (i + 1) ** 0.5) + + return -1 * (sum1 - term2 + term3) + + +class spot_setup(object): + """ + Setup for a simple example to run DDS Algorithm + """ + + def __init__(self): + self.params = None + self.objfunc = None + + def _objfunc_switcher(self, name): + """ + Set new parameter and objective function while setup is instanced in a test case + :param name: function name which overwrites initial objective function + :return: + """ + + if name == "ackley": + self.objfunc = ackley10 + self.params = [ + Uniform( + str(j), + -2, + 2, + 1.5, + 3.0, + -2, + 2, + doc=str(j) + " value of Rosenbrock function", + ) + for j in range(10) + ] + elif name == "griewank": + self.objfunc = griewank10 + self.params = [ + Uniform( + "d" + str(j), + -500, + 700, + 1.5, + 3.0, + -500, + 700, + doc=str(j) + "distinc parameter within a boundary", + as_int=True, + ) + for j in range(2) + ] + [ + Uniform( + "c" + str(j), + -500, + 700, + 1.5, + 3.0, + -500, + 700, + doc=str(j) + "continuous parameter within a boundary", + ) + for j in range(8) + ] + elif name == "cmf_style": + self.objfunc = ackley10 + self.params = [ + Uniform(0.5, 5.0, optguess=1.5, doc="saturated depth at beginning"), + Uniform( + 0.001, + 0.8, + optguess=0.1, + doc="porosity of matrix [m3 Pores / m3 Soil]", + ), + Uniform( + 1.0, + 240.0, + optguess=10.0, + doc="ssaturated conductivity of macropores [m/day]", + ), + Uniform(0.0001, 0.5, optguess=0.05, doc="macropore fraction [m3/m3]"), + Uniform( + 0.005, + 1.0, + optguess=0.05, + doc="mean distance between the macropores [m]", + ), + Uniform( + 0.0, + 1.0, + optguess=0.0, + doc="water content when matric potential pointing towards -infinity", + ), + Uniform( + 0.5, + 1.0, + optguess=0.99, + doc="wetness above which the parabolic extrapolation is used instead of VGM", + ), + Uniform( + 0.0, + 50, + optguess=0.1, + doc="exchange rate [1/day] for macropore-matrix-exchange", + ), + ] + + def parameters(self): + if self.params is None: + self.params = [ + Uniform( + "0", + -10, + 10, + 1.5, + 3.0, + -10, + 10, + doc="x value of Rosenbrock function", + ), + Uniform( + "1", + -10, + 10, + 1.5, + 3.0, + -10, + 10, + doc="y value of Rosenbrock function", + ), + Uniform( + "z", + -10, + 10, + 1.5, + 3.0, + -10, + 10, + doc="z value of Rosenbrock function", + ), + ] + return spotpy.parameter.generate(self.params) + + def simulation(self, vector): + x = np.array(vector) + # simulations = [sum(100.0 * (x[1:] - x[:-1] ** 2.0) ** 2.0 + (1 - x[:-1]) ** 2.0)] + simulations = x * np.random.rand(len(vector)) + # simulations = x * np.sum(vector) + return simulations + + def evaluation(self): + # observations = [0] + observations = [2, 3, 4] + return observations + + def objectivefunction(self, simulation, evaluation, params): + + if self.objfunc is None: + return -1 * rmse(evaluation, simulation) + else: + pars, names = params + return self.objfunc(pars) diff --git a/src/spotpy/examples/spot_setup_dtlz1.py b/src/spotpy/examples/spot_setup_dtlz1.py new file mode 100644 index 00000000..f51d68b1 --- /dev/null +++ b/src/spotpy/examples/spot_setup_dtlz1.py @@ -0,0 +1,62 @@ +import time + +import numpy as np + +import spotpy + + +def g1(x, k): + return 100 * ( + k + np.sum(np.square(x - 0.5) - np.cos(20 * np.pi * (x - 0.5)), axis=1) + ) + + +def dtlz1(x, n_var, n_obj): + + k = n_var - n_obj + 1 + + X, X_M = x[:, : n_obj - 1], x[:, n_obj - 1 :] + g = g1(X_M, k) + + f = [] + for i in range(0, n_obj): + _f = 0.5 * (1 + g) + _f *= np.prod(X[:, : X.shape[1] - i], axis=1) + if i > 0: + _f *= 1 - X[:, X.shape[1] - i] + f.append(_f) + + return f + + +class spot_setup(object): + def __init__(self, n_var=5, n_obj=3): + self.n_var = n_var + self.n_obj = n_obj + + self.params = [] + for i in range(self.n_var): + self.params.append(spotpy.parameter.Uniform(str(i), 0, 1)) + + def parameters(self): + return spotpy.parameter.generate(self.params) + + def simulation(self, vector): + vars = np.array(vector)[None] + sim = dtlz1(vars, n_var=self.n_var, n_obj=self.n_obj) + # time.sleep(0.1) + return sim + + def evaluation(self): + observations = [0] * self.n_obj + return observations + + def objectivefunction(self, simulation, evaluation): + obj = [] + for i, f in enumerate(simulation): + obj.append( + spotpy.objectivefunctions.mae( + evaluation=[evaluation[i]], simulation=[simulation[i]] + ) + ) + return obj diff --git a/src/spotpy/examples/spot_setup_griewank.py b/src/spotpy/examples/spot_setup_griewank.py new file mode 100644 index 00000000..2f0299a7 --- /dev/null +++ b/src/spotpy/examples/spot_setup_griewank.py @@ -0,0 +1,44 @@ +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +This example implements the Griewank function into SPOT. +""" +import numpy as np + +import spotpy + + +class spot_setup(object): + def __init__(self, dim=2): + self.dim = dim + self.params = [] + for i in range(self.dim): + self.params.append(spotpy.parameter.Uniform(str(i), -20, 20, 2, 4.0)) + + def parameters(self): + return spotpy.parameter.generate(self.params) + + def simulation(self, vector): + n = len(vector) + fr = 4000 + s = 0 + p = 1 + for j in range(n): + s = s + vector[j] ** 2 + for j in range(n): + p = p * np.cos(vector[j] / np.sqrt(j + 1)) + simulation = [s / fr - p + 1] + return simulation + + def evaluation(self): + observations = [0] + return observations + + def objectivefunction(self, simulation, evaluation): + objectivefunction = -spotpy.objectivefunctions.rmse( + evaluation=evaluation, simulation=simulation + ) + return objectivefunction diff --git a/src/spotpy/examples/spot_setup_hymod_exe.py b/src/spotpy/examples/spot_setup_hymod_exe.py new file mode 100644 index 00000000..562452af --- /dev/null +++ b/src/spotpy/examples/spot_setup_hymod_exe.py @@ -0,0 +1,98 @@ +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). +:author: Tobias Houska +This example implements the external hydrological model HYMOD into SPOTPY. +""" +import os + +# from shutil import rmtree +import sys +from distutils.dir_util import copy_tree, remove_tree + +import numpy as np + +import spotpy + + +class spot_setup(object): + def __init__(self, parallel="seq"): + + self.params = [ + spotpy.parameter.Uniform("cmax", low=1.0, high=500, optguess=412.33), + spotpy.parameter.Uniform("bexp", low=0.1, high=2.0, optguess=0.1725), + spotpy.parameter.Uniform("alpha", low=0.1, high=0.99, optguess=0.8127), + spotpy.parameter.Uniform("Ks", low=0.0, high=0.10, optguess=0.0404), + spotpy.parameter.Uniform("Kq", low=0.1, high=0.99, optguess=0.5592), + ] + + self.curdir = os.getcwd() + self.owd = os.path.realpath(__file__) + os.sep + ".." + self.hymod_path = self.owd + os.sep + "hymod_exe" + self.evals = list( + np.genfromtxt(self.hymod_path + os.sep + "bound.txt", skip_header=65)[:, 3] + )[:730] + self.Factor = 1944 * (1000 * 1000) / (1000 * 60 * 60 * 24) + self.parallel = parallel + + def parameters(self): + return spotpy.parameter.generate(self.params) + + def simulation(self, x): + + if self.parallel == "seq": + call = "" + elif self.parallel == "mpi": + # Running n parallel, care has to be taken when files are read or written + # Therefor we check the ID of the current computer core + call = str(int(os.environ["OMPI_COMM_WORLD_RANK"]) + 2) + # And generate a new folder with all underlying files + copy_tree(self.hymod_path, self.hymod_path + call) + + elif self.parallel == "mpc": + # Running n parallel, care has to be taken when files are read or written + # Therefor we check the ID of the current computer core + call = str(os.getpid()) + # And generate a new folder with all underlying files + copy_tree(self.hymod_path, self.hymod_path + call) + else: + raise "No call variable was assigned" + + os.chdir(self.hymod_path + call) + try: + params = open("Param.in", "w") + + for i in range(len(x)): + if i == len(x): + params.write(str(round(x[i], 5))) + else: + params.write(str(round(x[i], 5)) + " ") + params.close() + os.system("HYMODsilent.exe") + + # try: + SimRR = open("Q.out", "r") + + simulations = [] + for i in range(64): + SimRR.readline() + for i in range(730): + val = SimRR.readline() + simulations.append(float(val) * self.Factor) + SimRR.close() + except: + "Model has failed" + simulations = [np.nan] * 795 # Assign bad values - model might have crashed + os.chdir(self.curdir) + if self.parallel == "mpi" or self.parallel == "mpc": + remove_tree(self.hymod_path + call) + return simulations + + def evaluation(self): + return self.evals + + def objectivefunction(self, simulation, evaluation, params=None): + like = spotpy.objectivefunctions.nashsutcliffe( + evaluation, simulation + ) # Just an example, please choose an appropriate objective function depending on the used algorithm + return like diff --git a/src/spotpy/examples/spot_setup_hymod_python.py b/src/spotpy/examples/spot_setup_hymod_python.py new file mode 100644 index 00000000..f0144baf --- /dev/null +++ b/src/spotpy/examples/spot_setup_hymod_python.py @@ -0,0 +1,76 @@ +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +This example implements the python version of hymod into SPOTPY. +""" + + +import os + +from spotpy.examples.hymod_python.hymod import hymod +from spotpy.objectivefunctions import rmse +from spotpy.parameter import Uniform + + +class spot_setup(object): + cmax = Uniform(low=1.0, high=500, optguess=412.33) + bexp = Uniform(low=0.1, high=2.0, optguess=0.1725) + alpha = Uniform(low=0.1, high=0.99, optguess=0.8127) + Ks = Uniform(low=0.001, high=0.10, optguess=0.0404) + Kq = Uniform(low=0.1, high=0.99, optguess=0.5592) + # fake1 =spotpy.parameter.Uniform(low=0.1 , high=10, optguess=0.5592) + # fake2 =spotpy.parameter.Uniform(low=0.1 , high=10, optguess=0.5592) + + def __init__(self, obj_func=None): + # Just a way to keep this example flexible and applicable to various examples + self.obj_func = obj_func + # Transform [mm/day] into [l s-1], where 1.783 is the catchment area + self.Factor = 1.783 * 1000 * 1000 / (60 * 60 * 24) + # Load Observation data from file + self.PET, self.Precip = [], [] + self.date, self.trueObs = [], [] + # Find Path to Hymod on users system + self.owd = os.path.dirname(os.path.realpath(__file__)) + self.hymod_path = self.owd + os.sep + "hymod_python" + climatefile = open(self.hymod_path + os.sep + "hymod_input.csv", "r") + headerline = climatefile.readline()[:-1] + + # Read model forcing in working storage (this is done only ones) + if ";" in headerline: + self.delimiter = ";" + else: + self.delimiter = "," + self.header = headerline.split(self.delimiter) + for line in climatefile: + values = line.strip().split(self.delimiter) + self.date.append(str(values[0])) + self.Precip.append(float(values[1])) + self.PET.append(float(values[2])) + self.trueObs.append(float(values[3])) + climatefile.close() + + def simulation(self, x): + # Here the model is actualy startet with one paramter combination + data = hymod(self.Precip, self.PET, x[0], x[1], x[2], x[3], x[4]) + sim = [] + for val in data: + sim.append(val * self.Factor) + # The first year of simulation data is ignored (warm-up) + return sim[366:] + + def evaluation(self): + return self.trueObs[366:] + + def objectivefunction(self, simulation, evaluation, params=None): + # SPOTPY expects to get one or multiple values back, + # that define the performance of the model run + if not self.obj_func: + # This is used if not overwritten by user + like = rmse(evaluation, simulation) + else: + # Way to ensure flexible spot setup class + like = self.obj_func(evaluation, simulation) + return like diff --git a/src/spotpy/examples/spot_setup_hymod_python_pareto.py b/src/spotpy/examples/spot_setup_hymod_python_pareto.py new file mode 100644 index 00000000..a64e4a53 --- /dev/null +++ b/src/spotpy/examples/spot_setup_hymod_python_pareto.py @@ -0,0 +1,66 @@ +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +This example implements the python version of hymod into SPOTPY. +""" + + +import os + +import numpy as np + +import spotpy +from spotpy.examples.hymod_python.hymod import hymod + + +class spot_setup(object): + cmax = spotpy.parameter.Uniform(low=1.0, high=500, optguess=412.33) + bexp = spotpy.parameter.Uniform(low=0.1, high=2.0, optguess=0.1725) + alpha = spotpy.parameter.Uniform(low=0.1, high=0.99, optguess=0.8127) + Ks = spotpy.parameter.Uniform(low=0.0, high=0.10, optguess=0.0404) + Kq = spotpy.parameter.Uniform(low=0.1, high=0.99, optguess=0.5592) + + def __init__(self): + # Transform [mm/day] into [l s-1], where 1.783 is the catchment area + self.Factor = 1.783 * 1000 * 1000 / (60 * 60 * 24) + # Load Observation data from file + self.PET, self.Precip = [], [] + self.date, self.trueObs = [], [] + self.owd = os.path.dirname(os.path.realpath(__file__)) + self.hymod_path = self.owd + os.sep + "hymod_python" + climatefile = open(self.hymod_path + os.sep + "hymod_input.csv", "r") + headerline = climatefile.readline()[:-1] + + if ";" in headerline: + self.delimiter = ";" + else: + self.delimiter = "," + self.header = headerline.split(self.delimiter) + for line in climatefile: + values = line.strip().split(self.delimiter) + self.date.append(str(values[0])) + self.Precip.append(float(values[1])) + self.PET.append(float(values[2])) + self.trueObs.append(float(values[3])) + + climatefile.close() + + def simulation(self, x): + data = hymod(self.Precip, self.PET, x[0], x[1], x[2], x[3], x[4]) + sim = [] + for val in data: + sim.append(val * self.Factor) + return sim[366:] + + def evaluation(self): + return self.trueObs[366:] + + def objectivefunction(self, simulation, evaluation, params=None): + return [ + -abs(spotpy.objectivefunctions.bias(evaluation, simulation)), + spotpy.objectivefunctions.rsquared(evaluation, simulation), + spotpy.objectivefunctions.nashsutcliffe(evaluation, simulation), + ] diff --git a/src/spotpy/examples/spot_setup_hymod_unix.py b/src/spotpy/examples/spot_setup_hymod_unix.py new file mode 100644 index 00000000..e0a8ed01 --- /dev/null +++ b/src/spotpy/examples/spot_setup_hymod_unix.py @@ -0,0 +1,109 @@ +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +This example implements the external hydrological model HYMOD into SPOTPY. +""" +import numpy as np + +try: + import spotpy +except ImportError: + import sys + + sys.path.append(".") + import spotpy + +import multiprocessing as mp +import os + +# from shutil import rmtree +import sys +from distutils.dir_util import copy_tree, remove_tree + + +class spot_setup(object): + def __init__(self, parallel="seq"): + + self.params = [ + spotpy.parameter.Uniform("cmax", low=1.0, high=500, optguess=412.33), + spotpy.parameter.Uniform("bexp", low=0.1, high=2.0, optguess=0.1725), + spotpy.parameter.Uniform("alpha", low=0.1, high=0.99, optguess=0.8127), + spotpy.parameter.Uniform("Ks", low=0.0, high=0.10, optguess=0.0404), + spotpy.parameter.Uniform("Kq", low=0.1, high=0.99, optguess=0.5592), + ] + + self.curdir = os.getcwd() + self.owd = os.path.dirname(os.path.realpath(__file__)) + self.hymod_path = self.owd + os.sep + "hymod_unix" + self.evals = list( + np.genfromtxt(self.hymod_path + os.sep + "bound.txt", skip_header=65)[:, 3] + )[:730] + self.Factor = 1944 * (1000 * 1000) / (1000 * 60 * 60 * 24) + self.parallel = parallel + + def parameters(self): + return spotpy.parameter.generate(self.params) + + def simulation(self, x): + + if self.parallel == "seq": + call = "" + elif self.parallel == "mpi": + # Running n parallel, care has to be taken when files are read or written + # Therefor we check the ID of the current computer core + call = str(int(os.environ["OMPI_COMM_WORLD_RANK"]) + 2) + # And generate a new folder with all underlying files + copy_tree(self.hymod_path, self.hymod_path + call) + + elif self.parallel == "mpc": + # Running n parallel, care has to be taken when files are read or written + # Therefor we check the ID of the current computer core + call = str(os.getpid()) + # And generate a new folder with all underlying files + copy_tree(self.hymod_path, self.hymod_path + call) + else: + raise "No call variable was assigned" + + os.chdir(self.hymod_path + call) + try: + params = open("Param.in", "w") + + for i in range(len(x)): + if i == len(x): + params.write(str(round(x[i], 5))) + else: + params.write(str(round(x[i], 5)) + " ") + params.close() + + os.system( + "./hymod_%s.%s" % (sys.version_info.major, sys.version_info.minor) + ) + + SimRR = open("Q.out", "r") + + simulations = [] + for i in range(64): + SimRR.readline() + for i in range(730): + val = SimRR.readline() + simulations.append(float(val) * self.Factor) + SimRR.close() + except: + "Model has failed" + simulations = [np.nan] * 795 # Assign bad values - model might have crashed + os.chdir(self.curdir) + if self.parallel == "mpi" or self.parallel == "mpc": + remove_tree(self.hymod_path + call) + return simulations + + def evaluation(self): + return self.evals + + def objectivefunction(self, simulation, evaluation, params=None): + like = spotpy.objectivefunctions.nashsutcliffe( + evaluation, simulation + ) # Just an example, please choose an appropriate objective function depending on the used algorithm + return like diff --git a/src/spotpy/examples/spot_setup_rosenbrock.py b/src/spotpy/examples/spot_setup_rosenbrock.py new file mode 100644 index 00000000..2943e607 --- /dev/null +++ b/src/spotpy/examples/spot_setup_rosenbrock.py @@ -0,0 +1,50 @@ +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Optimization Tool for Python (SPOTPY). +:author: Tobias Houska + +This example implements the Rosenbrock function into a SPOTPY class. +""" + +import numpy as np + +from spotpy.objectivefunctions import rmse +from spotpy.parameter import Uniform + + +class spot_setup(object): + """ + A 3 dimensional implementation of the Rosenbrock function + + Result at (1,1,1) is 0. + """ + + x = Uniform(-10, 10, 1.5, 3.0, -10, 10, doc="x value of Rosenbrock function") + y = Uniform(-10, 10, 1.5, 3.0, -10, 10, doc="y value of Rosenbrock function") + z = Uniform(-10, 10, 1.5, 3.0, -10, 10, doc="z value of Rosenbrock function") + + def __init__(self, obj_func=None): + self.obj_func = obj_func + + def simulation(self, vector): + x = np.array(vector) + simulations = [ + sum(100.0 * (x[1:] - x[:-1] ** 2.0) ** 2.0 + (1 - x[:-1]) ** 2.0) + ] + return simulations + + def evaluation(self): + observations = [0] + return observations + + def objectivefunction(self, simulation, evaluation, params=None): + + # SPOTPY expects to get one or multiple values back, + # that define the performence of the model run + if not self.obj_func: + # This is used if not overwritten by user + like = rmse(evaluation, simulation) + else: + # Way to ensure on flexible spot setup class + like = self.obj_func(evaluation, simulation) + return like diff --git a/src/spotpy/examples/spot_setup_standardnormal.py b/src/spotpy/examples/spot_setup_standardnormal.py new file mode 100644 index 00000000..e756307f --- /dev/null +++ b/src/spotpy/examples/spot_setup_standardnormal.py @@ -0,0 +1,37 @@ +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +This example implements the Standard Normal function into SPOT. +""" +import numpy as np + +import spotpy + + +class spot_setup(object): + def __init__(self, mean=0, std=1): + self.params = [spotpy.parameter.Uniform("x", -5, 5, 1.5, 3.0)] + self.mean = mean + self.std = std + + def parameters(self): + return spotpy.parameter.generate(self.params) + + def simulation(self, x): + simulations = (1.0 / (self.std * np.sqrt(2 * np.pi))) ** ( + (-1.0 / 2.0) * (((x - self.mean) / self.std) ** 2) + ) + return simulations + + def evaluation(self): + observations = [0] + return observations + + def objectivefunction(self, simulation, evaluation): + objectivefunction = -spotpy.objectivefunctions.rmse( + evaluation=evaluation, simulation=simulation + ) + return objectivefunction diff --git a/spotpy/gui/__init__.py b/src/spotpy/gui/__init__.py similarity index 100% rename from spotpy/gui/__init__.py rename to src/spotpy/gui/__init__.py diff --git a/spotpy/gui/mpl.py b/src/spotpy/gui/mpl.py similarity index 77% rename from spotpy/gui/mpl.py rename to src/spotpy/gui/mpl.py index 09e3179f..7925106b 100644 --- a/spotpy/gui/mpl.py +++ b/src/spotpy/gui/mpl.py @@ -7,22 +7,19 @@ """ import sys -import matplotlib - -from matplotlib.widgets import Slider, Button -from matplotlib import pylab as plt - import time -from ..parameter import get_parameters_array, create_set +import matplotlib +from matplotlib import pylab as plt +from matplotlib.widgets import Button, Slider -if sys.version_info < (3, 5): - raise ImportError('spotpy.gui.mpl needs at least Python 3.5, you are running Python {}.{}.{}' - .format(*sys.version_info[:3])) - +from ..parameter import create_set, get_parameters_array -if matplotlib.__version__ < '2.1': - raise ImportError('Your matplotlib package is too old. Required >=2.1, you have ' + matplotlib.__version__) +if matplotlib.__version__ < "2.1": + raise ImportError( + "Your matplotlib package is too old. Required >=2.1, you have " + + matplotlib.__version__ + ) def as_scalar(val): @@ -70,7 +67,7 @@ def __init__(self, rect, wtype, *args, **kwargs): self.ax = plt.axes(rect) events = {} for k in list(kwargs.keys()): - if k.startswith('on_'): + if k.startswith("on_"): events[k] = kwargs.pop(k) self.object = wtype(self.ax, *args, **kwargs) for k in events: @@ -89,6 +86,7 @@ class ValueChanger: >>>from matplotlib.widgets import Slider >>>w = Widget([0,0,0.1,0.1], Slider, 'slider', 0, 100, on_changed=ValueChanger('a', d)) """ + def __init__(self, key, stor): self.stor = stor self.key = key @@ -123,8 +121,12 @@ def __init__(self, setup): """ self.fig = plt.figure(type(setup).__name__) self.ax = plt.axes([0.05, 0.1, 0.65, 0.85]) - self.button_run = Widget([0.75, 0.01, 0.1, 0.03], Button, 'Simulate', on_clicked=self.run) - self.button_clear = Widget([0.87, 0.01, 0.1, 0.03], Button, 'Clear plot', on_clicked=self.clear) + self.button_run = Widget( + [0.75, 0.01, 0.1, 0.03], Button, "Simulate", on_clicked=self.run + ) + self.button_clear = Widget( + [0.87, 0.01, 0.1, 0.03], Button, "Clear plot", on_clicked=self.clear + ) self.parameter_values = {} self.setup = setup self.sliders = self._make_widgets() @@ -152,16 +154,23 @@ def _make_widgets(self): Creates the sliders :return: """ - if hasattr(self, 'sliders'): + if hasattr(self, "sliders"): for s in self.sliders: s.ax.remove() sliders = [] - step = max(0.005, min(0.05, 0.8/len(self.parameter_array))) + step = max(0.005, min(0.05, 0.8 / len(self.parameter_array))) for i, row in enumerate(self.parameter_array): rect = [0.75, 0.9 - step * i, 0.2, step - 0.005] - s = Widget(rect, Slider, row['name'], row['minbound'], row['maxbound'], - valinit=row['optguess'], on_changed=ValueChanger(row['name'], self.parameter_values)) + s = Widget( + rect, + Slider, + row["name"], + row["minbound"], + row["maxbound"], + valinit=row["optguess"], + on_changed=ValueChanger(row["name"], self.parameter_values), + ) sliders.append(s) plt.draw() return sliders @@ -176,24 +185,28 @@ def clear(self, _=None): """ obs = self.setup.evaluation() self.ax.clear() - self.lines = list(self.ax.plot(obs, 'k:', label='Observation', zorder=2)) + self.lines = list(self.ax.plot(obs, "k:", label="Observation", zorder=2)) self.ax.legend() def run(self, _=None): """ Runs the model and plots the result """ - self.ax.set_title('Calculating...') + self.ax.set_title("Calculating...") plt.draw() time.sleep(0.001) parset = create_set(self.setup, **self.parameter_values) sim = self.setup.simulation(parset) objf = as_scalar(self.setup.objectivefunction(sim, self.setup.evaluation())) - label = ('{:0.4g}=M('.format(objf) - + ', '.join('{f}={v:0.4g}'.format(f=f, v=v) for f, v in zip(parset.name, parset)) - + ')') - self.lines.extend(self.ax.plot(sim, '-', label=label)) + label = ( + "{:0.4g}=M(".format(objf) + + ", ".join( + "{f}={v:0.4g}".format(f=f, v=v) for f, v in zip(parset.name, parset) + ) + + ")" + ) + self.lines.extend(self.ax.plot(sim, "-", label=label)) self.ax.legend() self.ax.set_title(type(self.setup).__name__) plt.draw() diff --git a/spotpy/hydrology/__init__.py b/src/spotpy/hydrology/__init__.py similarity index 95% rename from spotpy/hydrology/__init__.py rename to src/spotpy/hydrology/__init__.py index cfe25ae1..6d085d5d 100644 --- a/spotpy/hydrology/__init__.py +++ b/src/spotpy/hydrology/__init__.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). :author: Tobias Houska -''' +""" diff --git a/spotpy/hydrology/signatures.py b/src/spotpy/hydrology/signatures.py similarity index 93% rename from spotpy/hydrology/signatures.py rename to src/spotpy/hydrology/signatures.py index e7f7c987..10fb97ea 100644 --- a/spotpy/hydrology/signatures.py +++ b/src/spotpy/hydrology/signatures.py @@ -57,15 +57,17 @@ """ -import numpy as np import inspect import sys +import numpy as np + class SignatureMethod: """ Wraps a signature method from this module to access multiple variables """ + @classmethod def find_all(cls): """ @@ -76,7 +78,7 @@ def find_all(cls): current_module = sys.modules[__name__] methods = inspect.getmembers(current_module) - return [cls(m, name) for name, m in methods if name.startswith('get_')] + return [cls(m, name) for name, m in methods if name.startswith("get_")] @classmethod def run(cls, list_of_methods, data, measurements_per_day=1): @@ -98,10 +100,10 @@ def run(cls, list_of_methods, data, measurements_per_day=1): @classmethod def extract_from_doc(cls, token, doc): - for doc_line in doc.split('\n'): + for doc_line in doc.split("\n"): if doc_line.strip().startswith(token): - var_str = doc_line[len(token):].strip() - return [v.strip() for v in var_str.split(',')] + var_str = doc_line[len(token) :].strip() + return [v.strip() for v in var_str.split(",")] return [] def __init__(self, method, name): @@ -109,7 +111,7 @@ def __init__(self, method, name): self.name = name[4:] doc = inspect.getdoc(self.method) - self.variables = self.extract_from_doc(':return:', doc) or [self.name] + self.variables = self.extract_from_doc(":return:", doc) or [self.name] def __call__(self, data, measurements_per_day=1): """ @@ -125,8 +127,7 @@ def __call__(self, data, measurements_per_day=1): return [(self.variables[0], res)] def __repr__(self): - return 'Sig({n})->{v}'.format(n=self.name, v=', '.join(self.variables)) - + return "Sig({n})->{v}".format(n=self.name, v=", ".join(self.variables)) def remove_nan(data): @@ -176,9 +177,11 @@ def summarize(data, step, f): """ if len(data) < step: return np.array([f(data)]) - return np.fromiter((f(data[i:i+step]) - for i in range(0, len(data), step)), - count=len(data) // step, dtype=float) + return np.fromiter( + (f(data[i : i + step]) for i in range(0, len(data), step)), + count=len(data) // step, + dtype=float, + ) class Quantile(object): @@ -189,9 +192,12 @@ class Quantile(object): :return: Q_{} """ + def __init__(self, quantile): self.quantile = quantile - self.__doc__ = inspect.getdoc(type(self)).replace('', '{:0.4g}'.format(self.quantile)) + self.__doc__ = inspect.getdoc(type(self)).replace( + "", "{:0.4g}".format(self.quantile) + ) def __call__(self, data, measurements_per_day=None): """ @@ -204,7 +210,7 @@ def __call__(self, data, measurements_per_day=None): return np.percentile(remove_nan(data), 100 - self.quantile) def __repr__(self): - return 'q({:0.2f}%)'.format(self.quantile) + return "q({:0.2f}%)".format(self.quantile) get_q0_01 = Quantile(0.01) @@ -281,19 +287,19 @@ def get_sfdc(data, measurements_per_day=None): """ mean = get_mean(data) - Q33 = Quantile(33)(data)/mean - Q66 = Quantile(66)(data)/mean + Q33 = Quantile(33)(data) / mean + Q66 = Quantile(66)(data) / mean - # Determine if the fdc has a slope at this tage and return the + # Determine if the fdc has a slope at this tage and return the # corresponding values if Q33 == 0 and Q66 == 0: return 0 elif Q33 == 0 and not Q66 == 0: - return -np.log(Q66) / (2/3 - 1/3) + return -np.log(Q66) / (2 / 3 - 1 / 3) elif not Q33 == 0 and Q66 == 0: - return np.log(Q33) / (2/3 - 1/3) + return np.log(Q33) / (2 / 3 - 1 / 3) else: - return (np.log(Q33) - np.log(Q66)) / (2/3 - 1/3) + return (np.log(Q33) - np.log(Q66)) / (2 / 3 - 1 / 3) def calc_baseflow(data, measurements_per_day=1): @@ -307,15 +313,17 @@ def calc_baseflow(data, measurements_per_day=1): :param measurements_per_day: :return: The baseflow timeseries in the same resolution as data """ - period_length = 5 # days + period_length = 5 # days if measurements_per_day < 1: - raise ValueError('At least a daily measurement frequency is needed to calculate baseflow') + raise ValueError( + "At least a daily measurement frequency is needed to calculate baseflow" + ) # Remove NaN values data = fill_nan(data) def irange(seq): - """ Returns the indices of a sequence""" + """Returns the indices of a sequence""" return range(len(seq)) # Calculate daily mean @@ -324,7 +332,6 @@ def irange(seq): # Get minimum flow for each 5 day period (Step 1 in Gustard et al 1992) Q = summarize(daily_flow, period_length, np.min) - def is_baseflow(i, Q): """ Returns True if a 5 day period can be considered as baseflow @@ -332,17 +339,16 @@ def is_baseflow(i, Q): :param Q: 5day period minimum values :return: True if Q[i] is a baseflow """ - if 0 < i < len(Q)-1: + if 0 < i < len(Q) - 1: # The boolean expression after the or works with the assumption - # that flow values of 0 can always be considered as baseflow. + # that flow values of 0 can always be considered as baseflow. return (Q[i] * 0.9 < min(Q[i - 1], Q[i + 1])) or (Q[i] == 0) else: return False # Get each 5 day period index, where the baseflow condition is fullfilled # (Step 2 in Gustard et al 1992) - QB_pos = [i for i in irange(Q) - if is_baseflow(i, Q)] + QB_pos = [i for i in irange(Q) if is_baseflow(i, Q)] QB_raw = Q[QB_pos] # get interpolated values for each minflow timestep (Step 3) @@ -451,6 +457,7 @@ def get_qlf(data, measurements_per_day=1): def lowflow(value, mean): return value < 0.2 * mean + fq, md = flow_event(data, lowflow, np.mean(data)) return fq * measurements_per_day * 365, md / measurements_per_day @@ -547,7 +554,7 @@ def get_recession(data, measurements_per_day=None): q = fill_nan(data) # Only use median of flows above 0, to avoid mathmatical errors. - q = q / np.median(q[q>0]) + q = q / np.median(q[q > 0]) dqdt = np.diff(q) # Use only recession situation (dqdt < 0) q = q[:-1][dqdt < 0] @@ -556,7 +563,7 @@ def get_recession(data, measurements_per_day=None): b, t0 = np.polyfit(np.log(q), np.log(-dqdt), 1) - return b, 1/np.exp(t0), r2 + return b, 1 / np.exp(t0), r2 def get_zero_q_freq(data, measurements_per_day=None): @@ -567,5 +574,3 @@ def get_zero_q_freq(data, measurements_per_day=None): :return: ZERO_Q_FREQ """ return ((len(data) - np.count_nonzero(data)) / len(data)) * 100 - - diff --git a/spotpy/likelihoods.py b/src/spotpy/likelihoods.py similarity index 79% rename from spotpy/likelihoods.py rename to src/spotpy/likelihoods.py index b6545f06..d3cb2ee9 100644 --- a/spotpy/likelihoods.py +++ b/src/spotpy/likelihoods.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -''' +""" Copyright 2017 by Tobias Houska, Benjamin Manns This file is part of Statistical Parameter Estimation Tool (SPOTPY). :author: Benjamin Manns This module contains a framework to summarize the distance between the model simulations and corresponding observations by calculating likelihood values. We modified the formula so, that a best fit of model can be archived by maximizing negative likelihood to zero -''' +""" #### TEST RESULTS WITH DREAM and hymod.py #### # Dream has now a bunch of options. We tested only likelihoods, which do not need a additional parameter @@ -18,30 +18,34 @@ # ABCBoxcarLikelihood with Option 2 and 6 is good # logLikelihood with option 2 and 6 is good -import numpy as np import math import warnings +import numpy as np + class LikelihoodError(Exception): """ Define an own error class to know it is an error made by a likelihood calculation to warn the use for wrong inputs """ + pass def __generateMeaserror(data): return np.array(data) * 0.1 + def __calcSimpleDeviation_bias(data, comparedata, mu_h): __standartChecksBeforeStart(data, comparedata) d = np.array(data) c = np.array(comparedata) - #Adjust c for multiplicative bias parameter: + # Adjust c for multiplicative bias parameter: mu_t = np.exp(mu_h * c) Et = c * mu_t return d - Et + def __calcSimpleDeviation(data, comparedata): __standartChecksBeforeStart(data, comparedata) d = np.array(data) @@ -52,9 +56,13 @@ def __calcSimpleDeviation(data, comparedata): def __standartChecksBeforeStart(data, comparedata): # some standard checks if data.__len__() != comparedata.__len__(): - raise LikelihoodError("Simulation and observation data have not the same length") + raise LikelihoodError( + "Simulation and observation data have not the same length" + ) if data.__len__() == 0: - raise LikelihoodError("Data with no content can not be used as a foundation of calculation a likelihood") + raise LikelihoodError( + "Data with no content can not be used as a foundation of calculation a likelihood" + ) def __jitter_measerror_if_needed(fun_name, measerror): @@ -62,7 +70,8 @@ def __jitter_measerror_if_needed(fun_name, measerror): if size > 0: warnings.warn( "[" + fun_name + "] realized that there are distinct distributed values. " - "We jittered the values but the result can be far away from the truth.") + "We jittered the values but the result can be far away from the truth." + ) measerror[measerror == 0.0] = np.random.uniform(0.01, 0.1, size) return measerror @@ -95,15 +104,19 @@ def acf(data, lag): len = data.__len__() if len <= 0: - raise LikelihoodError("Data with no content can not be used to calc autokorrelation") + raise LikelihoodError( + "Data with no content can not be used to calc autokorrelation" + ) if lag is None or type(lag) != type(1): raise LikelihoodError("The lag musst be an integer") if lag > len: - raise LikelihoodError("The lag can not be bigger then the size of your data") + raise LikelihoodError( + "The lag can not be bigger then the size of your data" + ) m = np.mean(data) d = np.array(data) # R-Style numpy inline sum - return np.sum((d[lag:len] - m) * (d[0:len - lag] - m)) / len + return np.sum((d[lag:len] - m) * (d[0 : len - lag] - m)) / len @staticmethod def AR_1_Coeff(data): @@ -128,8 +141,8 @@ def AR_1_Coeff(data): def logLikelihood(data, comparedata, measerror=None): """ This formula is based on the gaussian likelihood: homo/heteroscedastic data error formula which can be used in both - cases if the data has a homo- or heteroscedastic data error. To archive numerical stability a log-transformation was done, - which derives following formula, as shown in formular 8 in: Vrugt 2016 Markov chain Monte Carlo + cases if the data has a homo- or heteroscedastic data error. To archive numerical stability a log-transformation was done, + which derives following formula, as shown in formular 8 in: Vrugt 2016 Markov chain Monte Carlo simulation using the DREAM software package: Theory, concepts, and Matlab implementation, EMS: @@ -159,8 +172,11 @@ def logLikelihood(data, comparedata, measerror=None): measerror = __jitter_measerror_if_needed("logLikelihood", measerror) # TODO: Maximize is done but in positive way (from negative to zero is hard) - return -data.__len__() / 2 * np.log(2 * np.pi) - np.nansum(np.log(measerror)) - 0.5 * np.sum( - ((data - comparedata) / measerror) ** 2) + return ( + -data.__len__() / 2 * np.log(2 * np.pi) + - np.nansum(np.log(measerror)) + - 0.5 * np.sum(((data - comparedata) / measerror) ** 2) + ) def gaussianLikelihoodMeasErrorOut(data, comparedata): @@ -187,7 +203,7 @@ def gaussianLikelihoodMeasErrorOut(data, comparedata): __standartChecksBeforeStart(data, comparedata) errorArr = np.array(__calcSimpleDeviation(data, comparedata)) - return -data.__len__() / 2 * np.log(np.sum(errorArr ** 2)) + return -data.__len__() / 2 * np.log(np.sum(errorArr**2)) def gaussianLikelihoodHomoHeteroDataError(data, comparedata, measerror=None): @@ -221,12 +237,16 @@ def gaussianLikelihoodHomoHeteroDataError(data, comparedata, measerror=None): if measerror is None: measerror = __generateMeaserror(data) measerror = np.array(measerror) - measerror = __jitter_measerror_if_needed("gaussianLikelihoodHomoHeteroDataError", measerror) + measerror = __jitter_measerror_if_needed( + "gaussianLikelihoodHomoHeteroDataError", measerror + ) # TODO Maximizing with negative to zero? # original: -np.prod((1 / (np.sqrt(2 * np.pi * measerror**2)))*np.exp(-0.5 * ((data-comparedata)/(measerror))**2)) return -np.sum( - (1 / (np.sqrt(2 * np.pi * measerror ** 2))) * np.exp(-0.5 * ((data - comparedata) / (measerror)) ** 2)) + (1 / (np.sqrt(2 * np.pi * measerror**2))) + * np.exp(-0.5 * ((data - comparedata) / (measerror)) ** 2) + ) def LikelihoodAR1WithC(data, comparedata, measerror=None, params=None): @@ -289,12 +309,15 @@ def LikelihoodAR1WithC(data, comparedata, measerror=None, params=None): if missingparams.__len__() > 0: raise LikelihoodError( "Unfortunately contains your param list not all parameters which are needed for this class." - "Following parameter are needed, too: " + str(missingparams)) + "Following parameter are needed, too: " + str(missingparams) + ) - phi = float(randomparset[parameternames == 'likelihood_phi']) + phi = float(randomparset[parameternames == "likelihood_phi"]) # Break the calculation if given parameter are not valid if abs(phi) >= 1: - warnings.warn("The parameter 'phi' should be real between -1 and 1 and is: " + str(phi)) + warnings.warn( + "The parameter 'phi' should be real between -1 and 1 and is: " + str(phi) + ) return np.NAN expect = np.nanmean(data) @@ -308,8 +331,16 @@ def LikelihoodAR1WithC(data, comparedata, measerror=None, params=None): sum_2 = np.sum(((errorArr[1:] - c - phi * errorArr[:-1]) / (measerror[1:])) ** 2) # TODO Its maximaized but maybe from negative to zero, is that possible? - return -(-(n / 2) * np.log(2 * np.pi) - 0.5 * np.log(measerror[0] ** 2 / (1 - phi ** 2)) - ( - (errorArr[0] - (c / (1 - phi))) ** 2 / (2 * measerror[0] ** 2 / (1 - phi ** 2))) - sum_1 - 0.5 * sum_2) + return -( + -(n / 2) * np.log(2 * np.pi) + - 0.5 * np.log(measerror[0] ** 2 / (1 - phi**2)) + - ( + (errorArr[0] - (c / (1 - phi))) ** 2 + / (2 * measerror[0] ** 2 / (1 - phi**2)) + ) + - sum_1 + - 0.5 * sum_2 + ) def LikelihoodAR1NoC(data, comparedata, measerror=None, params=None): @@ -361,15 +392,19 @@ def LikelihoodAR1NoC(data, comparedata, measerror=None, params=None): if missingparams.__len__() > 0: raise LikelihoodError( "Unfortunately contains your param list not all parameters which are needed for this class." - "Following parameter are needed, too: " + str(missingparams)) + "Following parameter are needed, too: " + str(missingparams) + ) parameternames = np.array(parameternames) randomparset = np.array(randomparset) - phi = float(randomparset[parameternames == 'likelihood_phi']) + phi = float(randomparset[parameternames == "likelihood_phi"]) # Break the calculation if given parameter are not valid if abs(phi) >= 1: - warnings.warn("The parameter 'phi' should be real between -1 and 1 and is: " + str(phi)) + warnings.warn( + "The parameter 'phi' should be real between -1 and 1 and is: " + + str(phi) + ) return np.NAN sum_1 = np.sum(np.log(measerror[1:])) @@ -377,8 +412,12 @@ def LikelihoodAR1NoC(data, comparedata, measerror=None, params=None): # TODO Maximizing with negative to zero? return -float( - -(n / 2) * np.log(2 * np.pi) + 0.5 * np.log(1 - phi ** 2) - 0.5 * (1 - phi ** 2) * (1 / measerror[0] ** 2) * \ - errorArr[0] ** 2 - sum_1 - 0.5 * sum_2) + -(n / 2) * np.log(2 * np.pi) + + 0.5 * np.log(1 - phi**2) + - 0.5 * (1 - phi**2) * (1 / measerror[0] ** 2) * errorArr[0] ** 2 + - sum_1 + - 0.5 * sum_2 + ) def generalizedLikelihoodFunction(data, comparedata, measerror=None, params=None): @@ -443,8 +482,14 @@ def generalizedLikelihoodFunction(data, comparedata, measerror=None, params=None comparedata = np.array(comparedata) measerror = __jitter_measerror_if_needed("generalizedLikelihoodFunction", measerror) - paramDependencies = ["likelihood_beta", "likelihood_xi", "likelihood_sigma0", "likelihood_sigma1", - "likelihood_phi1", "likelihood_muh"] + paramDependencies = [ + "likelihood_beta", + "likelihood_xi", + "likelihood_sigma0", + "likelihood_sigma1", + "likelihood_phi1", + "likelihood_muh", + ] if params is None: # for this params look into http://onlinelibrary.wiley.com/doi/10.1029/2009WR008933/epdf, page 5 @@ -452,7 +497,7 @@ def generalizedLikelihoodFunction(data, comparedata, measerror=None, params=None xi = np.random.uniform(0.01, 10, 1) sigma0 = np.random.uniform(0, 1, 1) sigma1 = np.random.uniform(0, 1, 1) - phi1 = np.random.uniform(0, .99, 1) + phi1 = np.random.uniform(0, 0.99, 1) muh = np.random.uniform(0, 100, 1) else: @@ -468,43 +513,72 @@ def generalizedLikelihoodFunction(data, comparedata, measerror=None, params=None if missingparams.__len__() > 0: raise LikelihoodError( "Unfortunately contains your param list not all parameters which are needed for this class." - "Following parameter are needed, too: " + str(missingparams)) + "Following parameter are needed, too: " + str(missingparams) + ) - beta = float(randomparset[np.where(parameternames == 'likelihood_beta')]) - xi = float(randomparset[np.where(parameternames == 'likelihood_xi')]) - sigma0 = float(randomparset[np.where(parameternames == 'likelihood_sigma0')]) - sigma1 = float(randomparset[parameternames == 'likelihood_sigma0']) - phi1 = float(randomparset[np.where(parameternames == 'likelihood_phi1')]) - muh = float(randomparset[np.where(parameternames == 'likelihood_muh')]) + beta = float(randomparset[np.where(parameternames == "likelihood_beta")]) + xi = float(randomparset[np.where(parameternames == "likelihood_xi")]) + sigma0 = float(randomparset[np.where(parameternames == "likelihood_sigma0")]) + sigma1 = float(randomparset[parameternames == "likelihood_sigma0"]) + phi1 = float(randomparset[np.where(parameternames == "likelihood_phi1")]) + muh = float(randomparset[np.where(parameternames == "likelihood_muh")]) # Break the calculation if given parameter are not valid if beta <= -1 or beta > 1: - warnings.warn("The parameter 'beta' should be greater then -1 and less equal 1 and is: " + str(beta)) + warnings.warn( + "The parameter 'beta' should be greater then -1 and less equal 1 and is: " + + str(beta) + ) return np.NAN if xi < 0.1 or xi > 10: - warnings.warn("The parameter 'xi' should be between 0.1 and 10 and is: " + str(xi)) + warnings.warn( + "The parameter 'xi' should be between 0.1 and 10 and is: " + str(xi) + ) return np.NAN if sigma0 < 0 or sigma0 > 1: - warnings.warn("The parameter 'sigma0' should be between 0 and 1 and is: " + str(sigma0)) + warnings.warn( + "The parameter 'sigma0' should be between 0 and 1 and is: " + + str(sigma0) + ) return np.NAN if sigma1 < 0 or sigma1 > 1: - warnings.warn("The parameter 'sigma1' should be between 0 and 1 and is: " + str(sigma1)) + warnings.warn( + "The parameter 'sigma1' should be between 0 and 1 and is: " + + str(sigma1) + ) return np.NAN if phi1 < 0 or phi1 > 1: - warnings.warn("The parameter 'phi1' should be between 0 and 1 and is: " + str(phi1)) + warnings.warn( + "The parameter 'phi1' should be between 0 and 1 and is: " + str(phi1) + ) return np.NAN if muh < 0 or muh > 100: - warnings.warn("The parameter 'muh' should be between 0 and 100 and is: " + str(muh)) + warnings.warn( + "The parameter 'muh' should be between 0 and 100 and is: " + str(muh) + ) return np.NAN try: - omegaBeta = np.sqrt(math.gamma(3 * (1 + beta) / 2)) / ((1 + beta) * np.sqrt(math.gamma((1 + beta) / 2) ** 3)) - M_1 = math.gamma(1 + beta) / (np.sqrt(math.gamma(3 * (1 + beta) / 2)) * np.sqrt(math.gamma((1 + beta) / 2))) + omegaBeta = np.sqrt(math.gamma(3 * (1 + beta) / 2)) / ( + (1 + beta) * np.sqrt(math.gamma((1 + beta) / 2) ** 3) + ) + M_1 = math.gamma(1 + beta) / ( + np.sqrt(math.gamma(3 * (1 + beta) / 2)) + * np.sqrt(math.gamma((1 + beta) / 2)) + ) M_2 = 1 - sigma_xi = np.sqrt(np.abs(float((M_2 - M_1 ** 2) * (xi ** 2 + xi ** (-2)) + 2 * M_1 ** 2 - M_2))) - cBeta = (math.gamma(3 * (1 + beta) / 2) / math.gamma((1 + beta) / 2)) ** (1 / (1 + beta)) + sigma_xi = np.sqrt( + np.abs( + float((M_2 - M_1**2) * (xi**2 + xi ** (-2)) + 2 * M_1**2 - M_2) + ) + ) + cBeta = (math.gamma(3 * (1 + beta) / 2) / math.gamma((1 + beta) / 2)) ** ( + 1 / (1 + beta) + ) except ValueError: - raise LikelihoodError("Please check your parameter input there is something wrong with the parameter") + raise LikelihoodError( + "Please check your parameter input there is something wrong with the parameter" + ) if xi != 0.0: mu_xi = M_1 * (xi - (xi ** (-1))) @@ -540,12 +614,18 @@ def generalizedLikelihoodFunction(data, comparedata, measerror=None, params=None sigmas = sigma0 + sigma1 * E if sigmas[sigmas <= 0.0].size > 0: - warnings.warn("Sorry, you comparedata have negative values. Maybe you model has some inaccurate" - " assumptions or there is another error." - " We cannot calculate this likelihood") + warnings.warn( + "Sorry, you comparedata have negative values. Maybe you model has some inaccurate" + " assumptions or there is another error." + " We cannot calculate this likelihood" + ) return np.NAN - return n * np.log(omegaBeta * (2 * sigma_xi) / np.abs(xi + (1 / xi))) - np.sum(np.log(sigmas)) - cBeta * sum_at + return ( + n * np.log(omegaBeta * (2 * sigma_xi) / np.abs(xi + (1 / xi))) + - np.sum(np.log(sigmas)) + - cBeta * sum_at + ) def LaplacianLikelihood(data, comparedata, measerror=None): @@ -580,7 +660,9 @@ def LaplacianLikelihood(data, comparedata, measerror=None): measerror = __jitter_measerror_if_needed("LaplacianLikelihood", measerror) # Log from negative value makes no sense at all - return -1 * np.sum(np.log(2 * np.abs(measerror))) - np.sum(np.abs(errArr) / measerror) + return -1 * np.sum(np.log(2 * np.abs(measerror))) - np.sum( + np.abs(errArr) / measerror + ) def SkewedStudentLikelihoodHomoscedastic(data, comparedata, measerror=None): @@ -617,10 +699,14 @@ def SkewedStudentLikelihoodHomoscedastic(data, comparedata, measerror=None): # TODO Maximizing with negative to zero? # Original: -np.prod(1 / (np.sqrt(2 * np.pi) * measerror) * np.exp(-1 * (res ** 2) / (2))) - return -np.sum((1 / (np.sqrt(2 * np.pi) * measerror) * np.exp(-1 * (res ** 2) / (2)))) + return -np.sum( + (1 / (np.sqrt(2 * np.pi) * measerror) * np.exp(-1 * (res**2) / (2))) + ) -def SkewedStudentLikelihoodHeteroscedastic(data, comparedata, measerror=None, params=None): +def SkewedStudentLikelihoodHeteroscedastic( + data, comparedata, measerror=None, params=None +): """ Under the assumption that the data are heteroscedastic, i.e. the they have for every measurement another error and that the residuals are non-Gaussian distributed we perform a likelihoodcalculation based on this formualar, having @@ -688,7 +774,9 @@ def SkewedStudentLikelihoodHeteroscedastic(data, comparedata, measerror=None, pa measerror = __generateMeaserror(data) measerror = np.array(measerror) - measerror = __jitter_measerror_if_needed("SkewedStudentLikelihoodHeteroscedastic", measerror) + measerror = __jitter_measerror_if_needed( + "SkewedStudentLikelihoodHeteroscedastic", measerror + ) diff = np.array(__calcSimpleDeviation(data, comparedata)) @@ -714,45 +802,69 @@ def SkewedStudentLikelihoodHeteroscedastic(data, comparedata, measerror=None, pa if missingparams.__len__() > 0: raise LikelihoodError( "Unfortunately contains your param list not all parameters which are needed for this class." - "Following parameter are needed, too: " + str(missingparams)) + "Following parameter are needed, too: " + str(missingparams) + ) - nu = randomparset[parameternames == 'likelihood_nu'][0] - k = randomparset[parameternames == 'likelihood_kappa'][0] - phi = randomparset[parameternames == 'likelihood_phi'][0] + nu = randomparset[parameternames == "likelihood_nu"][0] + k = randomparset[parameternames == "likelihood_kappa"][0] + phi = randomparset[parameternames == "likelihood_phi"][0] if abs(phi) > 1: warnings.warn( - "[SkewedStudentLikelihoodHeteroscedastic] The parameter 'phi' should be between -1 and 1 and is: " + str( - phi)) + "[SkewedStudentLikelihoodHeteroscedastic] The parameter 'phi' should be between -1 and 1 and is: " + + str(phi) + ) return np.NAN if nu <= 2: warnings.warn( - "[SkewedStudentLikelihoodHeteroscedastic] The parameter 'nu' should be greater then 2 and is: " + str( - nu)) + "[SkewedStudentLikelihoodHeteroscedastic] The parameter 'nu' should be greater then 2 and is: " + + str(nu) + ) return np.NAN if k <= 0: warnings.warn( - "[SkewedStudentLikelihoodHeteroscedastic] The parameter 'k' should be greater then 0 and is: " + str( - k)) + "[SkewedStudentLikelihoodHeteroscedastic] The parameter 'k' should be greater then 0 and is: " + + str(k) + ) return np.NAN - eta_all = diff[1:] - phi * diff[:-1] * np.sqrt(1 - phi ** 2) - c_1 = ((k ** 2 - 1 / (k ** 2)) * 2 * math.gamma((nu + 1) / 2) * np.sqrt(nu / (nu - 2)) * (nu - 2)) / ( - (k + (1 / k)) * math.gamma(nu / 2) * np.sqrt(np.pi * nu) * (nu - 1)) + eta_all = diff[1:] - phi * diff[:-1] * np.sqrt(1 - phi**2) + c_1 = ( + (k**2 - 1 / (k**2)) + * 2 + * math.gamma((nu + 1) / 2) + * np.sqrt(nu / (nu - 2)) + * (nu - 2) + ) / ((k + (1 / k)) * math.gamma(nu / 2) * np.sqrt(np.pi * nu) * (nu - 1)) - for_c2 = -1 * (c_1) ** 2 + (k ** 3 + 1 / k ** 3) / (k + 1 / k) + for_c2 = -1 * (c_1) ** 2 + (k**3 + 1 / k**3) / (k + 1 / k) c_2 = np.sqrt(for_c2) # TODO Maximizing with negative to zero? - return np.log(-np.prod((2 * c_2 * math.gamma((nu + 1) / 2) * np.sqrt(nu / (nu - 2))) / ( - (k + 1 / k) * math.gamma(nu / 2) * np.sqrt(np.pi * nu) * np.sqrt(1 - phi ** 2) * measerror[1:]) \ - * (1 + (1 / (nu - 2)) * ( - (c_1 + c_2 * eta_all) / (k ** (np.sign(c_1 + c_2 * eta_all)))) ** 2) ** ( - -(nu + 1) / 2))) - - -def SkewedStudentLikelihoodHeteroscedasticAdvancedARModel(data, comparedata, measerror=None, params=None): + return np.log( + -np.prod( + (2 * c_2 * math.gamma((nu + 1) / 2) * np.sqrt(nu / (nu - 2))) + / ( + (k + 1 / k) + * math.gamma(nu / 2) + * np.sqrt(np.pi * nu) + * np.sqrt(1 - phi**2) + * measerror[1:] + ) + * ( + 1 + + (1 / (nu - 2)) + * ((c_1 + c_2 * eta_all) / (k ** (np.sign(c_1 + c_2 * eta_all)))) ** 2 + ) + ** (-(nu + 1) / 2) + ) + ) + + +def SkewedStudentLikelihoodHeteroscedasticAdvancedARModel( + data, comparedata, measerror=None, params=None +): """ This function is based of the previos one, called `SkewedStudentLikelihoodHeteroscedastic`. We expand @@ -792,7 +904,9 @@ def SkewedStudentLikelihoodHeteroscedasticAdvancedARModel(data, comparedata, mea measerror = __generateMeaserror(data) measerror = np.array(measerror) - measerror = __jitter_measerror_if_needed("SkewedStudentLikelihoodHeteroscedasticAdvancedARModel", measerror) + measerror = __jitter_measerror_if_needed( + "SkewedStudentLikelihoodHeteroscedasticAdvancedARModel", measerror + ) res = np.array(__calcSimpleDeviation(data, comparedata)) @@ -816,49 +930,84 @@ def SkewedStudentLikelihoodHeteroscedasticAdvancedARModel(data, comparedata, mea if missingparams.__len__() > 0: raise LikelihoodError( "Unfortunately contains your param list not all parameters which are needed for this class." - "Following parameter are needed, too: " + str(missingparams)) + "Following parameter are needed, too: " + str(missingparams) + ) - nu = randomparset[parameternames == 'likelihood_nu'][0] - k = randomparset[parameternames == 'likelihood_kappa'][0] - phi = randomparset[parameternames == 'likelihood_phi'][0] + nu = randomparset[parameternames == "likelihood_nu"][0] + k = randomparset[parameternames == "likelihood_kappa"][0] + phi = randomparset[parameternames == "likelihood_phi"][0] if abs(phi) > 1: warnings.warn( - "[SkewedStudentLikelihoodHeteroscedasticAdvancedARModel] The parameter 'phi' should be between -1 and 1 and is: " + str( - phi)) + "[SkewedStudentLikelihoodHeteroscedasticAdvancedARModel] The parameter 'phi' should be between -1 and 1 and is: " + + str(phi) + ) return np.NAN if nu <= 2: warnings.warn( - "[SkewedStudentLikelihoodHeteroscedasticAdvancedARModel] The parameter 'nu' should be greater then 2 and is: " + str( - nu)) + "[SkewedStudentLikelihoodHeteroscedasticAdvancedARModel] The parameter 'nu' should be greater then 2 and is: " + + str(nu) + ) return np.NAN if k <= 0: warnings.warn( - "[SkewedStudentLikelihoodHeteroscedasticAdvancedARModel] The parameter 'k' should be greater then 0 and is: " + str( - k)) + "[SkewedStudentLikelihoodHeteroscedasticAdvancedARModel] The parameter 'k' should be greater then 0 and is: " + + str(k) + ) return np.NAN N = data.__len__() - eta_all = (res[1:] - phi * res[:-1] + phi / (N) * np.sum(res)) * np.sqrt(1 - phi ** 2) - - c_1 = ((k ** 2 - 1 / (k ** 2)) * 2 * math.gamma((nu + 1) / 2) * np.sqrt(nu / (nu - 2)) * (nu - 2)) / ( - (k + (1 / k)) * math.gamma(nu / 2) * np.sqrt(np.pi * nu) * (nu - 1)) - for_c2 = -1 * (c_1) ** 2 + (k ** 3 + 1 / k ** 3) / (k + 1 / k) + eta_all = (res[1:] - phi * res[:-1] + phi / (N) * np.sum(res)) * np.sqrt( + 1 - phi**2 + ) + + c_1 = ( + (k**2 - 1 / (k**2)) + * 2 + * math.gamma((nu + 1) / 2) + * np.sqrt(nu / (nu - 2)) + * (nu - 2) + ) / ((k + (1 / k)) * math.gamma(nu / 2) * np.sqrt(np.pi * nu) * (nu - 1)) + for_c2 = -1 * (c_1) ** 2 + (k**3 + 1 / k**3) / (k + 1 / k) c_2 = np.sqrt(for_c2) # TODO Maximizing with negative to zero? - datas = ((2 * c_2 * math.gamma((nu + 1) / 2) * np.sqrt(nu / (nu - 2))) / ( - (k + 1 / k) * math.gamma(nu / 2) * np.sqrt(np.pi * nu) * np.sqrt(1 - phi ** 2) * measerror[1:]) \ - * (1 + (1 / (nu - 2)) * ( - (c_1 + c_2 * eta_all) / (k ** (np.sign(c_1 + c_2 * eta_all)))) ** 2) ** ( - -(nu + 1) / 2)) - - return np.log(-np.prod((2 * c_2 * math.gamma((nu + 1) / 2) * np.sqrt(nu / (nu - 2))) / ( - (k + 1 / k) * math.gamma(nu / 2) * np.sqrt(np.pi * nu) * np.sqrt(1 - phi ** 2) * measerror[1:]) \ - * (1 + (1 / (nu - 2)) * ( - (c_1 + c_2 * eta_all) / (k ** (np.sign(c_1 + c_2 * eta_all)))) ** 2) ** ( - -(nu + 1) / 2))) + datas = ( + (2 * c_2 * math.gamma((nu + 1) / 2) * np.sqrt(nu / (nu - 2))) + / ( + (k + 1 / k) + * math.gamma(nu / 2) + * np.sqrt(np.pi * nu) + * np.sqrt(1 - phi**2) + * measerror[1:] + ) + * ( + 1 + + (1 / (nu - 2)) + * ((c_1 + c_2 * eta_all) / (k ** (np.sign(c_1 + c_2 * eta_all)))) ** 2 + ) + ** (-(nu + 1) / 2) + ) + + return np.log( + -np.prod( + (2 * c_2 * math.gamma((nu + 1) / 2) * np.sqrt(nu / (nu - 2))) + / ( + (k + 1 / k) + * math.gamma(nu / 2) + * np.sqrt(np.pi * nu) + * np.sqrt(1 - phi**2) + * measerror[1:] + ) + * ( + 1 + + (1 / (nu - 2)) + * ((c_1 + c_2 * eta_all) / (k ** (np.sign(c_1 + c_2 * eta_all)))) ** 2 + ) + ** (-(nu + 1) / 2) + ) + ) def NoisyABCGaussianLikelihood(data, comparedata, measerror=None): @@ -897,12 +1046,14 @@ def NoisyABCGaussianLikelihood(data, comparedata, measerror=None): size = sigmas[sigmas == 0.0].size if size > 0: warnings.warn( - "[NoisyABCGaussianLikelihood] reaslized that there are distinct distributed values. We jittered the values but the result can be far away from the truth.") + "[NoisyABCGaussianLikelihood] reaslized that there are distinct distributed values. We jittered the values but the result can be far away from the truth." + ) sigmas[sigmas == 0.0] = np.random.uniform(0.01, 0.1, size) if measerror == 0.0: warnings.warn( - "[NoisyABCGaussianLikelihood] reaslized that the mean of the measerror is zero and therefore is no likelihood calculation possible") + "[NoisyABCGaussianLikelihood] reaslized that the mean of the measerror is zero and therefore is no likelihood calculation possible" + ) return np.NAN m = data.__len__() @@ -910,18 +1061,24 @@ def NoisyABCGaussianLikelihood(data, comparedata, measerror=None): comparedata = np.array(comparedata) # The euclidean distance has a bit diffrent formula then the original paper showed - return -m / 2 * np.log(2 * np.pi) - m * np.log(measerror) - 0.5 * 1 / (measerror ** 2) * np.sqrt( - np.sum(((data - comparedata) / sigmas) ** 2)) + return ( + -m / 2 * np.log(2 * np.pi) + - m * np.log(measerror) + - 0.5 + * 1 + / (measerror**2) + * np.sqrt(np.sum(((data - comparedata) / sigmas) ** 2)) + ) def ABCBoxcarLikelihood(data, comparedata, measerror=None): """ A simple ABC likelihood function is the Boxcar likelihood given by the formular: - + .. math:: p = \\max_{i=1}^N(\\epsilon_j - \\rho(S(Y),S(Y(X)))). - + :math:`\\rho(S(Y),S(Y(X)))` is the eucledean distance. `Usage:` Maximizing the likelihood value guides to the best model. @@ -953,25 +1110,25 @@ def ABCBoxcarLikelihood(data, comparedata, measerror=None): def LimitsOfAcceptability(data, comparedata, measerror=None): """ - This calculation use the generalized likelihood uncertainty estimation by counting all Euclidean distances which are - smaller then the deviation of the measurement value. + This calculation use the generalized likelihood uncertainty estimation by counting all Euclidean distances which are + smaller then the deviation of the measurement value. - .. math:: + .. math:: - p=\\sum_{j=1}^m I(|\\rho(S_j(Y)-S_j(Y(X))| \\leq \\epsilon_j) + p=\\sum_{j=1}^m I(|\\rho(S_j(Y)-S_j(Y(X))| \\leq \\epsilon_j) - Variable :math:`I(a)` returns one if `a` is true, zero otherwise. + Variable :math:`I(a)` returns one if `a` is true, zero otherwise. - `Usage:` Maximizing the likelihood value guides to the best model. + `Usage:` Maximizing the likelihood value guides to the best model. - :param data: observed measurements as a numerical list - :type data: list - :param comparedata: simulated data from a model which should fit the original data somehow - :type comparedata: list - :param measerror: measurement errors of every data input, if nothing is given a standart calculation is done to simulate measurement errors - :type measerror: list - :return: the p value as a likelihood - :rtype: float + :param data: observed measurements as a numerical list + :type data: list + :param comparedata: simulated data from a model which should fit the original data somehow + :type comparedata: list + :param measerror: measurement errors of every data input, if nothing is given a standart calculation is done to simulate measurement errors + :type measerror: list + :return: the p value as a likelihood + :rtype: float """ __standartChecksBeforeStart(data, comparedata) if measerror is None: @@ -1017,7 +1174,8 @@ def InverseErrorVarianceShapingFactor(data, comparedata, G=10): errArr = np.nanvar(np.array(__calcSimpleDeviation(data, comparedata))) if errArr == 0.0: warnings.warn( - "[InverseErrorVarianceShapingFactor] reaslized that the variance in y(x)-y is zero and that makes no sence and also impossible to calculate the likelihood.") + "[InverseErrorVarianceShapingFactor] reaslized that the variance in y(x)-y is zero and that makes no sence and also impossible to calculate the likelihood." + ) return np.NAN else: # Gives an better convergence, so close values are more less and apart values are more great. @@ -1058,7 +1216,8 @@ def NashSutcliffeEfficiencyShapingFactor(data, comparedata, G=10): if np.nanvar(data) == 0.0: warnings.warn( - "[NashSutcliffeEfficiencyShapingFactor] reaslized that the variance of the data is zero. Thereforee is no likelihood calculation possible") + "[NashSutcliffeEfficiencyShapingFactor] reaslized that the variance of the data is zero. Thereforee is no likelihood calculation possible" + ) return np.NAN else: ratio = np.nanvar(errArr) / np.nanvar(data) @@ -1068,7 +1227,8 @@ def NashSutcliffeEfficiencyShapingFactor(data, comparedata, G=10): "[NashSutcliffeEfficiencyShapingFactor]: The ratio between residual variation and observation " "variation is bigger then one and therefore" "we can not calculate the liklihood. Please use another function which fits to this data and / or " - "model") + "model" + ) return np.NAN else: return G * np.log(1 - ratio) diff --git a/spotpy/objectivefunctions.py b/src/spotpy/objectivefunctions.py similarity index 73% rename from spotpy/objectivefunctions.py rename to src/spotpy/objectivefunctions.py index d5c7e626..47de88ff 100644 --- a/spotpy/objectivefunctions.py +++ b/src/spotpy/objectivefunctions.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). @@ -8,15 +8,18 @@ This tool holds functions for statistic analysis. It takes Python-lists and returns the objective function value of interest. -''' +""" -import numpy as np import logging -logging.basicConfig(format='%(levelname)s: %(module)s.%(funcName)s(): %(message)s') + +import numpy as np + +logging.basicConfig(format="%(levelname)s: %(module)s.%(funcName)s(): %(message)s") + def bias(evaluation, simulation): """ - Bias as shown in Gupta in Sorooshian (1998), Toward improved calibration of hydrologic models: + Bias as shown in Gupta in Sorooshian (1998), Toward improved calibration of hydrologic models: Multiple and noncommensurable measures of information, Water Resources Research .. math:: @@ -38,7 +41,9 @@ def bias(evaluation, simulation): return float(bias) else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan @@ -65,7 +70,9 @@ def pbias(evaluation, simulation): return 100 * (float(np.nansum(sim - obs)) / float(np.nansum(obs))) else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan @@ -75,7 +82,7 @@ def nashsutcliffe(evaluation, simulation): .. math:: - NSE = 1-\\frac{\\sum_{i=1}^{N}(e_{i}-s_{i})^2}{\\sum_{i=1}^{N}(e_{i}-\\bar{e})^2} + NSE = 1-\\frac{\\sum_{i=1}^{N}(e_{i}-s_{i})^2}{\\sum_{i=1}^{N}(e_{i}-\\bar{e})^2} :evaluation: Observed data to compared with simulation data. :type: list @@ -93,12 +100,14 @@ def nashsutcliffe(evaluation, simulation): mean_observed = np.nanmean(e) # compute numerator and denominator numerator = np.nansum((e - s) ** 2) - denominator = np.nansum((e - mean_observed)**2) + denominator = np.nansum((e - mean_observed) ** 2) # compute coefficient return 1 - (numerator / denominator) else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan @@ -118,16 +127,22 @@ def lognashsutcliffe(evaluation, simulation, epsilon=0): :epsilon: Value which is added to simulation and evaluation data to errors when simulation or evaluation data has zero values :type: float or list - + :return: log Nash-Sutcliffe model efficiency :rtype: float """ if len(evaluation) == len(simulation): - s, e = np.array(simulation)+epsilon, np.array(evaluation)+epsilon - return float(1 - sum((np.log(s) - np.log(e))**2) / sum((np.log(e) - np.mean(np.log(e)))**2)) + s, e = np.array(simulation) + epsilon, np.array(evaluation) + epsilon + return float( + 1 + - sum((np.log(s) - np.log(e)) ** 2) + / sum((np.log(e) - np.mean(np.log(e))) ** 2) + ) else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan @@ -145,14 +160,16 @@ def log_p(evaluation, simulation): :rtype: float """ scale = np.mean(evaluation) / 10 - if scale < .01: - scale = .01 + if scale < 0.01: + scale = 0.01 if len(evaluation) == len(simulation): y = (np.array(evaluation) - np.array(simulation)) / scale - normpdf = -y**2 / 2 - np.log(np.sqrt(2 * np.pi)) + normpdf = -(y**2) / 2 - np.log(np.sqrt(2 * np.pi)) return np.mean(normpdf) else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan @@ -177,7 +194,9 @@ def correlationcoefficient(evaluation, simulation): correlation_coefficient = np.corrcoef(evaluation, simulation)[0, 1] return correlation_coefficient else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan @@ -199,9 +218,11 @@ def rsquared(evaluation, simulation): :rtype: float """ if len(evaluation) == len(simulation): - return correlationcoefficient(evaluation, simulation)**2 + return correlationcoefficient(evaluation, simulation) ** 2 else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan @@ -225,10 +246,12 @@ def mse(evaluation, simulation): if len(evaluation) == len(simulation): obs, sim = np.array(evaluation), np.array(simulation) - mse = np.nanmean((obs - sim)**2) + mse = np.nanmean((obs - sim) ** 2) return mse else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan @@ -278,7 +301,9 @@ def mae(evaluation, simulation): mae = np.mean(np.abs(sim - obs)) return mae else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan @@ -286,7 +311,7 @@ def rrmse(evaluation, simulation): """ Relative Root Mean Squared Error - .. math:: + .. math:: RRMSE=\\frac{\\sqrt{\\frac{1}{N}\\sum_{i=1}^{N}(e_{i}-s_{i})^2}}{\\bar{e}} @@ -304,7 +329,9 @@ def rrmse(evaluation, simulation): rrmse = rmse(evaluation, simulation) / np.mean(evaluation) return rrmse else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan @@ -312,9 +339,9 @@ def agreementindex(evaluation, simulation): """ Agreement Index (d) developed by Willmott (1981) - .. math:: + .. math:: - d = 1 - \\frac{\\sum_{i=1}^{N}(e_{i} - s_{i})^2}{\\sum_{i=1}^{N}(\\left | s_{i} - \\bar{e} \\right | + \\left | e_{i} - \\bar{e} \\right |)^2} + d = 1 - \\frac{\\sum_{i=1}^{N}(e_{i} - s_{i})^2}{\\sum_{i=1}^{N}(\\left | s_{i} - \\bar{e} \\right | + \\left | e_{i} - \\bar{e} \\right |)^2} :evaluation: Observed data to compared with simulation data. @@ -328,11 +355,20 @@ def agreementindex(evaluation, simulation): """ if len(evaluation) == len(simulation): simulation, evaluation = np.array(simulation), np.array(evaluation) - Agreement_index = 1 - (np.sum((evaluation - simulation)**2)) / (np.sum( - (np.abs(simulation - np.mean(evaluation)) + np.abs(evaluation - np.mean(evaluation)))**2)) + Agreement_index = 1 - (np.sum((evaluation - simulation) ** 2)) / ( + np.sum( + ( + np.abs(simulation - np.mean(evaluation)) + + np.abs(evaluation - np.mean(evaluation)) + ) + ** 2 + ) + ) return Agreement_index else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan @@ -356,10 +392,12 @@ def covariance(evaluation, simulation): obs, sim = np.array(evaluation), np.array(simulation) obs_mean = np.mean(obs) sim_mean = np.mean(sim) - covariance = np.mean((obs - obs_mean)*(sim - sim_mean)) + covariance = np.mean((obs - obs_mean) * (sim - sim_mean)) return covariance else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan @@ -388,15 +426,17 @@ def decomposed_mse(evaluation, simulation): e_std = np.std(evaluation) s_std = np.std(simulation) - bias_squared = bias(evaluation, simulation)**2 - sdsd = (e_std - s_std)**2 + bias_squared = bias(evaluation, simulation) ** 2 + sdsd = (e_std - s_std) ** 2 lcs = 2 * e_std * s_std * (1 - correlationcoefficient(evaluation, simulation)) decomposed_mse = bias_squared + sdsd + lcs return decomposed_mse else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan @@ -404,13 +444,13 @@ def kge(evaluation, simulation, return_all=False): """ Kling-Gupta Efficiency - Corresponding paper: + Corresponding paper: Gupta, Kling, Yilmaz, Martinez, 2009, Decomposition of the mean squared error and NSE performance criteria: Implications for improving hydrological modelling output: kge: Kling-Gupta Efficiency optional_output: - cc: correlation + cc: correlation alpha: ratio of the standard deviation beta: ratio of the mean """ @@ -418,35 +458,45 @@ def kge(evaluation, simulation, return_all=False): cc = np.corrcoef(evaluation, simulation)[0, 1] alpha = np.std(simulation) / np.std(evaluation) beta = np.sum(simulation) / np.sum(evaluation) - kge = 1 - np.sqrt((cc - 1)**2 + (alpha - 1)**2 + (beta - 1)**2) + kge = 1 - np.sqrt((cc - 1) ** 2 + (alpha - 1) ** 2 + (beta - 1) ** 2) if return_all: return kge, cc, alpha, beta else: return kge else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan + def _spearmann_corr(x, y): """Separmann correlation coefficient""" col = [list(a) for a in zip(x, y)] xy = sorted(col, key=lambda x: x[0], reverse=False) # rang of x-value for i, row in enumerate(xy): - row.append(i+1) + row.append(i + 1) a = sorted(xy, key=lambda x: x[1], reverse=False) # rang of y-value for i, row in enumerate(a): - row.append(i+1) + row.append(i + 1) - MW_rank_x = np.nanmean(np.array(a)[:,2]) - MW_rank_y = np.nanmean(np.array(a)[:,3]) + MW_rank_x = np.nanmean(np.array(a)[:, 2]) + MW_rank_y = np.nanmean(np.array(a)[:, 3]) + + numerator = np.nansum( + [float((a[j][2] - MW_rank_x) * (a[j][3] - MW_rank_y)) for j in range(len(a))] + ) + denominator1 = np.sqrt( + np.nansum([(a[j][2] - MW_rank_x) ** 2.0 for j in range(len(a))]) + ) + denominator2 = np.sqrt( + np.nansum([(a[j][3] - MW_rank_x) ** 2.0 for j in range(len(a))]) + ) + return float(numerator / (denominator1 * denominator2)) - numerator = np.nansum([float((a[j][2]-MW_rank_x)*(a[j][3]-MW_rank_y)) for j in range(len(a))]) - denominator1 = np.sqrt(np.nansum([(a[j][2]-MW_rank_x)**2. for j in range(len(a))])) - denominator2 = np.sqrt(np.nansum([(a[j][3]-MW_rank_x)**2. for j in range(len(a))])) - return float(numerator/(denominator1*denominator2)) def kge_non_parametric(evaluation, simulation, return_all=False): """ @@ -457,53 +507,58 @@ def kge_non_parametric(evaluation, simulation, return_all=False): output: kge: Kling-Gupta Efficiency - + author: Nadine Maier and Tobias Houska optional_output: - cc: correlation + cc: correlation alpha: ratio of the standard deviation beta: ratio of the mean """ if len(evaluation) == len(simulation): - ## self-made formula + ## self-made formula cc = _spearmann_corr(evaluation, simulation) ### scipy-Version - #cc = stm.spearmanr(evaluation, simulation, axis=0)[0] + # cc = stm.spearmanr(evaluation, simulation, axis=0)[0] - ### pandas version - #a = pd.DataFrame({'eva': evaluation, 'sim': simulation}) - #cc = a.ix[:,1].corr(a.ix[:,0], method = 'spearman') + ### pandas version + # a = pd.DataFrame({'eva': evaluation, 'sim': simulation}) + # cc = a.ix[:,1].corr(a.ix[:,0], method = 'spearman') - fdc_sim = np.sort(simulation / (np.nanmean(simulation)*len(simulation))) - fdc_obs = np.sort(evaluation / (np.nanmean(evaluation)*len(evaluation))) + fdc_sim = np.sort(simulation / (np.nanmean(simulation) * len(simulation))) + fdc_obs = np.sort(evaluation / (np.nanmean(evaluation) * len(evaluation))) alpha = 1 - 0.5 * np.nanmean(np.abs(fdc_sim - fdc_obs)) - + beta = np.mean(simulation) / np.mean(evaluation) - kge = 1 - np.sqrt((cc - 1)**2 + (alpha - 1)**2 + (beta - 1)**2) + kge = 1 - np.sqrt((cc - 1) ** 2 + (alpha - 1) ** 2 + (beta - 1) ** 2) if return_all: return kge, cc, alpha, beta else: return kge else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan + def rsr(evaluation, simulation): """ - RMSE-observations standard deviation ratio + RMSE-observations standard deviation ratio - Corresponding paper: + Corresponding paper: Moriasi, Arnold, Van Liew, Bingner, Harmel, Veith, 2007, Model Evaluation Guidelines for Systematic Quantification of Accuracy in Watershed Simulations output: - rsr: RMSE-observations standard deviation ratio + rsr: RMSE-observations standard deviation ratio """ if len(evaluation) == len(simulation): rsr = rmse(evaluation, simulation) / np.std(evaluation) return rsr else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan @@ -514,8 +569,8 @@ def volume_error(evaluation, simulation): and observed runoff (i.e. long-term water balance). used in this paper: Reynolds, J.E., S. Halldin, C.Y. Xu, J. Seibert, and A. Kauffeldt. 2017. - “Sub-Daily Runoff Predictions Using Parameters Calibrated on the Basis of Data with a - Daily Temporal Resolution.” Journal of Hydrology 550 (July):399–411. + “Sub-Daily Runoff Predictions Using Parameters Calibrated on the Basis of Data with a + Daily Temporal Resolution.” Journal of Hydrology 550 (July):399–411. https://doi.org/10.1016/j.jhydrol.2017.05.012. .. math:: @@ -534,14 +589,32 @@ def volume_error(evaluation, simulation): ve = np.sum(simulation - evaluation) / np.sum(evaluation) return float(ve) else: - logging.warning("evaluation and simulation lists does not have the same length.") + logging.warning( + "evaluation and simulation lists does not have the same length." + ) return np.nan -_all_functions = [agreementindex, bias, correlationcoefficient, covariance, decomposed_mse, - kge, log_p, lognashsutcliffe, mae, mse, nashsutcliffe, pbias, rmse, rrmse, rsquared, - rsr, volume_error - ] +_all_functions = [ + agreementindex, + bias, + correlationcoefficient, + covariance, + decomposed_mse, + kge, + log_p, + lognashsutcliffe, + mae, + mse, + nashsutcliffe, + pbias, + rmse, + rrmse, + rsquared, + rsr, + volume_error, +] + def calculate_all_functions(evaluation, simulation): """ diff --git a/spotpy/parallel/__init__.py b/src/spotpy/parallel/__init__.py similarity index 95% rename from spotpy/parallel/__init__.py rename to src/spotpy/parallel/__init__.py index 5b874cef..31f76bd8 100644 --- a/spotpy/parallel/__init__.py +++ b/src/spotpy/parallel/__init__.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). :author: Philipp Kraft -''' +""" diff --git a/spotpy/parallel/mpi.py b/src/spotpy/parallel/mpi.py similarity index 85% rename from spotpy/parallel/mpi.py rename to src/spotpy/parallel/mpi.py index 1fbd77f1..d313ac97 100644 --- a/spotpy/parallel/mpi.py +++ b/src/spotpy/parallel/mpi.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). @@ -11,15 +11,11 @@ When an algorithm is constructed with parallel='mpi' the repeater of the algorithm as a ForEach object from this package. The .start() method seperates one master process from all the other processes that are used as workers. The master holds a list called "slots", -where the master notes which processes are busy with a job. When a new job should be sent to -a worker, the master looks for a free slot and sends the job to the corresponding worker +where the master notes which processes are busy with a job. When a new job should be sent to +a worker, the master looks for a free slot and sends the job to the corresponding worker process. -''' +""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from mpi4py import MPI @@ -27,6 +23,7 @@ class tag: """ This is just an enum to identify messages """ + job = 11 answer = 12 @@ -35,6 +32,7 @@ class PhaseChange(object): """ Object to identify a change of a simulation phase """ + def __init__(self, phase): self.phase = phase @@ -61,13 +59,13 @@ def __repr__(self): def __init__(self, process): """ Creates a repetition around a callable - + :param process: A callable to process the data """ self.comm = MPI.COMM_WORLD self.size = self.comm.Get_size() if self.size <= 1: - raise RuntimeError('Need at least two processes for parallelization') + raise RuntimeError("Need at least two processes for parallelization") self.rank = self.comm.Get_rank() self.process = process self.phase = None @@ -75,25 +73,25 @@ def __init__(self, process): if self.rank == 0: # The slots are a place for the master to remember which worker is doing something # Idle slots contain None - self.slots = [None] * (self.size-1) + self.slots = [None] * (self.size - 1) def is_master(self): """ - :return: True if this Repititor lives on the master process + :return: True if this Repititor lives on the master process """ return self.rank == 0 def is_worker(self): """ - - :return: True if self lives on a worker process + + :return: True if self lives on a worker process """ return self.rank > 0 def is_idle(self): """ - - :return: True, if all slots are empty + + :return: True, if all slots are empty """ assert self.is_master() return all(s is None for s in self.slots) @@ -101,7 +99,7 @@ def is_idle(self): def terminate(self): """ Sends a termination command to all workers - :return: + :return: """ assert self.is_master(), "Don't call terminate on worker" for i in range(1, self.size): @@ -110,8 +108,8 @@ def terminate(self): def setphase(self, phasename): """ Sends out to all workers that a new phase has come - :param phasename: - :return: + :param phasename: + :return: """ assert self.is_master() for i in range(1, self.size): @@ -120,7 +118,7 @@ def setphase(self, phasename): def __wait(self): """ - The loop where a worker is waiting for jobs. + The loop where a worker is waiting for jobs. Breaks when master sent StopIteration :return: Nothing, calls exit() """ @@ -140,10 +138,10 @@ def __wait(self): # Send the object back for processing it res = self.process(obj) self.comm.send(res, dest=0, tag=tag.answer) - + if callable(self.on_worker_terminate): self.on_worker_terminate() - + finally: exit() @@ -152,10 +150,10 @@ def __send(self, jobiter): The master uses this function to send jobs to the workers First it looks for a free slots, and then the jobs go there Used by __call__ - :param jobiter: An iterator over job arguments + :param jobiter: An iterator over job arguments :return: True if there are pending jobs """ - assert(self.is_master()) + assert self.is_master() for i, slot in enumerate(self.slots): # found a free slot if slot is None: @@ -165,7 +163,7 @@ def __send(self, jobiter): job = next(jobiter) self.slots[i] = job # Send slot job to destination rank - self.comm.send(job, dest=i+1, tag=tag.job) + self.comm.send(job, dest=i + 1, tag=tag.job) except StopIteration: return False return True @@ -173,8 +171,8 @@ def __send(self, jobiter): def start(self): """ Sepearates the master from the workers - Sends all workers into wait modus, the master will just proceed - :return: + Sends all workers into wait modus, the master will just proceed + :return: """ if self.is_worker(): self.__wait() @@ -182,9 +180,9 @@ def start(self): def __call__(self, jobs): """ Sends the jobs out to the workers and receives the results - :param jobs: an iterable of jobs to be sent to the workers. Each job contains + :param jobs: an iterable of jobs to be sent to the workers. Each job contains the args of the process function - :return: Yields the received result + :return: Yields the received result """ # Get the iterator for the iterable @@ -197,9 +195,9 @@ def __call__(self, jobs): # If slot is active if slot is not None: # Check if slot has data in the pipeline - if self.comm.Iprobe(source=i+1, tag=tag.answer): + if self.comm.Iprobe(source=i + 1, tag=tag.answer): # Receive data - data = self.comm.recv(source=i+1, tag=tag.answer) + data = self.comm.recv(source=i + 1, tag=tag.answer) # Free slot self.slots[i] = None yield data diff --git a/spotpy/parallel/mproc.py b/src/spotpy/parallel/mproc.py similarity index 86% rename from spotpy/parallel/mproc.py rename to src/spotpy/parallel/mproc.py index 21094b59..be322bb3 100644 --- a/spotpy/parallel/mproc.py +++ b/src/spotpy/parallel/mproc.py @@ -1,36 +1,36 @@ -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). :author: Philipp Kraft -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +""" import pathos.multiprocessing as mp process_count = None + class PhaseChange(object): """ Object to identify a change of a simulation phase """ + def __init__(self, phase): self.phase = phase - + + class ForEach(object): """ ForEach is a classes for multiprocessed work based on a generater object which is given if __call__ is called We using the pathos multiprocessing module and the orderd map function where results are saved until results in the given order are caluculated. We yielding back the result so a generator object is created. """ + def __init__(self, process): self.size = process_count or mp.cpu_count() self.process = process - self.phase=None + self.phase = None self.pool = mp.ProcessingPool(self.size) def is_idle(self): diff --git a/spotpy/parallel/sequential.py b/src/spotpy/parallel/sequential.py similarity index 59% rename from spotpy/parallel/sequential.py rename to src/spotpy/parallel/sequential.py index 85c9321f..8ea954dc 100644 --- a/spotpy/parallel/sequential.py +++ b/src/spotpy/parallel/sequential.py @@ -1,29 +1,30 @@ -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). :author: Philipp Kraft -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +""" + class ForEach(object): - def __init__(self,process): + def __init__(self, process): self.process = process - self.phase=None + self.phase = None + def is_idle(self): return True + def terminate(self): pass - def setphase(self,phasename): + + def setphase(self, phasename): self.phase = phasename + def start(self): pass - def __call__(self,jobs): + + def __call__(self, jobs): for job in jobs: data = self.process(job) yield data - diff --git a/spotpy/parallel/umproc.py b/src/spotpy/parallel/umproc.py similarity index 81% rename from spotpy/parallel/umproc.py rename to src/spotpy/parallel/umproc.py index e16bbed8..999de46c 100644 --- a/spotpy/parallel/umproc.py +++ b/src/spotpy/parallel/umproc.py @@ -1,32 +1,32 @@ -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). :author: Benjamin Manns -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +""" import pathos.multiprocessing as mp process_count = None + class PhaseChange(object): """ Object to identify a change of a simulation phase """ + def __init__(self, phase): self.phase = phase - + + class ForEach(object): """ ForEach is a classes for multiprocessed work based on a generater object which is given if __call__ is called We using the pathos multiprocessing module and the unordered map function where results are yield back while some processes are still running. """ + def __init__(self, process): self.size = process_count or mp.cpu_count() self.process = process @@ -42,17 +42,14 @@ def terminate(self): def start(self): pass - def setphase(self,phasename): - self.phase=phasename - + def setphase(self, phasename): + self.phase = phasename def f(self, job): data = self.process(job) return data - def __call__(self,jobs): + def __call__(self, jobs): results = self.pool.uimap(self.f, jobs) for i in results: yield i - - diff --git a/spotpy/parameter.py b/src/spotpy/parameter.py similarity index 76% rename from spotpy/parameter.py rename to src/spotpy/parameter.py index aeeb6396..9bb24be9 100644 --- a/spotpy/parameter.py +++ b/src/spotpy/parameter.py @@ -1,20 +1,17 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). :author: Philipp Kraft and Tobias Houska Contains classes to generate random parameter sets -''' -from __future__ import division, print_function, absolute_import -import numpy.random as rnd -import numpy as np -import sys +""" import copy - -if sys.version_info[0] >= 3: - unicode = str +import sys from itertools import cycle +import numpy as np +import numpy.random as rnd + class _ArgumentHelper(object): """ @@ -27,6 +24,7 @@ class _ArgumentHelper(object): the new way to define parameters as class properties in a setup class is ugly if a parameter name needs to be given. This class helps by checking keyword and arguments for their types. """ + def __init__(self, parent, *args, **kwargs): self.parent = parent self.classname = type(parent).__name__ @@ -34,7 +32,6 @@ def __init__(self, parent, *args, **kwargs): self.kwargs = kwargs.copy() self.processed_args = 0 - def name(self): """ A helper method for Base.__init__. @@ -50,16 +47,16 @@ def name(self): :return: name """ # Check if args[0] is string like (and exists) - if self.args and unicode(self.args[0]) == self.args[0]: + if self.args and str(self.args[0]) == self.args[0]: name = self.args.pop(0) self.processed_args += 1 # else get the name from the keywords - elif 'name' in self.kwargs: - name = self.kwargs.pop('name') + elif "name" in self.kwargs: + name = self.kwargs.pop("name") self.processed_args += 1 # or just do not use a name else: - name = '' + name = "" return name @@ -103,9 +100,8 @@ def attributes(self, names, raise_for_missing=None, as_dict=False): # If the algorithm did not find values for distribution parameters in args are kwargs, fail if missing and raise_for_missing: raise TypeError( - '{T} expected values for the parameters {m}'.format( - T=self.classname, - m=', '.join(missing) + "{T} expected values for the parameters {m}".format( + T=self.classname, m=", ".join(missing) ) ) # Return the name, the distribution parameter values, and a tuple of unprocessed args and kwargs @@ -133,7 +129,9 @@ def check_complete(self): """ total_args = len(self) + self.processed_args if len(self): - error = '{}: {} arguments where given but only {} could be used'.format(self.classname, total_args, self.processed_args) + error = "{}: {} arguments where given but only {} could be used".format( + self.classname, total_args, self.processed_args + ) raise TypeError(error) @@ -145,11 +143,12 @@ def _round_sig(x, sig=3): :return: rounded value """ from math import floor, log10 + # Check for zero to avoid math value error with log10(0.0) if abs(x) < 1e-12: return 0 else: - return round(x, sig-int(floor(log10(abs(x))))-1) + return round(x, sig - int(floor(log10(abs(x)))) - 1) class Base(object): @@ -170,11 +169,13 @@ def __init__(*args, **kwargs): The Uniform parameter class is the reference implementation. """ + __rndargs__ = () + def __init__(self, rndfunc, rndfuncname, *args, **kwargs): """ :name: Name of the parameter - :rndfunc: Function to draw a random number, + :rndfunc: Function to draw a random number, eg. the random functions from numpy.random using the rndargs :rndargs: tuple of the argument names for the random function eg. for uniform: ('low', 'high'). The values for the rndargs are retrieved @@ -190,18 +191,23 @@ def __init__(self, rndfunc, rndfuncname, *args, **kwargs): self.rndfunctype = rndfuncname arghelper = _ArgumentHelper(self, *args, **kwargs) self.name = arghelper.name() - arghelper.alias('default', 'optguess') + arghelper.alias("default", "optguess") self.rndargs = arghelper.attributes(type(self).__rndargs__, type(self).__name__) if self.rndfunc: # Get the standard arguments for the parameter or create them - param_args = arghelper.attributes(['step', 'optguess', 'minbound', 'maxbound'], as_dict=True) + param_args = arghelper.attributes( + ["step", "optguess", "minbound", "maxbound"], as_dict=True + ) # Draw one sample of size 1000 sample = self(size=1000) - self.step = param_args.get('step', _round_sig(np.percentile(sample, 50) - np.percentile(sample, 40))) - self.optguess = param_args.get('optguess', _round_sig(np.median(sample))) - self.minbound = param_args.get('minbound', _round_sig(np.min(sample))) - self.maxbound = param_args.get('maxbound', _round_sig(np.max(sample))) + self.step = param_args.get( + "step", + _round_sig(np.percentile(sample, 50) - np.percentile(sample, 40)), + ) + self.optguess = param_args.get("optguess", _round_sig(np.median(sample))) + self.minbound = param_args.get("minbound", _round_sig(np.min(sample))) + self.maxbound = param_args.get("maxbound", _round_sig(np.max(sample))) else: @@ -210,7 +216,7 @@ def __init__(self, rndfunc, rndfuncname, *args, **kwargs): self.minbound = 0.0 self.maxbound = 0.0 - self.description = arghelper.get('doc') + self.description = arghelper.get("doc") self.as_int = not not arghelper.get("as_int") arghelper.check_complete() @@ -225,56 +231,64 @@ def astuple(self): """ Returns a tuple of a realization and the other parameter properties """ - return self(), self.name, self.step, self.optguess, self.minbound, self.maxbound, self.as_int - + return ( + self(), + self.name, + self.step, + self.optguess, + self.minbound, + self.maxbound, + self.as_int, + ) + def __repr__(self): """ Returns a textual representation of the parameter """ - return "{tname}('{p.name}', {p.rndargs})".format(tname=type(self).__name__, p=self) + return "{tname}('{p.name}', {p.rndargs})".format( + tname=type(self).__name__, p=self + ) def __str__(self): """ Returns the description of the parameter, if available, else repr(self) - :return: + :return: """ - doc = vars(self).get('description') + doc = vars(self).get("description") if doc: - res = '{} ({})'.format(doc, repr(self)) - if sys.version_info.major == 2: - return res.encode('utf-8', errors='ignore') - else: - return res + res = "{} ({})".format(doc, repr(self)) + return res else: return repr(self) def __unicode__(self): - doc = vars(self).get('description') + doc = vars(self).get("description") if doc: - return u'{}({})'.format(unicode(doc), repr(self)) + return "{}({})".format(str(doc), repr(self)) else: return repr(self) - class Uniform(Base): """ A specialization of the Base parameter for uniform distributions """ - __rndargs__ = 'low', 'high' + + __rndargs__ = "low", "high" + def __init__(self, *args, **kwargs): """ :name: Name of the parameter :low: lower bound of the uniform distribution :high: higher bound of the uniform distribution - :step: (optional) number for step size required for some algorithms, + :step: (optional) number for step size required for some algorithms, eg. mcmc need a parameter of the variance for the next step default is median of rndfunc(*rndargs, size=1000) :optguess: (optional) number for start point of parameter - default is quantile(0.5) - quantile(0.4) of - rndfunc(*rndargs, size=1000) + default is quantile(0.5) - quantile(0.4) of + rndfunc(*rndargs, size=1000) """ - super(Uniform, self).__init__(rnd.uniform, 'Uniform', *args, **kwargs) + super(Uniform, self).__init__(rnd.uniform, "Uniform", *args, **kwargs) class List(Base): @@ -286,11 +300,13 @@ class List(Base): list_param() 1 """ - __rndargs__ = ('values', ) + + __rndargs__ = ("values",) + def __init__(self, *args, **kwargs): - self.repeat = kwargs.pop('repeat', False) - super(List, self).__init__(None, 'List', *args, **kwargs) - self.values, = self.rndargs + self.repeat = kwargs.pop("repeat", False) + super(List, self).__init__(None, "List", *args, **kwargs) + (self.values,) = self.rndargs # Hack to avoid skipping the first value. See __call__ function below. self.throwaway_first = True @@ -323,7 +339,7 @@ def __call__(self, size=None): try: return next(self.iterator) except StopIteration: - text = 'Number of repetitions is higher than the number of available parameter sets' + text = "Number of repetitions is higher than the number of available parameter sets" raise IndexError(text) def astuple(self): @@ -334,10 +350,11 @@ class Constant(Base): """ A specialization that produces always the same constant value """ - __rndargs__ = 'scalar', + + __rndargs__ = ("scalar",) def __init__(self, *args, **kwargs): - super(Constant, self).__init__(self, 'Constant', *args, **kwargs) + super(Constant, self).__init__(self, "Constant", *args, **kwargs) value = property(lambda self: self.rndargs[0]) @@ -360,87 +377,97 @@ class Normal(Base): """ A specialization of the Base parameter for normal distributions """ - __rndargs__ = 'mean', 'stddev' + + __rndargs__ = "mean", "stddev" + def __init__(self, *args, **kwargs): """ :name: Name of the parameter :mean: center of the normal distribution :stddev: variance of the normal distribution - :step: (optional) number for step size required for some algorithms, + :step: (optional) number for step size required for some algorithms, eg. mcmc need a parameter of the variance for the next step default is median of rndfunc(*rndargs, size=1000) :optguess: (optional) number for start point of parameter - default is quantile(0.5) - quantile(0.4) of - rndfunc(*rndargs, size=1000) + default is quantile(0.5) - quantile(0.4) of + rndfunc(*rndargs, size=1000) """ - super(Normal, self).__init__(rnd.normal, 'Normal', *args, **kwargs) + super(Normal, self).__init__(rnd.normal, "Normal", *args, **kwargs) class logNormal(Base): """ A specialization of the Base parameter for normal distributions """ - __rndargs__ = 'mean', 'sigma' + + __rndargs__ = "mean", "sigma" + def __init__(self, *args, **kwargs): """ :name: Name of the parameter :mean: Mean value of the underlying normal distribution :sigma: Standard deviation of the underlying normal distribution >0 - :step: (optional) number for step size required for some algorithms, + :step: (optional) number for step size required for some algorithms, eg. mcmc need a parameter of the variance for the next step default is median of rndfunc(*rndargs, size=1000) :optguess: (optional) number for start point of parameter - default is quantile(0.5) - quantile(0.4) of - rndfunc(*rndargs, size=1000) + default is quantile(0.5) - quantile(0.4) of + rndfunc(*rndargs, size=1000) """ - super(logNormal, self).__init__(rnd.lognormal, 'logNormal', *args, **kwargs) + super(logNormal, self).__init__(rnd.lognormal, "logNormal", *args, **kwargs) class Chisquare(Base): """ A specialization of the Base parameter for chisquare distributions """ - __rndargs__ = 'dt', + + __rndargs__ = ("dt",) + def __init__(self, *args, **kwargs): """ :name: Name of the parameter :dt: Number of degrees of freedom. - :step: (optional) number for step size required for some algorithms, + :step: (optional) number for step size required for some algorithms, eg. mcmc need a parameter of the variance for the next step default is median of rndfunc(*rndargs, size=1000) :optguess: (optional) number for start point of parameter - default is quantile(0.5) - quantile(0.4) of - rndfunc(*rndargs, size=1000) + default is quantile(0.5) - quantile(0.4) of + rndfunc(*rndargs, size=1000) """ - super(Chisquare, self).__init__(rnd.chisquare, 'Chisquare', *args, **kwargs) + super(Chisquare, self).__init__(rnd.chisquare, "Chisquare", *args, **kwargs) class Exponential(Base): """ A specialization of the Base parameter for exponential distributions """ - __rndargs__ = 'scale', + + __rndargs__ = ("scale",) def __init__(self, *args, **kwargs): """ :name: Name of the parameter :scale: The scale parameter, \beta = 1 divided by lambda. - :step: (optional) number for step size required for some algorithms, + :step: (optional) number for step size required for some algorithms, eg. mcmc need a parameter of the variance for the next step default is median of rndfunc(*rndargs, size=1000) :optguess: (optional) number for start point of parameter - default is quantile(0.5) - quantile(0.4) of + default is quantile(0.5) - quantile(0.4) of rndfunc(*rndargs, size=1000) """ - super(Exponential, self).__init__(rnd.exponential, 'Exponential', *args, **kwargs) + super(Exponential, self).__init__( + rnd.exponential, "Exponential", *args, **kwargs + ) class Gamma(Base): """ A specialization of the Base parameter for gamma distributions """ - __rndargs__ = 'shape', 'scale' + + __rndargs__ = "shape", "scale" def __init__(self, *args, **kwargs): """ @@ -455,7 +482,7 @@ def __init__(self, *args, **kwargs): rndfunc(*rndargs, size=1000) """ - super(Gamma, self).__init__(rnd.gamma, 'Gamma', *args, **kwargs) + super(Gamma, self).__init__(rnd.gamma, "Gamma", *args, **kwargs) class Wald(Base): @@ -463,60 +490,65 @@ class Wald(Base): A specialization of the Base parameter for Wald distributions """ - __rndargs__ = 'mean', 'scale' + __rndargs__ = "mean", "scale" + def __init__(self, *args, **kwargs): """ :name: Name of the parameter :mean: Shape of the distribution. :scale: Shape of the distribution. - :step: (optional) number for step size required for some algorithms, + :step: (optional) number for step size required for some algorithms, eg. mcmc need a parameter of the variance for the next step default is median of rndfunc(*rndargs, size=1000) :optguess: (optional) number for start point of parameter - default is quantile(0.5) - quantile(0.4) of + default is quantile(0.5) - quantile(0.4) of rndfunc(*rndargs, size=1000) """ - super(Wald, self).__init__(rnd.wald, 'Wald', *args, **kwargs) + super(Wald, self).__init__(rnd.wald, "Wald", *args, **kwargs) class Weibull(Base): """ A specialization of the Base parameter for Weibull distributions """ - __rndargs__ = 'a', + + __rndargs__ = ("a",) + def __init__(self, *args, **kwargs): """ :name: Name of the parameter :a: Shape of the distribution. - :step: (optional) number for step size required for some algorithms, + :step: (optional) number for step size required for some algorithms, eg. mcmc need a parameter of the variance for the next step default is median of rndfunc(*rndargs, size=1000) :optguess: (optional) number for start point of parameter - default is quantile(0.5) - quantile(0.4) of - rndfunc(*rndargs, size=1000) + default is quantile(0.5) - quantile(0.4) of + rndfunc(*rndargs, size=1000) """ - super(Weibull, self).__init__(rnd.weibull, 'Weibull', *args, **kwargs) + super(Weibull, self).__init__(rnd.weibull, "Weibull", *args, **kwargs) class Triangular(Base): """ A parameter sampling from a triangular distribution """ - __rndargs__ = 'left', 'mode', 'right' + + __rndargs__ = "left", "mode", "right" + def __init__(self, *args, **kwargs): """ :name: Name of the parameter :left: Lower limit of the parameter :mode: The value where the peak of the distribution occurs. :right: Upper limit, should be larger than `left`. - :step: (optional) number for step size required for some algorithms, + :step: (optional) number for step size required for some algorithms, eg. mcmc need a parameter of the variance for the next step default is median of rndfunc(*rndargs, size=1000) :optguess: (optional) number for start point of parameter - default is quantile(0.5) - quantile(0.4) of - rndfunc(*rndargs, size=1000) + default is quantile(0.5) - quantile(0.4) of + rndfunc(*rndargs, size=1000) """ - super(Triangular, self).__init__(rnd.triangular, 'Triangular', *args, **kwargs) + super(Triangular, self).__init__(rnd.triangular, "Triangular", *args, **kwargs) class ParameterSet(object): @@ -548,6 +580,7 @@ class ParameterSet(object): """ + def __init__(self, param_info): """ Creates a set of parameters from a parameter info array. @@ -562,7 +595,9 @@ def __init__(self, param_info): :param param_info: A record array containing the properties of the parameters of this set. """ - self.__lookup = dict(("p" + x if x.isdigit() else x, i) for i, x in enumerate(param_info['name'])) + self.__lookup = dict( + ("p" + x if x.isdigit() else x, i) for i, x in enumerate(param_info["name"]) + ) self.__info = param_info def __call__(self, *values, **kwargs): @@ -576,20 +611,22 @@ def __call__(self, *values, **kwargs): """ if values: if len(self.__info) != len(values): - raise ValueError('Given values do are not the same length as the parameter set') - self.__info['random'][:] = values + raise ValueError( + "Given values do are not the same length as the parameter set" + ) + self.__info["random"][:] = values for k in kwargs: try: - self.__info['random'][self.__lookup[k]] = kwargs[k] + self.__info["random"][self.__lookup[k]] = kwargs[k] except KeyError: - raise TypeError('{} is not a parameter of this set'.format(k)) + raise TypeError("{} is not a parameter of this set".format(k)) return self def __len__(self): - return len(self.__info['random']) + return len(self.__info["random"]) def __iter__(self): - return iter(self.__info['random']) + return iter(self.__info["random"]) def __getitem__(self, item): """ @@ -600,7 +637,7 @@ def __getitem__(self, item): """ if type(item) is str: item = self.__lookup[item] - return self.__info['random'][item] + return self.__info["random"][item] def __setitem__(self, key, value): """ @@ -610,21 +647,25 @@ def __setitem__(self, key, value): """ if key in self.__lookup: key = self.__lookup[key] - self.__info['random'][key] = value + self.__info["random"][key] = value def __getattr__(self, item): """ Provides the attribute access like print(ps.a) """ - if item.startswith('_'): - raise AttributeError('{} is not a member of this parameter set'.format(item)) + if item.startswith("_"): + raise AttributeError( + "{} is not a member of this parameter set".format(item) + ) elif item in self.__lookup: - return self.__info['random'][self.__lookup[item]] + return self.__info["random"][self.__lookup[item]] elif item in self.__info.dtype.names: return self.__info[item] else: - raise AttributeError('{} is not a member of this parameter set'.format(item)) + raise AttributeError( + "{} is not a member of this parameter set".format(item) + ) def __setattr__(self, key, value): """ @@ -632,20 +673,21 @@ def __setattr__(self, key, value): ps.a = 2 """ # Allow normal usage - if key.startswith('_') or key not in self.__lookup: + if key.startswith("_") or key not in self.__lookup: return object.__setattr__(self, key, value) else: - self.__info['random'][self.__lookup[key]] = value + self.__info["random"][self.__lookup[key]] = value def __str__(self): - return 'parameters({})'.format( - ', '.join('{}={:g}'.format(k, self.__info['random'][i]) - for i, k in enumerate(self.__info['name']) - ) + return "parameters({})".format( + ", ".join( + "{}={:g}".format(k, self.__info["random"][i]) + for i, k in enumerate(self.__info["name"]) + ) ) def __repr__(self): - return 'spotpy.parameter.ParameterSet()' + return "spotpy.parameter.ParameterSet()" def __dir__(self): """ @@ -654,24 +696,25 @@ def __dir__(self): :return: List of method names and fields """ - attrs = [attr for attr in vars(type(self)) if not attr.startswith('_')] - return attrs + list(self.__info['name']) + list(self.__info.dtype.names) + attrs = [attr for attr in vars(type(self)) if not attr.startswith("_")] + return attrs + list(self.__info["name"]) + list(self.__info.dtype.names) - def set_by_array(self,array): + def set_by_array(self, array): for i, a in enumerate(array): - self.__setitem__(i,a) + self.__setitem__(i, a) def copy(self): return ParameterSet(copy.deepcopy(self.__info)) + def get_classes(): keys = [] current_module = sys.modules[__name__] for key in dir(current_module): - if isinstance( getattr(current_module, key), type ): + if isinstance(getattr(current_module, key), type): keys.append(key) return keys - + def generate(parameters): """ @@ -679,11 +722,19 @@ def generate(parameters): is given as a structured array in the format the parameters function of a setup expects :parameters: A sequence of parameter objects """ - dtype = [('random', '=3.6, we keep - # the order. - if sys.version_info[:3] < (3, 6, 0): - # Sort the list parameters by name - parameters.sort(key=lambda p: p.name) - # Get the parameters list of setup if the parameter attribute is not a method: - if hasattr(setup, 'parameters') and not callable(setup.parameters): + if hasattr(setup, "parameters") and not callable(setup.parameters): # parameters is not callable, assume a list of parameter objects. # Generate the parameters array from it and append it to our list parameters.extend(setup.parameters) - return parameters \ No newline at end of file + return parameters diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index 5156db42..00000000 --- a/tests/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from . import testutils -from . import test_setup_parameters \ No newline at end of file diff --git a/tests/mpitest.sh b/tests/mpitest.sh deleted file mode 100644 index e6000aa6..00000000 --- a/tests/mpitest.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -for SAMPLER in abc demcz dream fast fscabc lhs mc mcmc mle rope sa # sceua -do - mpirun -c 6 python spotpy/examples/cli_cmf_lumped.py run -s $SAMPLER -n 10000 -p mpi -done diff --git a/tests/test_algorithms.py b/tests/test_algorithms.py index 3a565496..3fc9a566 100644 --- a/tests/test_algorithms.py +++ b/tests/test_algorithms.py @@ -1,142 +1,235 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska -''' +""" -import unittest -try: - import spotpy -except ImportError: - import sys - sys.path.append(".") - import spotpy -#from spotpy.examples.tutorial_padds import padds_spot_setup -from spotpy.examples.spot_setup_rosenbrock import spot_setup -from spotpy.examples.spot_setup_hymod_python import spot_setup as spot_setup_hymod -from spotpy.describe import describe import os +import unittest + import numpy as np -from spotpy.likelihoods import gaussianLikelihoodMeasErrorOut as GausianLike +import spotpy +from spotpy.describe import describe +from spotpy.examples.spot_setup_hymod_python import spot_setup as spot_setup_hymod +from spotpy.examples.spot_setup_rosenbrock import spot_setup +from spotpy.likelihoods import gaussianLikelihoodMeasErrorOut as GausianLike -#https://docs.python.org/3/library/unittest.html +# https://docs.python.org/3/library/unittest.html -class TestAlgorithms(unittest.TestCase): +class TestAlgorithms(unittest.TestCase): def multi_obj_func(self, evaluation, simulation, params=None): - #used to overwrite objective function in hymod example + # used to overwrite objective function in hymod example like1 = abs(spotpy.objectivefunctions.bias(evaluation, simulation)) like2 = spotpy.objectivefunctions.rmse(evaluation, simulation) - like3 = spotpy.objectivefunctions.rsquared(evaluation, simulation)*-1 + like3 = spotpy.objectivefunctions.rsquared(evaluation, simulation) * -1 return np.array([like1, like2, like3]) def setUp(self): # How many digits to match in case of floating point answers self.tolerance = 7 - #Create samplers for every algorithm: + # Create samplers for every algorithm: self.rep = 987 - self.timeout = 10 #Given in Seconds + self.timeout = 10 # Given in Seconds - self.parallel = os.environ.get('SPOTPY_PARALLEL', 'seq') + self.parallel = os.environ.get("SPOTPY_PARALLEL", "seq") self.dbformat = "ram" def test_mc(self): - sampler=spotpy.algorithms.mc(spot_setup(),parallel=self.parallel, dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.mc( + spot_setup(), + parallel=self.parallel, + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep) results = sampler.getdata() self.assertEqual(len(results), self.rep) def test_lhs(self): - sampler=spotpy.algorithms.lhs(spot_setup(),parallel=self.parallel, dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.lhs( + spot_setup(), + parallel=self.parallel, + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep) results = sampler.getdata() self.assertEqual(len(results), self.rep) def test_mle(self): - sampler=spotpy.algorithms.mle(spot_setup(),parallel=self.parallel, dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.mle( + spot_setup(), + parallel=self.parallel, + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep) results = sampler.getdata() self.assertEqual(len(results), self.rep) def test_mcmc(self): - sampler=spotpy.algorithms.mcmc(spot_setup(GausianLike),parallel=self.parallel, dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.mcmc( + spot_setup(GausianLike), + parallel=self.parallel, + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep) results = sampler.getdata() self.assertEqual(len(results), self.rep) def test_demcz(self): - sampler=spotpy.algorithms.demcz(spot_setup(GausianLike),parallel=self.parallel, dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.demcz( + spot_setup(GausianLike), + parallel=self.parallel, + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep, convergenceCriteria=0) results = sampler.getdata() self.assertEqual(len(results), self.rep) def test_dream(self): - sampler=spotpy.algorithms.dream(spot_setup(GausianLike),parallel=self.parallel, dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.dream( + spot_setup(GausianLike), + parallel=self.parallel, + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep, convergence_limit=0.9, runs_after_convergence=500) results = sampler.getdata() self.assertEqual(len(results), self.rep) def test_sceua(self): - sampler=spotpy.algorithms.sceua(spot_setup(),parallel=self.parallel, dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.sceua( + spot_setup(), + parallel=self.parallel, + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep) results = sampler.getdata() - self.assertLessEqual(len(results), self.rep) #Sceua save per definition not all sampled runs + self.assertLessEqual( + len(results), self.rep + ) # Sceua save per definition not all sampled runs def test_abc(self): - sampler=spotpy.algorithms.abc(spot_setup(),parallel=self.parallel, dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.abc( + spot_setup(), + parallel=self.parallel, + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep) results = sampler.getdata() self.assertEqual(len(results), self.rep) def test_fscabc(self): - sampler=spotpy.algorithms.fscabc(spot_setup(),parallel=self.parallel, dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.fscabc( + spot_setup(), + parallel=self.parallel, + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep) results = sampler.getdata() self.assertEqual(len(results), self.rep) def test_rope(self): - sampler=spotpy.algorithms.rope(spot_setup(),parallel=self.parallel, dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.rope( + spot_setup(), + parallel=self.parallel, + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep) results = sampler.getdata() self.assertEqual(len(results), self.rep) def test_sa(self): - sampler=spotpy.algorithms.sa(spot_setup(),parallel=self.parallel, dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.sa( + spot_setup(), + parallel=self.parallel, + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep) results = sampler.getdata() self.assertEqual(len(results), self.rep) - + def test_list(self): - #generate a List sampler input - sampler=spotpy.algorithms.mc(spot_setup(),parallel=self.parallel, dbname='Rosen', dbformat='csv', sim_timeout=self.timeout) + # generate a List sampler input + sampler = spotpy.algorithms.mc( + spot_setup(), + parallel=self.parallel, + dbname="Rosen", + dbformat="csv", + sim_timeout=self.timeout, + ) sampler.sample(self.rep) - sampler=spotpy.algorithms.list_sampler(spot_setup(),parallel=self.parallel, dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.list_sampler( + spot_setup(), + parallel=self.parallel, + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep) results = sampler.getdata() self.assertEqual(len(results), self.rep) def test_fast(self): - sampler=spotpy.algorithms.fast(spot_setup(),parallel=self.parallel, dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.fast( + spot_setup(), + parallel=self.parallel, + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep, M=5) results = sampler.getdata() - self.assertEqual(len(results), self.rep) #Si values should be returned + self.assertEqual(len(results), self.rep) # Si values should be returned def test_padds(self): - sampler=spotpy.algorithms.padds(spot_setup_hymod(self.multi_obj_func),parallel=self.parallel, dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) - sampler.sample(int(self.rep*0.5), metric='ones') + sampler = spotpy.algorithms.padds( + spot_setup_hymod(self.multi_obj_func), + parallel=self.parallel, + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) + sampler.sample(int(self.rep * 0.5), metric="ones") results = sampler.getdata() - self.assertEqual(len(results)+5, int(self.rep*0.5)) + self.assertEqual(len(results) + 5, int(self.rep * 0.5)) def test_nsgaii(self): - generations=20 + generations = 20 n_pop = 10 - sampler=spotpy.algorithms.NSGAII(spot_setup_hymod(self.multi_obj_func),parallel=self.parallel, dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) - sampler.sample(generations, n_obj= 3, n_pop = n_pop) + sampler = spotpy.algorithms.NSGAII( + spot_setup_hymod(self.multi_obj_func), + parallel=self.parallel, + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) + sampler.sample(generations, n_obj=3, n_pop=n_pop) results = sampler.getdata() - self.assertLessEqual(len(results), generations*n_pop) + self.assertLessEqual(len(results), generations * n_pop) @classmethod def tearDownClass(cls): @@ -146,5 +239,6 @@ def tearDownClass(cls): except FileNotFoundError: pass -if __name__ == '__main__': + +if __name__ == "__main__": unittest.main(exit=False) diff --git a/tests/test_analyser.py b/tests/test_analyser.py index 8f106dab..0dafe110 100644 --- a/tests/test_analyser.py +++ b/tests/test_analyser.py @@ -1,29 +1,27 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Benjamin Manns This file is part of Statistical Parameter Estimation Tool (SPOTPY). :author: Tobias Houska, Benjamin Manns -''' +""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals import matplotlib as mpl -mpl.use('Agg') -import matplotlib.pyplot as plt -import spotpy +mpl.use("Agg") +import os import unittest + +import matplotlib.pyplot as plt import numpy as np + +import spotpy import spotpy.analyser -import os -import sys -from spotpy.examples.spot_setup_rosenbrock import spot_setup as rosenbrock_setup from spotpy.examples.spot_setup_griewank import spot_setup as griewank_setup from spotpy.examples.spot_setup_hymod_python import spot_setup as hymod_setup -from spotpy.likelihoods import gaussianLikelihoodMeasErrorOut as GausianLike +from spotpy.examples.spot_setup_rosenbrock import spot_setup as rosenbrock_setup +from spotpy.likelihoods import gaussianLikelihoodMeasErrorOut as GausianLike + class TestAnalyser(unittest.TestCase): @classmethod @@ -33,100 +31,87 @@ def setUpClass(self): self.parallel = "seq" self.dbformat = "ram" self.timeout = 5 - self.fig_name = 'test_output.png' + self.fig_name = "test_output.png" - sampler = spotpy.algorithms.mc(rosenbrock_setup(), - sim_timeout=self.timeout) + sampler = spotpy.algorithms.mc(rosenbrock_setup(), sim_timeout=self.timeout) sampler.sample(self.rep) self.results = sampler.getdata() - sampler = spotpy.algorithms.mc(griewank_setup(), - sim_timeout=self.timeout) + sampler = spotpy.algorithms.mc(griewank_setup(), sim_timeout=self.timeout) sampler.sample(self.rep) self.griewank_results = sampler.getdata() - if sys.version_info >= (3, 6): # FAST is only fully operational under - #Python 3 - sampler = spotpy.algorithms.fast(rosenbrock_setup(), - sim_timeout=self.timeout) - sampler.sample(self.rep) - self.sens_results = sampler.getdata() - #Hymod resuts are empty with Python <3.6 - sampler = spotpy.algorithms.dream(hymod_setup(GausianLike), - sim_timeout=self.timeout) - self.r_hat = sampler.sample(self.rep) - self.hymod_results = sampler.getdata() - + + sampler = spotpy.algorithms.fast(rosenbrock_setup(), sim_timeout=self.timeout) + sampler.sample(self.rep) + self.sens_results = sampler.getdata() + # Hymod resuts are empty with Python <3.6 + sampler = spotpy.algorithms.dream( + hymod_setup(GausianLike), sim_timeout=self.timeout + ) + self.r_hat = sampler.sample(self.rep) + self.hymod_results = sampler.getdata() + def test_setUp(self): self.assertEqual(len(self.results), self.rep) self.assertEqual(len(self.griewank_results), self.rep) - if sys.version_info >= (3, 6): - self.assertEqual(len(self.hymod_results), self.rep) - self.assertEqual(len(self.sens_results), self.rep) - + self.assertEqual(len(self.hymod_results), self.rep) + self.assertEqual(len(self.sens_results), self.rep) + def test_get_parameters(self): - self.assertEqual(len(spotpy.analyser.get_parameters( - self.results)[0]), 3) + self.assertEqual(len(spotpy.analyser.get_parameters(self.results)[0]), 3) def test_get_parameternames(self): - self.assertEqual(spotpy.analyser.get_parameternames( - self.results - ),['x', 'y', 'z']) + self.assertEqual( + spotpy.analyser.get_parameternames(self.results), ["x", "y", "z"] + ) def test_get_parameter_fields(self): - self.assertEqual(len(spotpy.analyser.get_parameternames( - self.results - )), 3) + self.assertEqual(len(spotpy.analyser.get_parameternames(self.results)), 3) def test_get_minlikeindex(self): - minlikeindex = spotpy.analyser.get_minlikeindex( - self.results - ) + minlikeindex = spotpy.analyser.get_minlikeindex(self.results) self.assertEqual(len(minlikeindex), 2) - self.assertEqual(type(minlikeindex),type((1,1))) + self.assertEqual(type(minlikeindex), type((1, 1))) def test_get_maxlikeindex(self): - get_maxlikeindex = spotpy.analyser.get_maxlikeindex( - self.results - ) + get_maxlikeindex = spotpy.analyser.get_maxlikeindex(self.results) self.assertEqual(len(get_maxlikeindex), 2) - self.assertEqual(type(get_maxlikeindex),type((1,1))) + self.assertEqual(type(get_maxlikeindex), type((1, 1))) def test_get_like_fields(self): - get_like_fields = spotpy.analyser.get_like_fields( - self.results - ) + get_like_fields = spotpy.analyser.get_like_fields(self.results) self.assertEqual(len(get_like_fields), 1) - self.assertEqual(type(get_like_fields),type([])) + self.assertEqual(type(get_like_fields), type([])) def test_calc_like(self): calc_like = spotpy.analyser.calc_like( self.results, - rosenbrock_setup().evaluation(),spotpy.objectivefunctions.rmse) + rosenbrock_setup().evaluation(), + spotpy.objectivefunctions.rmse, + ) self.assertEqual(len(calc_like), len(self.results)) self.assertEqual(type(calc_like), type([])) def test_get_best_parameterset(self): get_best_parameterset_true = spotpy.analyser.get_best_parameterset( - self.results,True) + self.results, True + ) get_best_parameterset_false = spotpy.analyser.get_best_parameterset( - self.results, False) + self.results, False + ) self.assertEqual(len(get_best_parameterset_true[0]), 3) self.assertEqual(type(get_best_parameterset_true[0]), np.void) self.assertEqual(len(get_best_parameterset_false[0]), 3) self.assertEqual(type(get_best_parameterset_false[0]), np.void) def test_get_modelruns(self): - get_modelruns = spotpy.analyser.get_modelruns( - self.results - ) + get_modelruns = spotpy.analyser.get_modelruns(self.results) self.assertEqual(len(get_modelruns[0]), 1) self.assertEqual(type(get_modelruns[0]), np.void) def test_get_header(self): - get_header = spotpy.analyser.get_header( - self.results - ) + get_header = spotpy.analyser.get_header(self.results) self.assertEqual(len(get_header), 6) self.assertEqual(type(get_header), type(())) @@ -142,11 +127,10 @@ def test_get_parbounds(self): self.assertEqual(type(get_parbounds), type([])) def test_get_percentiles(self): - get_percentiles = spotpy.analyser.get_percentiles( - self.results) - self.assertEqual(len(get_percentiles),5) + get_percentiles = spotpy.analyser.get_percentiles(self.results) + self.assertEqual(len(get_percentiles), 5) self.assertEqual(type(get_percentiles[0][0]), type(np.abs(-1.0))) - self.assertEqual(type(get_percentiles),type(())) + self.assertEqual(type(get_percentiles), type(())) def test__geweke(self): sample1 = [] @@ -162,7 +146,7 @@ def test_plot_Geweke(self): sample1 = [] for a in self.results: sample1.append(a[0]) - spotpy.analyser.plot_Geweke(sample1,"sample1") + spotpy.analyser.plot_Geweke(sample1, "sample1") plt.savefig(self.fig_name) # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so @@ -170,83 +154,83 @@ def test_plot_Geweke(self): # the size self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) - def test_get_sensitivity_of_fast(self): - if sys.version_info >= (3, 6): - get_sensitivity_of_fast = spotpy.analyser.get_sensitivity_of_fast(self.sens_results) - self.assertEqual(len(get_sensitivity_of_fast), 2) - self.assertEqual(len(get_sensitivity_of_fast["S1"]), 3) - self.assertEqual(len(get_sensitivity_of_fast["ST"]), 3) - self.assertEqual(type(get_sensitivity_of_fast), type({})) + get_sensitivity_of_fast = spotpy.analyser.get_sensitivity_of_fast( + self.sens_results + ) + self.assertEqual(len(get_sensitivity_of_fast), 2) + self.assertEqual(len(get_sensitivity_of_fast["S1"]), 3) + self.assertEqual(len(get_sensitivity_of_fast["ST"]), 3) + self.assertEqual(type(get_sensitivity_of_fast), type({})) def test_get_simulation_fields(self): - get_simulation_fields = spotpy.analyser.get_simulation_fields( - self.results) - self.assertEqual(['simulation_0'],get_simulation_fields) + get_simulation_fields = spotpy.analyser.get_simulation_fields(self.results) + self.assertEqual(["simulation_0"], get_simulation_fields) def test_compare_different_objectivefunctions(self): - - sampler = spotpy.algorithms.lhs(rosenbrock_setup(), - sim_timeout=self.timeout) + sampler = spotpy.algorithms.lhs(rosenbrock_setup(), sim_timeout=self.timeout) sampler.sample(self.rep) - compare_different_objectivefunctions = spotpy.analyser.compare_different_objectivefunctions( - sampler.getdata()['like1'], self.results['like1']) - + compare_different_objectivefunctions = ( + spotpy.analyser.compare_different_objectivefunctions( + sampler.getdata()["like1"], self.results["like1"] + ) + ) - self.assertEqual(type(compare_different_objectivefunctions[1]),type(np.array([0.5])[0])) + self.assertEqual( + type(compare_different_objectivefunctions[1]), type(np.array([0.5])[0]) + ) def test_plot_parameter_uncertainty(self): - if sys.version_info >= (3, 6): - posterior = spotpy.analyser.get_posterior(self.hymod_results,percentage=10) - #assertAlmostEqual tests on after comma accuracy, therefor we divide both by 100 - self.assertAlmostEqual(len(posterior)/100, self.rep*0.001, 1) - self.assertEqual(type(posterior), type(np.array([]))) - spotpy.analyser.plot_parameter_uncertainty(posterior, - hymod_setup().evaluation(), - fig_name=self.fig_name) - - # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so - # we expecting a plot with some content without testing the structure of the plot, just - # the size - self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) + posterior = spotpy.analyser.get_posterior(self.hymod_results, percentage=10) + # assertAlmostEqual tests on after comma accuracy, therefor we divide both by 100 + self.assertAlmostEqual(len(posterior) / 100, self.rep * 0.001, 1) + self.assertEqual(type(posterior), type(np.array([]))) + spotpy.analyser.plot_parameter_uncertainty( + posterior, hymod_setup().evaluation(), fig_name=self.fig_name + ) + # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so + # we expecting a plot with some content without testing the structure of the plot, just + # the size + self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) def test_plot_fast_sensitivity(self): - if sys.version_info >= (3, 6): - spotpy.analyser.plot_fast_sensitivity(self.sens_results,fig_name=self.fig_name) - # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so - # we expecting a plot with some content without testing the structure of the plot, just - # the size - self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) - + spotpy.analyser.plot_fast_sensitivity(self.sens_results, fig_name=self.fig_name) + # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so + # we expecting a plot with some content without testing the structure of the plot, just + # the size + self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) + def test_plot_heatmap_griewank(self): - spotpy.analyser.plot_heatmap_griewank([self.griewank_results],["test"], - fig_name=self.fig_name) + spotpy.analyser.plot_heatmap_griewank( + [self.griewank_results], ["test"], fig_name=self.fig_name + ) # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so # we expecting a plot with some content without testing the structure of the plot, just # the size self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) - def test_plot_objectivefunction(self): - spotpy.analyser.plot_objectivefunction(self.results, - rosenbrock_setup().evaluation(), - fig_name=self.fig_name) - + spotpy.analyser.plot_objectivefunction( + self.results, rosenbrock_setup().evaluation(), fig_name=self.fig_name + ) + # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so # we expecting a plot with some content without testing the structure of the plot, just # the size self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) def test_plot_parametertrace_algorithms(self): - spotpy.analyser.plot_parametertrace_algorithms([self.griewank_results], - ["test_plot_parametertrace_algorithms"], - spot_setup=griewank_setup(), - fig_name=self.fig_name) + spotpy.analyser.plot_parametertrace_algorithms( + [self.griewank_results], + ["test_plot_parametertrace_algorithms"], + spot_setup=griewank_setup(), + fig_name=self.fig_name, + ) # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so # we expecting a plot with some content without testing the structure of the plot, just # the size @@ -254,7 +238,9 @@ def test_plot_parametertrace_algorithms(self): os.remove(self.fig_name) def test_plot_parametertrace(self): - spotpy.analyser.plot_parametertrace(self.griewank_results, ["0","1"], fig_name=self.fig_name) + spotpy.analyser.plot_parametertrace( + self.griewank_results, ["0", "1"], fig_name=self.fig_name + ) # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so # we expecting a plot with some content without testing the structure of the plot, just @@ -262,27 +248,29 @@ def test_plot_parametertrace(self): self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) def test_plot_posterior_parametertrace(self): - spotpy.analyser.plot_posterior_parametertrace(self.griewank_results, ["0","1"], fig_name=self.fig_name) - + spotpy.analyser.plot_posterior_parametertrace( + self.griewank_results, ["0", "1"], fig_name=self.fig_name + ) + # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so # we expecting a plot with some content without testing the structure of the plot, just # the size self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) - def test_plot_posterior(self): - if sys.version_info >= (3, 6): - spotpy.analyser.plot_posterior(self.hymod_results - , hymod_setup().evaluation(),fig_name=self.fig_name) - - # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so - # we expecting a plot with some content without testing the structure of the plot, just - # the size - self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) - + spotpy.analyser.plot_posterior( + self.hymod_results, hymod_setup().evaluation(), fig_name=self.fig_name + ) + + # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so + # we expecting a plot with some content without testing the structure of the plot, just + # the size + self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) + def test_plot_bestmodelrun(self): - spotpy.analyser.plot_bestmodelrun(self.griewank_results, - griewank_setup().evaluation(), fig_name=self.fig_name) + spotpy.analyser.plot_bestmodelrun( + self.griewank_results, griewank_setup().evaluation(), fig_name=self.fig_name + ) # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so # we expecting a plot with some content without testing the structure of the plot, just @@ -291,21 +279,26 @@ def test_plot_bestmodelrun(self): os.remove(self.fig_name) def test_plot_bestmodelruns(self): - if sys.version_info >= (3, 6): - spotpy.analyser.plot_bestmodelruns( - [self.hymod_results[0:10],self.hymod_results[10:20]], hymod_setup().evaluation(), - dates=range(1, 1+len(hymod_setup().evaluation())), algorithms=["test", "test2"], - fig_name=self.fig_name) + spotpy.analyser.plot_bestmodelruns( + [self.hymod_results[0:10], self.hymod_results[10:20]], + hymod_setup().evaluation(), + dates=range(1, 1 + len(hymod_setup().evaluation())), + algorithms=["test", "test2"], + fig_name=self.fig_name, + ) - # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so - # we expecting a plot with some content without testing the structure of the plot, just - # the size - self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) + # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so + # we expecting a plot with some content without testing the structure of the plot, just + # the size + self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) def test_plot_objectivefunctiontraces(self): - spotpy.analyser.plot_objectivefunctiontraces([self.results], - [rosenbrock_setup().evaluation()], - ["test"],fig_name=self.fig_name) + spotpy.analyser.plot_objectivefunctiontraces( + [self.results], + [rosenbrock_setup().evaluation()], + ["test"], + fig_name=self.fig_name, + ) # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so # we expecting a plot with some content without testing the structure of the plot, just @@ -313,55 +306,54 @@ def test_plot_objectivefunctiontraces(self): self.assertGreaterEqual(os.path.getsize(self.fig_name), 6820) def test_plot_regression(self): - spotpy.analyser.plot_regression(self.results, rosenbrock_setup().evaluation(), - fig_name=self.fig_name) + spotpy.analyser.plot_regression( + self.results, rosenbrock_setup().evaluation(), fig_name=self.fig_name + ) # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so # we expecting a plot with some content without testing the structure of the plot, just # the size self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) - + def test_plot_parameterInteraction(self): # Test only untder Python 3.6 as lower versions results in a strange ValueError - if sys.version_info >= (3, 6): - spotpy.analyser.plot_parameterInteraction(self.results, - fig_name = self.fig_name) - - # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so - # we expecting a plot with some content without testing the structure of the plot, just - # the size - self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) - - + spotpy.analyser.plot_parameterInteraction(self.results, fig_name=self.fig_name) + + # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so + # we expecting a plot with some content without testing the structure of the plot, just + # the size + self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) + def test_plot_allmodelruns(self): - if sys.version_info >= (3, 6): - modelruns = [] - for run in self.hymod_results: - on_run = [] - for i in run: - on_run.append(i) - on_run = np.array(on_run)[:-7] - modelruns.append(on_run.tolist()) - spotpy.analyser.plot_allmodelruns(modelruns, hymod_setup().evaluation(), - dates=range(1, len(hymod_setup().evaluation()) + 1), - fig_name=self.fig_name) - - # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so - # we expecting a plot with some content without testing the structure of the plot, just - # the size - self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) + modelruns = [] + for run in self.hymod_results: + on_run = [] + for i in run: + on_run.append(i) + on_run = np.array(on_run)[:-7] + modelruns.append(on_run.tolist()) + spotpy.analyser.plot_allmodelruns( + modelruns, + hymod_setup().evaluation(), + dates=range(1, len(hymod_setup().evaluation()) + 1), + fig_name=self.fig_name, + ) + # approximately 8855 KB is the size of an empty matplotlib.pyplot.plot, so + # we expecting a plot with some content without testing the structure of the plot, just + # the size + self.assertGreaterEqual(os.path.getsize(self.fig_name), 6855) def test_plot_gelman_rubin(self): - if sys.version_info >= (3, 6): - spotpy.analyser.plot_gelman_rubin(self.hymod_results, self.r_hat, fig_name =self.fig_name) - self.assertGreaterEqual(abs(os.path.getsize(self.fig_name)), 100) - + spotpy.analyser.plot_gelman_rubin( + self.hymod_results, self.r_hat, fig_name=self.fig_name + ) + self.assertGreaterEqual(abs(os.path.getsize(self.fig_name)), 100) @classmethod def tearDownClass(self): - os.remove('test_output.png') + os.remove("test_output.png") -if __name__ == '__main__': +if __name__ == "__main__": unittest.main(exit=False) diff --git a/tests/test_cli.py b/tests/test_cli.py index 4aedeca1..70584c70 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,27 +1,26 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska, Philipp Kraft -''' +""" -from __future__ import division, print_function, unicode_literals import io +import os +import unittest from click.testing import CliRunner -from spotpy.cli import cli, run, get_config_from_file, get_sampler_from_string + from spotpy.algorithms import _algorithm +from spotpy.cli import cli, get_config_from_file, get_sampler_from_string, run from spotpy.examples.spot_setup_rosenbrock import spot_setup -import unittest - -import os class TestCLI(unittest.TestCase): def test_cli_config(self): runner = CliRunner() - result = runner.invoke(cli, ['run', '--config']) + result = runner.invoke(cli, ["run", "--config"]) if result.exception: raise result.exception self.assertEqual(result.exit_code, 0) @@ -29,32 +28,44 @@ def test_cli_config(self): def test_cli_run(self): setup = spot_setup() runner = CliRunner() - result = runner.invoke(run, ['-n 10', '-s', 'lhs', '-p', 'seq'], obj=setup) + result = runner.invoke(run, ["-n 10", "-s", "lhs", "-p", "seq"], obj=setup) self.assertEqual(result.exit_code, 0) def test_config_file(self): - config = dict(n='10', sampler='mc', parallel='seq') + config = dict(n="10", sampler="mc", parallel="seq") # 1. Create config file - with io.open('spotpy.conf', 'w', encoding='utf-8') as f: - f.write('# A comment to be ignored\n') - f.write('Some nonsense to fail\n') + with io.open("spotpy.conf", "w", encoding="utf-8") as f: + f.write("# A comment to be ignored\n") + f.write("Some nonsense to fail\n") for k, v in config.items(): - f.write('{} = {}\n'.format(k, v)) + f.write("{} = {}\n".format(k, v)) try: # 2. Read config file config_from_file = get_config_from_file() - self.assertDictEqual(config, config_from_file, 'Configuration from file is not the same as before') + self.assertDictEqual( + config, + config_from_file, + "Configuration from file is not the same as before", + ) finally: - os.unlink('spotpy.conf') + os.unlink("spotpy.conf") def test_sampler_from_string(self): - sampler_names = "abc|demcz|dream|fast|fscabc|lhs|mc|mcmc|mle|rope|sa|sceua".split('|') - samplers = [get_sampler_from_string(sampler_name) for sampler_name in sampler_names] - wrong_samplers = [n for n, c in zip(sampler_names, samplers) if not issubclass(c, _algorithm)] - self.assertFalse(wrong_samplers, 'Samplers not found from name: ' + ', '.join(wrong_samplers)) + sampler_names = ( + "abc|demcz|dream|fast|fscabc|lhs|mc|mcmc|mle|rope|sa|sceua".split("|") + ) + samplers = [ + get_sampler_from_string(sampler_name) for sampler_name in sampler_names + ] + wrong_samplers = [ + n for n, c in zip(sampler_names, samplers) if not issubclass(c, _algorithm) + ] + self.assertFalse( + wrong_samplers, "Samplers not found from name: " + ", ".join(wrong_samplers) + ) -if __name__ == '__main__': - unittest.main() \ No newline at end of file +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_database.py b/tests/test_database.py index 08a9d509..d12ae1a5 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1,22 +1,25 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska -''' -import unittest -import os +""" import glob -import spotpy -import spotpy.database as db +import os +import unittest + import numpy as np -#https://docs.python.org/3/library/unittest.html +import spotpy.database as db + +# https://docs.python.org/3/library/unittest.html + class MockSetup: """ Mock class to use the save function of a spotpy setup """ + def save(self, *args, **kwargs): pass @@ -24,9 +27,15 @@ def save(self, *args, **kwargs): class TestDatabase(unittest.TestCase): @classmethod def setUpClass(self): - self.parnames = ['x1', 'x2', 'x3', 'x4', 'x5'] + self.parnames = ["x1", "x2", "x3", "x4", "x5"] self.like = 0.213 - self.randompar = [175.21733934706367, 0.41669126598819262, 0.25265012080652388, 0.049706767415682945, 0.69674090782836173] + self.randompar = [ + 175.21733934706367, + 0.41669126598819262, + 0.25265012080652388, + 0.049706767415682945, + 0.69674090782836173, + ] self.simulations_multi = [] for i in range(5): @@ -43,7 +52,16 @@ def objf(self): return np.random.uniform(0, 1, 1)[0] def test_csv_multiline(self): - csv = db.get_datawriter('csv', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations_multi, chains=1, save_sim=True) + csv = db.get_datawriter( + "csv", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations_multi, + chains=1, + save_sim=True, + ) csv.save(self.like, self.randompar, self.simulations_multi) csv.save(self.like, self.randompar, self.simulations_multi) @@ -56,10 +74,18 @@ def test_csv_multiline(self): self.assertEqual(len(csvdata), 2) self.assertEqual(len(csv.header), 32) - def test_csv_multiline_false(self): # Save not Simulations - csv = db.get_datawriter('csv', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations_multi, chains=1, save_sim=False) + csv = db.get_datawriter( + "csv", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations_multi, + chains=1, + save_sim=False, + ) csv.save(self.like, self.randompar, self.simulations_multi) csv.save(self.like, self.randompar, self.simulations_multi) @@ -72,8 +98,16 @@ def test_csv_multiline_false(self): self.assertEqual(len(csv.header), 7) def test_csv_single(self): - csv = db.get_datawriter('csv', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations, - chains=1, save_sim=True) + csv = db.get_datawriter( + "csv", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations, + chains=1, + save_sim=True, + ) csv.save(self.like, self.randompar, self.simulations) csv.save(self.like, self.randompar, self.simulations) @@ -87,9 +121,14 @@ def test_csv_single(self): def test_csv_append(self): csv = db.get_datawriter( - 'csv', "UnitTest_tmp", - self.parnames, self.like, self.randompar, - simulations=self.simulations, chains=1, save_sim=True, + "csv", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations, + chains=1, + save_sim=True, ) csv.save(self.like, self.randompar, self.simulations) @@ -97,10 +136,15 @@ def test_csv_append(self): csv.finalize() csv_new = db.get_datawriter( - 'csv', "UnitTest_tmp", - self.parnames, self.like, self.randompar, - simulations=self.simulations, chains=1, save_sim=True, - dbappend=True + "csv", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations, + chains=1, + save_sim=True, + dbappend=True, ) csv_new.save(self.like, self.randompar, self.simulations) @@ -111,8 +155,16 @@ def test_csv_append(self): self.assertEqual(len(csvdata), 4) def test_csv_single_false(self): - csv = db.get_datawriter('csv', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations, - chains=1, save_sim=False) + csv = db.get_datawriter( + "csv", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations, + chains=1, + save_sim=False, + ) csv.save(self.like, self.randompar, self.simulations) csv.save(self.like, self.randompar, self.simulations) @@ -125,7 +177,16 @@ def test_csv_single_false(self): self.assertEqual(len(csv.header), 7) def test_hdf5_multiline(self): - hdf5 = db.get_datawriter('hdf5', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations_multi, chains=1, save_sim=True) + hdf5 = db.get_datawriter( + "hdf5", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations_multi, + chains=1, + save_sim=True, + ) hdf5.save(self.like, self.randompar, self.simulations_multi) hdf5.save(self.like, self.randompar, self.simulations_multi) @@ -138,10 +199,18 @@ def test_hdf5_multiline(self): self.assertEqual(len(hdf5data), 2) self.assertEqual(len(hdf5.header), 32) - def test_hdf5_multiline_false(self): # Save not Simulations - hdf5 = db.get_datawriter('hdf5', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations_multi, chains=1, save_sim=False) + hdf5 = db.get_datawriter( + "hdf5", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations_multi, + chains=1, + save_sim=False, + ) hdf5.save(self.like, self.randompar, self.simulations_multi) hdf5.save(self.like, self.randompar, self.simulations_multi) @@ -154,8 +223,16 @@ def test_hdf5_multiline_false(self): self.assertEqual(len(hdf5.header), 7) def test_hdf5_single(self): - hdf5 = db.get_datawriter('hdf5', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations, - chains=1, save_sim=True) + hdf5 = db.get_datawriter( + "hdf5", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations, + chains=1, + save_sim=True, + ) hdf5.save(self.like, self.randompar, self.simulations) hdf5.save(self.like, self.randompar, self.simulations) @@ -169,9 +246,14 @@ def test_hdf5_single(self): def test_hdf5_append(self): hdf5 = db.get_datawriter( - 'hdf5', "UnitTest_tmp", - self.parnames, self.like, self.randompar, - simulations=self.simulations, chains=1, save_sim=True, + "hdf5", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations, + chains=1, + save_sim=True, ) hdf5.save(self.like, self.randompar, self.simulations) @@ -179,10 +261,15 @@ def test_hdf5_append(self): hdf5.finalize() hdf5_new = db.get_datawriter( - 'hdf5', "UnitTest_tmp", - self.parnames, self.like, self.randompar, - simulations=self.simulations, chains=1, save_sim=True, - dbappend=True + "hdf5", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations, + chains=1, + save_sim=True, + dbappend=True, ) hdf5_new.save(self.like, self.randompar, self.simulations) @@ -192,10 +279,17 @@ def test_hdf5_append(self): hdf5data = hdf5_new.getdata() self.assertEqual(len(hdf5data), 4) - def test_hdf5_single_false(self): - hdf5 = db.get_datawriter('hdf5', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations, - chains=1, save_sim=False) + hdf5 = db.get_datawriter( + "hdf5", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations, + chains=1, + save_sim=False, + ) hdf5.save(self.like, self.randompar, self.simulations) hdf5.save(self.like, self.randompar, self.simulations) @@ -207,9 +301,17 @@ def test_hdf5_single_false(self): self.assertEqual(len(hdf5data), 2) self.assertEqual(len(hdf5.header), 7) - def test_sql_multiline(self): - sql = db.get_datawriter('sql', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations_multi, chains=1, save_sim=True) + sql = db.get_datawriter( + "sql", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations_multi, + chains=1, + save_sim=True, + ) sql.save(self.like, self.randompar, self.simulations_multi) sql.finalize() sqldata = sql.getdata() @@ -218,9 +320,17 @@ def test_sql_multiline(self): self.assertEqual(len(sqldata), 1) self.assertEqual(len(sql.header), 32) - def test_sql_multiline_false(self): - sql = db.get_datawriter('sql', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations_multi, chains=1, save_sim=False) + sql = db.get_datawriter( + "sql", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations_multi, + chains=1, + save_sim=False, + ) sql.save(self.like, self.randompar, self.simulations_multi) sql.finalize() sqldata = sql.getdata() @@ -230,8 +340,16 @@ def test_sql_multiline_false(self): self.assertEqual(len(sql.header), 7) def test_sql_single(self): - sql = db.get_datawriter('sql', "UnitTest_tmp", self.parnames, self.like, self.randompar, - simulations=self.simulations, chains=1, save_sim=True) + sql = db.get_datawriter( + "sql", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations, + chains=1, + save_sim=True, + ) sql.save(self.like, self.randompar, self.simulations) sql.finalize() sqldata = sql.getdata() @@ -241,8 +359,16 @@ def test_sql_single(self): self.assertEqual(len(sql.header), 12) def test_sql_single_false(self): - sql = db.get_datawriter('sql', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations, - chains=1, save_sim=False) + sql = db.get_datawriter( + "sql", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations, + chains=1, + save_sim=False, + ) sql.save(self.like, self.randompar, self.simulations) sql.finalize() @@ -253,7 +379,16 @@ def test_sql_single_false(self): self.assertEqual(len(sql.header), 7) def test_ram_multiline(self): - ram = db.get_datawriter('ram', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations_multi, chains=1, save_sim=True) + ram = db.get_datawriter( + "ram", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations_multi, + chains=1, + save_sim=True, + ) ram.save(self.like, self.randompar, self.simulations_multi) ram.finalize() @@ -265,7 +400,16 @@ def test_ram_multiline(self): self.assertEqual(len(ramdata.dtype), len(ram.header)) def test_ram_multiline_false(self): - ram = db.get_datawriter('ram', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations_multi, chains=1, save_sim=False) + ram = db.get_datawriter( + "ram", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations_multi, + chains=1, + save_sim=False, + ) ram.save(self.like, self.randompar, self.simulations_multi) ram.finalize() @@ -277,8 +421,16 @@ def test_ram_multiline_false(self): self.assertEqual(len(ram.header), 7) def test_ram_single(self): - ram = db.get_datawriter('ram', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations, - chains=1, save_sim=True) + ram = db.get_datawriter( + "ram", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations, + chains=1, + save_sim=True, + ) ram.save(self.like, self.randompar, self.simulations) ram.finalize() @@ -290,8 +442,16 @@ def test_ram_single(self): self.assertEqual(len(ram.header), 12) def test_ram_single_false(self): - ram = db.get_datawriter('ram', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations, - chains=1, save_sim=False) + ram = db.get_datawriter( + "ram", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations, + chains=1, + save_sim=False, + ) ram.save(self.like, self.randompar, self.simulations) ram.finalize() @@ -304,14 +464,27 @@ def test_ram_single_false(self): def test_not_existing_dbformat(self): with self.assertRaises(AttributeError): - _ = db.get_datawriter('xxx', "UnitTest_tmp", self.parnames, self.like, self.randompar, simulations=self.simulations, - chains=1, save_sim=True) + _ = db.get_datawriter( + "xxx", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations, + chains=1, + save_sim=True, + ) def test_noData(self): nodata = db.get_datawriter( - 'noData', "UnitTest_tmp", - self.parnames, np.array(self.like), self.randompar, - simulations=self.simulations, chains=1, save_sim=True + "noData", + "UnitTest_tmp", + self.parnames, + np.array(self.like), + self.randompar, + simulations=self.simulations, + chains=1, + save_sim=True, ) nodata.save(self.like, self.randompar, self.simulations) nodata.finalize() @@ -319,32 +492,47 @@ def test_noData(self): def test_custom(self): custom = db.get_datawriter( - 'custom', "UnitTest_tmp", - self.parnames, self.like, self.randompar, + "custom", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, setup=MockSetup(), - simulations=self.simulations, chains=1, save_sim=True + simulations=self.simulations, + chains=1, + save_sim=True, ) custom.save(self.like, self.randompar, self.simulations) custom.finalize() - self.assertEqual(custom.getdata(),None) + self.assertEqual(custom.getdata(), None) def test_custom_no_setup(self): with self.assertRaises(ValueError): _ = db.get_datawriter( - 'custom', "UnitTest_tmp", - self.parnames, self.like, self.randompar, - simulations=self.simulations, chains=1, save_sim=True + "custom", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, + simulations=self.simulations, + chains=1, + save_sim=True, ) def test_custom_wrong_setup(self): with self.assertRaises(AttributeError): _ = db.get_datawriter( - 'custom', "UnitTest_tmp", - self.parnames, self.like, self.randompar, + "custom", + "UnitTest_tmp", + self.parnames, + self.like, + self.randompar, setup=[], - simulations=self.simulations, chains=1, save_sim=True + simulations=self.simulations, + chains=1, + save_sim=True, ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_dds.py b/tests/test_dds.py index da6b0df5..1f786fb9 100644 --- a/tests/test_dds.py +++ b/tests/test_dds.py @@ -1,23 +1,18 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska, Benjamin Manns -''' +""" +import json +import os import unittest -import sys -import numpy as np -try: - import spotpy -except ImportError: - sys.path.append(".") - import spotpy +import numpy as np -import os +import spotpy from spotpy.examples.spot_setup_dds import spot_setup -import json # replaces numpy.random module in a way @@ -25,18 +20,22 @@ class FixedRandomizerEndOfDataException(Exception): pass -class FixedRandomizer(): +class FixedRandomizer: def __init__(self): self.debug = False self.uniform_counter = 0 self.normal_counter = 0 - self.uniform_list=list(np.loadtxt(os.path.dirname(__file__)+"/dds_tests/uniform_list.txt")) + self.uniform_list = list( + np.loadtxt(os.path.dirname(__file__) + "/dds_tests/uniform_list.txt") + ) - self.uniform_list*=3 + self.uniform_list *= 3 self.max_normal_counter = 10000 self.max_uniform_counter = 30000 - self.normal_list = list(np.loadtxt(os.path.dirname(__file__)+"/dds_tests/normal_list.txt")) + self.normal_list = list( + np.loadtxt(os.path.dirname(__file__) + "/dds_tests/normal_list.txt") + ) def rand(self, dim_x=1, dim_y=1): x = dim_x * [0] @@ -45,23 +44,25 @@ def rand(self, dim_x=1, dim_y=1): x[i] = self.uniform_list[self.uniform_counter] self.uniform_counter = self.uniform_counter + 1 if self.debug: - print("fixrand::rand() counter = "+str(self.uniform_counter)) + print("fixrand::rand() counter = " + str(self.uniform_counter)) else: - raise FixedRandomizerEndOfDataException("No more data left. Counter is: "+str(self.uniform_counter)) + raise FixedRandomizerEndOfDataException( + "No more data left. Counter is: " + str(self.uniform_counter) + ) if len(x) == 1: return x[0] else: return x - def randint(self,x_from,x_to): - vals = [j for j in range(x_from,x_to)] + def randint(self, x_from, x_to): + vals = [j for j in range(x_from, x_to)] vals_size = len(vals) if vals_size == 0: raise ValueError("x_to >= x_from") - fraq = 1. / vals_size + fraq = 1.0 / vals_size if self.uniform_counter < self.max_uniform_counter: q_uni = self.uniform_list[self.uniform_counter] - pos = np.int(np.floor(q_uni / fraq)) + pos = int(np.floor(q_uni / fraq)) self.uniform_counter += 1 if self.debug: print("fixrand::randint() counter = " + str(self.uniform_counter)) @@ -69,11 +70,11 @@ def randint(self,x_from,x_to): else: raise FixedRandomizerEndOfDataException("No more data left.") - def normal(self,loc,scale,size=1): + def normal(self, loc, scale, size=1): x = [] for j in range(size): if self.normal_counter < self.max_normal_counter: - x.append(self.normal_list[self.normal_counter]*scale + loc) + x.append(self.normal_list[self.normal_counter] * scale + loc) self.normal_counter += 1 if self.debug: print("fixrand::normal() counter = " + str(self.normal_counter)) @@ -94,7 +95,9 @@ def setUp(self): self.f_random = FixedRandomizer() def json_helper(self, run): - with open(os.path.dirname(__file__) + "/dds_tests/run_" + str(run) + ".json") as f: + with open( + os.path.dirname(__file__) + "/dds_tests/run_" + str(run) + ".json" + ) as f: data = json.load(f) return data @@ -135,36 +138,57 @@ def outside_bound(self, x_curr, min_bound, max_bound): def test_bounds(self): self.spot_setup._objfunc_switcher("cmf_style") - sampler = spotpy.algorithms.dds(self.spot_setup, parallel="seq", dbname='test_DDS', dbformat="csv", - sim_timeout=self.timeout) + sampler = spotpy.algorithms.dds( + self.spot_setup, + parallel="seq", + dbname="test_DDS", + dbformat="csv", + sim_timeout=self.timeout, + ) sampler._set_np_random(self.f_random) - results = sampler.sample(1000,1) - print("results",results[0]['sbest']) + results = sampler.sample(1000, 1) + print("results", results[0]["sbest"]) print(sampler.min_bound, sampler.max_bound) - self.outside_bound(results[0]['sbest'], sampler.min_bound, sampler.max_bound) + self.outside_bound(results[0]["sbest"], sampler.min_bound, sampler.max_bound) def run_a_dds(self, run): original_result = self.json_helper(run) - self.spot_setup._objfunc_switcher(original_result['objfunc']) + self.spot_setup._objfunc_switcher(original_result["objfunc"]) - sampler = spotpy.algorithms.dds(self.spot_setup, parallel="seq", dbname='test_DDS', dbformat="csv", - sim_timeout=self.timeout,r=original_result["r_val"]) + sampler = spotpy.algorithms.dds( + self.spot_setup, + parallel="seq", + dbname="test_DDS", + dbformat="csv", + sim_timeout=self.timeout, + r=original_result["r_val"], + ) sampler._set_np_random(self.f_random) if original_result.get("s_initial") is not None: # if a parameter initialisation is given, test this: - results = sampler.sample(original_result["evatrials"], - original_result["trial_runs"], x_initial=original_result["s_initial"]) + results = sampler.sample( + original_result["evatrials"], + original_result["trial_runs"], + x_initial=original_result["s_initial"], + ) else: - results = sampler.sample(original_result["evatrials"], - original_result["trial_runs"]) + results = sampler.sample( + original_result["evatrials"], original_result["trial_runs"] + ) for t in range(original_result["trial_runs"]): - print(results[t]["objfunc_val"], -1*original_result["results"][t]["objfunc_val"]) - self.assertAlmostEqual(results[t]["objfunc_val"], -1*original_result["results"][t]["objfunc_val"], - delta=0.000001) + print( + results[t]["objfunc_val"], + -1 * original_result["results"][t]["objfunc_val"], + ) + self.assertAlmostEqual( + results[t]["objfunc_val"], + -1 * original_result["results"][t]["objfunc_val"], + delta=0.000001, + ) py_sbest = results[t]["sbest"] matlb_sbest = original_result["results"][t]["sbest"] for k in range(len(py_sbest)): @@ -175,26 +199,52 @@ def run_a_dds(self, run): matlb_trial_initial = original_result["results"][t]["trial_initial"] for k in range(len(py_sbest)): print(t, k, py_trial_initial[k], matlb_trial_initial[k]) - self.assertAlmostEqual(py_trial_initial[k], matlb_trial_initial[k], delta=0.0001) + self.assertAlmostEqual( + py_trial_initial[k], matlb_trial_initial[k], delta=0.0001 + ) def test_own_initial_out_of_borders_ackley_1(self): self.spot_setup._objfunc_switcher("ackley") - sampler = spotpy.algorithms.dds(self.spot_setup, parallel="seq", dbname='test_DDS', dbformat="csv", - sim_timeout=self.timeout) - self.assertRaises(ValueError,sampler.sample,1000, x_initial=np.random.uniform(-2, 2, 9) + [3]) + sampler = spotpy.algorithms.dds( + self.spot_setup, + parallel="seq", + dbname="test_DDS", + dbformat="csv", + sim_timeout=self.timeout, + ) + self.assertRaises( + ValueError, + sampler.sample, + 1000, + x_initial=np.random.uniform(-2, 2, 9) + [3], + ) def test_own_initial_too_lees(self): self.spot_setup._objfunc_switcher("ackley") - sampler = spotpy.algorithms.dds(self.spot_setup, parallel="seq", dbname='test_DDS', dbformat="csv", - sim_timeout=self.timeout) - self.assertRaises(ValueError, sampler.sample, 1000, x_initial=np.random.uniform(-2, 2, 9)) + sampler = spotpy.algorithms.dds( + self.spot_setup, + parallel="seq", + dbname="test_DDS", + dbformat="csv", + sim_timeout=self.timeout, + ) + self.assertRaises( + ValueError, sampler.sample, 1000, x_initial=np.random.uniform(-2, 2, 9) + ) def test_own_initial_too_much(self): self.spot_setup._objfunc_switcher("ackley") - sampler = spotpy.algorithms.dds(self.spot_setup, parallel="seq", dbname='test_DDS', dbformat="csv", - sim_timeout=self.timeout) - self.assertRaises(ValueError, sampler.sample, 1000, x_initial=np.random.uniform(-2, 2, 11)) - - -if __name__ == '__main__': + sampler = spotpy.algorithms.dds( + self.spot_setup, + parallel="seq", + dbname="test_DDS", + dbformat="csv", + sim_timeout=self.timeout, + ) + self.assertRaises( + ValueError, sampler.sample, 1000, x_initial=np.random.uniform(-2, 2, 11) + ) + + +if __name__ == "__main__": unittest.main() diff --git a/tests/test_describe.py b/tests/test_describe.py index 5035963b..ab441985 100644 --- a/tests/test_describe.py +++ b/tests/test_describe.py @@ -1,27 +1,24 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska, Philipp Kraft -''' - -from __future__ import unicode_literals, absolute_import, division, print_function -import sys +""" +import inspect import unittest -sys.path.insert(0, '.') +from io import StringIO import spotpy from spotpy.parameter import Uniform -from io import StringIO -import inspect class SpotSetup(object): """ Just a fun setup using non ASCIÎ letters to challenge describe in Python 2 """ - a = Uniform(-1, 1, doc='α parameter') - beta = Uniform(-1, 1, doc='β parameter') + + a = Uniform(-1, 1, doc="α parameter") + beta = Uniform(-1, 1, doc="β parameter") @staticmethod def simulation(par): @@ -37,12 +34,11 @@ def objectivefunction(simulation, evaluation): class TestDescribe(unittest.TestCase): - def test_setup(self): t = spotpy.describe.setup(SpotSetup()) io = StringIO() io.write(t) - self.assertGreaterEqual(io.getvalue().count('\n'), 6) + self.assertGreaterEqual(io.getvalue().count("\n"), 6) def sampler_test(self, sampler=None): if not sampler: @@ -50,50 +46,60 @@ def sampler_test(self, sampler=None): t = spotpy.describe.describe(sampler) io = StringIO() io.write(t) - line_count = io.getvalue().count('\n') - self.assertGreaterEqual(line_count, 14, '<{}> description to short ({} lines, but >14 expected)' - .format(sampler, line_count)) + line_count = io.getvalue().count("\n") + self.assertGreaterEqual( + line_count, + 14, + "<{}> description to short ({} lines, but >14 expected)".format( + sampler, line_count + ), + ) def test_mc_sampler(self): - sampler = spotpy.algorithms.mc(spot_setup=SpotSetup(), dbformat='ram', dbname='äöü') + sampler = spotpy.algorithms.mc( + spot_setup=SpotSetup(), dbformat="ram", dbname="äöü" + ) self.sampler_test(sampler) def test_rope_sampler(self): - sampler = spotpy.algorithms.rope(spot_setup=SpotSetup(), dbformat='ram', dbname='äöü') + sampler = spotpy.algorithms.rope( + spot_setup=SpotSetup(), dbformat="ram", dbname="äöü" + ) self.sampler_test(sampler) - def test_all_algorithms(self): for sname, scls in inspect.getmembers(spotpy.algorithms, inspect.isclass): - if not sname.startswith('_'): + if not sname.startswith("_"): model = SpotSetup() - sampler = scls(spot_setup=model, dbformat='ram', dbname='äöü') + sampler = scls(spot_setup=model, dbformat="ram", dbname="äöü") self.sampler_test(sampler) -if sys.version_info > (3, 5): - - class TestDescribeRst(unittest.TestCase): - def test_setup_rst(self): - setup = SpotSetup() - rst = spotpy.describe.rst(setup) +class TestDescribeRst(unittest.TestCase): + def test_setup_rst(self): + setup = SpotSetup() + rst = spotpy.describe.rst(setup) - def test_sampler_rst(self): - for sname, scls in inspect.getmembers(spotpy.algorithms, inspect.isclass): - if not sname.startswith('_'): - model = SpotSetup() - sampler = scls(spot_setup=model, dbformat='ram', dbname='äöü') - rst = spotpy.describe.rst(sampler) + def test_sampler_rst(self): + for sname, scls in inspect.getmembers(spotpy.algorithms, inspect.isclass): + if not sname.startswith("_"): + model = SpotSetup() + sampler = scls(spot_setup=model, dbformat="ram", dbname="äöü") + rst = spotpy.describe.rst(sampler) - rst.append('This is a test for ``rst.append().``\n' * 10, 'Appending', 1) - rst.append_math(r'c = \sqrt{a^2 + b^2}') - rst.append(title='Image', titlelevel=2) - rst.append_image('https://img.shields.io/travis/thouska/spotpy/master.svg', - target='https://travis-ci.org/thouska/spotpy', - width='200px') + rst.append( + "This is a test for ``rst.append().``\n" * 10, "Appending", 1 + ) + rst.append_math(r"c = \sqrt{a^2 + b^2}") + rst.append(title="Image", titlelevel=2) + rst.append_image( + "https://img.shields.io/travis/thouska/spotpy/master.svg", + target="https://travis-ci.org/thouska/spotpy", + width="200px", + ) - rst.as_html() + rst.as_html() -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_fast.py b/tests/test_fast.py index 33aa7734..f76069d2 100644 --- a/tests/test_fast.py +++ b/tests/test_fast.py @@ -1,38 +1,36 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska -''' +""" import unittest -import sys - -try: - import spotpy -except ImportError: - sys.path.append(".") - import spotpy +import spotpy from spotpy.examples.spot_setup_hymod_python import spot_setup -# Test only untder Python 3 as lower versions results in a strange fft error -if sys.version_info >= (3, 6): - class TestFast(unittest.TestCase): - def setUp(self): +class TestFast(unittest.TestCase): + def setUp(self): + + self.spot_setup = spot_setup() + self.rep = 200 # REP must be a multiply of amount of parameters which are in 7 if using hymod + self.timeout = 10 # Given in Seconds - self.spot_setup = spot_setup() - self.rep = 200 # REP must be a multiply of amount of parameters which are in 7 if using hymod - self.timeout = 10 # Given in Seconds + def test_fast(self): + sampler = spotpy.algorithms.fast( + self.spot_setup, + parallel="seq", + dbname="test_FAST", + dbformat="ram", + sim_timeout=self.timeout, + ) + results = [] + sampler.sample(self.rep) + results = sampler.getdata() + self.assertEqual(200, len(results)) - def test_fast(self): - sampler = spotpy.algorithms.fast(self.spot_setup, parallel="seq", dbname='test_FAST', dbformat="ram", - sim_timeout=self.timeout) - results = [] - sampler.sample(self.rep) - results = sampler.getdata() - self.assertEqual(200,len(results)) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_gui_mpl.py b/tests/test_gui_mpl.py index 8889f953..be81fc6e 100644 --- a/tests/test_gui_mpl.py +++ b/tests/test_gui_mpl.py @@ -1,53 +1,94 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska, Philipp Kraft -''' +""" import unittest + import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") + +import inspect import sys -if sys.version_info >= (3, 5) and matplotlib.__version__ >= '2.1': +import numpy as np + +from spotpy import parameter +from spotpy.gui.mpl import GUI + + +class SpotSetupBase(object): + """ + The base for a number of test cases. + Each Test case should have for parameters a,b,c,d and + the sum of the parameters should be zero + """ + + def simulation(self, par): + return [par.a + par.b + par.c + par.d] + + def evaluation(self): + return [0] + + def objectivefunction(self, simulation, evaluation): + return np.abs(simulation[0] - evaluation[0]) + + @classmethod + def get_derived(cls): + """ + Returns a list of all derived classes in this module + """ + module = sys.modules[__name__] + + def predicate(mcls): + return inspect.isclass(mcls) and issubclass(mcls, cls) and mcls is not cls + + return [mcls for cname, mcls in inspect.getmembers(module, predicate)] + + def __repr__(self): + return "{}()".format(type(self).__name__) + - sys.path.append(".") +class SpotSetupMixedParameterFunction(SpotSetupBase): + """ + A Test case with two parameters as class parameters (a,b) + and 2 given from the parameter function + """ - try: - import spotpy - except ImportError: - import spotpy + a = parameter.Uniform(0, 1) + b = parameter.Uniform(1, 2) - from spotpy.gui.mpl import GUI - from .test_setup_parameters import SpotSetupMixedParameterFunction as Setup + def parameters(self): + return parameter.generate([parameter.Uniform(name, -1, 1) for name in "cd"]) - class TestGuiMpl(unittest.TestCase): +class TestGuiMpl(unittest.TestCase): + def test_setup(self): + setup = SpotSetupMixedParameterFunction() + with GUI(setup) as gui: + self.assertTrue(hasattr(gui, "setup")) - def test_setup(self): - setup = Setup() - with GUI(setup) as gui: - self.assertTrue(hasattr(gui, 'setup')) + def test_sliders(self): + setup = SpotSetupMixedParameterFunction() + with GUI(setup) as gui: + self.assertEqual(len(gui.sliders), 4) - def test_sliders(self): - setup = Setup() - with GUI(setup) as gui: - self.assertEqual(len(gui.sliders), 4) + def test_clear(self): + setup = SpotSetupMixedParameterFunction() + with GUI(setup) as gui: + gui.clear() + self.assertEqual(len(gui.lines), 1) - def test_clear(self): - setup = Setup() - with GUI(setup) as gui: - gui.clear() - self.assertEqual(len(gui.lines), 1) + def test_run(self): + setup = SpotSetupMixedParameterFunction() + with GUI(setup) as gui: + gui.clear() + gui.run() + self.assertEqual(len(gui.lines), 2) - def test_run(self): - setup = Setup() - with GUI(setup) as gui: - gui.clear() - gui.run() - self.assertEqual(len(gui.lines), 2) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_likelihood.py b/tests/test_likelihood.py index b07f7beb..5edd38ad 100644 --- a/tests/test_likelihood.py +++ b/tests/test_likelihood.py @@ -7,20 +7,13 @@ This code tests the likelihood framework and present all existing function. """ -import numpy as np - -try: - import spotpy -except ImportError: - import sys +import unittest - sys.path.append(".") - import spotpy +import numpy as np -import unittest +import spotpy from spotpy.likelihoods import LikelihoodError - # We use all available likelihood functions. The pydoc of every function tells, if we can add a # parameter `param` to the function which includes model parameter. The `param` must be None or a tuple with values # and names. If `param` is None, the needed values are calculated by the function itself. @@ -29,65 +22,96 @@ class TestLikelihood(unittest.TestCase): def setUp(self): np.random.seed(12) - self.normal_data, self.normal_comparedata = np.random.normal(1500, 2530, 20), np.random.normal(15, 25, 20) - self.binom_data, self.binom_comparedata = np.random.binomial(20, 0.1, 20), np.random.binomial(20, 0.1, 20) + self.normal_data, self.normal_comparedata = np.random.normal( + 1500, 2530, 20 + ), np.random.normal(15, 25, 20) + self.binom_data, self.binom_comparedata = np.random.binomial( + 20, 0.1, 20 + ), np.random.binomial(20, 0.1, 20) self.do_print = True def test_logLikelihood(self): - l_normal = spotpy.likelihoods.logLikelihood(self.normal_data, self.normal_comparedata) + l_normal = spotpy.likelihoods.logLikelihood( + self.normal_data, self.normal_comparedata + ) self.assertGreaterEqual(np.abs(l_normal), 900) - self.assertEqual(type(np.float(l_normal)), type(np.float(1))) + self.assertEqual(type(float(l_normal)), type(float(1))) if self.do_print: print("logLikelihood: " + str(l_normal)) - l_binom = spotpy.likelihoods.logLikelihood(self.binom_data, self.binom_comparedata) + l_binom = spotpy.likelihoods.logLikelihood( + self.binom_data, self.binom_comparedata + ) self.assertGreaterEqual(np.abs(l_binom), 900) - self.assertEqual(type(np.float(l_binom)), type(np.float(1))) + self.assertEqual(type(float(l_binom)), type(float(1))) if self.do_print: print("logLikelihood: " + str(l_binom)) def test_gaussianLikelihoodMeasErrorOut(self): - l_normal = spotpy.likelihoods.gaussianLikelihoodMeasErrorOut(self.normal_data, self.normal_comparedata) + l_normal = spotpy.likelihoods.gaussianLikelihoodMeasErrorOut( + self.normal_data, self.normal_comparedata + ) self.assertGreaterEqual(-40, l_normal) - self.assertEqual(type(np.float(l_normal)), type(np.float(1))) + self.assertEqual(type(float(l_normal)), type(float(1))) if self.do_print: print("gaussianLikelihoodMeasErrorOut: " + str(l_normal)) - l_binom = spotpy.likelihoods.gaussianLikelihoodMeasErrorOut(self.binom_data, self.binom_comparedata) + l_binom = spotpy.likelihoods.gaussianLikelihoodMeasErrorOut( + self.binom_data, self.binom_comparedata + ) self.assertGreaterEqual(-40, l_binom) - self.assertEqual(type(np.float(l_binom)), type(np.float(1))) + self.assertEqual(type(float(l_binom)), type(float(1))) if self.do_print: print("gaussianLikelihoodMeasErrorOut: " + str(l_binom)) def test_gaussianLikelihoodHomoHeteroDataError(self): - l_normal = spotpy.likelihoods.gaussianLikelihoodHomoHeteroDataError(self.normal_data, self.normal_comparedata) + l_normal = spotpy.likelihoods.gaussianLikelihoodHomoHeteroDataError( + self.normal_data, self.normal_comparedata + ) self.assertGreaterEqual(5, np.abs(l_normal)) - self.assertEqual(type(np.float(l_normal)), type(np.float(1))) + self.assertEqual(type(float(l_normal)), type(float(1))) if self.do_print: print("gaussianLikelihoodHomoHeteroDataError: " + str(l_normal)) - l_binom = spotpy.likelihoods.gaussianLikelihoodHomoHeteroDataError(self.binom_data, self.binom_comparedata) + l_binom = spotpy.likelihoods.gaussianLikelihoodHomoHeteroDataError( + self.binom_data, self.binom_comparedata + ) self.assertGreaterEqual(10, np.abs(l_binom)) - self.assertEqual(type(np.float(l_binom)), type(np.float(1))) + self.assertEqual(type(float(l_binom)), type(float(1))) if self.do_print: print("gaussianLikelihoodHomoHeteroDataError: " + str(l_binom)) def test_LikelihoodAR1NoC(self): l_list = [] - l_list.append(spotpy.likelihoods.LikelihoodAR1NoC(self.normal_data, self.normal_comparedata, - params=([0.98], ["likelihood_phi"]))) + l_list.append( + spotpy.likelihoods.LikelihoodAR1NoC( + self.normal_data, + self.normal_comparedata, + params=([0.98], ["likelihood_phi"]), + ) + ) try: - l_list.append(spotpy.likelihoods.LikelihoodAR1NoC(self.normal_data, self.normal_comparedata, - params=([], []))) + l_list.append( + spotpy.likelihoods.LikelihoodAR1NoC( + self.normal_data, self.normal_comparedata, params=([], []) + ) + ) except LikelihoodError as e: print("LikelihoodError occurred: " + str(e)) - l_list.append(spotpy.likelihoods.LikelihoodAR1NoC(self.normal_data, self.normal_comparedata, - params=([1.1], ["likelihood_phi"]))) + l_list.append( + spotpy.likelihoods.LikelihoodAR1NoC( + self.normal_data, + self.normal_comparedata, + params=([1.1], ["likelihood_phi"]), + ) + ) - l_list.append(spotpy.likelihoods.LikelihoodAR1NoC(self.binom_data, self.binom_data)) + l_list.append( + spotpy.likelihoods.LikelihoodAR1NoC(self.binom_data, self.binom_data) + ) for l in l_list: self.assertNotEqual(None, l) @@ -97,17 +121,34 @@ def test_LikelihoodAR1NoC(self): def test_LikelihoodAR1WithC(self): l_normal_list = [] try: - l_normal_list.append(spotpy.likelihoods.LikelihoodAR1WithC(self.normal_data, self.normal_comparedata, - params=([], []))) + l_normal_list.append( + spotpy.likelihoods.LikelihoodAR1WithC( + self.normal_data, self.normal_comparedata, params=([], []) + ) + ) except LikelihoodError as e: print("Likelihood Error occurred " + str(e)) - l_normal_list.append(spotpy.likelihoods.LikelihoodAR1WithC(self.normal_data, self.normal_comparedata, - params=([0.98], ["likelihood_phi"]))) - l_normal_list.append(spotpy.likelihoods.LikelihoodAR1WithC(self.normal_data, self.normal_comparedata, - params=([1.1], ["likelihood_phi"]))) + l_normal_list.append( + spotpy.likelihoods.LikelihoodAR1WithC( + self.normal_data, + self.normal_comparedata, + params=([0.98], ["likelihood_phi"]), + ) + ) + l_normal_list.append( + spotpy.likelihoods.LikelihoodAR1WithC( + self.normal_data, + self.normal_comparedata, + params=([1.1], ["likelihood_phi"]), + ) + ) - l_normal_list.append(spotpy.likelihoods.LikelihoodAR1WithC(self.binom_data, self.binom_comparedata)) + l_normal_list.append( + spotpy.likelihoods.LikelihoodAR1WithC( + self.binom_data, self.binom_comparedata + ) + ) for l_normal in l_normal_list: self.assertNotEqual(None, l_normal) @@ -116,74 +157,133 @@ def test_LikelihoodAR1WithC(self): def test_generalizedLikelihoodFunction(self): size = 1000 - data, comparedata = np.random.normal(1500, 2530, size), np.random.normal(355, 25, size) - - param_list = ["likelihood_beta", "likelihood_xi", "likelihood_sigma0", "likelihood_sigma1", "likelihood_phi1", - "likelihood_muh"] + data, comparedata = np.random.normal(1500, 2530, size), np.random.normal( + 355, 25, size + ) + + param_list = [ + "likelihood_beta", + "likelihood_xi", + "likelihood_sigma0", + "likelihood_sigma1", + "likelihood_phi1", + "likelihood_muh", + ] l_normal_list = [] - l_normal_list.append(spotpy.likelihoods.generalizedLikelihoodFunction(data, comparedata, params= - ([-0.09, 1, 0.5, 0.567, 0.98, 57.32], param_list))) + l_normal_list.append( + spotpy.likelihoods.generalizedLikelihoodFunction( + data, + comparedata, + params=([-0.09, 1, 0.5, 0.567, 0.98, 57.32], param_list), + ) + ) try: - l_normal_list.append(spotpy.likelihoods.generalizedLikelihoodFunction(data, comparedata, params= - ([], []))) + l_normal_list.append( + spotpy.likelihoods.generalizedLikelihoodFunction( + data, comparedata, params=([], []) + ) + ) except LikelihoodError as e: print("Likelihood Error occurred " + str(e)) - l_normal_list.append(spotpy.likelihoods.generalizedLikelihoodFunction(data, comparedata, params= - ([2, 1, 0.5, 0.567, 0.98, 57.32], param_list))) - l_normal_list.append(spotpy.likelihoods.generalizedLikelihoodFunction(data, comparedata, params= - ([-0.09, 11, 0.5, 0.567, 0.98, 57.32], param_list))) - l_normal_list.append(spotpy.likelihoods.generalizedLikelihoodFunction(data, comparedata, params= - ([-0.09, 1, 1.5, 0.567, 0.98, 57.32], param_list))) - l_normal_list.append(spotpy.likelihoods.generalizedLikelihoodFunction(data, comparedata, params= - ([-0.09, 1, 0.5, 1.567, 0.98, 57.32], param_list))) - l_normal_list.append(spotpy.likelihoods.generalizedLikelihoodFunction(data, comparedata, params= - ([-0.09, 1, 0.5, 0.567, 2.98, 57.32], param_list))) - l_normal_list.append(spotpy.likelihoods.generalizedLikelihoodFunction(data, comparedata, params= - ([-0.09, 1, 0.5, 0.567, 0.98, 101], param_list))) - l_normal_list.append(spotpy.likelihoods.generalizedLikelihoodFunction(data, comparedata, params= - ([-0.09, 0.0, 0.5, 0.567, 0.98, 101], param_list))) + l_normal_list.append( + spotpy.likelihoods.generalizedLikelihoodFunction( + data, comparedata, params=([2, 1, 0.5, 0.567, 0.98, 57.32], param_list) + ) + ) + l_normal_list.append( + spotpy.likelihoods.generalizedLikelihoodFunction( + data, + comparedata, + params=([-0.09, 11, 0.5, 0.567, 0.98, 57.32], param_list), + ) + ) + l_normal_list.append( + spotpy.likelihoods.generalizedLikelihoodFunction( + data, + comparedata, + params=([-0.09, 1, 1.5, 0.567, 0.98, 57.32], param_list), + ) + ) + l_normal_list.append( + spotpy.likelihoods.generalizedLikelihoodFunction( + data, + comparedata, + params=([-0.09, 1, 0.5, 1.567, 0.98, 57.32], param_list), + ) + ) + l_normal_list.append( + spotpy.likelihoods.generalizedLikelihoodFunction( + data, + comparedata, + params=([-0.09, 1, 0.5, 0.567, 2.98, 57.32], param_list), + ) + ) + l_normal_list.append( + spotpy.likelihoods.generalizedLikelihoodFunction( + data, + comparedata, + params=([-0.09, 1, 0.5, 0.567, 0.98, 101], param_list), + ) + ) + l_normal_list.append( + spotpy.likelihoods.generalizedLikelihoodFunction( + data, + comparedata, + params=([-0.09, 0.0, 0.5, 0.567, 0.98, 101], param_list), + ) + ) for l_normal in l_normal_list: self.assertNotEqual(None, l_normal) - self.assertEqual(type(np.float(l_normal)), type(np.float(1))) + self.assertEqual(type(float(l_normal)), type(float(1))) if self.do_print: print("generalizedLikelihoodFunction: " + str(l_normal)) - l_binom = spotpy.likelihoods.generalizedLikelihoodFunction(self.binom_data, self.binom_comparedata) + l_binom = spotpy.likelihoods.generalizedLikelihoodFunction( + self.binom_data, self.binom_comparedata + ) self.assertNotEqual(None, l_binom) self.assertGreaterEqual(-10000, l_binom) - self.assertEqual(type(np.float(l_binom)), type(np.float(1))) + self.assertEqual(type(float(l_binom)), type(float(1))) if self.do_print: print("generalizedLikelihoodFunction: " + str(l_binom)) def test_LaplacianLikelihood(self): - l_normal = spotpy.likelihoods.LaplacianLikelihood(self.normal_data, self.normal_comparedata) + l_normal = spotpy.likelihoods.LaplacianLikelihood( + self.normal_data, self.normal_comparedata + ) self.assertNotEqual(None, l_normal) - self.assertEqual(type(np.float(l_normal)), type(np.float(1))) + self.assertEqual(type(float(l_normal)), type(float(1))) if self.do_print: print("LaplacianLikelihood: " + str(l_normal)) - l_binom = spotpy.likelihoods.LaplacianLikelihood(self.binom_data, self.binom_comparedata) + l_binom = spotpy.likelihoods.LaplacianLikelihood( + self.binom_data, self.binom_comparedata + ) self.assertNotEqual(None, l_normal) - self.assertEqual(type(np.float(l_binom)), type(np.float(1))) + self.assertEqual(type(float(l_binom)), type(float(1))) if self.do_print: print("LaplacianLikelihood: " + str(l_binom)) def test_SkewedStudentLikelihoodHomoscedastic(self): - l_normal = spotpy.likelihoods.SkewedStudentLikelihoodHomoscedastic(self.normal_data, self.normal_comparedata) + l_normal = spotpy.likelihoods.SkewedStudentLikelihoodHomoscedastic( + self.normal_data, self.normal_comparedata + ) self.assertGreaterEqual(12, np.abs(l_normal)) - self.assertEqual(type(np.float(l_normal)), type(np.float(1))) + self.assertEqual(type(float(l_normal)), type(float(1))) if self.do_print: print("SkewedStudentLikelihoodHomoscedastic: " + str(l_normal)) - l_binom = spotpy.likelihoods.SkewedStudentLikelihoodHomoscedastic(self.binom_data, self.binom_comparedata) + l_binom = spotpy.likelihoods.SkewedStudentLikelihoodHomoscedastic( + self.binom_data, self.binom_comparedata + ) self.assertGreaterEqual(17, np.abs(l_binom)) - self.assertEqual(type(np.float(l_binom)), type(np.float(1))) + self.assertEqual(type(float(l_binom)), type(float(1))) if self.do_print: print("SkewedStudentLikelihoodHomoscedastic: " + str(l_binom)) @@ -191,37 +291,57 @@ def test_SkewedStudentLikelihoodHeteroscedastic(self): l_normal_list = [] paramDependencies = ["likelihood_nu", "likelihood_kappa", "likelihood_phi"] l_normal_list.append( - spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedastic(self.normal_data, self.normal_comparedata, - params=([2.4, 0.15, 0.87], paramDependencies))) + spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedastic( + self.normal_data, + self.normal_comparedata, + params=([2.4, 0.15, 0.87], paramDependencies), + ) + ) try: l_normal_list.append( - spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedastic(self.normal_data, self.normal_comparedata, - params=([], []))) + spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedastic( + self.normal_data, self.normal_comparedata, params=([], []) + ) + ) except LikelihoodError as e: print("An error occurred: " + str(e)) l_normal_list.append( - spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedastic(self.normal_data, self.normal_comparedata, - params=([1, 0.15, 1.87], paramDependencies))) + spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedastic( + self.normal_data, + self.normal_comparedata, + params=([1, 0.15, 1.87], paramDependencies), + ) + ) l_normal_list.append( - spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedastic(self.normal_data, self.normal_comparedata, - params=([1, 0.15, 0.87], paramDependencies))) + spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedastic( + self.normal_data, + self.normal_comparedata, + params=([1, 0.15, 0.87], paramDependencies), + ) + ) l_normal_list.append( - spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedastic(self.normal_data, self.normal_comparedata, - params=([1, -0.15, 0.87], paramDependencies))) + spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedastic( + self.normal_data, + self.normal_comparedata, + params=([1, -0.15, 0.87], paramDependencies), + ) + ) for l_normal in l_normal_list: if not np.isnan(l_normal): self.assertGreaterEqual(-100, l_normal) - self.assertEqual(type(np.float(l_normal)), type(np.float(1))) + self.assertEqual(type(float(l_normal)), type(float(1))) if self.do_print: print("SkewedStudentLikelihoodHeteroscedastic: " + str(l_normal)) - l_binom = spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedastic(self.binom_data, self.binom_comparedata) + l_binom = spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedastic( + self.binom_data, self.binom_comparedata + ) if not np.isnan(l_binom): self.assertGreaterEqual(-100, l_binom) - self.assertEqual(type(np.float(l_binom)), type(np.float(1))) + self.assertEqual(type(float(l_binom)), type(float(1))) if self.do_print: print("SkewedStudentLikelihoodHeteroscedastic: " + str(l_binom)) @@ -229,147 +349,204 @@ def test_SkewedStudentLikelihoodHeteroscedasticAdvancedARModel(self): l_normal_list = [] params = ["likelihood_nu", "likelihood_kappa", "likelihood_phi"] - l_normal_list.append(spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel( - self.normal_data, self.normal_comparedata, params=([4, 43, 0.4], params))) + l_normal_list.append( + spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel( + self.normal_data, self.normal_comparedata, params=([4, 43, 0.4], params) + ) + ) try: - l_normal_list.append(spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel( - self.normal_data, self.normal_comparedata, params=([], []))) + l_normal_list.append( + spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel( + self.normal_data, self.normal_comparedata, params=([], []) + ) + ) except LikelihoodError as e: print("Likelihood Error occurred " + str(e)) - l_normal_list.append(spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel( - self.normal_data, self.normal_comparedata, params=([4, 43, 2.4], params))) - l_normal_list.append(spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel( - self.normal_data, self.normal_comparedata, params=([1, 43, 0.4], params))) - l_normal_list.append(spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel( - self.normal_data, self.normal_comparedata, params=([4, -3, 0.4], params))) + l_normal_list.append( + spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel( + self.normal_data, self.normal_comparedata, params=([4, 43, 2.4], params) + ) + ) + l_normal_list.append( + spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel( + self.normal_data, self.normal_comparedata, params=([1, 43, 0.4], params) + ) + ) + l_normal_list.append( + spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel( + self.normal_data, self.normal_comparedata, params=([4, -3, 0.4], params) + ) + ) for l_normal in l_normal_list: self.assertNotEqual(None, l_normal) - self.assertEqual(type(np.float(l_normal)), type(np.float(1))) + self.assertEqual(type(float(l_normal)), type(float(1))) if self.do_print: - print("SkewedStudentLikelihoodHeteroscedasticAdvancedARModel: " + str(l_normal)) + print( + "SkewedStudentLikelihoodHeteroscedasticAdvancedARModel: " + + str(l_normal) + ) - l_binom = spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel( - self.normal_data, self.normal_comparedata) + l_binom = ( + spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel( + self.normal_data, self.normal_comparedata + ) + ) self.assertNotEqual(None, l_binom) - self.assertEqual(type(np.float(l_binom)), type(np.float(1))) + self.assertEqual(type(float(l_binom)), type(float(1))) if self.do_print: - print("SkewedStudentLikelihoodHeteroscedasticAdvancedARModel: " + str(l_binom)) + print( + "SkewedStudentLikelihoodHeteroscedasticAdvancedARModel: " + str(l_binom) + ) def test_NoisyABCGaussianLikelihood(self): - l_normal = spotpy.likelihoods.NoisyABCGaussianLikelihood(self.normal_data, self.normal_comparedata) + l_normal = spotpy.likelihoods.NoisyABCGaussianLikelihood( + self.normal_data, self.normal_comparedata + ) self.assertNotEqual(None, l_normal) - self.assertEqual(type(np.float(l_normal)), type(np.float(1))) + self.assertEqual(type(float(l_normal)), type(float(1))) if self.do_print: print("NoisyABCGaussianLikelihood: " + str(l_normal)) - l_binom = spotpy.likelihoods.NoisyABCGaussianLikelihood(self.binom_data, self.binom_data, - measerror=[0.0]) + l_binom = spotpy.likelihoods.NoisyABCGaussianLikelihood( + self.binom_data, self.binom_data, measerror=[0.0] + ) self.assertNotEqual(None, l_binom) - self.assertEqual(type(np.float(l_binom)), type(np.float(1))) + self.assertEqual(type(float(l_binom)), type(float(1))) if self.do_print: print("NoisyABCGaussianLikelihood: " + str(l_binom)) def test_ABCBoxcarLikelihood(self): - l_normal = spotpy.likelihoods.ABCBoxcarLikelihood(self.normal_data, self.normal_comparedata) + l_normal = spotpy.likelihoods.ABCBoxcarLikelihood( + self.normal_data, self.normal_comparedata + ) self.assertNotEqual(None, l_normal) self.assertNotEqual(np.nan, l_normal) - self.assertEqual(type(np.float(l_normal)), type(np.float(1))) + self.assertEqual(type(float(l_normal)), type(float(1))) if self.do_print: print("ABCBoxcarLikelihood: " + str(l_normal)) - l_binom = spotpy.likelihoods.ABCBoxcarLikelihood(self.binom_data, self.binom_comparedata) + l_binom = spotpy.likelihoods.ABCBoxcarLikelihood( + self.binom_data, self.binom_comparedata + ) self.assertNotEqual(None, l_binom) self.assertNotEqual(np.nan, l_binom) - self.assertEqual(type(np.float(l_binom)), type(np.float(1))) + self.assertEqual(type(float(l_binom)), type(float(1))) if self.do_print: print("ABCBoxcarLikelihood: " + str(l_binom)) def test_LimitsOfAcceptability(self): - l_normal = spotpy.likelihoods.LimitsOfAcceptability(self.normal_data, self.normal_comparedata) + l_normal = spotpy.likelihoods.LimitsOfAcceptability( + self.normal_data, self.normal_comparedata + ) self.assertEqual(12, l_normal) self.assertNotEqual(None, l_normal) - self.assertEqual(type(np.int(l_normal)), type(int(1))) + self.assertEqual(type(int(l_normal)), type(int(1))) if self.do_print: print("LimitsOfAcceptability: " + str(l_normal)) - l_binom = spotpy.likelihoods.LimitsOfAcceptability(self.binom_data, self.binom_comparedata) + l_binom = spotpy.likelihoods.LimitsOfAcceptability( + self.binom_data, self.binom_comparedata + ) self.assertEqual(5, l_binom) self.assertNotEqual(None, l_binom) - self.assertEqual(type(np.int(l_binom)), type(int(1))) + self.assertEqual(type(int(l_binom)), type(int(1))) if self.do_print: print("LimitsOfAcceptability: " + str(l_binom)) def test_InverseErrorVarianceShapingFactor(self): - l_normal = spotpy.likelihoods.InverseErrorVarianceShapingFactor(self.normal_data, self.normal_comparedata) + l_normal = spotpy.likelihoods.InverseErrorVarianceShapingFactor( + self.normal_data, self.normal_comparedata + ) self.assertGreaterEqual(-10, l_normal) - self.assertEqual(type(np.float(l_normal)), type(np.float(1))) + self.assertEqual(type(float(l_normal)), type(float(1))) if self.do_print: print("inverseErrorVarianceShapingFactor: " + str(l_normal)) - l_binom = spotpy.likelihoods.InverseErrorVarianceShapingFactor(self.binom_data, self.binom_comparedata) + l_binom = spotpy.likelihoods.InverseErrorVarianceShapingFactor( + self.binom_data, self.binom_comparedata + ) self.assertGreaterEqual(-10, l_binom) - self.assertEqual(type(np.float(l_binom)), type(np.float(1))) + self.assertEqual(type(float(l_binom)), type(float(1))) if self.do_print: print("inverseErrorVarianceShapingFactor: " + str(l_binom)) def test_ExponentialTransformErrVarShapingFactor(self): - l_binom = spotpy.likelihoods.ExponentialTransformErrVarShapingFactor(self.binom_data, self.binom_comparedata) + l_binom = spotpy.likelihoods.ExponentialTransformErrVarShapingFactor( + self.binom_data, self.binom_comparedata + ) self.assertGreaterEqual(-30, l_binom) - self.assertEqual(type(np.float(l_binom)), type(np.float(1))) + self.assertEqual(type(float(l_binom)), type(float(1))) if self.do_print: print("inverseErrorVarianceShapingFactor: " + str(l_binom)) - l_gauss = spotpy.likelihoods.ExponentialTransformErrVarShapingFactor(self.normal_data, self.normal_comparedata) + l_gauss = spotpy.likelihoods.ExponentialTransformErrVarShapingFactor( + self.normal_data, self.normal_comparedata + ) self.assertGreaterEqual(-30, l_gauss) - self.assertEqual(type(np.float(l_gauss)), type(np.float(1))) + self.assertEqual(type(float(l_gauss)), type(float(1))) if self.do_print: print("inverseErrorVarianceShapingFactor: " + str(l_gauss)) def test_NashSutcliffeEfficiencyShapingFactor(self): l_normal_list = [] - l_normal_list.append(spotpy.likelihoods.NashSutcliffeEfficiencyShapingFactor(self.normal_data, - self.normal_comparedata)) + l_normal_list.append( + spotpy.likelihoods.NashSutcliffeEfficiencyShapingFactor( + self.normal_data, self.normal_comparedata + ) + ) - l_normal_list.append(spotpy.likelihoods.NashSutcliffeEfficiencyShapingFactor(self.normal_data, - self.normal_data)) + l_normal_list.append( + spotpy.likelihoods.NashSutcliffeEfficiencyShapingFactor( + self.normal_data, self.normal_data + ) + ) - l_normal_list.append(spotpy.likelihoods.NashSutcliffeEfficiencyShapingFactor(self.binom_data, - self.binom_comparedata)) + l_normal_list.append( + spotpy.likelihoods.NashSutcliffeEfficiencyShapingFactor( + self.binom_data, self.binom_comparedata + ) + ) try: - l_normal_list.append(spotpy.likelihoods.NashSutcliffeEfficiencyShapingFactor([], - [])) + l_normal_list.append( + spotpy.likelihoods.NashSutcliffeEfficiencyShapingFactor([], []) + ) except LikelihoodError as e: print("Likelihood Error occurred: " + str(e)) try: - l_normal_list.append(spotpy.likelihoods.NashSutcliffeEfficiencyShapingFactor([1], - [])) + l_normal_list.append( + spotpy.likelihoods.NashSutcliffeEfficiencyShapingFactor([1], []) + ) except LikelihoodError as e: print("Likelihood Error occurred " + str(e)) for l_normal in l_normal_list: self.assertNotEqual(None, l_normal) - self.assertEqual(type(np.float(l_normal)), type(np.float(1))) + self.assertEqual(type(float(l_normal)), type(float(1))) if self.do_print: print("NashSutcliffeEfficiencyShapingFactor: " + str(l_normal)) def test_sumOfAbsoluteErrorResiduals(self): - l_normal = spotpy.likelihoods.sumOfAbsoluteErrorResiduals(self.normal_data, self.normal_comparedata) + l_normal = spotpy.likelihoods.sumOfAbsoluteErrorResiduals( + self.normal_data, self.normal_comparedata + ) self.assertGreaterEqual(7, np.abs(np.abs(l_normal) - 10)) - self.assertEqual(type(np.float(l_normal)), type(np.float(1))) + self.assertEqual(type(float(l_normal)), type(float(1))) if self.do_print: print("sumOfAbsoluteErrorResiduals: " + str(l_normal)) - l_binom = spotpy.likelihoods.sumOfAbsoluteErrorResiduals(self.binom_data, self.binom_comparedata) + l_binom = spotpy.likelihoods.sumOfAbsoluteErrorResiduals( + self.binom_data, self.binom_comparedata + ) self.assertGreaterEqual(7, np.abs(np.abs(l_binom) - 10)) - self.assertEqual(type(np.float(l_binom)), type(np.float(1))) + self.assertEqual(type(float(l_binom)), type(float(1))) if self.do_print: print("sumOfAbsoluteErrorResiduals: " + str(l_binom)) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_list_and_constant_parameters_with_samplers.py b/tests/test_list_and_constant_parameters_with_samplers.py index 744c22d3..95e28cc1 100644 --- a/tests/test_list_and_constant_parameters_with_samplers.py +++ b/tests/test_list_and_constant_parameters_with_samplers.py @@ -1,22 +1,18 @@ -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska -''' +""" +import inspect import unittest +from itertools import cycle + import numpy as np -import inspect -try: - import spotpy -except ImportError: - import sys - sys.path.append(".") - import spotpy +import spotpy from spotpy import parameter from spotpy.objectivefunctions import rmse -from itertools import cycle class Rosenbrock(object): @@ -28,7 +24,12 @@ class Rosenbrock(object): def simulation(self, vector): vector = np.array(vector) - simulations = [sum(100.0 * (vector[1:] - vector[:-1] ** 2.0) ** 2.0 + (1 - vector[:-1]) ** 2.0)] + simulations = [ + sum( + 100.0 * (vector[1:] - vector[:-1] ** 2.0) ** 2.0 + + (1 - vector[:-1]) ** 2.0 + ) + ] return simulations def evaluation(self): @@ -46,10 +47,18 @@ class RosenbrockWithConstant(Rosenbrock): Result at (1,1,1) is 0. """ - c = parameter.Constant(0, doc='Constant offset, should stay 0') - x = parameter.Uniform(-10, 10, 1.5, 3.0, -10, 10, doc='x value of Rosenbrock function') - y = parameter.Uniform(-10, 10, 1.5, 3.0, -10, 10, doc='y value of Rosenbrock function') - z = parameter.Uniform(-10, 10, 1.5, 3.0, -10, 10, doc='z value of Rosenbrock function') + + c = parameter.Constant(0, doc="Constant offset, should stay 0") + x = parameter.Uniform( + -10, 10, 1.5, 3.0, -10, 10, doc="x value of Rosenbrock function" + ) + y = parameter.Uniform( + -10, 10, 1.5, 3.0, -10, 10, doc="y value of Rosenbrock function" + ) + z = parameter.Uniform( + -10, 10, 1.5, 3.0, -10, 10, doc="z value of Rosenbrock function" + ) + class RosenbrockWithList(Rosenbrock): """ @@ -57,18 +66,25 @@ class RosenbrockWithList(Rosenbrock): Result at (1,1,1) is 0. """ - l = parameter.List(np.arange(0, 10), repeat=True, doc='list parameter for testing') - x = parameter.Uniform(-10, 10, 1.5, 3.0, -10, 10, doc='x value of Rosenbrock function') - y = parameter.Uniform(-10, 10, 1.5, 3.0, -10, 10, doc='y value of Rosenbrock function') - z = parameter.Uniform(-10, 10, 1.5, 3.0, -10, 10, doc='z value of Rosenbrock function') + l = parameter.List(np.arange(0, 10), repeat=True, doc="list parameter for testing") + x = parameter.Uniform( + -10, 10, 1.5, 3.0, -10, 10, doc="x value of Rosenbrock function" + ) + y = parameter.Uniform( + -10, 10, 1.5, 3.0, -10, 10, doc="y value of Rosenbrock function" + ) + z = parameter.Uniform( + -10, 10, 1.5, 3.0, -10, 10, doc="z value of Rosenbrock function" + ) def get_all_samplers(): def use(cl): # Check if class name starts with an exclusion term - return (inspect.isclass(cl) and - not any([cl.__name__.startswith(ex) for ex in ('_', 'list')])) + return inspect.isclass(cl) and not any( + [cl.__name__.startswith(ex) for ex in ("_", "list")] + ) return inspect.getmembers(spotpy.algorithms, use) @@ -81,18 +97,22 @@ def setUp(self): def sampler_with_constant(self, sampler_class): - sampler = sampler_class(self.setup_constant(), dbformat='ram', save_sim=False) + sampler = sampler_class(self.setup_constant(), dbformat="ram", save_sim=False) sampler.sample(self.rep) - self.assertTrue(all(line[1] == 0 for line in sampler.datawriter.ram), - msg='Parameter c == 0 not true in all lines with sampler {}'.format(sampler)) + self.assertTrue( + all(line[1] == 0 for line in sampler.datawriter.ram), + msg="Parameter c == 0 not true in all lines with sampler {}".format( + sampler + ), + ) def sampler_with_list(self, sampler_class, valid=False): - sampler = sampler_class(self.setup_list(), dbformat='ram', save_sim=False) + sampler = sampler_class(self.setup_list(), dbformat="ram", save_sim=False) sampler.sample(self.rep) - iterator = cycle(np.arange(0,10)) + iterator = cycle(np.arange(0, 10)) for i in range(self.rep): - self.assertEqual(sampler.datawriter.ram[i][1],next(iterator)) + self.assertEqual(sampler.datawriter.ram[i][1], next(iterator)) def test_abc_sampler(self): self.sampler_with_constant(spotpy.algorithms.abc) @@ -121,7 +141,7 @@ def test_lhs_sampler(self): def test_mc_sampler(self): self.sampler_with_constant(spotpy.algorithms.mc) - self.sampler_with_list(spotpy.algorithms.mc, valid = True) + self.sampler_with_list(spotpy.algorithms.mc, valid=True) def test_mcmc_sampler(self): self.sampler_with_constant(spotpy.algorithms.mcmc) @@ -148,5 +168,6 @@ def test_sceua_sampler(self): with self.assertRaises(TypeError): self.sampler_with_list(spotpy.algorithms.sceua) -if __name__ == '__main__': - unittest.main() \ No newline at end of file + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_objectivefunctions.py b/tests/test_objectivefunctions.py index ac8b29d9..012cde53 100644 --- a/tests/test_objectivefunctions.py +++ b/tests/test_objectivefunctions.py @@ -1,20 +1,17 @@ -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska, Benjamin Manns -''' +""" import unittest -try: - import spotpy -except ImportError: - import sys - sys.path.append(".") - import spotpy -from spotpy import objectivefunctions as of + import numpy as np -#https://docs.python.org/3/library/unittest.html +from spotpy import objectivefunctions as of + +# https://docs.python.org/3/library/unittest.html + class TestObjectiveFunctions(unittest.TestCase): @@ -54,8 +51,7 @@ def test_lognashsutcliffe_for_invalid_obs_is_nan(self): self.assertTrue(np.isnan(res)) def test_log_p_with_default_scale(self): - """If the mean of the evaluation function is <0.01, it gets reset to 0.01 - """ + """If the mean of the evaluation function is <0.01, it gets reset to 0.01""" res = of.log_p(self.evaluation, self.simulation) self.assertAlmostEqual(res, -13135.8578574, self.tolerance) @@ -74,20 +70,20 @@ def test_correlationcoefficient_for_perfect_positive_is_one(self): res = of.correlationcoefficient(self.evaluation, self.evaluation) self.assertAlmostEqual(res, 1, self.tolerance) - res = of.correlationcoefficient(self.evaluation, 2*self.evaluation) + res = of.correlationcoefficient(self.evaluation, 2 * self.evaluation) self.assertAlmostEqual(res, 1, self.tolerance) - res = of.correlationcoefficient(self.evaluation, 0.5*self.evaluation) + res = of.correlationcoefficient(self.evaluation, 0.5 * self.evaluation) self.assertAlmostEqual(res, 1, self.tolerance) def test_correlationcoefficient_for_perfect_negative_is_minus_one(self): res = of.correlationcoefficient(self.evaluation, -self.evaluation) self.assertAlmostEqual(res, -1, self.tolerance) - res = of.correlationcoefficient(self.evaluation, -2*self.evaluation) + res = of.correlationcoefficient(self.evaluation, -2 * self.evaluation) self.assertAlmostEqual(res, -1, self.tolerance) - res = of.correlationcoefficient(self.evaluation, -0.5*self.evaluation) + res = of.correlationcoefficient(self.evaluation, -0.5 * self.evaluation) self.assertAlmostEqual(res, -1, self.tolerance) def test_rsquared_random(self): @@ -99,19 +95,19 @@ def test_rsquared_perfect_corr_is_1(self): res = of.rsquared(self.evaluation, self.evaluation) self.assertAlmostEqual(res, 1, self.tolerance) - res = of.rsquared(self.evaluation, 2*self.evaluation) + res = of.rsquared(self.evaluation, 2 * self.evaluation) self.assertAlmostEqual(res, 1, self.tolerance) - res = of.rsquared(self.evaluation, 0.5*self.evaluation) + res = of.rsquared(self.evaluation, 0.5 * self.evaluation) self.assertAlmostEqual(res, 1, self.tolerance) res = of.rsquared(self.evaluation, -self.evaluation) self.assertAlmostEqual(res, 1, self.tolerance) - res = of.rsquared(self.evaluation, -2*self.evaluation) + res = of.rsquared(self.evaluation, -2 * self.evaluation) self.assertAlmostEqual(res, 1, self.tolerance) - res = of.rsquared(self.evaluation, -0.5*self.evaluation) + res = of.rsquared(self.evaluation, -0.5 * self.evaluation) self.assertAlmostEqual(res, 1, self.tolerance) def test_mse(self): @@ -147,7 +143,7 @@ def test_rrmse_with_self_is_zero(self): self.assertAlmostEqual(res, 0.0, self.tolerance) def test_rrmse_with_obs_mean_zero_is_inf(self): - #FIXME: Currently failing because rrmse returns np.inf + # FIXME: Currently failing because rrmse returns np.inf evaluation = [-5, -4, -3, -2, -1, 1, 2, 3, 4, 5] res = of.rrmse(evaluation, self.simulation) self.assertTrue(np.isinf(res)) @@ -177,7 +173,12 @@ def test_kge(self): self.assertAlmostEqual(res, -0.92083174734809159, self.tolerance) def test_kge_return_all(self): - expected = (-0.92083174734809159, -0.1105109772757096, 0.95721520413458061, -0.56669379018786747) + expected = ( + -0.92083174734809159, + -0.1105109772757096, + 0.95721520413458061, + -0.56669379018786747, + ) res = of.kge(self.evaluation, self.simulation, return_all=True) for exp, actual in zip(expected, res): self.assertAlmostEqual(actual, exp, self.tolerance) @@ -187,7 +188,12 @@ def test_kge_non_parametric(self): self.assertAlmostEqual(res, -0.84274521306792427, self.tolerance) def test_kge_non_parametric_return_all(self): - expected = (-0.8427452130679243, 0.030303030303030304, 0.970533493046538, -0.5666937901878675) + expected = ( + -0.8427452130679243, + 0.030303030303030304, + 0.970533493046538, + -0.5666937901878675, + ) res = of.kge_non_parametric(self.evaluation, self.simulation, return_all=True) for exp, actual in zip(expected, res): self.assertAlmostEqual(actual, exp, self.tolerance) @@ -213,8 +219,10 @@ def test_length_mismatch_return_nan(self): for func in all_funcs: res = func([0], [0, 1]) - self.assertTrue(np.isnan(res), "Expected np.nan in length mismatch, Got {}".format(res)) + self.assertTrue( + np.isnan(res), "Expected np.nan in length mismatch, Got {}".format(res) + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 58868643..91fb2378 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -1,45 +1,63 @@ # -*- coding: utf-8 -*- -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska -''' +""" import unittest + import spotpy -import numpy as np from spotpy.examples.spot_setup_rosenbrock import spot_setup -from spotpy.describe import describe -import os + class TestParallel(unittest.TestCase): @classmethod def setUpClass(self): # How many digits to match in case of floating point answers self.tolerance = 7 - #Create samplers for every algorithm: + # Create samplers for every algorithm: self.rep = 21 - self.timeout = 10 #Given in Seconds + self.timeout = 10 # Given in Seconds self.dbformat = "ram" def test_seq(self): - sampler=spotpy.algorithms.mc(spot_setup(),parallel='seq', dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.mc( + spot_setup(), + parallel="seq", + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep) results = sampler.getdata() self.assertEqual(len(results), self.rep) def test_mpc(self): - sampler=spotpy.algorithms.mc(spot_setup(),parallel='mpc', dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.mc( + spot_setup(), + parallel="mpc", + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep) results = sampler.getdata() self.assertEqual(len(results), self.rep) def test_umpc(self): - sampler=spotpy.algorithms.mc(spot_setup(),parallel='umpc', dbname='Rosen', dbformat=self.dbformat, sim_timeout=self.timeout) + sampler = spotpy.algorithms.mc( + spot_setup(), + parallel="umpc", + dbname="Rosen", + dbformat=self.dbformat, + sim_timeout=self.timeout, + ) sampler.sample(self.rep) results = sampler.getdata() self.assertEqual(len(results), self.rep) -if __name__ == '__main__': + +if __name__ == "__main__": unittest.main(exit=False) diff --git a/tests/test_parameter.py b/tests/test_parameter.py index 87e45fdd..12c63052 100644 --- a/tests/test_parameter.py +++ b/tests/test_parameter.py @@ -1,31 +1,48 @@ -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska, Philipp Kraft -''' +""" +# Import inspect to scan spotpy.parameter for all Parameter classes +import inspect import unittest -import sys -try: - import spotpy -except ImportError: - sys.path.append(".") - import spotpy -from spotpy import parameter + import numpy as np -# Import inspect to scan spotpy.parameter for all Parameter classes -import inspect -from .testutils import repeat -# https://docs.python.org/3/library/unittest.html +import spotpy +from spotpy import parameter -class TestListParameterDistribution(unittest.TestCase): +def repeat(times): + """Repeats a single test the given number of times + + Usage: + @repeat(5) + def test_foo(self): + self.assertTrue(self.bar == self.baz) + + The above test will execute 5 times + Reference: https://stackoverflow.com/a/13606054/4014685 + """ + + def repeatHelper(f): + def func_repeat_executor(*args, **kwargs): + for i in range(0, times): + f(*args, **kwargs) + args[0].setUp() + + return func_repeat_executor + + return repeatHelper + + +class TestListParameterDistribution(unittest.TestCase): def setUp(self): self.values = [1, 2, 3, 4, 5] - self.list_param = parameter.List('test', self.values) - self.list_param_repeat = parameter.List('test2', self.values, repeat=True) + self.list_param = parameter.List("test", self.values) + self.list_param_repeat = parameter.List("test2", self.values, repeat=True) def test_list_is_callable(self): self.assertTrue(callable(self.list_param), "List instance should be callable") @@ -103,37 +120,50 @@ def test_uniform_has_correct_statistics(self): unif = parameter.Uniform("test", 0, 1) # Generate 10k random numbers nums = [unif() for _ in range(10000)] - self.assertAlmostEqual(np.mean(nums), (1 - 0)/2.0, self.tolerance, "Mean of Unif(0, 1) should be 1/2") - self.assertAlmostEqual(np.var(nums), 1.0/12, self.tolerance, "Variance of Unif(0, 1) should be 1/12") + self.assertAlmostEqual( + np.mean(nums), + (1 - 0) / 2.0, + self.tolerance, + "Mean of Unif(0, 1) should be 1/2", + ) + self.assertAlmostEqual( + np.var(nums), + 1.0 / 12, + self.tolerance, + "Variance of Unif(0, 1) should be 1/12", + ) class TestConstantParameterDistribution(unittest.TestCase): - def setUp(self): self.const = parameter.Constant("test", 10) def test_constant_is_callable(self): - self.assertTrue(callable(self.const), "Constant param instance should be callable") + self.assertTrue( + callable(self.const), "Constant param instance should be callable" + ) def test_constant_gives_only_constants(self): nums = set([self.const() for _ in range(1000)]) - self.assertEqual(len(nums), 1, 'All results must be the same') - self.assertEqual(nums.pop(), 10, 'All results must be 10') + self.assertEqual(len(nums), 1, "All results must be the same") + self.assertEqual(nums.pop(), 10, "All results must be 10") def test_parameter_settings(self): p = parameter.generate([self.const])[0] - self.assertEqual(p['random'], 10, 'Random value must always be 10') - self.assertEqual(p['optguess'], 10, 'Optguess value must always be 10') - self.assertEqual(p['maxbound'], 10, 'maxbound value must always be 10') - self.assertEqual(p['minbound'], 10, 'minbound value must always be 10') - self.assertEqual(p['step'], 0, 'step value must always be 0') + self.assertEqual(p["random"], 10, "Random value must always be 10") + self.assertEqual(p["optguess"], 10, "Optguess value must always be 10") + self.assertEqual(p["maxbound"], 10, "maxbound value must always be 10") + self.assertEqual(p["minbound"], 10, "minbound value must always be 10") + self.assertEqual(p["step"], 0, "step value must always be 0") def test_find_constant_parameters(self): - flex = parameter.Uniform('flexible', 0, 1) + flex = parameter.Uniform("flexible", 0, 1) p = parameter.generate([flex, self.const]) constant_parameters = parameter.find_constant_parameters(p) - self.assertFalse(constant_parameters[0], 'Flexible parameter detected as constant') - self.assertTrue(constant_parameters[1], 'Constant parameter not detected') + self.assertFalse( + constant_parameters[0], "Flexible parameter detected as constant" + ) + self.assertTrue(constant_parameters[1], "Constant parameter not detected") class TestNormalParameterDistribution(unittest.TestCase): @@ -153,8 +183,12 @@ def test_normal_processes_non_keyword_args(self): @repeat(5) def test_normal_has_correct_statistics(self): nums = [self.norm() for _ in range(10000)] - self.assertAlmostEqual(np.mean(nums), 5, self.tolerance, "Mean of Norm(5, 10) should be 5") - self.assertAlmostEqual(np.std(nums), 10, self.tolerance, "SD of Norm(5, 10) should be 10") + self.assertAlmostEqual( + np.mean(nums), 5, self.tolerance, "Mean of Norm(5, 10) should be 5" + ) + self.assertAlmostEqual( + np.std(nums), 10, self.tolerance, "SD of Norm(5, 10) should be 10" + ) class TestLogNormalParameterDistribution(unittest.TestCase): @@ -166,7 +200,9 @@ def setUp(self): self.log_norm = parameter.logNormal("test", mean=5, sigma=10) def test_normal_is_callable(self): - self.assertTrue(callable(self.log_norm), "Log Normal param instance should be callable") + self.assertTrue( + callable(self.log_norm), "Log Normal param instance should be callable" + ) def test_normal_processes_non_keyword_args(self): _ = parameter.logNormal("test", 0, 1) @@ -175,8 +211,18 @@ def test_normal_processes_non_keyword_args(self): def test_normal_has_correct_statistics(self): nums = [self.log_norm() for _ in range(10000)] log_nums = np.log(nums) - self.assertAlmostEqual(np.mean(log_nums), 5, self.tolerance, "Mean of Log(LogNorm(5, 10)) should be 5") - self.assertAlmostEqual(np.std(log_nums), 10, self.tolerance, "SD of Log(LogNorm(5, 10)) should be 10") + self.assertAlmostEqual( + np.mean(log_nums), + 5, + self.tolerance, + "Mean of Log(LogNorm(5, 10)) should be 5", + ) + self.assertAlmostEqual( + np.std(log_nums), + 10, + self.tolerance, + "SD of Log(LogNorm(5, 10)) should be 10", + ) class TestChiSquareParameterDistribution(unittest.TestCase): @@ -188,7 +234,9 @@ def setUp(self): self.chisq = parameter.Chisquare("test", dt=self.df) def test_chisq_is_callable(self): - self.assertTrue(callable(self.chisq), "Chisquare param instance should be callable") + self.assertTrue( + callable(self.chisq), "Chisquare param instance should be callable" + ) def test_chisq_processes_non_keyword_args(self): _ = parameter.Chisquare("test", 5) @@ -196,10 +244,18 @@ def test_chisq_processes_non_keyword_args(self): @repeat(5) def test_chisq_has_correct_statistics(self): nums = [self.chisq() for _ in range(10000)] - self.assertAlmostEqual(np.mean(nums), self.df, self.tolerance, - "Mean of Chisquare({df}) should be {df}".format(df=self.df)) - self.assertAlmostEqual(np.std(nums), np.sqrt(2*self.df), self.tolerance, - "SD of Chisquare({df}) should be sqrt(2*{df})".format(df=self.df)) + self.assertAlmostEqual( + np.mean(nums), + self.df, + self.tolerance, + "Mean of Chisquare({df}) should be {df}".format(df=self.df), + ) + self.assertAlmostEqual( + np.std(nums), + np.sqrt(2 * self.df), + self.tolerance, + "SD of Chisquare({df}) should be sqrt(2*{df})".format(df=self.df), + ) class TestExponentialParameterDistribution(unittest.TestCase): @@ -211,7 +267,9 @@ def setUp(self): self.exp = parameter.Exponential("test", scale=self.beta) def test_exp_is_callable(self): - self.assertTrue(callable(self.exp), "Exponential param instance should be callable") + self.assertTrue( + callable(self.exp), "Exponential param instance should be callable" + ) def test_exp_processes_non_keyword_args(self): _ = parameter.Exponential("test", self.beta) @@ -219,10 +277,18 @@ def test_exp_processes_non_keyword_args(self): @repeat(5) def test_exp_has_correct_statistics(self): nums = [self.exp() for _ in range(10000)] - self.assertAlmostEqual(np.mean(nums), self.beta, self.tolerance, - "Mean of Exponential({beta}) should be {beta}".format(beta=self.beta)) - self.assertAlmostEqual(np.std(nums), self.beta, self.tolerance, - "SD of Exponential({beta}) should be {beta}".format(beta=self.beta)) + self.assertAlmostEqual( + np.mean(nums), + self.beta, + self.tolerance, + "Mean of Exponential({beta}) should be {beta}".format(beta=self.beta), + ) + self.assertAlmostEqual( + np.std(nums), + self.beta, + self.tolerance, + "SD of Exponential({beta}) should be {beta}".format(beta=self.beta), + ) class TestGammaParameterDistribution(unittest.TestCase): @@ -243,12 +309,24 @@ def test_gamma_processes_non_keyword_args(self): @repeat(5) def test_gamma_has_correct_statistics(self): nums = [self.gamma() for _ in range(10000)] - expected_mean = self.shape*self.scale - expected_sd = np.sqrt(self.shape*self.scale*self.scale) - self.assertAlmostEqual(np.mean(nums), expected_mean, self.tolerance, - "Mean of Gamma({}, {}) should be {}".format(self.shape, self.scale, expected_mean)) - self.assertAlmostEqual(np.std(nums), expected_sd, self.tolerance, - "SD of Gamma({}, {}) should be {}".format(self.shape, self.scale, expected_sd)) + expected_mean = self.shape * self.scale + expected_sd = np.sqrt(self.shape * self.scale * self.scale) + self.assertAlmostEqual( + np.mean(nums), + expected_mean, + self.tolerance, + "Mean of Gamma({}, {}) should be {}".format( + self.shape, self.scale, expected_mean + ), + ) + self.assertAlmostEqual( + np.std(nums), + expected_sd, + self.tolerance, + "SD of Gamma({}, {}) should be {}".format( + self.shape, self.scale, expected_sd + ), + ) class TestWaldParameterDistribution(unittest.TestCase): @@ -269,10 +347,22 @@ def test_wald_processes_non_keyword_args(self): def test_wald_has_correct_statistics(self): nums = [self.wald() for _ in range(40000)] expected_sd = np.sqrt(self.mean**3 / self.scale) - self.assertAlmostEqual(np.mean(nums), self.mean, self.tolerance, - "Mean of Wald({}, {}) should be {}".format(self.mean, self.scale, self.mean)) - self.assertAlmostEqual(np.std(nums), expected_sd, self.tolerance, - "SD of Wald({}, {}) should be {}".format(self.mean, self.scale, expected_sd)) + self.assertAlmostEqual( + np.mean(nums), + self.mean, + self.tolerance, + "Mean of Wald({}, {}) should be {}".format( + self.mean, self.scale, self.mean + ), + ) + self.assertAlmostEqual( + np.std(nums), + expected_sd, + self.tolerance, + "SD of Wald({}, {}) should be {}".format( + self.mean, self.scale, expected_sd + ), + ) class TestWeibullParameterDistribution(unittest.TestCase): @@ -284,17 +374,27 @@ def setUp(self): self.weibull = parameter.Weibull("test", a=self.a) def test_weibull_is_callable(self): - self.assertTrue(callable(self.weibull), "Weibull param instance should be callable") + self.assertTrue( + callable(self.weibull), "Weibull param instance should be callable" + ) def test_weibull_processes_non_keyword_args(self): _ = parameter.Weibull("test", self.a) def test_weibull_has_correct_statistics(self): nums = [self.weibull() for _ in range(10000)] - self.assertAlmostEqual(np.mean(nums), 0.918169, self.tolerance, - "Mean of Weibull({}) should be {}".format(self.a, 0.918169)) - self.assertAlmostEqual(np.std(nums), 0.0442300, self.tolerance, - "SD of Weibull({}) should be {}".format(self.a, 0.0442300)) + self.assertAlmostEqual( + np.mean(nums), + 0.918169, + self.tolerance, + "Mean of Weibull({}) should be {}".format(self.a, 0.918169), + ) + self.assertAlmostEqual( + np.std(nums), + 0.0442300, + self.tolerance, + "SD of Weibull({}) should be {}".format(self.a, 0.0442300), + ) class TestTriangularParameterDistribution(unittest.TestCase): @@ -303,21 +403,45 @@ class TestTriangularParameterDistribution(unittest.TestCase): def setUp(self): self.a, self.c, self.b = 0, 2, 5 - self.triangular = parameter.Triangular("test", left=self.a, mode=self.c, right=self.b) + self.triangular = parameter.Triangular( + "test", left=self.a, mode=self.c, right=self.b + ) def test_triangular_is_callable(self): - self.assertTrue(callable(self.triangular), "Triangular param instance should be callable") + self.assertTrue( + callable(self.triangular), "Triangular param instance should be callable" + ) def test_triangular_has_correct_statistics(self): nums = [self.triangular() for _ in range(10000)] expected_mean = (self.a + self.b + self.c) / 3 - expected_sd = np.sqrt((self.a**2 + self.b**2 + self.c**2 - self.a*self.c - self.a*self.b - self.b*self.c)/18) - self.assertAlmostEqual(np.mean(nums), expected_mean, self.tolerance, - "Mean of Triangular({}, {}, {}) should be {}" - .format(self.a, self.c, self.b, expected_mean)) - self.assertAlmostEqual(np.std(nums), expected_sd, self.tolerance, - "SD of Triangular({}, {}, {}) should be {}" - .format(self.a, self.c, self.b, expected_sd)) + expected_sd = np.sqrt( + ( + self.a**2 + + self.b**2 + + self.c**2 + - self.a * self.c + - self.a * self.b + - self.b * self.c + ) + / 18 + ) + self.assertAlmostEqual( + np.mean(nums), + expected_mean, + self.tolerance, + "Mean of Triangular({}, {}, {}) should be {}".format( + self.a, self.c, self.b, expected_mean + ), + ) + self.assertAlmostEqual( + np.std(nums), + expected_sd, + self.tolerance, + "SD of Triangular({}, {}, {}) should be {}".format( + self.a, self.c, self.b, expected_sd + ), + ) class TestParameterArguments(unittest.TestCase): @@ -339,36 +463,62 @@ def test_correct_with_default_step(self): for cl, args in zip(self.classes, self.rndargs): p_no_name = cl(*args) p_with_name = cl(cl.__name__, *args) - self.assertTrue(10 <= p_no_name.optguess < 20, 'Optguess out of boundaries') - self.assertTrue(10 <= p_with_name.optguess < 20, 'Optguess out of boundaries') - self.assertTrue(p_no_name.step < 10, 'Step to large') - self.assertTrue(p_with_name.step < 10, 'Step to large') + self.assertTrue(10 <= p_no_name.optguess < 20, "Optguess out of boundaries") + self.assertTrue( + 10 <= p_with_name.optguess < 20, "Optguess out of boundaries" + ) + self.assertTrue(p_no_name.step < 10, "Step to large") + self.assertTrue(p_with_name.step < 10, "Step to large") def test_correct_with_extra_args(self): for cl, args in zip(self.classes, self.rndargs): p_no_name = cl(*args, step=1, default=12) p_with_name = cl(cl.__name__, *args, step=1, default=12) - self.assertTrue(p_no_name.optguess == 12, - 'Optguess not found from default (name={})'.format(repr(p_no_name.name))) - self.assertTrue(p_with_name.optguess == 12, - 'Optguess not found from default (name={})'.format(repr(p_with_name.name))) - self.assertTrue(p_no_name.step == 1, - 'Step overridden by class (name={})'.format(repr(p_no_name.name))) - self.assertTrue(p_with_name.step == 1, - 'Step overridden by class (name={})'.format(repr(p_with_name.name))) + self.assertTrue( + p_no_name.optguess == 12, + "Optguess not found from default (name={})".format( + repr(p_no_name.name) + ), + ) + self.assertTrue( + p_with_name.optguess == 12, + "Optguess not found from default (name={})".format( + repr(p_with_name.name) + ), + ) + self.assertTrue( + p_no_name.step == 1, + "Step overridden by class (name={})".format(repr(p_no_name.name)), + ) + self.assertTrue( + p_with_name.step == 1, + "Step overridden by class (name={})".format(repr(p_with_name.name)), + ) def test_minbound_zero(self): - param = spotpy.parameter.Normal(name="parname", mean=0.6, stddev=0.2, optguess=0.6, minbound=0, maxbound=1) + param = spotpy.parameter.Normal( + name="parname", mean=0.6, stddev=0.2, optguess=0.6, minbound=0, maxbound=1 + ) self.assertEqual(param.minbound, 0) def test_maxbound_zero(self): - param = spotpy.parameter.Normal(name="parname", mean=-0.6, stddev=0.2, optguess=-0.6, minbound=-1, maxbound=0) + param = spotpy.parameter.Normal( + name="parname", + mean=-0.6, + stddev=0.2, + optguess=-0.6, + minbound=-1, + maxbound=0, + ) self.assertEqual(param.maxbound, 0) def test_optguess_zero(self): - param = spotpy.parameter.Normal(name="parname", mean=0.1, stddev=0.2, optguess=0.0, minbound=-1, maxbound=1) + param = spotpy.parameter.Normal( + name="parname", mean=0.1, stddev=0.2, optguess=0.0, minbound=-1, maxbound=1 + ) self.assertEqual(param.optguess, 0) + def make_args(pcls): """ Returns an args tuple for the parameter class pcls. @@ -379,16 +529,16 @@ def make_args(pcls): def get_classes(): """ Get all classes from spotpy.parameter module, except special cases - """ + """ + def predicate(cls): - return (inspect.isclass(cls) - and cls not in [parameter.Base, parameter.List] - and issubclass(cls, parameter.Base)) - - return [[cname, cls] - for cname, cls in - inspect.getmembers(parameter, predicate) - ] + return ( + inspect.isclass(cls) + and cls not in [parameter.Base, parameter.List] + and issubclass(cls, parameter.Base) + ) + + return [[cname, cls] for cname, cls in inspect.getmembers(parameter, predicate)] class TestParameterClasses(unittest.TestCase): @@ -398,8 +548,14 @@ class TestParameterClasses(unittest.TestCase): def test_classes_available(self): classes = get_classes() - self.assertGreaterEqual(len(classes), 1, 'No parameter classes found in spotpy.parameter') - self.assertIn('Uniform', [n for n, c in classes], 'Parameter class spotpy.parameter.Uniform not found') + self.assertGreaterEqual( + len(classes), 1, "No parameter classes found in spotpy.parameter" + ) + self.assertIn( + "Uniform", + [n for n, c in classes], + "Parameter class spotpy.parameter.Uniform not found", + ) def test_create_posargs(self): """ @@ -416,14 +572,17 @@ def test_create_posargs(self): # Check the names self.assertEqual(p_name.name, cname) # Check name is empty, when no name is given - self.assertEqual(p_no_name.name, '') + self.assertEqual(p_no_name.name, "") # Test Parameter character for both for p in [p_name, p_no_name]: self.assertTrue(callable(p)) self.assertGreater(p(), -np.inf) - self.assertTrue(p.step == 0.01, - '{} did not receive step as the correct value, check number of arguments' - .format(cls.__name__)) + self.assertTrue( + p.step == 0.01, + "{} did not receive step as the correct value, check number of arguments".format( + cls.__name__ + ), + ) def test_create_kwargs(self): """ @@ -438,14 +597,18 @@ def test_create_kwargs(self): p_no_name = cls(step=0.01, **kwargs) # Check the names self.assertEqual(p_name.name, cname) - self.assertEqual(p_no_name.name, '') + self.assertEqual(p_no_name.name, "") # Test Parameter character for both for p in [p_name, p_no_name]: self.assertTrue(callable(p)) self.assertGreater(p(), -np.inf) - self.assertEqual(p.step, 0.01, - '{} did not receive step as the correct value, check number of arguments' - .format(cls.__name__)) + self.assertEqual( + p.step, + 0.01, + "{} did not receive step as the correct value, check number of arguments".format( + cls.__name__ + ), + ) def test_too_many_args(self): """ @@ -453,7 +616,7 @@ def test_too_many_args(self): """ for cname, cls in get_classes(): # Implicit definition of step in args - step_args = make_args(cls) + (1, ) + step_args = make_args(cls) + (1,) with self.assertRaises(TypeError): _ = cls(*step_args, step=1) with self.assertRaises(TypeError): @@ -472,5 +635,5 @@ def test_too_few_args(self): _ = cls(cls.__name__, *args[:-1], step=1) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_setup_parameters.py b/tests/test_setup_parameters.py index 0f89ce4f..87ab237e 100644 --- a/tests/test_setup_parameters.py +++ b/tests/test_setup_parameters.py @@ -1,21 +1,19 @@ -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska, Philipp Kraft Tests the various possibilities to create and use parameters Focus especially the usage of parameters as class attributes -''' +""" +import inspect import sys import unittest -try: - import spotpy -except ImportError: - sys.path.append(".") - import spotpy -from spotpy import parameter + import numpy as np -import inspect + +import spotpy +from spotpy import parameter class SpotSetupBase(object): @@ -40,13 +38,14 @@ def get_derived(cls): Returns a list of all derived classes in this module """ module = sys.modules[__name__] + def predicate(mcls): return inspect.isclass(mcls) and issubclass(mcls, cls) and mcls is not cls + return [mcls for cname, mcls in inspect.getmembers(module, predicate)] def __repr__(self): - return '{}()'.format(type(self).__name__) - + return "{}()".format(type(self).__name__) class SpotSetupClassAttributes(SpotSetupBase): @@ -54,6 +53,7 @@ class SpotSetupClassAttributes(SpotSetupBase): A Test case with two parameters as class parameters (a,b) and 2 given from the parameter function """ + a = parameter.Uniform(-1, 1) b = parameter.Uniform(-1, 1) c = parameter.Uniform(-1, 1) @@ -64,26 +64,31 @@ class SpotSetupParameterFunction(SpotSetupBase): """ A Test case with 4 parameters given from the parameter function """ + def parameters(self): - return parameter.generate([parameter.Uniform(name, -1, 1) for name in 'abcd']) + return parameter.generate([parameter.Uniform(name, -1, 1) for name in "abcd"]) + class SpotSetupMixedParameterFunction(SpotSetupBase): """ A Test case with two parameters as class parameters (a,b) and 2 given from the parameter function """ + a = parameter.Uniform(0, 1) b = parameter.Uniform(1, 2) + def parameters(self): - return parameter.generate([parameter.Uniform(name, -1, 1) for name in 'cd']) + return parameter.generate([parameter.Uniform(name, -1, 1) for name in "cd"]) class SpotSetupParameterList(SpotSetupBase): """ A Test case with 4 parameters given from the parameter list """ + def __init__(self): - self.parameters = [parameter.Uniform(name, -1, 1) for name in 'abcd'] + self.parameters = [parameter.Uniform(name, -1, 1) for name in "abcd"] class SpotSetupMixedParameterList(SpotSetupBase): @@ -91,11 +96,12 @@ class SpotSetupMixedParameterList(SpotSetupBase): A Test case with two parameters as class parameters (a,b) and 2 given from the parameter function """ + a = parameter.Uniform(0, 1) b = parameter.Uniform(1, 2) def parameters(self): - return parameter.generate([parameter.Uniform(name, -1, 1) for name in 'cd']) + return parameter.generate([parameter.Uniform(name, -1, 1) for name in "cd"]) class TestParameterSet(unittest.TestCase): @@ -124,7 +130,7 @@ def test_iter(self): def test_getitem(self): values = [1] * len(self.ps) self.ps(*values) - self.assertEqual(self.ps['a'], 1.0) + self.assertEqual(self.ps["a"], 1.0) self.assertEqual(self.ps[0], 1.0) def test_getattr(self): @@ -135,7 +141,11 @@ def test_getattr(self): _ = self.ps.__x self.assertEqual(self.ps.a, 1.0) - self.assertEqual(list(self.ps.random), list(self.ps), 'Access to random variable does not equal list of names') + self.assertEqual( + list(self.ps.random), + list(self.ps), + "Access to random variable does not equal list of names", + ) with self.assertRaises(AttributeError): _ = self.ps.x @@ -150,19 +160,23 @@ def test_dir(self): attrs = dir(self.ps) for param in self.ps.name: - self.assertIn(param, attrs, 'Attribute {} not found in {}'.format(param, self.ps)) - for prop in ['maxbound', 'minbound', 'name', 'optguess', 'random', 'step']: - self.assertIn(prop, attrs, 'Property {} not found in {}'.format(prop, self.ps)) + self.assertIn( + param, attrs, "Attribute {} not found in {}".format(param, self.ps) + ) + for prop in ["maxbound", "minbound", "name", "optguess", "random", "step"]: + self.assertIn( + prop, attrs, "Property {} not found in {}".format(prop, self.ps) + ) def test_str(self): values = [1] * len(self.ps) self.ps(*values) - self.assertEqual(str(self.ps), 'parameters(a=1, b=1, c=1, d=1)') + self.assertEqual(str(self.ps), "parameters(a=1, b=1, c=1, d=1)") def test_repr(self): values = [1] * len(self.ps) self.ps(*values) - self.assertEqual(repr(self.ps), 'spotpy.parameter.ParameterSet()') + self.assertEqual(repr(self.ps), "spotpy.parameter.ParameterSet()") class TestSetupVariants(unittest.TestCase): @@ -174,15 +188,25 @@ def test_exists(self): self.assertGreater(len(self.objects), 0) def parameter_count_test(self, o): - params = parameter.create_set(o, valuetype='optguess') - param_names = ','.join(pn for pn in params.name) - self.assertEqual(len(params), 4, '{} should have 4 parameters, but found only {} ({})' - .format(o, len(params), param_names)) - self.assertEqual(param_names, 'a,b,c,d', '{} Parameter names should be "a,b,c,d" but got "{}"' - .format(o, param_names)) + params = parameter.create_set(o, valuetype="optguess") + param_names = ",".join(pn for pn in params.name) + self.assertEqual( + len(params), + 4, + "{} should have 4 parameters, but found only {} ({})".format( + o, len(params), param_names + ), + ) + self.assertEqual( + param_names, + "a,b,c,d", + '{} Parameter names should be "a,b,c,d" but got "{}"'.format( + o, param_names + ), + ) def make_sampler(self, o, algo=spotpy.algorithms.mc): - sampler = algo(spot_setup=o, dbformat='ram') + sampler = algo(spot_setup=o, dbformat="ram") sampler.sample(100) def test_parameter_class(self): @@ -193,10 +217,10 @@ def test_parameter_function(self): def test_parameter_list(self): self.parameter_count_test(SpotSetupParameterList()) - + def test_parameter_mixed_list(self): self.parameter_count_test(SpotSetupMixedParameterList()) - + def test_parameter_mixed_function(self): self.parameter_count_test(SpotSetupMixedParameterFunction()) @@ -217,7 +241,8 @@ def test_dream_sampler(self): self.make_sampler(o, spotpy.algorithms.dream) def test_fscabc_sampler(self): - for o in self.objects: self.make_sampler(o, spotpy.algorithms.fscabc) + for o in self.objects: + self.make_sampler(o, spotpy.algorithms.fscabc) def test_lhs_sampler(self): for o in self.objects: @@ -232,7 +257,8 @@ def test_mcmc_sampler(self): self.make_sampler(o, spotpy.algorithms.mcmc) def test_mle_sampler(self): - for o in self.objects: self.make_sampler(o, spotpy.algorithms.mle) + for o in self.objects: + self.make_sampler(o, spotpy.algorithms.mle) def test_rope_sampler(self): for o in self.objects: @@ -247,6 +273,5 @@ def test_sceua_sampler(self): self.make_sampler(o, spotpy.algorithms.sceua) -if __name__ == '_main__': +if __name__ == "_main__": unittest.main(verbosity=3) - diff --git a/tests/test_signatures.py b/tests/test_signatures.py index 802c2638..c52cf5d2 100644 --- a/tests/test_signatures.py +++ b/tests/test_signatures.py @@ -1,29 +1,23 @@ -''' +""" Copyright (c) 2018 by Tobias Houska This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). :author: Tobias Houska, Philipp Kraft -''' +""" import unittest + import numpy as np -try: - import spotpy -except ImportError: - import sys - sys.path.append(".") - import spotpy from spotpy.hydrology.signatures import SignatureMethod -import spotpy.hydrology as sig -class TestSignatures(unittest.TestCase): +class TestSignatures(unittest.TestCase): def setUp(self): np.random.seed(0) rain = np.random.normal(-1, 5, size=3650) rain[rain < 0] = 0.0 - runoff = np.zeros(rain.shape, np.float) + runoff = np.zeros(rain.shape, float) stor = 0.0 for i, prec in enumerate(rain): runoff[i] = 0.1 * stor @@ -38,8 +32,9 @@ def test_signatures(self): sig_result = SignatureMethod.run(sbm_list, self.runoff, 1) for name, value in sig_result: - self.assertNotEqual(value, np.nan, '{} returned no value'.format(name)) + self.assertNotEqual(value, np.nan, "{} returned no value".format(name)) + -if __name__ == '__main__': +if __name__ == "__main__": unittest.main(verbosity=3) diff --git a/tests/testutils.py b/tests/testutils.py deleted file mode 100644 index 2cfd67ef..00000000 --- a/tests/testutils.py +++ /dev/null @@ -1,29 +0,0 @@ -''' -Copyright (c) 2018 by Tobias Houska -This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). -:author: Tobias Houska, Benjamin Manns - -Utility functions for unit test execution -''' - -def repeat(times): - """Repeats a single test the given number of times - - Usage: - @repeat(5) - def test_foo(self): - self.assertTrue(self.bar == self.baz) - - The above test will execute 5 times - - Reference: https://stackoverflow.com/a/13606054/4014685 - """ - def repeatHelper(f): - def func_repeat_executor(*args, **kwargs): - for i in range(0, times): - f(*args, **kwargs) - args[0].setUp() - - return func_repeat_executor - - return repeatHelper diff --git a/tutorials/3dplot.py b/tutorials/3dplot.py new file mode 100644 index 00000000..341faf0f --- /dev/null +++ b/tutorials/3dplot.py @@ -0,0 +1,58 @@ +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +This file shows how to make 3d surface plots. +""" +import matplotlib.pyplot as plt +from matplotlib import cm +from matplotlib.ticker import FormatStrFormatter, LinearLocator +from mpl_toolkits.mplot3d import Axes3D +from mpl_toolkits.mplot3d.art3d import Line3DCollection, Poly3DCollection +from numpy import * + +import spotpy + +fig = plt.figure(figsize=(10, 10)) +ax = fig.gca(projection="3d") +# +# Plot Rosenbrock surface +X = arange(-30, 30, 0.05) +Y = arange(-30, 30, 0.05) +X, Y = meshgrid(X, Y) + +# from spot_setup_rosenbrock import spot_setup +# from spot_setup_griewank import spot_setup +from spotpy.examples.spot_setup_ackley import spot_setup + +Z = np.zeros(X.shape) +for i in range(X.shape[0]): + for j in range(X.shape[1]): + sim = spot_setup().simulation([X[i, j], Y[i, j]]) + like = spotpy.objectivefunctions.rmse(sim, [0]) + Z[i, j] = like + + +surf_Rosen = ax.plot_surface(X, Y, Z, rstride=5, linewidth=0, cmap=cm.rainbow) +ax.set_xlabel("x") +ax.set_ylabel("y") +ax.set_zlabel("RMSE") +plt.tight_layout() +plt.savefig("Griewank3d.tif", dpi=300) + + +# surf_Rosen = ax.plot_surface(X_Rosen, Y_Rosen, Z_Rosen, rstride=1, cstride=1, +# cmap=cm.coolwarm, linewidth=0, antialiased=False, alpha = 0.3) + +# Adjust axes +# ax.set_zlim(0, 600) +# ax.zaxis.set_major_locator(LinearLocator(5)) +# ax.zaxis.set_major_formatter(FormatStrFormatter('%.0f')) + +# Report minimum +# print 'Minimum location', v0_ori, '\nMinimum value', Rosenbrock(v0_ori), '\nNumber of function evaluations', f_evals + +# Render plot +plt.show() diff --git a/spotpy/examples/MyOwnDatabase.txt b/tutorials/MyOwnDatabase.txt similarity index 100% rename from spotpy/examples/MyOwnDatabase.txt rename to tutorials/MyOwnDatabase.txt diff --git a/spotpy/examples/NSGA2.csv b/tutorials/NSGA2.csv similarity index 100% rename from spotpy/examples/NSGA2.csv rename to tutorials/NSGA2.csv diff --git a/spotpy/examples/cli_hymod.py b/tutorials/cli_hymod.py similarity index 67% rename from spotpy/examples/cli_hymod.py rename to tutorials/cli_hymod.py index 082edfd0..82be868a 100644 --- a/spotpy/examples/cli_hymod.py +++ b/tutorials/cli_hymod.py @@ -3,12 +3,10 @@ """ -from __future__ import division, print_function, unicode_literals from spotpy.cli import main from spotpy.examples.spot_setup_hymod_python import spot_setup - -if __name__ == '__main__': +if __name__ == "__main__": setup = spot_setup() main(setup) diff --git a/tutorials/dds/__init__.py b/tutorials/dds/__init__.py new file mode 100644 index 00000000..f107b27a --- /dev/null +++ b/tutorials/dds/__init__.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" +Copyright (c) 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +:paper: Houska, T., Kraft, P., Chamorro-Chavez, A. and Breuer, L.: +SPOTting Model Parameters Using a Ready-Made Python Package, +PLoS ONE, 10(12), e0145180, doi:10.1371/journal.pone.0145180, 2015. + +This package enables the comprehensive use of different Bayesian and Heuristic calibration +techniques in one Framework. It comes along with an algorithms folder for the +sampling and an analyser class for the plotting of results by the sampling. + +:dependencies: - Numpy >1.8 (http://www.numpy.org/) + - Pandas >0.13 (optional) (http://pandas.pydata.org/) + - Matplotlib >1.4 (optional) (http://matplotlib.org/) + - CMF (optional) (http://fb09-pasig.umwelt.uni-giessen.de:8000/) + - mpi4py (optional) (http://mpi4py.scipy.org/) + - pathos (optional) (https://pypi.python.org/pypi/pathos/) + - sqlite3 (optional) (https://pypi.python.org/pypi/sqlite3/) + + :help: For specific questions, try to use the documentation website at: + http://fb09-pasig.umwelt.uni-giessen.de/spotpy/ + +For general things about parameter optimization techniques have a look at: +https://github.com/CamDavidsonPilon/Probabilistic-Programming-and-Bayesian-Methods-for-Hackers/tree/master/ + +Please cite our paper, if you are using SPOTPY. +""" diff --git a/spotpy/examples/dds/benchmark_dds.py b/tutorials/dds/benchmark_dds.py similarity index 54% rename from spotpy/examples/dds/benchmark_dds.py rename to tutorials/dds/benchmark_dds.py index 16e928af..5eb1173e 100644 --- a/spotpy/examples/dds/benchmark_dds.py +++ b/tutorials/dds/benchmark_dds.py @@ -1,13 +1,9 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from pprint import pprint -import numpy as np -import matplotlib.pylab as plt import json - import time +from pprint import pprint + +import matplotlib.pylab as plt +import numpy as np try: import spotpy @@ -19,15 +15,13 @@ from spotpy.examples.spot_setup_hymod_python import spot_setup - - spot_setup = spot_setup() # Create samplers for every algorithm: results = [] benchmarks_dict = [] -benchmarks_duration = {"dds":[], "sceua":[], "dds_like":[],"sceua_like":[]} +benchmarks_duration = {"dds": [], "sceua": [], "dds_like": [], "sceua_like": []} reps = [300, 1000, 3000, 4000, 5000, 10000] @@ -39,34 +33,47 @@ dbformat = "csv" start = time.time() - dds_sampler = spotpy.algorithms.DDS(spot_setup, parallel=parallel, dbname='DDS', dbformat=dbformat, sim_timeout=timeout) + dds_sampler = spotpy.algorithms.DDS( + spot_setup, + parallel=parallel, + dbname="DDS", + dbformat=dbformat, + sim_timeout=timeout, + ) dds_sampler.sample(rep, trials=1) results.append(dds_sampler.getdata()) dds_elapsed = time.time() - start start = time.time() - sceua_sampler = spotpy.algorithms.sceua(spot_setup, parallel=parallel, dbname='SCEUA', dbformat=dbformat, - sim_timeout=timeout, alt_objfun=None) + sceua_sampler = spotpy.algorithms.sceua( + spot_setup, + parallel=parallel, + dbname="SCEUA", + dbformat=dbformat, + sim_timeout=timeout, + alt_objfun=None, + ) sceua_sampler.sample(rep) results.append(sceua_sampler.getdata()) sceua_elapsed = time.time() - start - print("#########################################") - #print(dds_elapsed, dds_sampler.status.params) + # print(dds_elapsed, dds_sampler.status.params) print(sceua_elapsed, sceua_sampler.status.params) - benchmarks_dict.append({ - "rep": rep, - "dds_time": dds_elapsed, - "sceua_time": sceua_elapsed, - "dds_like": dds_sampler.status.objectivefunction, - "sceua_like": sceua_sampler.status.objectivefunction, - "dds_param": list(dds_sampler.status.params), - "sceua_param": list(sceua_sampler.status.params) - }) + benchmarks_dict.append( + { + "rep": rep, + "dds_time": dds_elapsed, + "sceua_time": sceua_elapsed, + "dds_like": dds_sampler.status.objectivefunction, + "sceua_like": sceua_sampler.status.objectivefunction, + "dds_param": list(dds_sampler.status.params), + "sceua_param": list(sceua_sampler.status.params), + } + ) benchmarks_duration["dds"].append(dds_elapsed) benchmarks_duration["sceua"].append(sceua_elapsed) benchmarks_duration["sceua_like"].append(sceua_sampler.status.objectivefunction) @@ -81,10 +88,13 @@ def autolabel(rects): """ for rect in rects: height = rect.get_height() - ax.text(rect.get_x() + rect.get_width()/2., 1.05*height, - '%d' % int(height), - ha='center', va='bottom') - + ax.text( + rect.get_x() + rect.get_width() / 2.0, + 1.05 * height, + "%d" % int(height), + ha="center", + va="bottom", + ) fig = plt.figure(figsize=(10, 6)) @@ -95,12 +105,13 @@ def autolabel(rects): X = np.arange(len(benchmarks_duration["dds"])) -dds_plot = ax.bar(x_pos, benchmarks_duration["dds_like"], color = 'b', width = 0.45) -sceua_plot = ax.bar([j+0.45 for j in x_pos], benchmarks_duration["sceua_like"], color = 'g', width = 0.45) - -#dds_plot = ax.bar(x_pos, benchmarks_duration["dds"], color = 'b', width = 0.45) -#sceua_plot = ax.bar([j+0.45 for j in x_pos], benchmarks_duration["sceua"], color = 'g', width = 0.45) +dds_plot = ax.bar(x_pos, benchmarks_duration["dds_like"], color="b", width=0.45) +sceua_plot = ax.bar( + [j + 0.45 for j in x_pos], benchmarks_duration["sceua_like"], color="g", width=0.45 +) +# dds_plot = ax.bar(x_pos, benchmarks_duration["dds"], color = 'b', width = 0.45) +# sceua_plot = ax.bar([j+0.45 for j in x_pos], benchmarks_duration["sceua"], color = 'g', width = 0.45) plt.xticks(x_pos, rep_labels) @@ -114,4 +125,3 @@ def autolabel(rects): plt.show() plt.savefig("MPI_TEST") # - diff --git a/spotpy/examples/dds/dds_parallel.py b/tutorials/dds/dds_parallel.py similarity index 72% rename from spotpy/examples/dds/dds_parallel.py rename to tutorials/dds/dds_parallel.py index 15af96d0..20f894c3 100644 --- a/spotpy/examples/dds/dds_parallel.py +++ b/tutorials/dds/dds_parallel.py @@ -1,14 +1,11 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -import numpy as np -import sys -import os -import matplotlib.pylab as plt import json +import os +import sys import time +import matplotlib.pylab as plt +import numpy as np + try: import spotpy except ImportError: @@ -32,7 +29,9 @@ parallel = "mpi" dbformat = "csv" start = time.time() -dds_sampler = spotpy.algorithms.dds(spot_setup, parallel=parallel, dbname='DDS', dbformat=dbformat, sim_timeout=timeout) +dds_sampler = spotpy.algorithms.dds( + spot_setup, parallel=parallel, dbname="DDS", dbformat=dbformat, sim_timeout=timeout +) dds_sampler.sample(rep, trials=1) dds_elapsed = time.time() - start print(dds_elapsed) @@ -43,4 +42,4 @@ print(benchmarks_duration) -json.dump(benchmarks_duration, open(json_path,"w")) \ No newline at end of file +json.dump(benchmarks_duration, open(json_path, "w")) diff --git a/spotpy/examples/dds/dds_parallel_data.json b/tutorials/dds/dds_parallel_data.json similarity index 100% rename from spotpy/examples/dds/dds_parallel_data.json rename to tutorials/dds/dds_parallel_data.json diff --git a/tutorials/dds/dds_parallel_plot.py b/tutorials/dds/dds_parallel_plot.py new file mode 100644 index 00000000..3a2ec8a0 --- /dev/null +++ b/tutorials/dds/dds_parallel_plot.py @@ -0,0 +1,71 @@ +import json + +import matplotlib as mp +import matplotlib.pylab as plt +import numpy as np + +data_normalizer = mp.colors.Normalize() +color_map = mp.colors.LinearSegmentedColormap( + "my_map", + { + "red": [(0, 1.0, 1.0), (1.0, 0.5, 0.5)], + "green": [(0, 0.5, 0.5), (1.0, 0, 0)], + "blue": [(0, 0.50, 0.5), (1.0, 0, 0)], + }, +) + + +def autolabel(ax, rects): + """ + Attach a text label above each bar displaying its height + """ + for rect in rects: + height = rect.get_height() + ax.text( + rect.get_x() + rect.get_width() / 2.0, + 1.05 * height, + "%f" % height, + ha="center", + va="bottom", + ) + + +def subplot(data, name, ylabel): + fig = plt.figure(figsize=(20, 6)) + ax = plt.subplot(111) + rep_labels = [str(j) for j in reps] + x_pos = [i for i, _ in enumerate(rep_labels)] + X = np.arange(len(data)) + ax_plot = ax.bar(x_pos, data, color=color_map(data_normalizer(data)), width=0.45) + + plt.xticks(x_pos, rep_labels) + plt.xlabel("Repetitions") + plt.ylabel(ylabel) + + autolabel(ax, ax_plot) + plt.savefig(name + ".png") + + +# fmt: off +parallel_data = json.loads( + '{"dds_duration": [' + '1.1293659210205078, 3.254117250442505, 4.888171672821045, 18.719818592071533, ' + '34.56907820701599, 169.47716689109802, 337.86882615089417, 1644.955144405365, ' + '3348.948029756546], ' + '"rep": [30, 500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000], ' + '"dds_like": [' + '-8384.884435178812, -8269.480874403698, -8268.453892284442, -8268.51195094138, ' + '-8269.65509041187, -8268.1421690868, -8267.791798085422, -8267.79178644684, -8268.141980514703]}' +) +# fmt: on +reps = parallel_data["rep"] +subplot( + parallel_data["dds_duration"], + "DDS_PARALLEL_DURATION_all", + "Duration of Run in Seconds", +) +subplot( + parallel_data["dds_like"], + "DDS_PARALLEL_OBJECTIVEFUNCTION_all", + "Best Objective Function Value", +) diff --git a/spotpy/examples/dds/dds_parallel_run.py b/tutorials/dds/dds_parallel_run.py similarity index 73% rename from spotpy/examples/dds/dds_parallel_run.py rename to tutorials/dds/dds_parallel_run.py index c97126d2..047b532a 100644 --- a/spotpy/examples/dds/dds_parallel_run.py +++ b/tutorials/dds/dds_parallel_run.py @@ -1,8 +1,9 @@ -import subprocess import os +import subprocess + path = os.path.abspath(os.path.dirname(__file__)) -for r in [500,1000,5000,10000,50000,100000,500000]: +for r in [500, 1000, 5000, 10000, 50000, 100000, 500000]: args = ["mpirun", "-c 6", "python", path + "/dds_parallel.py", str(r)] print(args) subprocess.run(args) - exit(8) \ No newline at end of file + exit(8) diff --git a/tutorials/getting_started.py b/tutorials/getting_started.py new file mode 100644 index 00000000..446835a9 --- /dev/null +++ b/tutorials/getting_started.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +This class holds the example code from the getting_started web-documention. +""" +# Getting started + +# To start your experience with SPOT you need to have SPOT installed. Please see the [Installation chapter](index.md) for further details. +# To use SPOT we have to import it and use one of the pre-build examples: +import spotpy # Load the SPOT package into your working storage +from spotpy.examples.spot_setup_rosenbrock import ( + spot_setup, # Import the two dimensional Rosenbrock example +) + +# The example comes along with parameter boundaries, the Rosenbrock function, the optimal value of the function and RMSE as a likelihood. +# So we can directly start to analyse the Rosenbrock function with one of the algorithms. We start with a simple Monte Carlo sampling: +if __name__ == "__main__": + # Give Monte Carlo algorithm the example setup and saves results in a RosenMC.csv file + # spot_setup.slow = True + sampler = spotpy.algorithms.mc(spot_setup(), dbname="RosenMC", dbformat="ram") + + # Now we can sample with the implemented Monte Carlo algortihm: + sampler.sample(10000) # Sample 100.000 parameter combinations + results = sampler.getdata() + # Now we want to have a look at the results. First we want to know, what the algorithm has done during the 10.000 iterations: + # spot.analyser.plot_parametertrace(results) # Use the analyser to show the parameter trace + spotpy.analyser.plot_parameterInteraction(results) + posterior = spotpy.analyser.get_posterior(results) + spotpy.analyser.plot_parameterInteraction(posterior) + # spotpy.analyser.plot_posterior_parametertrace(results, threshold=0.9) + + print(spotpy.analyser.get_best_parameterset(results)) diff --git a/tutorials/gui_hymod.py b/tutorials/gui_hymod.py new file mode 100644 index 00000000..a9f8d7f5 --- /dev/null +++ b/tutorials/gui_hymod.py @@ -0,0 +1,30 @@ +""" +Shows the usage of the matplotlib GUI + +Needs at least Python 3.5 +""" + + +import spotpy +from spotpy.examples.spot_setup_hymod_python import spot_setup +from spotpy.gui.mpl import GUI +from spotpy.objectivefunctions import rmse + +if __name__ == "__main__": + setup_class = spot_setup(rmse) + + # Select number of maximum allowed repetitions + rep = 10000 + + # Create the SCE-UA sampler of spotpy, alt_objfun is set to None to force SPOTPY + # to jump into the def objectivefunction in the spot_setup class (default is + # spotpy.objectivefunctions.rmse) + sampler = spotpy.algorithms.sceua( + setup_class, dbname="SCEUA_hymod", dbformat="csv", alt_objfun=None + ) + + # Start the sampler, one can specify ngs, kstop, peps and pcento id desired + # sampler.sample(rep,ngs=10, kstop=50, peps=0.1, pcento=0.1) + + gui = GUI(spot_setup()) + gui.show() diff --git a/spotpy/examples/plot_nsgaii_tutorial.py b/tutorials/plot_nsgaii_tutorial.py similarity index 50% rename from spotpy/examples/plot_nsgaii_tutorial.py rename to tutorials/plot_nsgaii_tutorial.py index ceae0593..98f8b5dd 100644 --- a/spotpy/examples/plot_nsgaii_tutorial.py +++ b/tutorials/plot_nsgaii_tutorial.py @@ -1,10 +1,9 @@ -import numpy as np import sys + import matplotlib.pyplot as plt -from mpl_toolkits.mplot3d import Axes3D +import numpy as np import pandas as pd - - +from mpl_toolkits.mplot3d import Axes3D # user config @@ -15,7 +14,7 @@ first = None n_pop = 50 -# output calibration +# output calibration df = pd.read_csv("NSGA2.csv") @@ -23,28 +22,26 @@ if last: - df = df.iloc[-last:,:] + df = df.iloc[-last:, :] elif first: - df = df.iloc[:first,:] + df = df.iloc[:first, :] else: pass - # plot objective functions fig = plt.figure() -for i,name in enumerate(df.columns[:n_obj]): - ax = fig.add_subplot(n_obj,1,i +1) - df.loc[::5,name].plot(lw=0.5,figsize=(18,8),ax = ax,color="black") +for i, name in enumerate(df.columns[:n_obj]): + ax = fig.add_subplot(n_obj, 1, i + 1) + df.loc[::5, name].plot(lw=0.5, figsize=(18, 8), ax=ax, color="black") plt.title(name) plt.show() - -x,y,z = df.iloc[-n_pop:,0],df.iloc[-n_pop:,1],df.iloc[-n_pop:,2] +x, y, z = df.iloc[-n_pop:, 0], df.iloc[-n_pop:, 1], df.iloc[-n_pop:, 2] fig = plt.figure() -ax = fig.add_subplot(111, projection='3d') -ax.scatter(x,y,z,marker="o") +ax = fig.add_subplot(111, projection="3d") +ax.scatter(x, y, z, marker="o") ax.set_xlabel("pbias") ax.set_ylabel("rmse") ax.set_zlabel("rsquared") @@ -52,11 +49,8 @@ # plot parameters fig = plt.figure() -for i,name in enumerate(df.columns[n_obj:8]): - ax = fig.add_subplot(5,1,i +1) - df.loc[:,name].plot(lw=0.5,figsize=(18,8),ax = ax,color="black") +for i, name in enumerate(df.columns[n_obj:8]): + ax = fig.add_subplot(5, 1, i + 1) + df.loc[:, name].plot(lw=0.5, figsize=(18, 8), ax=ax, color="black") plt.title(name) plt.show() - - - diff --git a/tutorials/tutorial_Parameterlist_iterator.py b/tutorials/tutorial_Parameterlist_iterator.py new file mode 100644 index 00000000..670e5439 --- /dev/null +++ b/tutorials/tutorial_Parameterlist_iterator.py @@ -0,0 +1,52 @@ +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +This example implements the Rosenbrock function into SPOT. +""" +import numpy as np + +import spotpy + + +class spot_setup(object): + def __init__(self): + self.slow = 1000 + self.params = [ + spotpy.parameter.List( + "x", [1, 2, 3, 4, 5, 6, 7, 8, 9, 0] + ), # Give possible x values as a List + spotpy.parameter.List( + "y", [0, 1, 2, 5, 6, 7, 8, 9, 0, 1] + ), # Give possible y values as a List + ] + + def parameters(self): + return spotpy.parameter.generate(self.params) + + def simulation(self, vector): + x = np.array(vector) + for i in range(self.slow): + s = np.sin(i) + simulations = [ + sum(100.0 * (x[1:] - x[:-1] ** 2.0) ** 2.0 + (1 - x[:-1]) ** 2.0) + ] + return simulations + + def evaluation(self): + observations = [0] + return observations + + def objectivefunction(self, simulation, evaluation): + objectivefunction = -spotpy.objectivefunctions.rmse(evaluation, simulation) + return objectivefunction + + +sampler = spotpy.algorithms.mc( + spot_setup(), dbname="Iterator_example", dbformat="csv" +) # Parameter lists can be sampled with MC +sampler.sample( + 10 +) # Choose equaly or less repetitions as you have parameters in your List diff --git a/tutorials/tutorial_ackley.py b/tutorials/tutorial_ackley.py new file mode 100644 index 00000000..02223561 --- /dev/null +++ b/tutorials/tutorial_ackley.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +This class holds the example code from the ackley tutorial web-documention. +""" +import spotpy +from spotpy.examples.spot_setup_ackley import spot_setup + +# Create samplers for every algorithm: +results = [] +spot_setup = spot_setup() +rep = 5000 + +sampler = spotpy.algorithms.mc(spot_setup, dbname="ackleyMC", dbformat="csv") +sampler.sample(rep) +results.append(sampler.getdata()) +# +sampler = spotpy.algorithms.lhs(spot_setup, dbname="ackleyLHS", dbformat="csv") +sampler.sample(rep) +results.append(sampler.getdata()) + +sampler = spotpy.algorithms.mle(spot_setup, dbname="ackleyMLE", dbformat="csv") +sampler.sample(rep) +results.append(sampler.getdata()) + +sampler = spotpy.algorithms.mcmc(spot_setup, dbname="ackleyMCMC", dbformat="csv") +sampler.sample(rep) +results.append(sampler.getdata()) + +sampler = spotpy.algorithms.sceua(spot_setup, dbname="ackleySCEUA", dbformat="csv") +sampler.sample(rep, ngs=2) +results.append(sampler.getdata()) + +sampler = spotpy.algorithms.sa(spot_setup, dbname="ackleySA", dbformat="csv") +sampler.sample(rep) +results.append(sampler.getdata()) + +sampler = spotpy.algorithms.demcz(spot_setup, dbname="ackleyDEMCz", dbformat="csv") +sampler.sample(rep, nChains=30) +results.append(sampler.getdata()) +# +sampler = spotpy.algorithms.rope(spot_setup, dbname="ackleyROPE", dbformat="csv") +sampler.sample(rep) +results.append(sampler.getdata()) + + +algorithms = ["MC", "LHS", "MLE", "MCMC", "SCEUA", "SA", "DEMCz", "ROPE"] +results = [] +for algorithm in algorithms: + results.append(spotpy.analyser.load_csv_results("ackley" + algorithm)) + + +evaluation = spot_setup.evaluation() + +spotpy.analyser.plot_objectivefunctiontraces(results, evaluation, algorithms) diff --git a/spotpy/examples/tutorial_cmf_lumped.py b/tutorials/tutorial_cmf_lumped.py similarity index 58% rename from spotpy/examples/tutorial_cmf_lumped.py rename to tutorials/tutorial_cmf_lumped.py index a79904fa..9a49cd83 100644 --- a/spotpy/examples/tutorial_cmf_lumped.py +++ b/tutorials/tutorial_cmf_lumped.py @@ -5,21 +5,20 @@ """ -from __future__ import division, print_function, unicode_literals -import sys import datetime - -import spotpy import os +import sys +import spotpy from spotpy.examples.spot_setup_cmf_lumped import SingleStorage + def parallel(): """ Returns 'mpi', if this code runs with MPI, else returns 'seq' :return: """ - return 'mpi' if 'OMPI_COMM_WORLD_SIZE' in os.environ else 'seq' + return "mpi" if "OMPI_COMM_WORLD_SIZE" in os.environ else "seq" def get_runs(default=1): @@ -29,9 +28,9 @@ def get_runs(default=1): :return: int """ # Get number of runs - if 'SPOTPYRUNS' in os.environ: + if "SPOTPYRUNS" in os.environ: # from environment - return int(os.environ['SPOTPYRUNS']) + return int(os.environ["SPOTPYRUNS"]) elif len(sys.argv) > 1: # from command line return int(sys.argv[1]) @@ -40,21 +39,23 @@ def get_runs(default=1): return default -if __name__ == '__main__': +if __name__ == "__main__": # Create the model - model = SingleStorage(datetime.datetime(1980, 1, 1), - datetime.datetime(1985, 12, 31)) + model = SingleStorage( + datetime.datetime(1980, 1, 1), datetime.datetime(1985, 12, 31) + ) runs = get_runs(default=100) # Create the sampler - sampler = spotpy.algorithms.mc(model, - parallel=parallel(), - dbname=model.dbname, dbformat='csv', - save_sim=True, sim_timeout=300) + sampler = spotpy.algorithms.mc( + model, + parallel=parallel(), + dbname=model.dbname, + dbformat="csv", + save_sim=True, + sim_timeout=300, + ) # Now we can sample with the implemented Monte Carlo algorithm: sampler.sample(runs) - - - diff --git a/spotpy/examples/tutorial_dds_hymod.py b/tutorials/tutorial_dds_hymod.py similarity index 51% rename from spotpy/examples/tutorial_dds_hymod.py rename to tutorials/tutorial_dds_hymod.py index 7eb52eb2..12c530c1 100644 --- a/spotpy/examples/tutorial_dds_hymod.py +++ b/tutorials/tutorial_dds_hymod.py @@ -1,99 +1,103 @@ # -*- coding: utf-8 -*- -''' +""" Copyright 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). :author: Tobias Houska This class holds example code how to use the dream algorithm -''' +""" import numpy as np + try: import spotpy except ImportError: import sys + sys.path.append(".") import spotpy -from spotpy.examples.spot_setup_hymod_python import spot_setup import matplotlib.pyplot as plt +from spotpy.examples.spot_setup_hymod_python import spot_setup if __name__ == "__main__": - parallel ='seq' # Runs everthing in sequential mode - np.random.seed(2000) # Makes the results reproduceable - + parallel = "seq" # Runs everthing in sequential mode + np.random.seed(2000) # Makes the results reproduceable + # Initialize the Hymod example # In this case, we tell the setup which algorithm we want to use, so # we can use this exmaple for different algorithms - spot_setup=spot_setup(spotpy.objectivefunctions.nashsutcliffe) - - #Select number of maximum allowed repetitions - rep=1000 - + spot_setup = spot_setup(spotpy.objectivefunctions.nashsutcliffe) + + # Select number of maximum allowed repetitions + rep = 1000 + # Create the SCE-UA sampler of spotpy, alt_objfun is set to None to force SPOTPY # to jump into the def objectivefunction in the spot_setup class (default is - # spotpy.objectivefunctions.rmse) - sampler=spotpy.algorithms.dds(spot_setup, dbname='DDS_hymod', dbformat='csv') - - #Start the sampler, one can specify ngs, kstop, peps and pcento id desired + # spotpy.objectivefunctions.rmse) + sampler = spotpy.algorithms.dds(spot_setup, dbname="DDS_hymod", dbformat="csv") + + # Start the sampler, one can specify ngs, kstop, peps and pcento id desired sampler.sample(rep) results = sampler.getdata() - - fig= plt.figure(1,figsize=(9,5)) - plt.plot(results['like1']) + + fig = plt.figure(1, figsize=(9, 5)) + plt.plot(results["like1"]) plt.show() - plt.ylabel('Objective function') - plt.xlabel('Iteration') - fig.savefig('hymod_objectivefunction.png',dpi=300) - - # Example plot to show the parameter distribution ###### + plt.ylabel("Objective function") + plt.xlabel("Iteration") + fig.savefig("hymod_objectivefunction.png", dpi=300) + + # Example plot to show the parameter distribution ###### def plot_parameter_trace(ax, results, parameter): - ax.plot(results['par'+parameter.name],'.') + ax.plot(results["par" + parameter.name], ".") ax.set_ylabel(parameter.name) ax.set_ylim(parameter.minbound, parameter.maxbound) - + def plot_parameter_histogram(ax, results, parameter): - #chooses the last 20% of the sample - ax.hist(results['par'+parameter.name][int(len(results)*0.9):], - bins =np.linspace(parameter.minbound,parameter.maxbound,20)) - ax.set_ylabel('Density') + # chooses the last 20% of the sample + ax.hist( + results["par" + parameter.name][int(len(results) * 0.9) :], + bins=np.linspace(parameter.minbound, parameter.maxbound, 20), + ) + ax.set_ylabel("Density") ax.set_xlim(parameter.minbound, parameter.maxbound) - - fig= plt.figure(2,figsize=(9,9)) - - ax1 = plt.subplot(5,2,1) + + fig = plt.figure(2, figsize=(9, 9)) + + ax1 = plt.subplot(5, 2, 1) plot_parameter_trace(ax1, results, spot_setup.cmax) - - ax2 = plt.subplot(5,2,2) - plot_parameter_histogram(ax2,results, spot_setup.cmax) - - ax3 = plt.subplot(5,2,3) + + ax2 = plt.subplot(5, 2, 2) + plot_parameter_histogram(ax2, results, spot_setup.cmax) + + ax3 = plt.subplot(5, 2, 3) plot_parameter_trace(ax3, results, spot_setup.bexp) - - ax4 = plt.subplot(5,2,4) + + ax4 = plt.subplot(5, 2, 4) plot_parameter_histogram(ax4, results, spot_setup.bexp) - - ax5 = plt.subplot(5,2,5) + + ax5 = plt.subplot(5, 2, 5) plot_parameter_trace(ax5, results, spot_setup.alpha) - - ax6 = plt.subplot(5,2,6) + + ax6 = plt.subplot(5, 2, 6) plot_parameter_histogram(ax6, results, spot_setup.alpha) - ax7 = plt.subplot(5,2,7) + ax7 = plt.subplot(5, 2, 7) plot_parameter_trace(ax7, results, spot_setup.Ks) - - ax8 = plt.subplot(5,2,8) + + ax8 = plt.subplot(5, 2, 8) plot_parameter_histogram(ax8, results, spot_setup.Ks) - ax9 = plt.subplot(5,2,9) + ax9 = plt.subplot(5, 2, 9) plot_parameter_trace(ax9, results, spot_setup.Kq) - ax9.set_xlabel('Iterations') - - ax10 = plt.subplot(5,2,10) + ax9.set_xlabel("Iterations") + + ax10 = plt.subplot(5, 2, 10) plot_parameter_histogram(ax10, results, spot_setup.Kq) - ax10.set_xlabel('Parameter range') - + ax10.set_xlabel("Parameter range") + plt.show() - fig.savefig('hymod_parameters.png',dpi=300) \ No newline at end of file + fig.savefig("hymod_parameters.png", dpi=300) diff --git a/tutorials/tutorial_dream_hymod.py b/tutorials/tutorial_dream_hymod.py new file mode 100644 index 00000000..ee2d17d6 --- /dev/null +++ b/tutorials/tutorial_dream_hymod.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +This class holds example code how to use the dream algorithm +""" + +import matplotlib.pyplot as plt +import numpy as np + +import spotpy +from spotpy.analyser import plot_parameter_trace, plot_posterior_parameter_histogram + +# from spotpy.examples.spot_setup_hymod_exe import spot_setup +from spotpy.examples.spot_setup_hymod_python import spot_setup +from spotpy.likelihoods import gaussianLikelihoodMeasErrorOut as GausianLike + +if __name__ == "__main__": + parallel = "seq" + # Initialize the Hymod example (will only work on Windows systems) + # spot_setup=spot_setup(parallel=parallel) + spot_setup = spot_setup(GausianLike) + + # Create the Dream sampler of spotpy, alt_objfun is set to None to force SPOTPY + # to jump into the def objectivefunction in the spot_setup class (default is + # spotpy.objectivefunctions.log_p) + + # Select number of maximum repetitions + rep = 5000 + + # Select seven chains and set the Gelman-Rubin convergence limit + delta = 3 + nChains = 7 + convergence_limit = 1.2 + + # Other possible settings to modify the DREAM algorithm, for details see Vrugt (2016) + c = 0.1 + nCr = 3 + eps = 10e-6 + runs_after_convergence = 100 + acceptance_test_option = 6 + + sampler = spotpy.algorithms.dream( + spot_setup, dbname="DREAM_hymod", dbformat="csv", parallel=parallel + ) + r_hat = sampler.sample( + rep, + nChains, + nCr, + delta, + c, + eps, + convergence_limit, + runs_after_convergence, + acceptance_test_option, + ) + + # Load the results gained with the dream sampler, stored in DREAM_hymod.csv + results = spotpy.analyser.load_csv_results("DREAM_hymod") + # Get fields with simulation data + fields = [word for word in results.dtype.names if word.startswith("sim")] + + # Example plot to show remaining parameter uncertainty # + fig = plt.figure(figsize=(9, 6)) + ax = plt.subplot(1, 1, 1) + q5, q25, q75, q95 = [], [], [], [] + for field in fields: + q5.append( + np.percentile(results[field][-100:-1], 2.5) + ) # ALl 100 runs after convergence + q95.append( + np.percentile(results[field][-100:-1], 97.5) + ) # ALl 100 runs after convergence + ax.plot(q5, color="dimgrey", linestyle="solid") + ax.plot(q95, color="dimgrey", linestyle="solid") + ax.fill_between( + np.arange(0, len(q5), 1), + list(q5), + list(q95), + facecolor="dimgrey", + zorder=0, + linewidth=0, + label="simulation uncertainty", + ) + ax.plot(spot_setup.evaluation(), color="red", markersize=2, label="data") + ax.set_ylim(-50, 450) + ax.set_xlim(0, 729) + ax.set_ylabel("Discharge [l s-1]") + ax.set_xlabel("Days") + ax.legend() + fig.savefig("DREAM_simulation_uncertainty_Hymod.png", dpi=150) + ######################################################### + + # Example plot to show the convergence ################# + spotpy.analyser.plot_gelman_rubin(results, r_hat, fig_name="DREAM_r_hat.png") + ######################################################## + + # Example plot to show the parameter distribution ###### + parameters = spotpy.parameter.get_parameters_array(spot_setup) + + fig, ax = plt.subplots(nrows=5, ncols=2) + fig.set_figheight(9) + fig.set_figwidth(9) + for par_id in range(len(parameters)): + plot_parameter_trace(ax[par_id][0], results, parameters[par_id]) + plot_posterior_parameter_histogram(ax[par_id][1], results, parameters[par_id]) + + ax[-1][0].set_xlabel("Iterations") + ax[-1][1].set_xlabel("Parameter range") + + plt.show() + fig.tight_layout() + fig.savefig("DREAM_parameter_uncertainty_Hymod.png", dpi=300) + ####################################################### diff --git a/spotpy/examples/tutorial_fast_hymod.py b/tutorials/tutorial_fast_hymod.py similarity index 66% rename from spotpy/examples/tutorial_fast_hymod.py rename to tutorials/tutorial_fast_hymod.py index 4c13fc72..57e89345 100644 --- a/spotpy/examples/tutorial_fast_hymod.py +++ b/tutorials/tutorial_fast_hymod.py @@ -1,38 +1,41 @@ # -*- coding: utf-8 -*- -''' +""" Copyright 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). :author: Tobias Houska This class holds example code how to use the FAST algorithm -''' +""" -import spotpy -#from spotpy.examples.spot_setup_hymod_python import spot_setup -from spotpy.examples.spot_setup_rosenbrock import spot_setup import numpy as np +import spotpy + +# from spotpy.examples.spot_setup_hymod_python import spot_setup +from spotpy.examples.spot_setup_rosenbrock import spot_setup if __name__ == "__main__": - parallel ='seq' + parallel = "seq" # Initialize the Hymod example spot_setup = spot_setup() - - #Select number of maximum repetitions + + # Select number of maximum repetitions # CHeck out https://spotpy.readthedocs.io/en/latest/Sensitivity_analysis_with_FAST/ # How to determine an appropriate number of repetitions rep = 2245 - - #Start a sensitivity analysis - sampler = spotpy.algorithms.fast(spot_setup, dbname='FAST_hymod', dbformat='csv', db_precision=np.float32) + + # Start a sensitivity analysis + sampler = spotpy.algorithms.fast( + spot_setup, dbname="FAST_hymod", dbformat="csv", db_precision=np.float32 + ) sampler.sample(rep) - + # Load the results gained with the fast sampler, stored in FAST_hymod.csv - results = spotpy.analyser.load_csv_results('FAST_hymod') - + results = spotpy.analyser.load_csv_results("FAST_hymod") + # Example plot to show the sensitivity index of each parameter spotpy.analyser.plot_fast_sensitivity(results, number_of_sensitiv_pars=3) - - # Example to get the sensitivity index of each parameter + + # Example to get the sensitivity index of each parameter SI = spotpy.analyser.get_sensitivity_of_fast(results) diff --git a/tutorials/tutorial_griewank.py b/tutorials/tutorial_griewank.py new file mode 100644 index 00000000..91d0e5be --- /dev/null +++ b/tutorials/tutorial_griewank.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +This class holds the example code from the Griewank tutorial web-documention. +""" +import spotpy +from spotpy.examples.spot_setup_griewank import spot_setup + +# Create samplers for every algorithm: +results = [] +spot_setup = spot_setup() +rep = 5000 + +sampler = spotpy.algorithms.mc(spot_setup, dbname="GriewankMC", dbformat="csv") +sampler.sample(rep) +results.append(sampler.getdata()) + +sampler = spotpy.algorithms.lhs(spot_setup, dbname="GriewankLHS", dbformat="csv") +sampler.sample(rep) +results.append(sampler.getdata()) + +sampler = spotpy.algorithms.mle(spot_setup, dbname="GriewankMLE", dbformat="csv") +sampler.sample(rep) +results.append(sampler.getdata()) + +sampler = spotpy.algorithms.mcmc(spot_setup, dbname="GriewankMCMC", dbformat="csv") +sampler.sample(rep) +results.append(sampler.getdata()) + +sampler = spotpy.algorithms.sceua(spot_setup, dbname="GriewankSCEUA", dbformat="csv") +sampler.sample(rep, ngs=4) +results.append(sampler.getdata()) + +sampler = spotpy.algorithms.sa(spot_setup, dbname="GriewankSA", dbformat="csv") +sampler.sample(rep) +results.append(sampler.getdata()) + +sampler = spotpy.algorithms.demcz(spot_setup, dbname="GriewankDEMCz", dbformat="csv") +sampler.sample(rep, nChains=4) +results.append(sampler.getdata()) + +sampler = spotpy.algorithms.rope(spot_setup, dbname="GriewankROPE", dbformat="csv") +sampler.sample(rep) +results.append(sampler.getdata()) + + +algorithms = ["MC", "LHS", "MLE", "MCMC", "SCEUA", "SA", "DEMCz", "ROPE"] +# results=[] +# for algorithm in algorithms: +# results.append(spot.analyser.load_csv_results('Griewank'+algorithm)) + + +evaluation = spot_setup.evaluation() + +spotpy.analyser.plot_heatmap_griewank(results, algorithms) diff --git a/tutorials/tutorial_likelihood.py b/tutorials/tutorial_likelihood.py new file mode 100644 index 00000000..1343d88d --- /dev/null +++ b/tutorials/tutorial_likelihood.py @@ -0,0 +1,553 @@ +# -*- coding: utf-8 -*- +""" +Copyright (c) 2017 by Benjamin Manns +This file is part of Statistical Parameter Estimation Tool (SPOTPY). +:author: Tobias Houska, Benjamin Manns + +This code shows how to use the likelihood framework and present all existing function. +""" + +import numpy as np + +import spotpy + +# First we use all available likelihood functions just alone. The pydoc of every function tells, if we can add a +# parameter `param` to the function which includes model parameter. The `param` must be None or a tuple with values +# and names. If `param` is None, the needed values are calculated by the function itself. + +data, comparedata = np.random.normal(1500, 2530, 20), np.random.normal(15, 25, 20) + +l = spotpy.likelihoods.logLikelihood(data, comparedata) +print("logLikelihood: " + str(l)) + +l = spotpy.likelihoods.gaussianLikelihoodMeasErrorOut(data, comparedata) +print("gaussianLikelihoodMeasErrorOut: " + str(l)) + +l = spotpy.likelihoods.gaussianLikelihoodHomoHeteroDataError(data, comparedata) +print("gaussianLikelihoodHomoHeteroDataError: " + str(l)) + +# Here are examples where functions get `params` +l = spotpy.likelihoods.LikelihoodAR1NoC( + data, comparedata, params=([0.98], ["likelihood_phi"]) +) +print("LikelihoodAR1NoC: " + str(l)) + +l = spotpy.likelihoods.LikelihoodAR1WithC(data, comparedata) +print("LikelihoodAR1WithC: " + str(l)) + +l = spotpy.likelihoods.generalizedLikelihoodFunction( + data, + comparedata, + params=( + [ + np.random.uniform(-0.99, 1, 1), + np.random.uniform(0.1, 10, 1), + np.random.uniform(0, 1, 1), + np.random.uniform(0, 1, 0), + np.random.uniform(0, 0.99, 1), + np.random.uniform(0, 100, 1), + ], + [ + "likelihood_beta", + "likelihood_xi", + "likelihood_sigma0", + "likelihood_sigma1", + "likelihood_phi1", + "likelihood_muh", + ], + ), +) +print("generalizedLikelihoodFunction: " + str(l)) + +l = spotpy.likelihoods.LaplacianLikelihood(data, comparedata) +print("LaplacianLikelihood: " + str(l)) + +l = spotpy.likelihoods.SkewedStudentLikelihoodHomoscedastic(data, comparedata) +print("SkewedStudentLikelihoodHomoscedastic: " + str(l)) + +l = spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedastic(data, comparedata) +print("SkewedStudentLikelihoodHeteroscedastic: " + str(l)) + +l = spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel( + data, + comparedata, + params=( + [ + np.random.uniform(2.01, 100, 1), + np.random.uniform(0.01, 100, 1), + np.random.uniform(-0.99, 0.99, 1), + ], + ["likelihood_nu", "likelihood_kappa", "likelihood_phi"], + ), +) + +print("SkewedStudentLikelihoodHeteroscedasticAdvancedARModel: " + str(l)) + +l = spotpy.likelihoods.NoisyABCGaussianLikelihood(data, comparedata) +print("NoisyABCGaussianLikelihood: " + str(l)) + +l = spotpy.likelihoods.ABCBoxcarLikelihood(data, comparedata) +print("ABCBoxcarLikelihood: " + str(l)) + +l = spotpy.likelihoods.LimitsOfAcceptability(data, comparedata) +print("LimitsOfAcceptability: " + str(l)) + +l = spotpy.likelihoods.InverseErrorVarianceShapingFactor(data, comparedata) +print("inverseErrorVarianceShapingFactor: " + str(l)) + +l = spotpy.likelihoods.ExponentialTransformErrVarShapingFactor(data, comparedata) +print("inverseErrorVarianceShapingFactor: " + str(l)) + +l = spotpy.likelihoods.NashSutcliffeEfficiencyShapingFactor(data, comparedata) +print("NashSutcliffeEfficiencyShapingFactor: " + str(l)) + +l = spotpy.likelihoods.sumOfAbsoluteErrorResiduals(data, comparedata) +print("sumOfAbsoluteErrorResiduals: " + str(l)) + + +# We also can use the likelihood functions in an algorithmus. We will need a setup class like this + + +class spot_setup_gauss(object): + def __init__(self): + self.params = [ + # Original mean: 12, sd:23 + spotpy.parameter.Uniform("mean", -20, 20, 2, 3.0, -20, 20), + spotpy.parameter.Uniform("sd", 1, 30, 2, 3.01, 1, 30), + # Some likelihood function need additional parameter, look them up in the documentation + # spotpy.parameter.Uniform('likelihood_nu', 2.01, 100, 1.5, 3.0, -10, 10), + # spotpy.parameter.Uniform('likelihood_kappa', 0.01, 100, 1.5, 3.0, -10, 10), + # spotpy.parameter.Uniform('likelihood_phi', -.99, .99, 0.1, 0.1, 0.1, 0.1), + # spotpy.parameter.Uniform('likelihood_beta', -.99, .99, 1.5, 3.0, -10, 10), + # spotpy.parameter.Uniform('likelihood_xsi', 0.11, 10, 1.5, 3.0, -10, 10), + # spotpy.parameter.Uniform('likelihood_sigma0', 0, 1, 1.5, 3.0, -10, 10), + # spotpy.parameter.Uniform('likelihood_sigma1', 0, 1, 1.5, 3.0, -10, 10), + # spotpy.parameter.Uniform('likelihood_phi1', 0, .99, 1.5, 3.0, -10, 10), + # spotpy.parameter.Uniform('likelihood_muh', 0, 100, 1.5, 3.0, -10, 10) + ] + + def parameters(self): + return spotpy.parameter.generate(self.params) + + def simulation(self, vector): + # x = np.random.randint(-100, 100, size=201) + # simulations= [sum(100.0 * (x[1:] - x[:-1] **2.0) **2.0 + (1 - x[:-1]) **2.0)] + import json + import subprocess + + output = subprocess.check_output( + "/Library/Frameworks/R.framework/Versions/3.1/Resources/Rscript /Users/" + "Karlson/Documents/03_PRIVAT/Privat_UNI/Uni_JOB/Job_AGRAR/develop/spotpy/" + f"spotpy/likelihood_test/myR_2.R {vector[0]} {vector[1]}", + shell=True, + ) + # print("this parameters: "+str(vector[0])+" "+str(vector[1])) + + output = output.decode("utf-8") + back = json.loads(output) + + simulations = back + + return simulations + + def evaluation(self): + + # Gauss with mean=12 and sd=23 + # fmt: off + observations = [ + 1.537678, 7.615278, 33.54329, 12.09963, 24.69595, 7.033905, 21.30595, -17.77526, -7.09708, + 36.80745, 10.68426, -42.7048, -21.01126, -6.314566, 38.01058, -15.79536, -17.69119, -4.482229, + 12.30351, -30.54512, 8.468925, 27.44369, 37.20623, -8.753253, 39.40037, 29.03273, 0.5257918, + 25.98343, -16.09876, 6.430084, 4.755722, -10.38204, 7.97673, -37.55442, 58.04988, 20.41361, + 32.13943, 30.37884, 6.898094, 13.32948, -14.5311, 25.0606, 25.81364, 25.82836, 47.70208, + 31.1919, 24.74743, 18.21143, 10.67086, 47.29963, 40.3718, 39.21012, 6.774497, -4.244702, + 13.45878, 33.80645, 15.64674, -6.277918, 36.83417, 23.13524, 11.85227, 31.38894, 13.00289, + 6.47117, 19.31257, -11.23233, 21.07677, 14.96549, 9.952789, 23.54167, 46.7991, 47.64822, + 7.33875, 17.64916, 38.79842, 11.75935, 17.70734, 15.64669, -6.890646, 3.710825, 44.42125, + -20.1855, 24.32393, 56.55909, 33.02915, 5.173076, -24.00348, 16.62815, -21.64049, 18.2159, + 41.69109, -31.26055, 36.9492, 2.780838, -4.519057, -24.71357, 20.63503, 17.08391, 26.23503, + 12.82442, 22.13652, 21.21188, 47.99579, 44.52914, -0.5511025, 55.47107, -15.12694, 2.884632, + 7.361032, 12.66143, 37.38807, 53.63648, 9.114074, 12.68311, -6.890102, 32.40405, 22.93079, + 1.498509, 22.68785, 29.71565, 21.42051, -9.961459, -10.22352, -28.16017, 14.14882, 9.64758, + -5.821728, -21.93086, 19.94631, 16.29195, 28.87528, 25.81239, 52.44341, 5.229822, -17.92572, + 11.85504, 17.21691, 17.19854, -6.37061, 16.1524, -13.08297, 13.45471, 9.43481, -2.177022, + 17.46416, 2.647446, 16.77834, 20.77827, 49.37553, 8.435563, -13.85352, 17.06572, 5.550149, + 2.674943, -21.95848, 11.82037, 30.51478, 8.334891, -17.1576, -16.82652, 50.31279, -31.05799, + 52.69635, 22.11049, -43.32149, -13.5348, -5.125771, 1.801732, 19.30368, 14.94216, -19.32855, + 20.75345, 21.03398, 5.430688, -3.163607, 10.1321, -12.9058, -13.77746, 25.02506, -7.187246, + 39.93147, 5.330449, 6.705344, -16.47149, -34.20934, 28.66261, -6.420032, 4.682751, -9.622858, + 17.95173, 3.316576, 14.6763, 13.84716, -20.52151, 24.35037, 28.57057, 18.17082, 26.14141, + 72.05923, -12.29775,26.83472 + ] + # fmt: on + return observations + + def objectivefunction( + self, simulation=simulation, evaluation=evaluation, params=None + ): + # Some functions do not need a `param` attribute, you will see that in the documentation or if an error occur. + # objectivefunction = spotpy.likelihoods.LimitsOfAcceptability(evaluation, simulation,params=params) + # objectivefunction = spotpy.likelihoods.NoisyABCGaussianLikelihood(evaluation, simulation) + # objectivefunction = spotpy.likelihoods.LimitsOfAcceptability(evaluation, simulation) + objectivefunction = spotpy.objectivefunctions.rmse( + simulation=simulation, evaluation=evaluation + ) + # print(objectivefunction) + + return objectivefunction + + +class spot_setup_wald(object): + def __init__(self): + self.params = [ + # See https://de.wikipedia.org/wiki/Inverse_Normalverteilung + # Original 12,23 + spotpy.parameter.Uniform("lambda", 1, 1000, 0.1, 3.0, -20, 20), + # spotpy.parameter.Uniform('nu', 1, 30, 2, 3.01, 1, 30), + ] + + def parameters(self): + return spotpy.parameter.generate(self.params) + + def simulation(self, vector): + import json + import subprocess + + output = subprocess.check_output( + "/Library/Frameworks/R.framework/Versions/3.1/Resources/Rscript /Users/Karlson/" + "Documents/03_PRIVAT/Privat_UNI/Uni_JOB/Job_AGRAR/develop/spotpy/spotpy/" + f"likelihood_test/myR_WALD.R {vector[0]}", + shell=True, + ) + # print("this parameters: "+str(vector[0])+" "+str(vector[1])) + + output = output.decode("utf-8") + back = json.loads(output) + + simulations = back + + return simulations + + def evaluation(self): + # Wald distrubtion with lambda=42 (nu = 1 as it is original inverse Gauß) + # fmt: off + observations = [ + 0.8215101, 1.050744, 1.068614, 0.9237615, 1.134586, 0.8342905, 0.9203649, + 0.8423139, 1.016296, 0.819583, 0.7727125, 1.049373, 0.9064652, 1.129859, + 0.7390692, 0.7807588, 0.9512094, 0.751157, 0.8342608, 0.9535379, 0.8855571, + 0.8164966, 0.9859118, 0.9663425, 0.9168434, 1.096442, 1.075291, 0.7939873, + 0.8371087, 0.8899696, 0.8223036, 0.9441274, 1.251677, 0.9946841, 0.9688333, + 0.8584872, 1.118507, 1.092399, 0.9389445, 1.320034, 1.05912, 0.8073291, + 0.9718409, 0.9993603, 1.009801, 1.191749, 1.096261, 0.9104541, 1.135112, + 1.024141, 0.68865, 1.117724, 1.071344, 0.9730503, 1.03879, 0.9040554, + 1.226641, 1.090904, 0.9188659, 0.9516232, 1.111537, 0.7868174, 1.03979, + 0.8529991, 1.546705, 0.9973017, 0.9056773, 1.020306, 0.8666091, 0.8227436, + 1.107373, 1.240635, 0.8642053, 1.012499, 0.8189009, 0.9112955, 0.9133874, + 0.764895, 0.9954879, 1.016124, 1.135945, 1.210386, 0.8935554, 1.133396, + 0.8744451, 1.27583, 0.9399524, 1.109192, 1.024147, 1.010473, 0.823447, + 1.063746, 1.587057, 1.25963, 1.075372, 0.9058057, 1.149925, 0.8951753, + 0.8786255, 0.7846421, 1.089261, 1.155204, 0.9162714, 1.091259, 1.012252, + 0.9860885, 1.000627, 0.8831002, 0.9736084, 1.020061, 1.099529, 0.8503705, + 1.092692, 1.018956, 0.9597126, 0.9760877, 0.8305396, 1.010439, 0.8659965, + 1.216233, 1.15933, 0.8352535, 1.086288, 1.085791, 1.215822, 1.455505, + 0.8928623, 1.227453, 0.9778177, 1.248284, 0.678555, 0.9379088, 1.076307, + 1.081512, 1.056103, 0.9475012, 1.11073, 1.216543, 1.409434, 0.8973831, + 0.7879291, 1.039925, 0.9954887, 0.8129037, 1.088005, 1.010168, 0.9842995, + 1.034264, 0.9122271, 1.128363, 1.331232, 1.206762, 1.134155, 1.166505, + 1.154047, 1.054108, 1.07734, 0.8397705, 0.9748741, 1.133629, 0.9498966, + 0.9889976, 1.023417, 0.9424091, 0.9424539, 1.246194, 0.9413468, 1.15047, + 0.7332654, 1.496362, 0.828069, 0.7696388, 0.918564, 0.8388578, 0.9455839, + 0.8227491, 0.9551339, 0.963993, 1.051606, 1.013032, 0.8144458, 1.07049, + 1.029506, 0.7699333, 0.9409208, 1.341655, 1.023382, 0.9868176, 0.9950876, + 0.954334, 0.957515, 1.136036, 1.265562, 0.9722909, 0.7632513, 0.8805661, + 0.8904488, 1.052702, 1.036818, 0.9569595, 0.9428334 + ] + # fmt: on + return observations + + def objectivefunction( + self, simulation=simulation, evaluation=evaluation, params=None + ): + objectivefunction = spotpy.likelihoods.NoisyABCGaussianLikelihood( + evaluation, simulation + ) + + return objectivefunction + + +class spot_setup_ar_1_students_t_res(object): + def __init__(self): + self.params = [ + # For students - Skew + spotpy.parameter.Uniform("likelihood_kappa", 1, 1, 1, 1, 1, 1), + spotpy.parameter.Uniform( + "likelihood_phi", -0.99, 0.99, 0.1, 3.0, -0.99, 0.99 + ), + spotpy.parameter.Uniform("likelihood_nu", 2.1, 100, 2, 3.01, 2.1, 10), + ] + + def parameters(self): + return spotpy.parameter.generate(self.params) + + def simulation(self, vector): + import json + import subprocess + + # parameter 0:phi, 1:nu + output = subprocess.check_output( + "/Library/Frameworks/R.framework/Versions/3.1/Resources/Rscript /Users/" + "Karlson/Documents/03_PRIVAT/Privat_UNI/Uni_JOB/Job_AGRAR/develop/spotpy/" + f"spotpy/likelihood_test/myR_AR1_Student-t-res.R {vector[0]} {vector[1]}", + shell=True, + ) + # print("this parameters: "+str(vector[0])+" "+str(vector[1]) + " "+str(vector[2])) + + output = output.decode("utf-8") + back = json.loads(output) + + simulations = back + + return simulations + + def evaluation(self): + # AR1 with student-t distributed residuals with phi=42 and df=7 + # fmt: off + observations = [ + -0.806554, -1.036565, -1.054924, 1.207057, -0.2267287, -0.6241089, -0.9865154, + 0.004914617, 0.5435943, -0.2529109, -0.220569, -1.085152, -1.446932, -0.2271839, + -0.9695579, -1.042956, -1.677816, -0.3111386, 0.2023657, 1.681282, 0.3311756, + -2.011093, -0.05581758, 1.792977, 1.469434, -0.4426333, 0.8043387, 1.093684, + 1.99667, -0.1171755, 0.5409204, -1.036451, -0.03801139, 0.6125889, 0.09342139, + 1.977853, 0.5931005, 2.947713, -1.51345, 0.152891, -1.079158, -1.466529, + -1.504046, 2.181459, -1.966612, 0.3770235, -0.7682167, -0.2559416, 1.215604, + 0.9032672, 0.7817497, -1.634914, -0.1532924, -1.329244, -3.425294, -0.9132035, + -0.1814598, -0.3703962, 0.8464317, 0.3035075, -0.1755196, -0.7057517, -0.6808711, + -1.221144, 0.04788332, 0.7737842, 0.9854636, 0.7845628, 3.702611, 3.388055, + 1.097186, 0.672756, -0.3338004, 0.2702849, -0.6202132, -2.228129, -0.7818483, + 0.155476, 0.657133, 0.5652424, 0.0008205475, 0.4045611, 0.7796379, 0.4325643, + -0.354133, -1.166525, -0.9882422, 0.4509809, -0.4848221, 0.1715434, -1.196337, + 0.4085583, -0.04952907, 1.035795, 0.9496666, 0.3619785, 0.8666738, 0.285121, + -1.432747, -1.939233, -0.1684185, 0.003022823, -0.4922612, 0.5042833, 0.09296572, + -1.455734, -0.213669, 1.063985, 0.2004293, -1.649053, 0.200737, -0.105006, + -2.633298, -3.161435, -0.5550424, -1.713277, -0.5169007, -0.3834445, -0.2791567, + 1.262414, 0.06332764, 1.640421, 0.3144967, -0.01392552, -1.151283, -1.268196, + -1.880887, 1.010745, 0.2109186, 2.752468, 0.09242852, -2.878202, -1.477239, + -2.243395, -0.6384147, -0.6950003, -1.686313, -0.6945071, 1.603874, 0.09509077, + -0.09816484, 1.150218, 2.324576, 1.504623, -1.063881, 1.330131, 2.611619, + 2.494136, 0.6338899, 1.004135, 0.1381033, -0.360101, 1.124663, 1.78177, + 1.603268, 1.325218, 2.4939, 2.153747, 1.97724, 1.18353, -0.7253327, + 0.823499, 0.4020309, 0.6603788, 0.129381, -1.321761, 1.939023, 0.6186619, + 0.9888297, 1.118437, 0.2581724, 1.385318, 1.997055, -0.8354623, 0.441386, + -0.5992372, -2.595201, 1.591507, -0.7960406, 0.2250709, 0.6408333, -0.06189073, + 0.103896, -0.4334433, 0.1433027, -0.6207327, 0.2989117, -0.2577758, -0.5267562, + -3.398514, 0.1546277, 1.012887, 0.06408349, 1.258883, 0.5752389, -1.694637, + -0.7765886, 2.41088, 0.2273703, -1.2156, -1.232374 + ] + # fmt: on + return observations + + def objectivefunction( + self, simulation=simulation, evaluation=evaluation, params=None + ): + objectivefunction = ( + spotpy.likelihoods.SkewedStudentLikelihoodHeteroscedasticAdvancedARModel( + evaluation, simulation, params=params + ) + ) + if np.isinf(objectivefunction): + objectivefunction = 0.0 + print("Class log " + str(objectivefunction)) + return objectivefunction + + +class spot_setup_ar_1_gauss_res(object): + def __init__(self): + self.params = [ + spotpy.parameter.Uniform( + "likelihood_phi", -0.99, 0.99, 0.2, 3.0, -0.99, 0.99 + ), + ] + + def parameters(self): + return spotpy.parameter.generate(self.params) + + def simulation(self, vector): + import json + import subprocess + + # parameter 0:phi + output = subprocess.check_output( + "/Library/Frameworks/R.framework/Versions/3.1/Resources/Rscript /Users/" + "Karlson/Documents/03_PRIVAT/Privat_UNI/Uni_JOB/Job_AGRAR/develop/spotpy/" + f"spotpy/likelihood_test/myR_AR1_Norm_res.R {vector[0]}", + shell=True, + ) + print("this parameters: " + str(vector[0])) + + output = output.decode("utf-8") + back = json.loads(output) + + simulations = back + + return simulations + + def evaluation(self): + # AR1 with normal distributed residuals with phi=0.428 and mean=0,sd=1 + # fmt: off + observations = [ + -2.522743, 1.408725, 1.946646, 0.888204, 0.7897667, 0.6112302, 2.290427, + 2.868624, 1.556995, 0.05254887, 0.7461225, 0.3719867, 0.979471, -1.190781, + 0.4436523, 2.236696, 0.2605191, 0.673389, 0.7251472, 0.6608128, -0.7443824, + 0.371268, 0.02141081, 0.4607711, -0.5507639, -1.005326, 0.7367659, 0.5269135, + 0.1166365, -0.5970824, -0.7521213, 0.367761, 0.1125972, -0.0795236, 0.1165288, + 0.6486639, 1.932282, 1.033983, -0.9876154, 0.1796362, -0.3983238, 0.01101198, + 0.9050182, -0.6080171, 0.08849208, -1.81438, -0.07913209, -0.009007132, -1.160989, + -1.297887, 1.419709, -1.066902, -1.270009, -1.030414, -2.38863, -0.2904009, + 0.7489411, -0.1846965, -1.433198, -1.145359, -0.04856701, 1.087533, 0.7987545, + 0.641762, -0.378111, -1.019192, 0.2837018, 2.259854, 0.4158938, 1.451425, + 0.9710148, 1.325311, 0.04831358, -0.2373003, 0.09663009, -2.557514, -0.9230433, + -0.6250428, -0.6359625, -0.2501693, 1.096061, 1.564296, 1.956924, -0.4511307, + -0.521957, -0.3384552, -0.3905848, -1.05168, 1.266555, 1.835193, 1.168423, + -0.5542428, -0.7080179, 0.8867539, 0.9273763, 0.9104679, 1.761588, 2.650327, + 0.549592, 1.152438, 0.1226201, -0.1466213, 0.6201685, -0.244967, 1.728424, + -0.1991486, 0.4715499, 2.541504, 2.055189, 0.5441279, 1.075167, 0.6381296, + 0.7177508, 1.106246, -0.6729018, -2.086599, -1.199925, 0.7879157, 0.01633025, + -0.5845763, 0.4181293, -2.16246, -2.197336, -2.209761, -1.856266, -1.021362, + 0.6899624, -0.898831, 0.3702167, 1.202066, 0.5307163, -1.183152, -1.305882, + -1.34188, 0.9997525, -1.611335, 0.4367853, 1.24461, 2.474933, 0.9325948, + 2.022836, -0.6440014, -1.253051, 0.2687869, -1.139736, 0.1336643, -1.383847, + 0.3793314, -0.7788819, 0.8646879, 1.433118, 1.026648, 1.31162, -0.8718095, + 0.3493136, -0.2196879, 0.3182434, -0.7072177, 0.6959091, -0.5620885, 0.8712038, + 0.6010974, -0.007788187, -0.1008797, -0.5404524, -0.6134115, 2.287364, -0.04623299, + -1.30409, 0.6175855, -1.475111, 0.2202588, 0.5428336, 0.3996769, 0.1243578, + 0.3912388, 1.346204, 0.6947638, -0.5677999, 0.3445474, -0.7952659, 1.144519, + 1.850942, -0.312677, -1.347769, -0.6379291, 0.1777509, 0.1736661, -0.6718341, + 0.8210482, -0.4401946, 0.9218458, -0.2263964, 0.2356263, -0.6045727, 0.124017, + 0.6440486, 1.095587, 0.4844389, -0.4179212, 1.385794 + ] + # fmt: on + return observations + + def objectivefunction( + self, simulation=simulation, evaluation=evaluation, params=None + ): + objectivefunction = spotpy.likelihoods.LikelihoodAR1NoC( + evaluation, simulation, params=params + ) + + return objectivefunction + + +class spot_setup_generalizedGauss(object): + def __init__(self): + self.params = [ + spotpy.parameter.Uniform( + "likelihood_phi1", 0.01, 0.99, 3.0, 3.0, 0.01, 0.99 + ), + spotpy.parameter.Uniform("likelihood_xi", 0.1, 10, 3, 3, 0.1, 10), + spotpy.parameter.Uniform( + "likelihood_beta", 0.1, 0.99, 3, 3.0, -0.99, 0.99 + ), # positive - simulation in R says it! + spotpy.parameter.Uniform("likelihood_sigma0", 0, 1, 0.1, 3.0, 0, 1), + spotpy.parameter.Uniform("likelihood_sigma1", 0, 1, 0.1, 3.0, 0, 1), + spotpy.parameter.Uniform("likelihood_muh", 0, 100, 0.1, 3.0, 0, 100), + ] + + def parameters(self): + return spotpy.parameter.generate(self.params) + + def simulation(self, vector): + import json + import subprocess + + # we need a skew exponential power to add xi and beta in it which is the model for the error residuals of AR1 Model + + output = subprocess.check_output( + "/Library/Frameworks/R.framework/Versions/3.1/Resources/Rscript /Users/" + "Karlson/Documents/03_PRIVAT/Privat_UNI/Uni_JOB/Job_AGRAR/develop/spotpy/" + f"spotpy/likelihood_test/myR_AR1_SEP_res.R {vector[0]} {vector[1]} {vector[2]}", + shell=True, + ) + # print("this parameters: "+str(vector[0])+" "+str(vector[1]) + " "+str(vector[2])) + + output = output.decode("utf-8") + back = json.loads(output) + if back[0] == "NA": + back = np.repeat(np.NAN, back.__len__()).tolist() + simulations = back + + return simulations + + def evaluation(self): + # AR1 with SEP distribution phi=0.142 xsi=3, beta=9.5 + # [ 0.99 1.28841626 0.99 0.43366888 0.38087079 72.72585542] + # fmt: off + observations = [ + 1.367689, -0.320067, -0.04381581, 0.6023338, 0.9038274, 1.034441, -0.3475758, + -0.6766884, 0.375266, 0.3902351, 1.41773, 1.146159, 1.142029, -0.5467857, + 1.132456, 0.05771065, -0.01329709, 1.245674, 1.262945, 0.5637976, -0.6106627, + -0.1347206, 0.4439383, -0.1191365, 0.6781304, -0.4293178, 0.1856715, 0.4008803, + 1.34489, 0.9124905, 1.237749, 0.5098399, 0.8364595, -0.4464507, 0.6631744, + 0.2039722, -0.05081068, 0.7299973, 0.8854515, 1.180466, -0.4876658, 0.7830223, + -0.4316994, 1.099141, 0.5340687, 0.8495034, 0.8779076, 0.6735508, -0.3102846, + 0.2900948, -0.05825545, -0.941212, 1.025862, 0.7281562, -0.361788, 0.6388547, + 1.038316, 1.343974, 0.8034503, 1.39158, 0.5718842, 0.4621339, 0.828369, + 1.091347, 0.9504174, 1.100152, 0.4411185, 1.178236, 1.528249, 1.311347, + 1.011896, 0.6851925, 1.102152, 1.191884, -0.3198258, 1.023772, 1.021118, + 0.351345, -0.7778747, 0.3130401, 0.8374449, -0.2165474, 0.6511311, 0.9294736, + 1.007714, 1.124968, 1.122693, 1.053253, 1.064944, -0.3810931, -0.7520672, + 0.07500417, -0.6589652, 0.9858736, -0.3338579, 0.5976432, 1.065922, -0.1717056, + -0.7935736, -0.2154963, 0.1094597, 0.9271599, 0.8882699, -0.204626, -0.1153957, + -0.03153619, 1.145353, 0.1135476, -0.0652023, -0.3510398, -0.8471455, -0.7796421, + -0.2307928, -0.2594656, 0.8092929, 0.4113968, -0.188539, 1.19418, 0.8070983, + 0.9118222, 1.071649, 1.051291, -0.8035766, 0.8092788, -0.1163294, -0.02733921, + -0.6852544, -0.408636, -0.08736997, 0.4578535, 0.7799243, 0.4268271, 0.5604364, + -0.1452668, -0.1654945, -0.483941, 0.1326935, 0.4563893, 0.7192259, 0.7154398, + 1.120117, 0.3798121, -0.1878339, -0.5358535, -0.5510031, 0.0233894, 0.07423701, + 0.9234318, -0.1600513, 1.372747, 0.6790618, 0.8772782, -0.1664986, 0.9622479, + 0.9873338, 0.2296364, -1.002397, -0.2306121, -0.1446204, 0.31194, -0.1342179, + 0.08709208, -0.2634807, -0.5770442, 0.5588156, -0.4229277, -0.8920537, -0.3130578, + 0.9963966, -0.1462349, 0.2177117, 1.019684, 0.3005968, 0.7721734, -0.1104061, + -0.7366346, 0.9962065, 0.4035172, 1.175904, 1.103031, -0.742134, -0.3189378, + 0.5614889, 0.4403444, -0.6407969, -0.4805289, 0.2666954, 0.8946856, 0.200042, + 0.700875, -0.3121022, -0.4439033, -0.7696692, 0.6370263, 1.367421, 0.8487393, + 0.1497969, -0.4690384, 1.088799, 0.0210073, 0.3703306 + ] + # fmt: on + return observations + + def objectivefunction( + self, simulation=simulation, evaluation=evaluation, params=None + ): + objectivefunction = spotpy.likelihoods.generalizedLikelihoodFunction( + evaluation, simulation, params=params + ) + if np.isinf(objectivefunction): + objectivefunction = 0.0 + print("Class log " + str(objectivefunction)) + return objectivefunction + + +# And now we can start an algorithm to find the best possible data + + +results = [] +spot_setup = spot_setup_gauss() +rep = 1000 + +# TODO alt_objfun immer wieder anschalten sonst Trick 17! +sampler = spotpy.algorithms.lhs( + spot_setup, dbname="RosenMC", dbformat="csv", alt_objfun=None +) +sampler.sample(rep) +results.append(sampler.getdata()) + +import pickle + +pickle.dump(results, open("mle_LimitsOfAcceptability_v2.p", "wb")) + +# from matplotlib import pyplot as plt +# plt.plot(results) +# plt.show() +# plt.plot(results['like1']) +# plt.show() diff --git a/spotpy/examples/tutorial_listsampler.py b/tutorials/tutorial_listsampler.py similarity index 80% rename from spotpy/examples/tutorial_listsampler.py rename to tutorials/tutorial_listsampler.py index d7198f83..3964fa1c 100644 --- a/spotpy/examples/tutorial_listsampler.py +++ b/tutorials/tutorial_listsampler.py @@ -1,59 +1,59 @@ -''' +""" Copyright 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). :author: Tobias Houska This example implements the Rosenbrock function into SPOT. -''' +""" import spotpy from spotpy.examples.spot_setup_hymod_python import spot_setup - if __name__ == "__main__": # 1 We start classical by perfroming a sensitivity analysis - parallel ='seq' + parallel = "seq" # 1.1 Initialize the Hymod example spot_setup = spot_setup() - - #S 1.2 elect number of maximum repetitions + + # S 1.2 elect number of maximum repetitions rep = 1345 - + # 1.3 Start a sensitivity analysis - sampler = spotpy.algorithms.fast(spot_setup, dbname='FAST_hymod', dbformat='csv') + sampler = spotpy.algorithms.fast(spot_setup, dbname="FAST_hymod", dbformat="csv") sampler.sample(rep) ########################################################### - + # 2 Special part begins - - # 2.1 Lets assume something went wrong with the model or you want to use + + # 2.1 Lets assume something went wrong with the model or you want to use # the same parameter set again. Then the list_sampler is what you need. # It is also thinkable that you want to change something in this database # and restart. - # The list_sampler takes a spotpy database as input, reads the paramater + # The list_sampler takes a spotpy database as input, reads the paramater # in it and starts the model again: - + # 2.2 Use the generated database to reuse the parameters: - sampler_new = spotpy.algorithms.list_sampler(spot_setup, dbname='FAST_hymod', dbformat='csv') - + sampler_new = spotpy.algorithms.list_sampler( + spot_setup, dbname="FAST_hymod", dbformat="csv" + ) + # 2.3 Start the sampler with the according number of repetitions sampler_new.sample(rep) - + # 2.4 Load the result, the new database will have added the word "list" - results = spotpy.analyser.load_csv_results('FAST_hymodlist') - - # 2.5 Please mind that these results contains the same data as we have used - # the same spot_setup. However we could have also used another spot_setup, + results = spotpy.analyser.load_csv_results("FAST_hymodlist") + + # 2.5 Please mind that these results contains the same data as we have used + # the same spot_setup. However we could have also used another spot_setup, # which takes the same parameters. ############################################################# - + # 3.0 Classical part starts again - + # 3.1 Example plot to show the sensitivity index of each parameter from t spotpy.analyser.plot_fast_sensitivity(results, number_of_sensitiv_pars=3) - - # 3.2 Example to get the sensitivity index of each parameter + + # 3.2 Example to get the sensitivity index of each parameter SI = spotpy.analyser.get_sensitivity_of_fast(results) ############################################################# - \ No newline at end of file diff --git a/spotpy/examples/tutorial_nsgaii.py b/tutorials/tutorial_nsgaii.py similarity index 50% rename from spotpy/examples/tutorial_nsgaii.py rename to tutorials/tutorial_nsgaii.py index ccc096aa..a89694a9 100644 --- a/spotpy/examples/tutorial_nsgaii.py +++ b/tutorials/tutorial_nsgaii.py @@ -1,20 +1,15 @@ #!/usr/bin/env python # coding: utf-8 # -*- coding: utf-8 -*- -''' +""" Copyright 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). :author: Tobias Houska This class holds example code how to use the nsgaii algorithm -''' +""" -from __future__ import division, print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals try: import spotpy @@ -24,17 +19,19 @@ sys.path.append(".") import spotpy -import spotpy.algorithms import unittest -from spotpy.examples.spot_setup_hymod_python import spot_setup + import matplotlib.pyplot as plt import numpy as np +import spotpy.algorithms +from spotpy.examples.spot_setup_hymod_python import spot_setup + def multi_obj_func(evaluation, simulation): """ This function is used to overwrite objective function in hymod example - + Parameters ---------- evaluation : array @@ -50,129 +47,144 @@ def multi_obj_func(evaluation, simulation): """ like1 = abs(spotpy.objectivefunctions.pbias(evaluation, simulation)) like2 = spotpy.objectivefunctions.rmse(evaluation, simulation) - like3 = spotpy.objectivefunctions.rsquared(evaluation, simulation)*-1 + like3 = spotpy.objectivefunctions.rsquared(evaluation, simulation) * -1 return [like1, like2, like3] + if __name__ == "__main__": - - generations=40 + generations = 40 n_pop = 20 skip_duplicates = True - - sp_setup=spot_setup(obj_func= multi_obj_func) - sampler = spotpy.algorithms.NSGAII(spot_setup=sp_setup, dbname='NSGA2', dbformat="csv") - - sampler.sample(generations, n_obj= 3, n_pop = n_pop) + sp_setup = spot_setup(obj_func=multi_obj_func) + sampler = spotpy.algorithms.NSGAII( + spot_setup=sp_setup, dbname="NSGA2", dbformat="csv" + ) + + sampler.sample(generations, n_obj=3, n_pop=n_pop) results = sampler.getdata() - - fig= plt.figure(1,figsize=(9,5)) - plt.plot(results['like1']) + + fig = plt.figure(1, figsize=(9, 5)) + plt.plot(results["like1"]) plt.show() - plt.ylabel('Objective function') - plt.xlabel('Iteration') - fig.savefig('hymod_objectivefunction.png',dpi=300) - - # Example plot to show the parameter distribution ###### + plt.ylabel("Objective function") + plt.xlabel("Iteration") + fig.savefig("hymod_objectivefunction.png", dpi=300) + + # Example plot to show the parameter distribution ###### def plot_parameter_trace(ax, results, parameter): - ax.plot(results['par'+parameter.name],'.') + ax.plot(results["par" + parameter.name], ".") ax.set_ylabel(parameter.name) ax.set_ylim(parameter.minbound, parameter.maxbound) - + def plot_parameter_histogram(ax, results, parameter): - #chooses the last 20% of the sample - ax.hist(results['par'+parameter.name][int(len(results)*0.9):], - bins =np.linspace(parameter.minbound,parameter.maxbound,20)) - ax.set_ylabel('Density') + # chooses the last 20% of the sample + ax.hist( + results["par" + parameter.name][int(len(results) * 0.9) :], + bins=np.linspace(parameter.minbound, parameter.maxbound, 20), + ) + ax.set_ylabel("Density") ax.set_xlim(parameter.minbound, parameter.maxbound) - - fig= plt.figure(2,figsize=(9,9)) - - ax1 = plt.subplot(5,2,1) + + fig = plt.figure(2, figsize=(9, 9)) + + ax1 = plt.subplot(5, 2, 1) plot_parameter_trace(ax1, results, spot_setup.cmax) - - ax2 = plt.subplot(5,2,2) - plot_parameter_histogram(ax2,results, spot_setup.cmax) - - ax3 = plt.subplot(5,2,3) + + ax2 = plt.subplot(5, 2, 2) + plot_parameter_histogram(ax2, results, spot_setup.cmax) + + ax3 = plt.subplot(5, 2, 3) plot_parameter_trace(ax3, results, spot_setup.bexp) - - ax4 = plt.subplot(5,2,4) + + ax4 = plt.subplot(5, 2, 4) plot_parameter_histogram(ax4, results, spot_setup.bexp) - - ax5 = plt.subplot(5,2,5) + + ax5 = plt.subplot(5, 2, 5) plot_parameter_trace(ax5, results, spot_setup.alpha) - - ax6 = plt.subplot(5,2,6) + + ax6 = plt.subplot(5, 2, 6) plot_parameter_histogram(ax6, results, spot_setup.alpha) - ax7 = plt.subplot(5,2,7) + ax7 = plt.subplot(5, 2, 7) plot_parameter_trace(ax7, results, spot_setup.Ks) - - ax8 = plt.subplot(5,2,8) + + ax8 = plt.subplot(5, 2, 8) plot_parameter_histogram(ax8, results, spot_setup.Ks) - ax9 = plt.subplot(5,2,9) + ax9 = plt.subplot(5, 2, 9) plot_parameter_trace(ax9, results, spot_setup.Kq) - ax9.set_xlabel('Iterations') - - ax10 = plt.subplot(5,2,10) + ax9.set_xlabel("Iterations") + + ax10 = plt.subplot(5, 2, 10) plot_parameter_histogram(ax10, results, spot_setup.Kq) - ax10.set_xlabel('Parameter range') - + ax10.set_xlabel("Parameter range") + plt.show() - fig.savefig('hymod_parameters.png',dpi=300) - + fig.savefig("hymod_parameters.png", dpi=300) # Example plot to show remaining parameter uncertainty # - fields=[word for word in results.dtype.names if word.startswith('sim')] - fig= plt.figure(3, figsize=(16,9)) - ax11 = plt.subplot(1,1,1) - q5,q25,q75,q95=[],[],[],[] + fields = [word for word in results.dtype.names if word.startswith("sim")] + fig = plt.figure(3, figsize=(16, 9)) + ax11 = plt.subplot(1, 1, 1) + q5, q25, q75, q95 = [], [], [], [] for field in fields: - q5.append(np.percentile(results[field][-100:-1],2.5)) - q95.append(np.percentile(results[field][-100:-1],97.5)) - ax11.plot(q5,color='dimgrey',linestyle='solid') - ax11.plot(q95,color='dimgrey',linestyle='solid') - ax11.fill_between(np.arange(0,len(q5),1),list(q5),list(q95),facecolor='dimgrey',zorder=0, - linewidth=0,label='parameter uncertainty') - ax11.plot(sp_setup.evaluation(),'r.',label='data') - ax11.set_ylim(-50,450) - ax11.set_xlim(0,729) + q5.append(np.percentile(results[field][-100:-1], 2.5)) + q95.append(np.percentile(results[field][-100:-1], 97.5)) + ax11.plot(q5, color="dimgrey", linestyle="solid") + ax11.plot(q95, color="dimgrey", linestyle="solid") + ax11.fill_between( + np.arange(0, len(q5), 1), + list(q5), + list(q95), + facecolor="dimgrey", + zorder=0, + linewidth=0, + label="parameter uncertainty", + ) + ax11.plot(sp_setup.evaluation(), "r.", label="data") + ax11.set_ylim(-50, 450) + ax11.set_xlim(0, 729) ax11.legend() - fig.savefig('python_hymod.png',dpi=300) + fig.savefig("python_hymod.png", dpi=300) ######################################################### - - x,y,z = results['like1'][-n_pop:],results['like2'][-n_pop:],results['like3'][-n_pop:] + x, y, z = ( + results["like1"][-n_pop:], + results["like2"][-n_pop:], + results["like3"][-n_pop:], + ) fig = plt.figure(4) - ax12 = fig.add_subplot(111, projection='3d') - ax12.scatter(x,y,z,marker="o") + ax12 = fig.add_subplot(111, projection="3d") + ax12.scatter(x, y, z, marker="o") ax12.set_xlabel("pbias") ax12.set_ylabel("rmse") ax12.set_zlabel("rsquared") plt.show() - class Test_NSGAII(unittest.TestCase): def multi_obj_func(evaluation, simulation): - #used to overwrite objective function in hymod example + # used to overwrite objective function in hymod example like1 = abs(spotpy.objectivefunctions.pbias(evaluation, simulation)) like2 = spotpy.objectivefunctions.rmse(evaluation, simulation) - like3 = spotpy.objectivefunctions.rsquared(evaluation, simulation)*-1 + like3 = spotpy.objectivefunctions.rsquared(evaluation, simulation) * -1 return [like1, like2, like3] def setUp(self): - generations=40 + generations = 40 n_pop = 20 skip_duplicates = True - self.sp_setup=spot_setup(obj_func= multi_obj_func) - self.sampler = spotpy.algorithms.NSGAII(self.sp_setup, dbname='NSGA2', dbformat="csv") - self.sampler.sample(generations, n_obj= 3, n_pop = n_pop, skip_duplicates = skip_duplicates) + self.sp_setup = spot_setup(obj_func=multi_obj_func) + self.sampler = spotpy.algorithms.NSGAII( + self.sp_setup, dbname="NSGA2", dbformat="csv" + ) + self.sampler.sample( + generations, n_obj=3, n_pop=n_pop, skip_duplicates=skip_duplicates + ) def test_sampler_output(self): self.assertEqual(200, len(self.sampler.getdata())) diff --git a/tutorials/tutorial_nsgaii_dltz1.py b/tutorials/tutorial_nsgaii_dltz1.py new file mode 100644 index 00000000..675b629c --- /dev/null +++ b/tutorials/tutorial_nsgaii_dltz1.py @@ -0,0 +1,64 @@ +import sys + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from mpl_toolkits.mplot3d import Axes3D + +import spotpy +from spotpy.examples.spot_setup_dtlz1 import spot_setup + +if __name__ == "__main__": + # Create samplers for every algorithm: + results = [] + n_obj = 3 + spot_setup = spot_setup(n_var=5, n_obj=n_obj) + generations = 10 + n_pop = 30 + skip_duplicates = False + + sampler = spotpy.algorithms.NSGAII( + spot_setup=spot_setup, dbname="NSGA2", dbformat="csv", save_sim=True + ) + sampler.sample(generations, n_obj=3, n_pop=n_pop, skip_duplicates=skip_duplicates) + + last = None + first = None + + # output calibration + + df = pd.read_csv("NSGA2.csv") + + df["like3"] = df.like3 * -1 + + if last: + df = df.iloc[-last:, :] + elif first: + df = df.iloc[:first, :] + else: + pass + + # plot objective functions + fig = plt.figure() + for i, name in enumerate(df.columns[:n_obj]): + ax = fig.add_subplot(n_obj, 1, i + 1) + df.loc[::5, name].plot(lw=0.5, figsize=(18, 8), ax=ax, color="black") + plt.title(name) + plt.show() + + x, y, z = df.iloc[-n_pop:, 0], df.iloc[-n_pop:, 1], df.iloc[-n_pop:, 2] + fig = plt.figure() + ax = fig.add_subplot(111, projection="3d") + ax.scatter(x, y, z, marker="o") + ax.set_xlabel("x") + ax.set_ylabel("y") + ax.set_zlabel("z") + plt.show() + + # plot parameters + fig = plt.figure() + for i, name in enumerate(df.columns[n_obj:8]): + ax = fig.add_subplot(5, 1, i + 1) + df.loc[:, name].plot(lw=0.5, figsize=(18, 8), ax=ax, color="black") + plt.title(name) + plt.show() diff --git a/spotpy/examples/tutorial_own_database.py b/tutorials/tutorial_own_database.py similarity index 71% rename from spotpy/examples/tutorial_own_database.py rename to tutorials/tutorial_own_database.py index cc6b7947..9c2386c7 100644 --- a/spotpy/examples/tutorial_own_database.py +++ b/tutorials/tutorial_own_database.py @@ -1,16 +1,13 @@ -''' +""" Copyright 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). :author: Tobias Houska This example implements the Rosenbrock function into SPOT. -''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +""" import numpy as np + import spotpy from spotpy.objectivefunctions import rmse @@ -18,17 +15,19 @@ class spot_setup(object): a = spotpy.parameter.Uniform(low=0, high=1) b = spotpy.parameter.Uniform(low=0, high=1) - + def __init__(self): - + self.db_headers = ["obj_functions", "parameters", "simulations"] - self.database = open('MyOwnDatabase.txt', 'w') + self.database = open("MyOwnDatabase.txt", "w") self.database.write("\t".join(self.db_headers) + "\n") def simulation(self, vector): x = np.array(vector) - simulations = [sum(100.0*(x[1:] - x[:-1]**2.0)**2.0 + (1 - x[:-1])**2.0)] + simulations = [ + sum(100.0 * (x[1:] - x[:-1] ** 2.0) ** 2.0 + (1 - x[:-1]) ** 2.0) + ] return simulations def evaluation(self): @@ -42,13 +41,16 @@ def objectivefunction(self, simulation, evaluation): def save(self, objectivefunctions, parameter, simulations, *args, **kwargs): param_str = "\t".join((str(p) for p in parameter)) sim_str = "\t".join((str(s) for s in simulations)) - line = "\t".join([str(objectivefunctions), param_str, sim_str]) + '\n' + line = "\t".join([str(objectivefunctions), param_str, sim_str]) + "\n" self.database.write(line) + if __name__ == "__main__": spot_setup = spot_setup() - + # set dbformat to custom and spotpy will return results in spot_setup.save function - sampler = spotpy.algorithms.mc(spot_setup, dbformat='custom') - sampler.sample(100) # Choose equal or less repetitions as you have parameters in your List - spot_setup.database.close() # Close the created txt file + sampler = spotpy.algorithms.mc(spot_setup, dbformat="custom") + sampler.sample( + 100 + ) # Choose equal or less repetitions as you have parameters in your List + spot_setup.database.close() # Close the created txt file diff --git a/tutorials/tutorial_padds_hymod.py b/tutorials/tutorial_padds_hymod.py new file mode 100644 index 00000000..591ea5a9 --- /dev/null +++ b/tutorials/tutorial_padds_hymod.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +This class holds example code how to use the dream algorithm +""" + +import numpy as np + +try: + import spotpy +except ImportError: + import sys + + sys.path.append(".") + import spotpy + +import matplotlib.pyplot as plt + +from spotpy.examples.spot_setup_hymod_python import spot_setup + + +def multi_obj_func(evaluation, simulation): + # used to overwrite objective function in hymod example + like1 = abs(spotpy.objectivefunctions.pbias(evaluation, simulation)) + like2 = spotpy.objectivefunctions.rmse(evaluation, simulation) + like3 = spotpy.objectivefunctions.rsquared(evaluation, simulation) * -1 + return np.array([like1, like2, like3]) + + +if __name__ == "__main__": + parallel = "seq" # Runs everthing in sequential mode + np.random.seed(2000) # Makes the results reproduceable + + # Initialize the Hymod example + # In this case, we tell the setup which algorithm we want to use, so + # we can use this exmaple for different algorithms + spot_setup = spot_setup(multi_obj_func) + + # Select number of maximum allowed repetitions + rep = 2000 + + # Create the PADDS sampler of spotpy, alt_objfun is set to None to force SPOTPY + # to jump into the def objectivefunction in the spot_setup class (default is + # spotpy.objectivefunctions.rmse) + sampler = spotpy.algorithms.padds(spot_setup, dbname="padds_hymod", dbformat="csv") + + # Start the sampler, one can specify metric if desired + sampler.sample(rep, metric="ones") + + # Load the results gained with the sceua sampler, stored in padds_hymod.csv + # results = spotpy.analyser.load_csv_results('padds_hymod') + results = sampler.getdata() + + # from pprint import pprint + # #pprint(results) + # pprint(results['chain']) + + for likno in range(1, 4): + fig_like1 = plt.figure(1, figsize=(9, 5)) + plt.plot(results["like" + str(likno)]) + plt.show() + fig_like1.savefig( + "hymod_padds_objectivefunction_" + str(likno) + ".png", dpi=300 + ) + + fig, ax = plt.subplots(3) + for likenr in range(3): + ax[likenr].plot(results["like" + str(likenr + 1)]) + ax[likenr].set_ylabel("like" + str(likenr + 1)) + fig.savefig("hymod_padds_objectivefunction.png", dpi=300) + + # Example plot to show the parameter distribution ###### + def plot_parameter_trace(ax, results, parameter): + ax.plot(results["par" + parameter.name], ".") + ax.set_ylabel(parameter.name) + ax.set_ylim(parameter.minbound, parameter.maxbound) + + def plot_parameter_histogram(ax, results, parameter): + # chooses the last 20% of the sample + ax.hist( + results["par" + parameter.name][int(len(results) * 0.9) :], + bins=np.linspace(parameter.minbound, parameter.maxbound, 20), + ) + ax.set_ylabel("Density") + ax.set_xlim(parameter.minbound, parameter.maxbound) + + fig = plt.figure(2, figsize=(9, 9)) + + ax1 = plt.subplot(5, 2, 1) + plot_parameter_trace(ax1, results, spot_setup.cmax) + + ax2 = plt.subplot(5, 2, 2) + plot_parameter_histogram(ax2, results, spot_setup.cmax) + + ax3 = plt.subplot(5, 2, 3) + plot_parameter_trace(ax3, results, spot_setup.bexp) + + ax4 = plt.subplot(5, 2, 4) + plot_parameter_histogram(ax4, results, spot_setup.bexp) + + ax5 = plt.subplot(5, 2, 5) + plot_parameter_trace(ax5, results, spot_setup.alpha) + + ax6 = plt.subplot(5, 2, 6) + plot_parameter_histogram(ax6, results, spot_setup.alpha) + + ax7 = plt.subplot(5, 2, 7) + plot_parameter_trace(ax7, results, spot_setup.Ks) + + ax8 = plt.subplot(5, 2, 8) + plot_parameter_histogram(ax8, results, spot_setup.Ks) + + ax9 = plt.subplot(5, 2, 9) + plot_parameter_trace(ax9, results, spot_setup.Kq) + ax9.set_xlabel("Iterations") + + ax10 = plt.subplot(5, 2, 10) + plot_parameter_histogram(ax10, results, spot_setup.Kq) + ax10.set_xlabel("Parameter range") + + plt.show() + fig.savefig("hymod_parameters.png", dpi=300) + + # Example plot to show remaining parameter uncertainty # + fields = [word for word in results.dtype.names if word.startswith("sim")] + fig = plt.figure(3, figsize=(16, 9)) + ax11 = plt.subplot(1, 1, 1) + q5, q25, q75, q95 = [], [], [], [] + for field in fields: + q5.append(np.percentile(results[field][-100:-1], 2.5)) + q95.append(np.percentile(results[field][-100:-1], 97.5)) + ax11.plot(q5, color="dimgrey", linestyle="solid") + ax11.plot(q95, color="dimgrey", linestyle="solid") + ax11.fill_between( + np.arange(0, len(q5), 1), + list(q5), + list(q95), + facecolor="dimgrey", + zorder=0, + linewidth=0, + label="parameter uncertainty", + ) + ax11.plot(spot_setup.evaluation(), "r.", label="data") + ax11.set_ylim(-50, 450) + ax11.set_xlim(0, 729) + ax11.legend() + fig.savefig("python_hymod.png", dpi=300) + ######################################################### diff --git a/spotpy/examples/tutorial_parallel_computing_hymod.py b/tutorials/tutorial_parallel_computing_hymod.py similarity index 56% rename from spotpy/examples/tutorial_parallel_computing_hymod.py rename to tutorials/tutorial_parallel_computing_hymod.py index 553cd3bb..d9121e61 100644 --- a/spotpy/examples/tutorial_parallel_computing_hymod.py +++ b/tutorials/tutorial_parallel_computing_hymod.py @@ -1,36 +1,36 @@ # -*- coding: utf-8 -*- -''' +""" Copyright 2015 by Tobias Houska This file is part of Statistical Parameter Estimation Tool (SPOTPY). :author: Tobias Houska This class holds example code how to use the dream algorithm -''' +""" try: import spotpy except ImportError: import sys + sys.path.append(".") import spotpy -#from spotpy.examples.spot_setup_hymod_python import spot_setup +# from spotpy.examples.spot_setup_hymod_python import spot_setup import sys -# When you are using parallel = 'mpc' you need to have -#the if __name__ == "__main__": line in your script +# When you are using parallel = 'mpc' you need to have +# the if __name__ == "__main__": line in your script if __name__ == "__main__": rep = 200 - #If you are working on a windows computer, this will be True - if 'win' in sys.platform: - parallel ='mpc' + # If you are working on a windows computer, this will be True + if "win" in sys.platform: + parallel = "mpc" from spotpy.examples.spot_setup_hymod_exe import spot_setup - - # If not you probably have a Mac or Unix system. Then save this file and start it - # from your terminal with "mpirun -c 20 python your_script.py" + # If not you probably have a Mac or Unix system. Then save this file and start it + # from your terminal with "mpirun -c 20 python your_script.py" else: - parallel = 'mpi' + parallel = "mpi" from spotpy.examples.spot_setup_hymod_unix import spot_setup # Initialize the Hymod example (this runs on Windows systems only) @@ -38,16 +38,17 @@ # your spot_setup class to run in parallel # If your model in def simulation reads any files, make sure, they are # unique for each cpu. Otherwise things get messed up... - spot_setup=spot_setup(parallel=parallel) - + spot_setup = spot_setup(parallel=parallel) + # Initialize a sampler that is suited for parallel computing (all except MLE, MCMC and SA) - sampler=spotpy.algorithms.mc(spot_setup, dbname='Parallel_hymod', dbformat='csv', - parallel=parallel) + sampler = spotpy.algorithms.mc( + spot_setup, dbname="Parallel_hymod", dbformat="csv", parallel=parallel + ) # Sample in parlallel sampler.sample(rep) - + # Load results from file - results = spotpy.analyser.load_csv_results('Parallel_hymod') + results = spotpy.analyser.load_csv_results("Parallel_hymod") print(len(results)) # Plot best model run - #spotpy.analyser.plot_bestmodelrun(results,spot_setup.evaluation()) \ No newline at end of file + # spotpy.analyser.plot_bestmodelrun(results,spot_setup.evaluation()) diff --git a/tutorials/tutorial_rosenbrock.py b/tutorials/tutorial_rosenbrock.py new file mode 100644 index 00000000..fd40a3de --- /dev/null +++ b/tutorials/tutorial_rosenbrock.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +""" +Copyright (c) 2018 by Tobias Houska +This file is part of Statistical Parameter Optimization Tool for Python(SPOTPY). +:author: Tobias Houska + +This file holds the example code from the Rosenbrock tutorial web-documention. +""" + + +try: + import spotpy +except ImportError: + import sys + + sys.path.append(".") + import spotpy + +from spotpy.describe import describe +from spotpy.examples.spot_setup_rosenbrock import spot_setup + +# Create samplers for every algorithm: +results = [] +rep = 1000 +timeout = 10 # Given in Seconds + +parallel = "seq" +dbformat = "csv" + +# Bayesian algorithms should be run with a likelihood function +bayesian_likelihood_func = spotpy.likelihoods.gaussianLikelihoodMeasErrorOut + + +sampler = spotpy.algorithms.mc( + spot_setup(), + parallel=parallel, + dbname="RosenMC", + dbformat=dbformat, + sim_timeout=timeout, +) +print(describe(sampler)) +sampler.sample(rep) +results.append(sampler.getdata()) + + +sampler = spotpy.algorithms.lhs( + spot_setup(), + parallel=parallel, + dbname="RosenLHS", + dbformat=dbformat, + sim_timeout=timeout, +) +sampler.sample(rep) +results.append(sampler.getdata()) + + +sampler = spotpy.algorithms.mle( + spot_setup(obj_func=bayesian_likelihood_func), + parallel=parallel, + dbname="RosenMLE", + dbformat=dbformat, + sim_timeout=timeout, +) +sampler.sample(rep) +results.append(sampler.getdata()) + + +sampler = spotpy.algorithms.sceua( + spot_setup(), + parallel=parallel, + dbname="RosenSCEUA", + dbformat=dbformat, + sim_timeout=timeout, +) +sampler.sample(rep, ngs=4) +results.append(sampler.getdata()) + + +sampler = spotpy.algorithms.sa( + spot_setup(obj_func=bayesian_likelihood_func), + parallel=parallel, + dbname="RosenSA", + dbformat=dbformat, + sim_timeout=timeout, +) +sampler.sample(rep) +results.append(sampler.getdata()) + + +sampler = spotpy.algorithms.rope( + spot_setup(), + parallel=parallel, + dbname="RosenROPE", + dbformat=dbformat, + sim_timeout=timeout, +) +sampler.sample(rep) +results.append(sampler.getdata()) + + +sampler = spotpy.algorithms.abc( + spot_setup(), + parallel=parallel, + dbname="RosenABC", + dbformat=dbformat, + sim_timeout=timeout, +) +sampler.sample(rep) +results.append(sampler.getdata()) + + +sampler = spotpy.algorithms.fscabc( + spot_setup(), + parallel=parallel, + dbname="RosenFSABC", + dbformat=dbformat, + sim_timeout=timeout, +) +sampler.sample(rep) +results.append(sampler.getdata()) + + +sampler = spotpy.algorithms.mcmc( + spot_setup(obj_func=bayesian_likelihood_func), + parallel=parallel, + dbname="RosenMCMC", + dbformat=dbformat, + sim_timeout=timeout, +) +sampler.sample(rep) +results.append(sampler.getdata()) + + +sampler = spotpy.algorithms.demcz( + spot_setup(obj_func=bayesian_likelihood_func), + parallel=parallel, + dbname="RosenDEMCz", + dbformat=dbformat, + sim_timeout=timeout, +) +sampler.sample(rep, nChains=4) +results.append(sampler.getdata()) + +sampler = spotpy.algorithms.dream( + spot_setup(obj_func=bayesian_likelihood_func), + parallel=parallel, + dbname="RosenDREAM", + dbformat=dbformat, + sim_timeout=timeout, +) +sampler.sample(rep, nChains=4) +results.append(sampler.getdata()) + + +print(results[0].dtype) # Check for Travis: Get the last sampled parameter for x +evaluation = spot_setup().evaluation() + +# Example how to plot the data +algorithms = [ + "mc", + "lhs", + "mle", + "sceua", + "sa", + "rope", + "abc", + "fscabc", + "mcmc", + "demcz", + "dream", +] +spotpy.analyser.plot_parametertrace_algorithms(results, algorithms, spot_setup()) diff --git a/tutorials/tutorial_sceua_hymod.py b/tutorials/tutorial_sceua_hymod.py new file mode 100644 index 00000000..73e89386 --- /dev/null +++ b/tutorials/tutorial_sceua_hymod.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +""" +Copyright 2015 by Tobias Houska +This file is part of Statistical Parameter Estimation Tool (SPOTPY). + +:author: Tobias Houska + +This class holds example code how to use the dream algorithm +""" + +import matplotlib.pyplot as plt +import numpy as np + +import spotpy +from spotpy.examples.spot_setup_hymod_python import spot_setup + +if __name__ == "__main__": + parallel = "seq" # Runs everthing in sequential mode + np.random.seed(2000) # Makes the results reproduceable + + # Initialize the Hymod example + # In this case, we tell the setup which algorithm we want to use, so + # we can use this exmaple for different algorithms + spot_setup = spot_setup(spotpy.objectivefunctions.rmse) + + # Select number of maximum allowed repetitions + rep = 5000 + sampler = spotpy.algorithms.sceua(spot_setup, dbname="SCEUA_hymod", dbformat="csv") + + # Start the sampler, one can specify ngs, kstop, peps and pcento id desired + sampler.sample(rep, ngs=7, kstop=3, peps=0.1, pcento=0.1) + + # Load the results gained with the sceua sampler, stored in SCEUA_hymod.csv + results = spotpy.analyser.load_csv_results("SCEUA_hymod") + + # Plot how the objective function was minimized during sampling + fig = plt.figure(1, figsize=(9, 6)) + plt.plot(results["like1"]) + plt.show() + plt.ylabel("RMSE") + plt.xlabel("Iteration") + fig.savefig("SCEUA_objectivefunctiontrace.png", dpi=150) + + # Plot the best model run + # Find the run_id with the minimal objective function value + bestindex, bestobjf = spotpy.analyser.get_minlikeindex(results) + + # Select best model run + best_model_run = results[bestindex] + + # Filter results for simulation results + fields = [word for word in best_model_run.dtype.names if word.startswith("sim")] + best_simulation = list(best_model_run[fields]) + + fig = plt.figure(figsize=(9, 6)) + ax = plt.subplot(1, 1, 1) + ax.plot( + best_simulation, + color="black", + linestyle="solid", + label="Best objf.=" + str(bestobjf), + ) + ax.plot(spot_setup.evaluation(), "r.", markersize=3, label="Observation data") + plt.xlabel("Number of Observation Points") + plt.ylabel("Discharge [l s-1]") + plt.legend(loc="upper right") + fig.savefig("SCEUA_best_modelrun.png", dpi=150) diff --git a/tutorials/tutorial_signatures.py b/tutorials/tutorial_signatures.py new file mode 100644 index 00000000..9ff1614e --- /dev/null +++ b/tutorials/tutorial_signatures.py @@ -0,0 +1,403 @@ +# -*- coding: utf-8 -*- +""" +Copyright (c) 2017 by Benjamin Manns +This file is part of Statistical Parameter Estimation Tool (SPOTPY). +:author: Benjamin Manns + +This code shows you, how to use the hydroligcal signatures. They can also be implemented in the def objective function. +""" + +from pprint import pprint + +import spotpy.signatures as sig +from spotpy.examples.spot_setup_hymod import spot_setup + +print("INFO: For this example you need the folder >hymod< in the examples folder") + +spot_setup = spot_setup() +parameterset = spot_setup.parameters()["random"] +simulation = spot_setup.simulation(parameterset) +observation = spot_setup.evaluation() + + +# Beispiele zum einfachen Copy & Paste + +print(sig.getMeanFlow(simulation, observation, mode="get_signature")) +print(sig.getMeanFlow(simulation, observation, mode="get_raw_data")) +print(sig.getMeanFlow(simulation, observation, mode="calc_Dev")) + +print(sig.getMedianFlow(simulation, observation, mode="get_signature")) +print(sig.getMedianFlow(simulation, observation, mode="get_raw_data")) +print(sig.getMedianFlow(simulation, observation, mode="calc_Dev")) + +print(sig.getSkewness(simulation, observation, mode="get_signature")) +print(sig.getSkewness(simulation, observation, mode="get_raw_data")) +print(sig.getSkewness(simulation, observation, mode="calc_Dev")) + +print(sig.getCoeffVariation(simulation, observation, mode="get_signature")) +print(sig.getCoeffVariation(simulation, observation, mode="get_raw_data")) +print(sig.getCoeffVariation(simulation, observation, mode="calc_Dev")) + +print(sig.getQ001(simulation, observation, mode="get_signature")) +print(sig.getQ001(simulation, observation, mode="get_raw_data")) +print(sig.getQ001(simulation, observation, mode="calc_Dev")) + +print(sig.getQ01(simulation, observation, mode="get_signature")) +print(sig.getQ01(simulation, observation, mode="get_raw_data")) +print(sig.getQ01(simulation, observation, mode="calc_Dev")) + +print(sig.getQ1(simulation, observation, mode="get_signature")) +print(sig.getQ1(simulation, observation, mode="get_raw_data")) +print(sig.getQ1(simulation, observation, mode="calc_Dev")) + +print(sig.getQ5(simulation, observation, mode="get_signature")) +print(sig.getQ5(simulation, observation, mode="get_raw_data")) +print(sig.getQ5(simulation, observation, mode="calc_Dev")) + +print(sig.getQ10(simulation, observation, mode="get_signature")) +print(sig.getQ10(simulation, observation, mode="get_raw_data")) +print(sig.getQ10(simulation, observation, mode="calc_Dev")) + +print(sig.getQ20(simulation, observation, mode="get_signature")) +print(sig.getQ20(simulation, observation, mode="get_raw_data")) +print(sig.getQ20(simulation, observation, mode="calc_Dev")) + +print(sig.getQ85(simulation, observation, mode="get_signature")) +print(sig.getQ85(simulation, observation, mode="get_raw_data")) +print(sig.getQ85(simulation, observation, mode="calc_Dev")) + +print(sig.getQ95(simulation, observation, mode="get_signature")) +print(sig.getQ95(simulation, observation, mode="get_raw_data")) +print(sig.getQ95(simulation, observation, mode="calc_Dev")) + +print(sig.getQ99(simulation, observation, mode="get_signature")) +print(sig.getQ99(simulation, observation, mode="get_raw_data")) +print(sig.getQ99(simulation, observation, mode="calc_Dev")) + +print(sig.getSlopeFDC(simulation, observation, mode="get_signature")) +print(sig.getSlopeFDC(simulation, observation, mode="get_raw_data")) +print(sig.getSlopeFDC(simulation, observation, mode="calc_Dev")) + +try: + import matplotlib.pyplot as plt + import pandas as pd + + timespanlen = simulation.__len__() + ddd = pd.date_range("2015-01-01 11:00", freq="5min", periods=timespanlen) + dd_daily = pd.date_range("2015-05-01", periods=timespanlen) + + print( + sig.getAverageFloodOverflowPerSection( + simulation, + observation, + mode="get_signature", + datetime_series=dd_daily, + threshold_value=1, + ) + ) + print( + sig.getAverageFloodOverflowPerSection( + simulation, + observation, + mode="get_raw_data", + datetime_series=dd_daily, + threshold_value=1, + ) + ) + print( + sig.getAverageFloodOverflowPerSection( + simulation, + observation, + mode="calc_Dev", + datetime_series=dd_daily, + threshold_value=1, + ) + ) + + print( + sig.getAverageFloodFrequencyPerSection( + simulation, + observation, + datetime_series=dd_daily, + threshold_value=1, + mode="get_signature", + ) + ) + print( + sig.getAverageFloodFrequencyPerSection( + simulation, + observation, + datetime_series=dd_daily, + threshold_value=1, + mode="get_raw_data", + ) + ) + + # If you want to plot the raw data, simple do: + vals = sig.getAverageFloodFrequencyPerSection( + simulation, + observation, + datetime_series=dd_daily, + threshold_value=1, + mode="get_raw_data", + ) + vals.plot() + plt.show() + + print( + sig.getAverageFloodFrequencyPerSection( + simulation, + observation, + datetime_series=dd_daily, + threshold_value=1, + mode="calc_Dev", + ) + ) + + print( + sig.getAverageFloodDuration( + simulation, + observation, + datetime_series=dd_daily, + threshold_value=3, + mode="get_signature", + ) + ) + print( + sig.getAverageFloodDuration( + simulation, + observation, + datetime_series=dd_daily, + threshold_value=3, + mode="get_raw_data", + ) + ) + print( + sig.getAverageFloodDuration( + simulation, + observation, + datetime_series=dd_daily, + threshold_value=3, + mode="calc_Dev", + ) + ) + + print( + sig.getAverageBaseflowUnderflowPerSection( + simulation, + observation, + datetime_series=dd_daily, + threshold_value=4, + mode="get_signature", + ) + ) + print( + sig.getAverageBaseflowUnderflowPerSection( + simulation, + observation, + datetime_series=dd_daily, + threshold_value=4, + mode="get_raw_data", + ) + ) + print( + sig.getAverageBaseflowUnderflowPerSection( + simulation, + observation, + datetime_series=dd_daily, + threshold_value=4, + mode="calc_Dev", + ) + ) + + print( + sig.getAverageBaseflowFrequencyPerSection( + simulation, + observation, + datetime_series=dd_daily, + threshold_value=3, + mode="get_signature", + ) + ) + print( + sig.getAverageBaseflowFrequencyPerSection( + simulation, + observation, + datetime_series=ddd, + threshold_value=5, + mode="get_raw_data", + ) + ) + print( + sig.getAverageBaseflowFrequencyPerSection( + simulation, + observation, + datetime_series=dd_daily, + threshold_value=3, + mode="calc_Dev", + ) + ) + + print( + sig.getAverageBaseflowDuration( + simulation, + observation, + datetime_series=dd_daily, + threshold_value=3, + mode="get_signature", + ) + ) + print( + sig.getAverageBaseflowDuration( + simulation, + observation, + datetime_series=ddd, + threshold_value=5, + mode="get_raw_data", + ) + ) + print( + sig.getAverageBaseflowDuration( + simulation, + observation, + datetime_series=dd_daily, + threshold_value=3, + mode="calc_Dev", + ) + ) + + print( + sig.getFloodFrequency( + simulation, + observation, + datetime_series=pd.date_range("2015-05-01", periods=timespanlen), + threshold_value=3, + mode="get_signature", + ) + ) + print( + sig.getFloodFrequency( + simulation, + observation, + datetime_series=pd.date_range("2015-05-01", periods=timespanlen), + threshold_value=3, + mode="get_raw_data", + ) + ) + print( + sig.getFloodFrequency( + simulation, + observation, + datetime_series=pd.date_range("2015-05-01", periods=timespanlen), + threshold_value=3, + mode="calc_Dev", + ) + ) + + print( + sig.getBaseflowFrequency( + simulation, + observation, + datetime_series=pd.date_range("2015-05-01", periods=timespanlen), + threshold_value=3, + mode="get_signature", + ) + ) + print( + sig.getBaseflowFrequency( + simulation, + observation, + datetime_series=pd.date_range( + "2015-05-01", freq="5min", periods=timespanlen + ), + threshold_value=3, + mode="get_raw_data", + ) + ) + print( + sig.getBaseflowFrequency( + simulation, + observation, + datetime_series=pd.date_range("2015-05-01", periods=timespanlen), + threshold_value=3, + mode="calc_Dev", + ) + ) + + print( + sig.getLowFlowVar( + simulation, + observation, + datetime_series=pd.date_range("2015-05-01", periods=timespanlen), + mode="get_signature", + ) + ) + print( + sig.getLowFlowVar( + simulation, + observation, + datetime_series=pd.date_range("2015-05-01", periods=timespanlen), + mode="get_raw_data", + ) + ) + print( + sig.getLowFlowVar( + simulation, + observation, + datetime_series=pd.date_range("2015-05-01", periods=timespanlen), + mode="calc_Dev", + ) + ) + + print( + sig.getHighFlowVar( + simulation, + observation, + datetime_series=pd.date_range("2015-05-01", periods=timespanlen), + mode="get_signature", + ) + ) + print( + sig.getHighFlowVar( + simulation, + observation, + datetime_series=pd.date_range("2015-05-01", periods=timespanlen), + mode="get_raw_data", + ) + ) + print( + sig.getHighFlowVar( + simulation, + observation, + datetime_series=pd.date_range("2015-05-01", periods=timespanlen), + mode="calc_Dev", + ) + ) + + print( + sig.getBaseflowIndex( + simulation, + observation, + datetime_series=pd.date_range("2015-05-01", periods=timespanlen), + mode="get_signature", + ) + ) + print( + sig.getBaseflowIndex( + simulation, + observation, + datetime_series=pd.date_range("2015-05-01", periods=timespanlen), + mode="get_raw_data", + ) + ) + print( + sig.getBaseflowIndex( + simulation, + observation, + datetime_series=pd.date_range("2015-05-01", periods=timespanlen), + mode="calc_Dev", + ) + ) + +except ImportError: + print("Please install Pandas to use these signature functions")