From 04655a3d8238850dcbd2b572fa380e5348efab0a Mon Sep 17 00:00:00 2001 From: Adam Stewart Date: Fri, 18 Feb 2022 18:49:30 +0000 Subject: [PATCH 01/10] Updated docs and added docs versioning - Updated full mkdocs-material dependencies. - Added mike to use docs versioning. - Fixed docstrings for new Return parsing. --- docs/gen_doc_stubs.py | 2 +- docs/theme/main.html | 21 +- mkdocs.yml | 9 +- poetry.lock | 214 ++++++++++++++---- pyproject.toml | 13 +- vast_pipeline/context_processors.py | 2 +- vast_pipeline/converters.py | 4 +- vast_pipeline/image/main.py | 41 +++- vast_pipeline/image/utils.py | 29 ++- .../management/commands/initpiperun.py | 2 +- .../management/commands/runpipeline.py | 8 +- vast_pipeline/management/helpers.py | 6 +- vast_pipeline/pipeline/association.py | 84 ++++--- vast_pipeline/pipeline/config.py | 14 +- vast_pipeline/pipeline/errors.py | 8 +- vast_pipeline/pipeline/finalise.py | 6 +- vast_pipeline/pipeline/forced_extraction.py | 14 +- vast_pipeline/pipeline/loading.py | 4 +- vast_pipeline/pipeline/model_generator.py | 2 +- vast_pipeline/pipeline/new_sources.py | 102 +++++---- vast_pipeline/pipeline/pairs.py | 2 +- vast_pipeline/pipeline/utils.py | 158 ++++++------- vast_pipeline/plots.py | 7 +- vast_pipeline/signals.py | 3 + vast_pipeline/utils/auth.py | 8 +- vast_pipeline/utils/external_query.py | 44 ++-- vast_pipeline/utils/utils.py | 20 +- vast_pipeline/utils/view.py | 5 +- vast_pipeline/views.py | 57 +++-- 29 files changed, 566 insertions(+), 323 deletions(-) diff --git a/docs/gen_doc_stubs.py b/docs/gen_doc_stubs.py index 29249b125..bacdaa728 100755 --- a/docs/gen_doc_stubs.py +++ b/docs/gen_doc_stubs.py @@ -11,7 +11,7 @@ # See https://github.com/mkdocstrings/mkdocstrings/issues/141 # __init__ is there just to avoid building these. # TODO: Fix Django pages for mkdocs. -problem_files = ['serializers.py', 'views.py', '__init__.py', '_version.py'] +problem_files = ['__init__.py', '_version.py'] for path in Path("vast_pipeline").glob("**/*.py"): diff --git a/docs/theme/main.html b/docs/theme/main.html index 1b2de320d..4631d9c48 100644 --- a/docs/theme/main.html +++ b/docs/theme/main.html @@ -28,13 +28,17 @@ {% block announce %} - New in version 1.0.0: Documentation, add epoch mode and more 🙌. Check the release notes! + + + New in version 1.0.0: Documentation, add epoch mode and more 🙌. Check the release notes! + + {% endblock %} @@ -48,3 +52,10 @@ {% endblock libs %} + +{% block outdated %} + You're viewing an outdated or development version. + + Click here to go to latest stable release. + +{% endblock %} diff --git a/mkdocs.yml b/mkdocs.yml index a21e1c471..ce8230585 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -55,7 +55,8 @@ markdown_extensions: - pymdownx.caret - pymdownx.details - pymdownx.inlinehilite - - pymdownx.tabbed + - pymdownx.tabbed: + alternate_style: true - pymdownx.superfences - pymdownx.highlight - pymdownx.tabbed @@ -170,10 +171,10 @@ nav: - forms.py: reference/forms.md - models.py: reference/models.md - plots.py: reference/plots.md - # - serializers.py: reference/serializers.md + - serializers.py: reference/serializers.md - signals.py: reference/signals.md - urls.py: reference/urls.md - # - views.py: reference/views.md + - views.py: reference/views.md plugins: - search @@ -213,3 +214,5 @@ extra: name: vast-pipeline source on GitHub - icon: fontawesome/solid/globe link: https://vast-survey.org + version: + provider: mike diff --git a/poetry.lock b/poetry.lock index 6a878e3f8..0cd806207 100644 --- a/poetry.lock +++ b/poetry.lock @@ -88,6 +88,17 @@ all = ["mocpy (>=0.5.2)", "astropy-healpix", "boto3", "regions", "pyregion", "ap docs = ["sphinx-astropy (>=1.5)", "scipy"] test = ["pytest-astropy", "matplotlib", "jinja2", "flask", "pytest-dependency"] +[[package]] +name = "astunparse" +version = "1.6.3" +description = "An AST unparser for Python" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.6.1,<2.0" + [[package]] name = "attrs" version = "21.4.0" @@ -242,6 +253,14 @@ PyYAML = ">=3.10" tornado = ">=5.1" typing_extensions = ">=3.7.4" +[[package]] +name = "cached-property" +version = "1.5.2" +description = "A decorator for caching properties in classes." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "certifi" version = "2021.10.8" @@ -363,6 +382,14 @@ sdist = ["setuptools_rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] test = ["pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] +[[package]] +name = "csscompressor" +version = "0.9.5" +description = "A python port of YUI CSS Compressor" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "cycler" version = "0.11.0" @@ -818,6 +845,20 @@ python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" [package.extras] docs = ["sphinx"] +[[package]] +name = "griffe" +version = "0.12.5" +description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +cached_property = {version = "*", markers = "python_version < \"3.8\""} + +[package.extras] +async = ["aiofiles (>=0.7,<1.0)"] + [[package]] name = "gunicorn" version = "20.1.0" @@ -1168,6 +1209,24 @@ category = "dev" optional = false python-versions = ">=3.6" +[[package]] +name = "mike" +version = "1.1.2" +description = "Manage multiple versions of your MkDocs-powered documentation" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +jinja2 = "*" +mkdocs = ">=1.0" +pyyaml = ">=5.1" +verspec = "*" + +[package.extras] +dev = ["coverage", "flake8 (>=3.0)", "shtab"] +test = ["coverage", "flake8 (>=3.0)", "shtab"] + [[package]] name = "mkdocs" version = "1.2.3" @@ -1193,11 +1252,11 @@ i18n = ["babel (>=2.9.0)"] [[package]] name = "mkdocs-autorefs" -version = "0.2.1" +version = "0.3.1" description = "Automatically link across pages in MkDocs." category = "dev" optional = false -python-versions = ">=3.6,<4.0" +python-versions = ">=3.6.2,<4.0.0" [package.dependencies] Markdown = ">=3.3,<4.0" @@ -1216,7 +1275,7 @@ mkdocs = ">=1.0.3,<2.0.0" [[package]] name = "mkdocs-git-revision-date-localized-plugin" -version = "0.9.3" +version = "0.12.1" description = "Mkdocs plugin that enables displaying the localized date of the last git modification of a markdown file." category = "dev" optional = false @@ -1229,18 +1288,19 @@ mkdocs = ">=1.0" [[package]] name = "mkdocs-material" -version = "7.3.0" +version = "8.2.1" description = "A Material Design theme for MkDocs" category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.6" [package.dependencies] +jinja2 = ">=2.11.1" markdown = ">=3.2" -mkdocs = ">=1.2.2" +mkdocs = ">=1.2.3" mkdocs-material-extensions = ">=1.0" -Pygments = ">=2.4" -pymdown-extensions = ">=7.0" +pygments = ">=2.10" +pymdown-extensions = ">=9.0" [[package]] name = "mkdocs-material-extensions" @@ -1252,33 +1312,64 @@ python-versions = ">=3.6" [[package]] name = "mkdocs-minify-plugin" -version = "0.4.1" -description = "An MkDocs plugin to minify HTML and/or JS files prior to being written to disk" +version = "0.5.0" +description = "An MkDocs plugin to minify HTML, JS or CSS files prior to being written to disk" category = "dev" optional = false python-versions = ">=3.0" [package.dependencies] +csscompressor = ">=0.9.5" htmlmin = ">=0.1.4" jsmin = ">=3.0.0" mkdocs = ">=1.0.4" [[package]] name = "mkdocstrings" -version = "0.15.2" +version = "0.18.0" description = "Automatic documentation from sources, for MkDocs." category = "dev" optional = false -python-versions = ">=3.6,<4.0" +python-versions = ">=3.7" [package.dependencies] -Jinja2 = ">=2.11.1,<4.0" -Markdown = ">=3.3,<4.0" -MarkupSafe = ">=1.1,<3.0" -mkdocs = ">=1.1.1,<2.0.0" -mkdocs-autorefs = ">=0.1,<0.3" -pymdown-extensions = ">=6.3,<9.0" -pytkdocs = ">=0.2.0,<0.12.0" +Jinja2 = ">=2.11.1" +Markdown = ">=3.3" +MarkupSafe = ">=1.1" +mkdocs = ">=1.2" +mkdocs-autorefs = ">=0.3.1" +mkdocstrings-python = {version = ">=0.5.2", optional = true, markers = "extra == \"python\""} +mkdocstrings-python-legacy = ">=0.2" +pymdown-extensions = ">=6.3" + +[package.extras] +crystal = ["mkdocstrings-crystal (>=0.3.4)"] +python = ["mkdocstrings-python (>=0.5.2)"] +python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] + +[[package]] +name = "mkdocstrings-python" +version = "0.6.2" +description = "A Python handler for mkdocstrings." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +griffe = ">=0.11.1" +mkdocstrings = ">=0.18" + +[[package]] +name = "mkdocstrings-python-legacy" +version = "0.2.1" +description = "A legacy Python handler for mkdocstrings." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +mkdocstrings = ">=0.18" +pytkdocs = ">=0.14" [[package]] name = "msgpack" @@ -1677,11 +1768,11 @@ tests = ["pytest (>=6.0.0,<7.0.0)", "coverage[toml] (==5.0.4)"] [[package]] name = "pymdown-extensions" -version = "8.2" +version = "9.2" description = "Extension pack for Python Markdown." category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] Markdown = ">=3.2" @@ -1740,14 +1831,19 @@ postgresql = ["psycopg2"] [[package]] name = "pytkdocs" -version = "0.9.0" +version = "0.15.0" description = "Load Python objects documentation." category = "dev" optional = false -python-versions = ">=3.6,<4.0" +python-versions = ">=3.6.2" + +[package.dependencies] +astunparse = {version = ">=1.6", markers = "python_version < \"3.9\""} +cached-property = {version = ">=1.5", markers = "python_version < \"3.8\""} +typing-extensions = {version = ">=3.7", markers = "python_version < \"3.8\""} [package.extras] -tests = ["coverage (>=5.2.1,<6.0.0)", "invoke (>=1.4.1,<2.0.0)", "marshmallow (>=3.5.2,<4.0.0)", "mypy (>=0.782,<0.783)", "pydantic (>=1.5.1,<2.0.0)", "pytest (>=6.0.1,<7.0.0)", "pytest-cov (>=2.10.1,<3.0.0)", "pytest-randomly (>=3.4.1,<4.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=2.1.0,<3.0.0)"] +numpy-style = ["docstring_parser (>=0.7)"] [[package]] name = "pytz" @@ -2237,6 +2333,17 @@ brotli = ["brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "verspec" +version = "0.1.0" +description = "Flexible version handling" +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +test = ["coverage", "flake8 (>=3.7)", "mypy", "pretend", "pytest"] + [[package]] name = "watchdog" version = "2.1.6" @@ -2396,6 +2503,10 @@ astroquery = [ {file = "astroquery-0.4.5-py3-none-any.whl", hash = "sha256:7b78c5d0b3afad3c7c7646a784bc03c5be0e4d350dcca6b4897bcb804ffff89b"}, {file = "astroquery-0.4.5.tar.gz", hash = "sha256:20002f84b61fb11ceeae408a4cd23b379490e174054ec777f946df8a3f06be1b"}, ] +astunparse = [ + {file = "astunparse-1.6.3-py2.py3-none-any.whl", hash = "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8"}, + {file = "astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872"}, +] attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, @@ -2433,6 +2544,10 @@ blessed = [ bokeh = [ {file = "bokeh-2.3.3.tar.gz", hash = "sha256:a5fdcc181835561447fcc5a371300973fce4114692d5853addec284d1cdeb677"}, ] +cached-property = [ + {file = "cached-property-1.5.2.tar.gz", hash = "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130"}, + {file = "cached_property-1.5.2-py2.py3-none-any.whl", hash = "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0"}, +] certifi = [ {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, @@ -2539,6 +2654,9 @@ cryptography = [ {file = "cryptography-36.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:39bdf8e70eee6b1c7b289ec6e5d84d49a6bfa11f8b8646b5b3dfe41219153316"}, {file = "cryptography-36.0.1.tar.gz", hash = "sha256:53e5c1dc3d7a953de055d77bef2ff607ceef7a2aac0353b5d630ab67f7423638"}, ] +csscompressor = [ + {file = "csscompressor-0.9.5.tar.gz", hash = "sha256:afa22badbcf3120a4f392e4d22f9fff485c044a1feda4a950ecc5eba9dd31a05"}, +] cycler = [ {file = "cycler-0.11.0-py3-none-any.whl", hash = "sha256:3a27e95f763a428a739d2add979fa7494c912a32c17c4c38c4d5f082cad165a3"}, {file = "cycler-0.11.0.tar.gz", hash = "sha256:9c87405839a19696e837b3b818fed3f5f69f16f1eec1a1ad77e043dcea9c772f"}, @@ -2739,6 +2857,10 @@ greenlet = [ {file = "greenlet-1.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:013d61294b6cd8fe3242932c1c5e36e5d1db2c8afb58606c5a67efce62c1f5fd"}, {file = "greenlet-1.1.2.tar.gz", hash = "sha256:e30f5ea4ae2346e62cedde8794a56858a67b878dd79f7df76a0767e356b1744a"}, ] +griffe = [ + {file = "griffe-0.12.5-py3-none-any.whl", hash = "sha256:a92d949b756c709939725a2d5dfd50a7ac7ece0fb41e867720e20567213bcb74"}, + {file = "griffe-0.12.5.tar.gz", hash = "sha256:9ba7ce1742e8069eb9a67d3cce9b1967af0b46130d04e41c4d7e0c654c930e5c"}, +] gunicorn = [ {file = "gunicorn-20.1.0-py3-none-any.whl", hash = "sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e"}, {file = "gunicorn-20.1.0.tar.gz", hash = "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"}, @@ -2971,37 +3093,49 @@ mergedeep = [ {file = "mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307"}, {file = "mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8"}, ] +mike = [ + {file = "mike-1.1.2-py3-none-any.whl", hash = "sha256:4c307c28769834d78df10f834f57f810f04ca27d248f80a75f49c6fa2d1527ca"}, + {file = "mike-1.1.2.tar.gz", hash = "sha256:56c3f1794c2d0b5fdccfa9b9487beb013ca813de2e3ad0744724e9d34d40b77b"}, +] mkdocs = [ {file = "mkdocs-1.2.3-py3-none-any.whl", hash = "sha256:a1fa8c2d0c1305d7fc2b9d9f607c71778572a8b110fb26642aa00296c9e6d072"}, {file = "mkdocs-1.2.3.tar.gz", hash = "sha256:89f5a094764381cda656af4298727c9f53dc3e602983087e1fe96ea1df24f4c1"}, ] mkdocs-autorefs = [ - {file = "mkdocs-autorefs-0.2.1.tar.gz", hash = "sha256:b8156d653ed91356e71675ce1fa1186d2b2c2085050012522895c9aa98fca3e5"}, - {file = "mkdocs_autorefs-0.2.1-py3-none-any.whl", hash = "sha256:f301b983a34259df90b3fcf7edc234b5e6c7065bd578781e66fd90b8cfbe76be"}, + {file = "mkdocs-autorefs-0.3.1.tar.gz", hash = "sha256:12baad29359f468b44d980ed35b713715409097a1d8e3d0ef90962db95205eda"}, + {file = "mkdocs_autorefs-0.3.1-py3-none-any.whl", hash = "sha256:f0fd7c115eaafda7fb16bf5ff5d70eda55d7c0599eac64f8b25eacf864312a85"}, ] mkdocs-gen-files = [ {file = "mkdocs-gen-files-0.3.4.tar.gz", hash = "sha256:c69188486bdc1e74bd2b9b7ebbde9f9eb21052ae7762f1b35420cfbfc6d7122e"}, {file = "mkdocs_gen_files-0.3.4-py3-none-any.whl", hash = "sha256:07f43245c87a03cfb03884e767655c2a61def24d07e47fb3a8d26b1581524d6a"}, ] mkdocs-git-revision-date-localized-plugin = [ - {file = "mkdocs-git-revision-date-localized-plugin-0.9.3.tar.gz", hash = "sha256:8f936bdf912fc1943bbb8c6027e7de6a348f9197bdb07f85a62ff1c3ee38da11"}, - {file = "mkdocs_git_revision_date_localized_plugin-0.9.3-py3-none-any.whl", hash = "sha256:a2ad799752862c0a089e9af12d8aeb2b17680d6a8502fa449eeecc638fc98722"}, + {file = "mkdocs-git-revision-date-localized-plugin-0.12.1.tar.gz", hash = "sha256:29a77224fdee0f125347e32ce18cc6bec0899b92f109399fcdb07ad962b9c2ea"}, + {file = "mkdocs_git_revision_date_localized_plugin-0.12.1-py3-none-any.whl", hash = "sha256:2b05dcaa0700119d92462ada141be1353b1bf90e5e11331ea32ef61a3009c958"}, ] mkdocs-material = [ - {file = "mkdocs-material-7.3.0.tar.gz", hash = "sha256:07db0580fa96c3473aee99ec3fb4606a1a5a1e4f4467e64c0cd1ba8da5b6476e"}, - {file = "mkdocs_material-7.3.0-py2.py3-none-any.whl", hash = "sha256:b183c27dc0f44e631bbc32c51057f61a3e2ba8b3c1080e59f944167eeba9ff1d"}, + {file = "mkdocs-material-8.2.1.tar.gz", hash = "sha256:efcb3a24e1ef374268306cb8b4d3ba0db5d2c4ee4aa1fc4bcbd2087571b1d975"}, + {file = "mkdocs_material-8.2.1-py2.py3-none-any.whl", hash = "sha256:9496b26593c8a0f9bb18bda7f1b919c0bc16838217f38212dca47cc9006d7fe0"}, ] mkdocs-material-extensions = [ {file = "mkdocs-material-extensions-1.0.3.tar.gz", hash = "sha256:bfd24dfdef7b41c312ede42648f9eb83476ea168ec163b613f9abd12bbfddba2"}, {file = "mkdocs_material_extensions-1.0.3-py3-none-any.whl", hash = "sha256:a82b70e533ce060b2a5d9eb2bc2e1be201cf61f901f93704b4acf6e3d5983a44"}, ] mkdocs-minify-plugin = [ - {file = "mkdocs-minify-plugin-0.4.1.tar.gz", hash = "sha256:dd12c178853485281aac3ccdd7c85733462550961fa6cf6d2ce0fdad103c9c9e"}, - {file = "mkdocs_minify_plugin-0.4.1-py2-none-any.whl", hash = "sha256:8799170c9b3bcc169be60104b80f3a43f9eb585d4d50701e2cbf342d35a46487"}, + {file = "mkdocs-minify-plugin-0.5.0.tar.gz", hash = "sha256:32d9e8fbd89327a0f4f648f517297aad344c1bad64cfde110d059bd2f2780a6d"}, + {file = "mkdocs_minify_plugin-0.5.0-py2-none-any.whl", hash = "sha256:487c31ae6b8b3230f56910ce6bcf5c7e6ad9a8c4f51c720a4b989f30c2b0233f"}, ] mkdocstrings = [ - {file = "mkdocstrings-0.15.2-py3-none-any.whl", hash = "sha256:8d6cbe64c07ae66739010979ca01d49dd2f64d1a45009f089d217b9cd2a65e36"}, - {file = "mkdocstrings-0.15.2.tar.gz", hash = "sha256:c2fee9a3a644647c06eb2044fdfede1073adfd1a55bf6752005d3db10705fe73"}, + {file = "mkdocstrings-0.18.0-py3-none-any.whl", hash = "sha256:75e277f6a56a894a727efcf6d418d36cd43d4db7da9614c2dc23300e257d95ad"}, + {file = "mkdocstrings-0.18.0.tar.gz", hash = "sha256:01d8ab962fc1f388c9b15cbf8c078b8738f92adf983b626d74135aaee2bce33a"}, +] +mkdocstrings-python = [ + {file = "mkdocstrings-python-0.6.2.tar.gz", hash = "sha256:74a7c54dfd9ea7c62bfb4311087b717145887d7a4ece120a587e428dd7689517"}, + {file = "mkdocstrings_python-0.6.2-py3-none-any.whl", hash = "sha256:b4d4a5168f46b36bb58bdf1111faf3aa929b387c3cb6109b7777fd458aa44bc0"}, +] +mkdocstrings-python-legacy = [ + {file = "mkdocstrings-python-legacy-0.2.1.tar.gz", hash = "sha256:d8b032e1d1b4c75a601fcc5837249e2b2dc2cb21c9ad32a313ffaa54d5838242"}, + {file = "mkdocstrings_python_legacy-0.2.1-py3-none-any.whl", hash = "sha256:a46f772b24d9a41f0b4eaa27a4ae70195cacc76af462fce691d085e0b4fc8c78"}, ] msgpack = [ {file = "msgpack-1.0.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:96acc674bb9c9be63fa8b6dabc3248fdc575c4adc005c440ad02f87ca7edd079"}, @@ -3393,8 +3527,8 @@ pyjwt = [ {file = "PyJWT-2.3.0.tar.gz", hash = "sha256:b888b4d56f06f6dcd777210c334e69c737be74755d3e5e9ee3fe67dc18a0ee41"}, ] pymdown-extensions = [ - {file = "pymdown-extensions-8.2.tar.gz", hash = "sha256:b6daa94aad9e1310f9c64c8b1f01e4ce82937ab7eb53bfc92876a97aca02a6f4"}, - {file = "pymdown_extensions-8.2-py3-none-any.whl", hash = "sha256:141452d8ed61165518f2c923454bf054866b85cf466feedb0eb68f04acdc2560"}, + {file = "pymdown-extensions-9.2.tar.gz", hash = "sha256:ed8f69a18bc158f00cbf03abc536b88b6e541b7e699156501e767c48f81d8850"}, + {file = "pymdown_extensions-9.2-py3-none-any.whl", hash = "sha256:f2fa7d9317c672a419868c893c20a28fb7ed7fc60d4ec4774c35e01398ab330c"}, ] pyopenssl = [ {file = "pyOpenSSL-22.0.0-py2.py3-none-any.whl", hash = "sha256:ea252b38c87425b64116f808355e8da644ef9b07e429398bfece610f893ee2e0"}, @@ -3413,8 +3547,8 @@ python3-openid = [ {file = "python3_openid-3.2.0-py3-none-any.whl", hash = "sha256:6626f771e0417486701e0b4daff762e7212e820ca5b29fcc0d05f6f8736dfa6b"}, ] pytkdocs = [ - {file = "pytkdocs-0.9.0-py3-none-any.whl", hash = "sha256:12ed87d71b3518301c7b8c12c1a620e4b481a9d2fca1038aea665955000fad7f"}, - {file = "pytkdocs-0.9.0.tar.gz", hash = "sha256:c8c39acb63824f69c3f6f58b3aed6ae55250c35804b76fd0cba09d5c11be13da"}, + {file = "pytkdocs-0.15.0-py3-none-any.whl", hash = "sha256:d6b2aec34448ec89acb8c1c25062cc1e70c6b26395d46fc7ee753b7e5a4e736a"}, + {file = "pytkdocs-0.15.0.tar.gz", hash = "sha256:4b45af89d6fa5fa50f979b0f9f54539286b84e245c81991bb838149aa2d9d9c9"}, ] pytz = [ {file = "pytz-2021.3-py2.py3-none-any.whl", hash = "sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c"}, @@ -3807,6 +3941,10 @@ urllib3 = [ {file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"}, {file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"}, ] +verspec = [ + {file = "verspec-0.1.0-py3-none-any.whl", hash = "sha256:741877d5633cc9464c45a469ae2a31e801e6dbbaa85b9675d481cda100f11c31"}, + {file = "verspec-0.1.0.tar.gz", hash = "sha256:c4504ca697b2056cdb4bfa7121461f5a0e81809255b41c03dda4ba823637c01e"}, +] watchdog = [ {file = "watchdog-2.1.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:9693f35162dc6208d10b10ddf0458cc09ad70c30ba689d9206e02cd836ce28a3"}, {file = "watchdog-2.1.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:aba5c812f8ee8a3ff3be51887ca2d55fb8e268439ed44110d3846e4229eb0e8b"}, diff --git a/pyproject.toml b/pyproject.toml index 54741085e..4df7d6ec0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,11 +43,11 @@ datashader = "^0.13.0" channels = "^3.0.4" [tool.poetry.dev-dependencies] -mkdocs-material = "^7.1.4" -mkdocstrings = "^0.15.0" -mkdocs-minify-plugin = "^0.4.0" +mkdocs-material = "^8.0.0" +mkdocstrings = {extras = ["python"], version = "^0.18.0"} +mkdocs-minify-plugin = "^0" mkdocs-material-extensions = "^1.0.1" -mkdocs-gen-files = "^0.3.1" +mkdocs-gen-files = "^0" pygraphviz = "^1.7" ipdb = "^0.13.6" ipython = "^7.31.1" @@ -60,8 +60,9 @@ black = "^20.8b1" django-extensions = "^3.1.1" rope = "^0.18.0" django-stubs = "^1.9.0" -lightgallery = "^0.5" -mkdocs-git-revision-date-localized-plugin = "^0.9" +lightgallery = "^0" +mkdocs-git-revision-date-localized-plugin = "^0" +mike = "^1.1.2" [tool.poetry.extras] prod = ["gevent", "gunicorn"] diff --git a/vast_pipeline/context_processors.py b/vast_pipeline/context_processors.py index 35859a52a..1e807072b 100644 --- a/vast_pipeline/context_processors.py +++ b/vast_pipeline/context_processors.py @@ -28,7 +28,7 @@ def pipeline_version(request: HttpRequest) -> Dict[str, Optional[str]]: request (HttpRequest): The web server request. Returns: - Dict[str, str]: key-value pairs to add to the template context. + Key-value pairs to add to the template context. """ url: Optional[str] = None if not __version__.endswith("dev"): diff --git a/vast_pipeline/converters.py b/vast_pipeline/converters.py index 753bf321f..9c618aa88 100644 --- a/vast_pipeline/converters.py +++ b/vast_pipeline/converters.py @@ -71,7 +71,7 @@ class AngleConverter: value.""" regex = r"\d+(\.\d+)?\s?\w+" - def to_python(self, value) -> float: + def to_python(self, value: Angle) -> float: """ Return the decimal degrees from the coordinate input as an Angle object. @@ -83,7 +83,7 @@ def to_python(self, value) -> float: """ return Angle(value).deg - def to_url(self, value) -> str: + def to_url(self, value: Angle) -> str: """ Return the string format of an Angle object from the coordinate input. diff --git a/vast_pipeline/image/main.py b/vast_pipeline/image/main.py index 7fab7c296..4ecbb6823 100644 --- a/vast_pipeline/image/main.py +++ b/vast_pipeline/image/main.py @@ -168,10 +168,20 @@ def __set_img_attr_for_telescope(self, header): # get the time as Julian Datetime using Pandas function self.jd = self.datetime.to_julian_date() - def __get_img_coordinates(self, header, fits_naxis1, fits_naxis2): + def __get_img_coordinates( + self, header: fits.Header, fits_naxis1: str, fits_naxis2: str + ) -> None: """ - set the image attributes ra, dec, fov_bmin and fov_bmaj, radius - from the image file header + Set the image attributes ra, dec, fov_bmin and fov_bmaj, radius + from the image file header. + + Args: + header: The FITS header object. + fits_naxis1: The header keyword of the NAXIS1 to use. + fits_naxis2: The header keyword of the NAXIS2 to use. + + Returns: + None """ wcs = WCS(header, naxis=2) pix_centre = [[header[fits_naxis1] / 2., header[fits_naxis2] / 2.]] @@ -195,10 +205,22 @@ def __get_img_coordinates(self, header, fits_naxis1, fits_naxis2): # TODO: check calcs self.radius_pixels = usable_radius_pix - def __get_radius_pixels(self, header, fits_naxis1, fits_naxis2): + def __get_radius_pixels( + self, header: fits.Header, fits_naxis1: str, fits_naxis2: str + ) -> float: """ Return the radius (pixels) of the full image. - If the image is not a square/circle then the shortest radius will be returned. + + If the image is not a square/circle then the shortest radius will be + returned. + + Args: + header: The FITS header object. + fits_naxis1: The header keyword of the NAXIS1 to use. + fits_naxis2: The header keyword of the NAXIS2 to use. + + Returns: + The radius of the image in pixels. """ if self.entire_image: # a large circle that *should* include the whole image (and then some) @@ -208,13 +230,16 @@ def __get_radius_pixels(self, header, fits_naxis1, fits_naxis2): diameter = min(header[fits_naxis1], header[fits_naxis2]) return diameter / 2. - def __get_frequency(self, header): + def __get_frequency(self, header: fits.Header) -> None: """ Set some 'shortcut' variables for access to the frequency parameters in the FITS file header. - @param hdulist: hdulist to parse - @type hdulist: hdulist + Args: + header: The FITS header object. + + Returns: + None """ self.freq_eff = None self.freq_bw = None diff --git a/vast_pipeline/image/utils.py b/vast_pipeline/image/utils.py index b620bf0d7..4b85d5389 100644 --- a/vast_pipeline/image/utils.py +++ b/vast_pipeline/image/utils.py @@ -90,10 +90,18 @@ def calc_error_radius(ra, ra_err, dec, dec_err) -> float: def calc_condon_flux_errors( - row: pd.Series, theta_B: float, theta_b: float , alpha_maj1: float=2.5, - alpha_min1: float=0.5, alpha_maj2: float=0.5, alpha_min2:float =2.5, - alpha_maj3: float=1.5, alpha_min3: float=1.5, clean_bias: float=0.0, - clean_bias_error: float=0.0, frac_flux_cal_error: float=0.0, + row: pd.Series, + theta_B: float, + theta_b: float, + alpha_maj1: float = 2.5, + alpha_min1: float = 0.5, + alpha_maj2: float = 0.5, + alpha_min2: float = 2.5, + alpha_maj3: float = 1.5, + alpha_min3: float = 1.5, + clean_bias: float = 0.0, + clean_bias_error: float = 0.0, + frac_flux_cal_error: float = 0.0 ) -> Tuple[float, float, float, float, float, float, float]: """ The following code for this function taken from the TraP with a few @@ -109,7 +117,7 @@ def calc_condon_flux_errors( Args: row (pd.Series): - The row containing the componenet information from the Selavy + The row containing the component information from the Selavy component catalogue. theta_B (float): The major axis size of the restoring beam of the image (degrees). @@ -138,10 +146,13 @@ def calc_condon_flux_errors( TraP). Returns: - Tuple containing the following calculated values: - peak flux error, integrated flux error, major axis error, minor - axis error, position angle error, right ascension error and the - declination error. + Peak flux error (Jy). + Integrated flux error (Jy). + Major axis error (deg). + Minor axis error (deg). + Position angle error (deg). + Right ascension error (deg). + Declination error (deg). """ major = row.bmaj / 3600. # degrees diff --git a/vast_pipeline/management/commands/initpiperun.py b/vast_pipeline/management/commands/initpiperun.py index 9a7fb75f1..a44c57f35 100644 --- a/vast_pipeline/management/commands/initpiperun.py +++ b/vast_pipeline/management/commands/initpiperun.py @@ -44,7 +44,7 @@ def initialise_run( PipelineInitError: A directory named `run_name` already exists. Returns: - Run: The initialised pipeline Run Django model object. + The initialised pipeline Run Django model object. """ # check for duplicated run name p_run = Run.objects.filter(name__exact=run_name) diff --git a/vast_pipeline/management/commands/runpipeline.py b/vast_pipeline/management/commands/runpipeline.py index 2cad11481..3740857a5 100644 --- a/vast_pipeline/management/commands/runpipeline.py +++ b/vast_pipeline/management/commands/runpipeline.py @@ -68,8 +68,12 @@ def run_pipe( The previous status through the UI. Defaults to 'END'. Returns: - Boolean equal to `True` on a successful completion, or in cases of - failures a CommandError is returned. + Boolean equal to `True` on a successful completion. + + Raises: + CommandError: Raised if error occurs during processing. + PipelineConfigError: Raised if an error is found in the pipeline + config. ''' path = run_dj_obj.path if run_dj_obj else path_name # set up logging for running pipeline from UI diff --git a/vast_pipeline/management/helpers.py b/vast_pipeline/management/helpers.py index e6781b6aa..c2bbca310 100644 --- a/vast_pipeline/management/helpers.py +++ b/vast_pipeline/management/helpers.py @@ -12,7 +12,7 @@ logger = logging.getLogger(__name__) -def get_p_run_name(name: str, return_folder: bool=False) -> Tuple[str, str]: +def get_p_run_name(name: str, return_folder: bool = False) -> Tuple[str, str]: """ Determines the name of the pipeline run. Can also return the output folder if selected. @@ -22,8 +22,8 @@ def get_p_run_name(name: str, return_folder: bool=False) -> Tuple[str, str]: return_folder: When `True` the pipeline directory is also returned. Returns: - The name of the pipeline run. If return_folder is `True` then both - the name and directory are returned. + The name of the pipeline run (always returned). + The run directory (if `return_folder` is set to `True`). """ if '/' in name: folder = os.path.realpath(name) diff --git a/vast_pipeline/pipeline/association.py b/vast_pipeline/pipeline/association.py index b05516100..fd15adf18 100644 --- a/vast_pipeline/pipeline/association.py +++ b/vast_pipeline/pipeline/association.py @@ -78,8 +78,9 @@ def calc_de_ruiter(df: pd.DataFrame) -> np.ndarray: def one_to_many_basic( - skyc2_srcs: pd.DataFrame, sources_df: pd.DataFrame, - id_incr_par_assoc: int=0 + skyc2_srcs: pd.DataFrame, + sources_df: pd.DataFrame, + id_incr_par_assoc: int = 0 ) -> Tuple[pd.DataFrame, pd.DataFrame]: """ Finds and processes the one-to-many associations in the basic @@ -102,8 +103,10 @@ def one_to_many_basic( association Returns: - Tuple containging the updated 'skyc2_srcs' and 'sources_df' with - all one_to_many relation information added. + Updated 'skyc2_srcs' with all the one_to_many relation information + added. + Updated 'sources_df' with all the one_to_many relation information + added. """ # select duplicated in 'source' field in skyc2_srcs, excluding -1 duplicated_skyc2 = skyc2_srcs.loc[ @@ -315,7 +318,7 @@ def one_to_many_advanced( temp_srcs: pd.DataFrame, sources_df: pd.DataFrame, method: str, - id_incr_par_assoc: int=0 + id_incr_par_assoc: int = 0 ) -> Tuple[pd.DataFrame, pd.DataFrame]: ''' Finds and processes the one-to-many associations in the advanced @@ -343,8 +346,10 @@ def one_to_many_advanced( Mainly useful for add mode with parallel association Returns: - Updated temp_srcs and sources_df with all one_to_many relation - information added. + Updated `temp_srcs` dataframe with all the one_to_many relation + information added. + Updated `sources_df` dataframe with all the one_to_many relation + information added. ''' # use only these columns for easy debugging of the dataframe cols = [ @@ -666,7 +671,7 @@ def many_to_one_advanced(temp_srcs: pd.DataFrame) -> pd.DataFrame: Args: temp_srcs: - The temporary associtation dataframe used through the advanced + The temporary association dataframe used through the advanced association process. Returns: @@ -786,9 +791,13 @@ def many_to_one_advanced(temp_srcs: pd.DataFrame) -> pd.DataFrame: def basic_association( - sources_df: pd.DataFrame, skyc1_srcs: pd.DataFrame, skyc1: SkyCoord, - skyc2_srcs: pd.DataFrame, skyc2: SkyCoord, limit: Angle, - id_incr_par_assoc: int=0 + sources_df: pd.DataFrame, + skyc1_srcs: pd.DataFrame, + skyc1: SkyCoord, + skyc2_srcs: pd.DataFrame, + skyc2: SkyCoord, + limit: Angle, + id_incr_par_assoc: int = 0 ) -> Tuple[pd.DataFrame, pd.DataFrame]: ''' The loop for basic source association that uses the astropy @@ -819,10 +828,10 @@ def basic_association( image are being used). Defaults to 0. Returns: - The output sources_df containing all input measurements along with the - association and relation information. - The output skyc1_srcs with updated with new sources from the - association. + The output `sources_df` containing all input measurements along with the + association and relation information. + The output `skyc1_srcs` with updated with new sources from the + association. ''' # match the new sources to the base # idx gives the index of the closest match in the base for skyc2 @@ -873,9 +882,15 @@ def basic_association( def advanced_association( - method: str, sources_df: pd.DataFrame, skyc1_srcs: pd.DataFrame, - skyc1: SkyCoord, skyc2_srcs: pd.DataFrame, skyc2: SkyCoord, - dr_limit: float, bw_max: float, id_incr_par_assoc: int=0 + method: str, + sources_df: pd.DataFrame, + skyc1_srcs: pd.DataFrame, + skyc1: SkyCoord, + skyc2_srcs: pd.DataFrame, + skyc2: SkyCoord, + dr_limit: float, + bw_max: float, + id_incr_par_assoc: int = 0 ) -> Tuple[pd.DataFrame, pd.DataFrame]: ''' The loop for advanced source association that uses the astropy @@ -912,10 +927,10 @@ def advanced_association( image are being used). Defaults to 0. Returns: - The output sources_df containing all input measurements along with the - association and relation information. - The output skyc1_srcs with updated with new sources from the - association. + The output `sources_df` containing all input measurements along with the + association and relation information. + The output `skyc1_srcs` with updated with new sources from the + association. ''' # read the needed sources fields # Step 1: get matches within semimajor axis of image. @@ -1034,10 +1049,19 @@ def advanced_association( return sources_df, skyc1_srcs -def association(images_df: pd.DataFrame, limit: Angle, dr_limit: float, - bw_limit: float, duplicate_limit: Angle, config: PipelineConfig, add_mode: bool, - previous_parquets: Dict[str, str], done_images_df: pd.DataFrame, - id_incr_par_assoc: int=0, parallel: bool=False) -> pd.DataFrame: +def association( + images_df: pd.DataFrame, + limit: Angle, + dr_limit: float, + bw_limit: float, + duplicate_limit: Angle, + config: PipelineConfig, + add_mode: bool, + previous_parquets: Dict[str, str], + done_images_df: pd.DataFrame, + id_incr_par_assoc: int = 0, + parallel: bool = False +) -> pd.DataFrame: ''' The main association function that does the common tasks between basic and advanced modes. @@ -1073,7 +1097,7 @@ def association(images_df: pd.DataFrame, limit: Angle, dr_limit: float, Returns: The output sources_df containing all input measurements along with the - association and relation information. + association and relation information. Raises: Exception: Raised if association method is not valid. @@ -1432,8 +1456,8 @@ def _correct_parallel_source_ids_add_mode( The start elem number for the new source ids. Returns: - The input df with corrected source ids and relations, and the new - start elem for the next group. + The input dataframe with corrected source ids and relations. + The new start elem for the next group. """ # When using add_mode the correction becomes easier with the increment # as there's a clear difference between old and new. @@ -1509,7 +1533,7 @@ def parallel_association( n_skyregion_groups: The number of sky region groups. Returns: - pd.DataFrame: The combined association results of the parallel + The combined association results of the parallel association with corrected source ids. """ logger.info( diff --git a/vast_pipeline/pipeline/config.py b/vast_pipeline/pipeline/config.py index 223421d6d..6c964ce57 100644 --- a/vast_pipeline/pipeline/config.py +++ b/vast_pipeline/pipeline/config.py @@ -212,10 +212,10 @@ def _create_input_epochs(input_files: List[str]) -> Dict[str, List[str]]: epochs. Args: - input_files (List[str]): the list of input file paths. + input_files: the list of input file paths. Returns: - Dict[str, List[str]]: the input file paths mapped to by unique epoch keys. + The input file paths mapped to by unique epoch keys. """ pad_width = len(str(len(input_files))) input_files_dict = { @@ -291,7 +291,7 @@ def _resolve_glob_expressions(input_files: yaml.YAML) -> List[str]: --- Returns: - List[str]: The resolved file paths in lexicographical order. + The resolved file paths in lexicographical order. """ file_list: List[str] = [] if input_files["glob"].is_sequence(): @@ -514,8 +514,8 @@ def check_prev_config_diff(self) -> bool: settings are the same (the requirement for add mode). Otherwise False is returned. Returns: - True if images are different but general settings are the same, otherwise - False is returned. + `True` if images are different but general settings are the same, + otherwise `False` is returned. """ prev_config = PipelineConfig.from_file( os.path.join(self["run"]["path"], "config_prev.yaml"), @@ -538,7 +538,7 @@ def check_prev_config_diff(self) -> bool: return False return True - def image_opts(self) -> Dict: + def image_opts(self) -> Dict[str, Any]: """ Get the config options required for image ingestion only. Namely: @@ -548,7 +548,7 @@ def image_opts(self) -> Dict: - dec_uncertainty Returns: - Dict: the relevant key value pairs + The relevant key value pairs """ keys = [ "selavy_local_rms_fill_value", diff --git a/vast_pipeline/pipeline/errors.py b/vast_pipeline/pipeline/errors.py index 141eb62cd..2b527740d 100644 --- a/vast_pipeline/pipeline/errors.py +++ b/vast_pipeline/pipeline/errors.py @@ -2,6 +2,8 @@ Defines errors for the pipeline to return. """ +from typing import Optional + class PipelineError(Exception): """ Generic pipeline error @@ -10,7 +12,7 @@ class PipelineError(Exception): msg (str): The full error string to return. """ - def __init__(self, msg: str=None) -> None: + def __init__(self, msg: Optional[str] = None) -> None: """ Initialises the error. @@ -51,7 +53,7 @@ class PipelineConfigError(PipelineError): """ Error for issue in the pipeline configuration """ - def __init__(self, msg=None): + def __init__(self, msg: Optional[str] = None): """ Initialises the config error. @@ -65,7 +67,7 @@ class PipelineInitError(PipelineError): """ Error for issue in the pipeline initialisation """ - def __init__(self, msg=None): + def __init__(self, msg: Optional[str] = None): """ Initialises the init error. diff --git a/vast_pipeline/pipeline/finalise.py b/vast_pipeline/pipeline/finalise.py index 46cffbc4e..5520861fb 100644 --- a/vast_pipeline/pipeline/finalise.py +++ b/vast_pipeline/pipeline/finalise.py @@ -48,8 +48,8 @@ def calculate_measurement_pair_aggregate_metrics( Returns: Measurement pair aggregate metrics indexed by the source ID, `source`. - The metric columns are named: `vs_abs_significant_max_{flux_type}` and - `m_abs_significant_max_{flux_type}`. + The metric columns are named: `vs_abs_significant_max_{flux_type}` + and `m_abs_significant_max_{flux_type}`. """ check_df = measurement_pairs_df.query(f"abs(vs_{flux_type}) >= @min_vs") @@ -117,7 +117,7 @@ def final_operations( Returns: The number of sources contained in the pipeline (used in the next steps - of main.py). + of main.py). """ timer = StopWatch() diff --git a/vast_pipeline/pipeline/forced_extraction.py b/vast_pipeline/pipeline/forced_extraction.py index 495aed228..e585727f2 100644 --- a/vast_pipeline/pipeline/forced_extraction.py +++ b/vast_pipeline/pipeline/forced_extraction.py @@ -76,7 +76,7 @@ def get_data_from_parquet( Returns: Dictionary with prefix string, an interger max_id and a string with the - id of the image + id of the image. ''' if add_mode: image_name = file.split("/")[-2] @@ -126,7 +126,7 @@ def extract_from_image( Returns: Dictionary with input dataframe with added columns (flux_int, - flux_int_err, chi_squared_fit) and image name. + flux_int_err, chi_squared_fit) and image name. """ # create the skycoord obj to pass to the forced extraction # see usage https://github.com/dlakaplan/forced_phot @@ -182,7 +182,7 @@ def finalise_forced_dfs( Returns: Input dataframe with added columns island_id, component_id, - name, bmaj, bmin, pa, image_id, time. + name, bmaj, bmin, pa, image_id, time. """ # make up the measurements name from the image island_id and component_id df['island_id'] = np.char.add( @@ -243,8 +243,8 @@ def parallel_extraction( Returns: Dataframe with forced extracted measurements data, columns are - 'source_tmp_id', 'ra', 'dec', 'image', 'flux_peak', 'island_id', - 'component_id', 'name', 'flux_int', 'flux_int_err' + 'source_tmp_id', 'ra', 'dec', 'image', 'flux_peak', 'island_id', + 'component_id', 'name', 'flux_int', 'flux_int_err' """ # explode the lists in 'img_diff' column (this will make a copy of the df) out = ( @@ -475,8 +475,8 @@ def forced_extraction( run (used in add image mode). Returns: - The sources_df with the extracted sources added and n_forced is the - total number of forced measurements present in the run. + The `sources_df` with the extracted sources added. + The total number of forced measurements present in the run. """ logger.info( 'Starting force extraction step.' diff --git a/vast_pipeline/pipeline/loading.py b/vast_pipeline/pipeline/loading.py index 121429c32..4b8207297 100644 --- a/vast_pipeline/pipeline/loading.py +++ b/vast_pipeline/pipeline/loading.py @@ -81,7 +81,9 @@ def make_upload_images( Dictionary of configuration options for the image ingestion. Returns: - A list of image, sky region and band objects that have been uploaded. + A list of Image objects that have been uploaded. + A list of SkyRegion objects that have been uploaded. + A list of Band objects that have been uploaded. ''' timer = StopWatch() images = [] diff --git a/vast_pipeline/pipeline/model_generator.py b/vast_pipeline/pipeline/model_generator.py index 5548926e8..310babae0 100644 --- a/vast_pipeline/pipeline/model_generator.py +++ b/vast_pipeline/pipeline/model_generator.py @@ -28,7 +28,7 @@ def measurement_models_generator( Returns: An iterable generator object containing the yielded Measurement - objects. + objects. """ for i, row in meas_df.iterrows(): one_m = Measurement() diff --git a/vast_pipeline/pipeline/new_sources.py b/vast_pipeline/pipeline/new_sources.py index 2e40c4c2e..c5b751147 100644 --- a/vast_pipeline/pipeline/new_sources.py +++ b/vast_pipeline/pipeline/new_sources.py @@ -46,8 +46,8 @@ def gen_array_coords_from_wcs(coords: SkyCoord, wcs: WCS) -> np.ndarray: Returns: Array containing the x and y array coordinates of the input sky - coordinates, e.g.: - np.array([[x1, x2, x3], [y1, y2, y3]]) + coordinates, e.g.: + np.array([[x1, x2, x3], [y1, y2, y3]]) """ array_coords = wcs.world_to_array_index(coords) array_coords = np.array([ @@ -77,7 +77,7 @@ def get_image_rms_measurements( Returns: The group dataframe with the 'img_diff_true_rms' column added. The - column will contain 'NaN' entires for sources that fail. + column will contain 'NaN' entires for sources that fail. """ if len(group) == 0: # input dataframe is empty, nothing to do @@ -203,7 +203,7 @@ def parallel_get_rms_measurements( Returns: The original input dataframe with the 'img_diff_true_rms' column - added. The column will contain 'NaN' entires for sources that fail. + added. The column will contain 'NaN' entires for sources that fail. """ out = df[[ 'source', 'wavg_ra', 'wavg_dec', @@ -254,33 +254,7 @@ def new_sources( The sources found from the association step. missing_sources_df: The dataframe containing the 'missing detections' for each source. - +----------+----------------------------------+-----------+------------+ - | source | img_list | wavg_ra | wavg_dec | - |----------+----------------------------------+-----------+------------+ - | 278 | ['VAST_0127-73A.EPOCH01.I.fits'] | 22.2929 | -71.8717 | - | 702 | ['VAST_0127-73A.EPOCH01.I.fits'] | 28.8125 | -69.3547 | - | 776 | ['VAST_0127-73A.EPOCH01.I.fits'] | 31.8223 | -70.4674 | - | 844 | ['VAST_0127-73A.EPOCH01.I.fits'] | 17.3152 | -72.346 | - | 934 | ['VAST_0127-73A.EPOCH01.I.fits'] | 9.75754 | -72.9629 | - +----------+----------------------------------+-----------+------------+ - ------------------------------------------------------------------+ - skyreg_img_list | - ------------------------------------------------------------------+ - ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | - ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | - ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | - ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | - ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | - ------------------------------------------------------------------+ - ----------------------------------+ - img_diff | - ----------------------------------| - ['VAST_0127-73A.EPOCH08.I.fits'] | - ['VAST_0127-73A.EPOCH08.I.fits'] | - ['VAST_0127-73A.EPOCH08.I.fits'] | - ['VAST_0127-73A.EPOCH08.I.fits'] | - ['VAST_0127-73A.EPOCH08.I.fits'] | - ----------------------------------+ + See the source code comments for the layout of this dataframe. min_sigma: The minimum sigma value acceptable when compared to the minimum rms of the respective image. @@ -292,26 +266,54 @@ def new_sources( Returns: The original input dataframe with the 'img_diff_true_rms' column - added. The column will contain 'NaN' entires for sources that fail. - Columns: - source - source id, int. - img_list - list of images, List. - wavg_ra - weighted average RA, float. - wavg_dec - weighted average Dec, float. - skyreg_img_list - list of sky regions of images in img_list, List. - img_diff - The images missing from coverage, List. - primary - What should be the first image, str. - detection - The first detection image, str. - detection_time - Datetime of detection, datetime.datetime. - img_diff_time - Datetime of img_diff list, datetime.datetime. - img_diff_rms_min - Minimum rms of diff images, float. - img_diff_rms_median - Median rms of diff images, float. - img_diff_rms_path - rms path of diff images, str. - flux_peak - Flux peak of source (detection), float. - diff_sigma - SNR in differnce images (compared to minimum), float. - img_diff_true_rms - The true rms value from the diff images, float. - new_high_sigma - peak flux / true rms value, float. + added. The column will contain 'NaN' entires for sources that fail. + Columns: + source - source id, int. + img_list - list of images, List. + wavg_ra - weighted average RA, float. + wavg_dec - weighted average Dec, float. + skyreg_img_list - list of sky regions of images in img_list, List. + img_diff - The images missing from coverage, List. + primary - What should be the first image, str. + detection - The first detection image, str. + detection_time - Datetime of detection, datetime.datetime. + img_diff_time - Datetime of img_diff list, datetime.datetime. + img_diff_rms_min - Minimum rms of diff images, float. + img_diff_rms_median - Median rms of diff images, float. + img_diff_rms_path - rms path of diff images, str. + flux_peak - Flux peak of source (detection), float. + diff_sigma - SNR in differnce images (compared to minimum), float. + img_diff_true_rms - The true rms value from the diff images, float. + new_high_sigma - peak flux / true rms value, float. """ + # Missing sources df layout + # +----------+----------------------------------+-----------+------------+ + # | source | img_list | wavg_ra | wavg_dec | + # |----------+----------------------------------+-----------+------------+ + # | 278 | ['VAST_0127-73A.EPOCH01.I.fits'] | 22.2929 | -71.8717 | + # | 702 | ['VAST_0127-73A.EPOCH01.I.fits'] | 28.8125 | -69.3547 | + # | 776 | ['VAST_0127-73A.EPOCH01.I.fits'] | 31.8223 | -70.4674 | + # | 844 | ['VAST_0127-73A.EPOCH01.I.fits'] | 17.3152 | -72.346 | + # | 934 | ['VAST_0127-73A.EPOCH01.I.fits'] | 9.75754 | -72.9629 | + # +----------+----------------------------------+-----------+------------+ + # ------------------------------------------------------------------+ + # skyreg_img_list | + # ------------------------------------------------------------------+ + # ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | + # ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | + # ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | + # ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | + # ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | + # ------------------------------------------------------------------+ + # ----------------------------------+ + # img_diff | + # ----------------------------------| + # ['VAST_0127-73A.EPOCH08.I.fits'] | + # ['VAST_0127-73A.EPOCH08.I.fits'] | + # ['VAST_0127-73A.EPOCH08.I.fits'] | + # ['VAST_0127-73A.EPOCH08.I.fits'] | + # ['VAST_0127-73A.EPOCH08.I.fits'] | + # ----------------------------------+ timer = StopWatch() logger.info("Starting new source analysis.") diff --git a/vast_pipeline/pipeline/pairs.py b/vast_pipeline/pipeline/pairs.py index ade8b26f3..470cbab3b 100644 --- a/vast_pipeline/pipeline/pairs.py +++ b/vast_pipeline/pipeline/pairs.py @@ -54,7 +54,7 @@ def calculate_measurement_pair_metrics(df: pd.DataFrame) -> pd.DataFrame: flux_int_err, flux_peak, flux_peak_err, has_siblings. Returns: - pd.DataFrame: Measurement pairs and 2-epoch metrics. Will contain columns: + Measurement pairs and 2-epoch metrics. Will contain columns: source - the source ID id_a, id_b - the measurement IDs flux_int_a, flux_int_b - measurement integrated fluxes in mJy diff --git a/vast_pipeline/pipeline/utils.py b/vast_pipeline/pipeline/utils.py index fd0902188..80bd07ae6 100644 --- a/vast_pipeline/pipeline/utils.py +++ b/vast_pipeline/pipeline/utils.py @@ -117,8 +117,8 @@ def get_create_img(band_id: int, image: SelavyImage) -> Tuple[Image, bool]: image: The image object. Returns: - The resulting image django ORM object, and a bool value denoting if the image - already existed in the database. + The resulting image django ORM object. + `True` the image already existed in the database, `False` if not. """ images = Image.objects.filter(name__exact=image.name) exists = images.exists() @@ -175,8 +175,8 @@ def get_create_p_run( user: The Django user that launched the pipeline run. Returns: - The pipeline run object and a boolean object representing whether - the pipeline run already existed ('True') or not ('False'). + The pipeline run object. + Whether the pipeline run already existed ('True') or not ('False'). ''' p_run = Run.objects.filter(name__exact=name) if p_run: @@ -341,7 +341,7 @@ def _load_measurements( Returns: The measurements of the image with some extra values set ready for - association . + association. """ image_centre = SkyCoord( image.ra, @@ -401,7 +401,7 @@ def prep_skysrc_df( Returns: The measurements of the image(s) with some extra values set ready for - association and duplicates removed if necessary. + association and duplicates removed if necessary. ''' cols = [ 'id', @@ -480,7 +480,7 @@ def add_new_one_to_many_relations( Returns: The new related field for the source in question, containing the - appended ids. + appended ids. """ if source_ids is None: source_ids = pd.DataFrame() @@ -521,7 +521,7 @@ def add_new_many_to_one_relations(row: pd.Series) -> List[int]: Returns: The new related field for the source in question, containing the - appended ids. + appended ids. """ out = row['new_relations'].copy() @@ -541,7 +541,6 @@ def cross_join(left: pd.DataFrame, right: pd.DataFrame) -> pd.DataFrame: Returns: The resultant merged DataFrame. - """ return ( left.assign(key=1) @@ -567,7 +566,6 @@ def get_eta_metric( Returns: The calculated eta value. - ''' if row['n_meas'] == 1: return 0. @@ -595,7 +593,6 @@ def groupby_funcs(df: pd.DataFrame) -> pd.Series: Returns: Pandas series containing the calculated metrics of the source. - ''' # calculated average ra, dec, fluxes and metrics d = {} @@ -729,7 +726,7 @@ def calc_ave_coord(grp: pd.DataFrame) -> pd.Series: Returns: A pandas series containing the average coordinate along with the - image and epoch lists. + image and epoch lists. """ d = {} grp = grp.sort_values(by='datetime') @@ -752,7 +749,7 @@ def parallel_groupby_coord(df: pd.DataFrame) -> pd.DataFrame: Returns: The resulting average coordinate values and unique image and epoch - lists for each unique source (group). + lists for each unique source (group). """ col_dtype = { 'img_list': 'O', @@ -781,7 +778,9 @@ def get_rms_noise_image_values(rms_path: str) -> Tuple[float, float, float]: rms_path: The system path to the RMS FITS image. Returns: - The median, minimum and maximum values of the RMS image. + The median value of the RMS image. + The minimum value of the RMS image. + The maximum value of the RMS image. Raises: IOError: Raised when the RMS FITS file cannot be found. @@ -814,8 +813,8 @@ def get_image_list_diff(row: pd.Series) -> Union[List[str], int]: row: The row from the sources dataframe that is being iterated over. Returns: - A list of the images missing from the observed image list. Will be - returned as '-1' integer value if there are no missing images. + A list of the images missing from the observed image list. + A '-1' integer value if there are no missing images. """ out = list( filter(lambda arg: arg not in row['img_list'], row['skyreg_img_list']) @@ -856,7 +855,7 @@ def get_names_and_epochs(grp: pd.DataFrame) -> pd.Series: Returns: Pandas series containing the list object that contains the lists of the - image names, epochs and datetimes. + image names, epochs and datetimes. """ d = {} d['skyreg_img_epoch_list'] = [[[x, ], y, z] for x, y, z in zip( @@ -903,44 +902,46 @@ def get_src_skyregion_merged_df( sky region objects for the run loaded into a dataframe. Returns: - DataFrame containing missing image information. Output format: - +----------+----------------------------------+-----------+------------+ - | source | img_list | wavg_ra | wavg_dec | - |----------+----------------------------------+-----------+------------+ - | 278 | ['VAST_0127-73A.EPOCH01.I.fits'] | 22.2929 | -71.8717 | - | 702 | ['VAST_0127-73A.EPOCH01.I.fits'] | 28.8125 | -69.3547 | - | 844 | ['VAST_0127-73A.EPOCH01.I.fits'] | 17.3152 | -72.346 | - | 934 | ['VAST_0127-73A.EPOCH01.I.fits'] | 9.75754 | -72.9629 | - | 1290 | ['VAST_0127-73A.EPOCH01.I.fits'] | 20.8455 | -76.8269 | - +----------+----------------------------------+-----------+------------+ - ------------------------------------------------------------------+ - skyreg_img_list | - ------------------------------------------------------------------+ - ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | - ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | - ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | - ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | - ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | - ------------------------------------------------------------------+ - ----------------------------------+------------------------------+ - img_diff | primary | - ----------------------------------+------------------------------+ - ['VAST_0127-73A.EPOCH08.I.fits'] | VAST_0127-73A.EPOCH01.I.fits | - ['VAST_0127-73A.EPOCH08.I.fits'] | VAST_0127-73A.EPOCH01.I.fits | - ['VAST_0127-73A.EPOCH08.I.fits'] | VAST_0127-73A.EPOCH01.I.fits | - ['VAST_0127-73A.EPOCH08.I.fits'] | VAST_0127-73A.EPOCH01.I.fits | - ['VAST_0127-73A.EPOCH08.I.fits'] | VAST_0127-73A.EPOCH01.I.fits | - ----------------------------------+------------------------------+ - ------------------------------+--------------+ - detection | in_primary | - ------------------------------+--------------| - VAST_0127-73A.EPOCH01.I.fits | True | - VAST_0127-73A.EPOCH01.I.fits | True | - VAST_0127-73A.EPOCH01.I.fits | True | - VAST_0127-73A.EPOCH01.I.fits | True | - VAST_0127-73A.EPOCH01.I.fits | True | - ------------------------------+--------------+ + DataFrame containing missing image information (see source code for + dataframe format). """ + # Output format: + # +----------+----------------------------------+-----------+------------+ + # | source | img_list | wavg_ra | wavg_dec | + # |----------+----------------------------------+-----------+------------+ + # | 278 | ['VAST_0127-73A.EPOCH01.I.fits'] | 22.2929 | -71.8717 | + # | 702 | ['VAST_0127-73A.EPOCH01.I.fits'] | 28.8125 | -69.3547 | + # | 844 | ['VAST_0127-73A.EPOCH01.I.fits'] | 17.3152 | -72.346 | + # | 934 | ['VAST_0127-73A.EPOCH01.I.fits'] | 9.75754 | -72.9629 | + # | 1290 | ['VAST_0127-73A.EPOCH01.I.fits'] | 20.8455 | -76.8269 | + # +----------+----------------------------------+-----------+------------+ + # ------------------------------------------------------------------+ + # skyreg_img_list | + # ------------------------------------------------------------------+ + # ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | + # ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | + # ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | + # ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | + # ['VAST_0127-73A.EPOCH01.I.fits', 'VAST_0127-73A.EPOCH08.I.fits'] | + # ------------------------------------------------------------------+ + # ----------------------------------+------------------------------+ + # img_diff | primary | + # ----------------------------------+------------------------------+ + # ['VAST_0127-73A.EPOCH08.I.fits'] | VAST_0127-73A.EPOCH01.I.fits | + # ['VAST_0127-73A.EPOCH08.I.fits'] | VAST_0127-73A.EPOCH01.I.fits | + # ['VAST_0127-73A.EPOCH08.I.fits'] | VAST_0127-73A.EPOCH01.I.fits | + # ['VAST_0127-73A.EPOCH08.I.fits'] | VAST_0127-73A.EPOCH01.I.fits | + # ['VAST_0127-73A.EPOCH08.I.fits'] | VAST_0127-73A.EPOCH01.I.fits | + # ----------------------------------+------------------------------+ + # ------------------------------+--------------+ + # detection | in_primary | + # ------------------------------+--------------| + # VAST_0127-73A.EPOCH01.I.fits | True | + # VAST_0127-73A.EPOCH01.I.fits | True | + # VAST_0127-73A.EPOCH01.I.fits | True | + # VAST_0127-73A.EPOCH01.I.fits | True | + # VAST_0127-73A.EPOCH01.I.fits | True | + # ------------------------------+--------------+ logger.info("Creating ideal source coverage df...") merged_timer = StopWatch() @@ -1100,7 +1101,7 @@ def _get_skyregion_relations( Returns: A list of other sky regions (including self) that are within the - 'xtr_radius' of the sky region in the row. + 'xtr_radius' of the sky region in the row. ''' target = SkyCoord( row['centre_ra'], @@ -1141,13 +1142,13 @@ def group_skyregions(df: pd.DataFrame) -> pd.DataFrame: Returns: The sky region group of each skyregion id. - +----+----------------+ - | | skyreg_group | - |----+----------------| - | 2 | 1 | - | 3 | 1 | - | 1 | 2 | - +----+----------------+ + +----+----------------+ + | | skyreg_group | + |----+----------------| + | 2 | 1 | + | 3 | 1 | + | 1 | 2 | + +----+----------------+ """ sr_coords = SkyCoord( df['centre_ra'], @@ -1242,20 +1243,22 @@ def get_parallel_assoc_image_df( +----+----------------+ Returns: - Dataframe containing the merged images and skyreg_id and skyreg_group. - +----+-------------------------------+-------------+----------------+ - | | image | skyreg_id | skyreg_group | - |----+-------------------------------+-------------+----------------| - | 0 | VAST_2118+00A.EPOCH01.I.fits | 2 | 1 | - | 1 | VAST_2118-06A.EPOCH01.I.fits | 3 | 1 | - | 2 | VAST_0127-73A.EPOCH01.I.fits | 1 | 2 | - | 3 | VAST_2118-06A.EPOCH03x.I.fits | 3 | 1 | - | 4 | VAST_2118-06A.EPOCH02.I.fits | 3 | 1 | - | 5 | VAST_2118-06A.EPOCH05x.I.fits | 3 | 1 | - | 6 | VAST_2118-06A.EPOCH06x.I.fits | 3 | 1 | - | 7 | VAST_0127-73A.EPOCH08.I.fits | 1 | 2 | - +----+-------------------------------+-------------+----------------+ + Dataframe containing the merged images and skyreg_id and skyreg_group + (see source code for output format). """ + # Output format + # +----+-------------------------------+-------------+----------------+ + # | | image | skyreg_id | skyreg_group | + # |----+-------------------------------+-------------+----------------| + # | 0 | VAST_2118+00A.EPOCH01.I.fits | 2 | 1 | + # | 1 | VAST_2118-06A.EPOCH01.I.fits | 3 | 1 | + # | 2 | VAST_0127-73A.EPOCH01.I.fits | 1 | 2 | + # | 3 | VAST_2118-06A.EPOCH03x.I.fits | 3 | 1 | + # | 4 | VAST_2118-06A.EPOCH02.I.fits | 3 | 1 | + # | 5 | VAST_2118-06A.EPOCH05x.I.fits | 3 | 1 | + # | 6 | VAST_2118-06A.EPOCH06x.I.fits | 3 | 1 | + # | 7 | VAST_0127-73A.EPOCH08.I.fits | 1 | 2 | + # +----+-------------------------------+-------------+----------------+ skyreg_ids = [i.skyreg_id for i in images] images_df = pd.DataFrame({ @@ -1456,7 +1459,8 @@ def reconstruct_associtaion_dfs( and 'measurement_pairs'. Returns: - The reconstructed sources_df and skyc1_srs dataframes. + The reconstructed `sources_df` dataframe. + The reconstructed `skyc1_srs` dataframes. """ prev_associations = pd.read_parquet(previous_parquet_paths['associations']) @@ -1673,7 +1677,7 @@ def write_parquets( run_path: directory to save parquets to. Returns: - Sky regions as pandas DataFrame + Sky regions as pandas DataFrame. """ # write images parquet file under pipeline run folder images_df = pd.DataFrame(map(lambda x: x.__dict__, images)) diff --git a/vast_pipeline/plots.py b/vast_pipeline/plots.py index 08cb9c800..ec4f5cb59 100644 --- a/vast_pipeline/plots.py +++ b/vast_pipeline/plots.py @@ -327,8 +327,11 @@ def fit_eta_v( use_peak_flux: Use peak fluxes for the analysis instead of integrated fluxes, defaults to 'False'. - Returns: Tuple containing the eta_fit_mean, eta_fit_sigma, v_fit_mean - and the v_fit_sigma. + Returns: + The mean of the eta fit. + The sigma of the eta fit. + The mean of the v fit. + The sigma of the v fit. """ if use_peak_flux: diff --git a/vast_pipeline/signals.py b/vast_pipeline/signals.py index 24bfb60b6..e00912f85 100644 --- a/vast_pipeline/signals.py +++ b/vast_pipeline/signals.py @@ -24,6 +24,9 @@ def delete_orphans_for_run(sender: Type[Run], instance: Run, using: str, **kwarg Model instance to be deleted. using: Database alias. + + Returns: + None """ image_orphans = ( Image.objects.annotate(num_runs=Count("run")) diff --git a/vast_pipeline/utils/auth.py b/vast_pipeline/utils/auth.py index eade66f29..74c3b8744 100644 --- a/vast_pipeline/utils/auth.py +++ b/vast_pipeline/utils/auth.py @@ -31,8 +31,8 @@ def create_admin_user(uid: int, response: Dict, details: Dict, user: User, Social auth user model object Returns: - return a dictionary with the Django User object in it or empty if - no action are taken + A dictionary with the Django User object in it or empty if + no action is taken. """ # assume github-org backend, add # if other backend are implemented @@ -88,8 +88,8 @@ def load_github_avatar(response: Dict, social: UserSocialAuth, *args, Arbitrary keyword arguments. Returns: - return a dictionary with the Social auth user object in it or empty if - no action are taken + A dictionary with the Social auth user object in it or empty if + no action is taken. """ # assume github-org backend, add # if other backend are implemented diff --git a/vast_pipeline/utils/external_query.py b/vast_pipeline/utils/external_query.py index e71691d3e..136463719 100644 --- a/vast_pipeline/utils/external_query.py +++ b/vast_pipeline/utils/external_query.py @@ -19,13 +19,13 @@ def simbad(coord: SkyCoord, radius: Angle) -> List[Dict[str, Any]]: Returns: A list of dicts, where each dict is a query result row with the following keys: - - object_name: the name of the astronomical object. - - database: the source of the result, i.e. SIMBAD. - - separation_arcsec: separation to the query coordinate in arcsec. - - otype: object type. - - otype_long: long form of the object type. - - ra_hms: RA coordinate string in hms format. - - dec_dms: Dec coordinate string in ±dms format. + - object_name: the name of the astronomical object. + - database: the source of the result, i.e. SIMBAD. + - separation_arcsec: separation to the query coordinate in arcsec. + - otype: object type. + - otype_long: long form of the object type. + - ra_hms: RA coordinate string in hms format. + - dec_dms: Dec coordinate string in ±dms format. """ CustomSimbad = Simbad() CustomSimbad.add_votable_fields( @@ -78,13 +78,13 @@ def ned(coord: SkyCoord, radius: Angle) -> List[Dict[str, Any]]: Returns: A list of dicts, where each dict is a query result row with the following keys: - - object_name: the name of the astronomical object. - - database: the source of the result, i.e. NED. - - separation_arcsec: separation to the query coordinate in arcsec. - - otype: object type. - - otype_long: long form of the object type. - - ra_hms: RA coordinate string in hms format. - - dec_dms: Dec coordinate string in ±dms format. + - object_name: the name of the astronomical object. + - database: the source of the result, i.e. NED. + - separation_arcsec: separation to the query coordinate in arcsec. + - otype: object type. + - otype_long: long form of the object type. + - ra_hms: RA coordinate string in hms format. + - dec_dms: Dec coordinate string in ±dms format. """ # NED API doesn't supply the long-form object types. # Copied from https://ned.ipac.caltech.edu/Documents/Guides/Database @@ -197,14 +197,14 @@ def tns(coord: SkyCoord, radius: Angle) -> List[Dict[str, Any]]: Returns: A list of dicts, where each dict is a query result row with the following keys: - - object_name: the name of the transient. - - database: the source of the result, i.e. TNS. - - separation_arcsec: separation to the query coordinate in arcsec. - - otype: object type. - - otype_long: long form of the object type. Not given by TNS, will always be - an empty string. - - ra_hms: RA coordinate string in hms format. - - dec_dms: Dec coordinate string in ±dms format. + - object_name: the name of the transient. + - database: the source of the result, i.e. TNS. + - separation_arcsec: separation to the query coordinate in arcsec. + - otype: object type. + - otype_long: long form of the object type. Not given by TNS, will always be + an empty string. + - ra_hms: RA coordinate string in hms format. + - dec_dms: Dec coordinate string in ±dms format. """ TNS_API_URL = "https://www.wis-tns.org/api/" headers = { diff --git a/vast_pipeline/utils/utils.py b/vast_pipeline/utils/utils.py index 3d3bd4ff7..f012ea8f6 100644 --- a/vast_pipeline/utils/utils.py +++ b/vast_pipeline/utils/utils.py @@ -211,7 +211,9 @@ def eq_to_cart(ra: float, dec: float) -> Tuple[float, float, float]: dec: The declination coordinate, in degrees, to convert. Returns: - The cartesian coordinates. + The cartesian x coordinate. + The cartesian y coordinate. + The cartesian z coordinate. """ # TODO: This part of the code can probably be removed along with the # storage of these coodinates on the image. @@ -231,7 +233,8 @@ def equ2gal(ra: float, dec: float) -> Tuple[float, float]: dec (float): Declination in units of degrees. Returns: - Tuple (float, float): Galactic longitude and latitude in degrees. + Galactic longitude in degrees. + Galactic latitude in degrees. """ c = SkyCoord(np.float(ra), np.float(dec), unit=(u.deg, u.deg), frame="icrs") l = c.galactic.l.deg @@ -249,7 +252,8 @@ def gal2equ(l: float, b: float) -> Tuple[float, float]: b (float): Galactic latitude in degrees. Returns: - Tuple (float, float): Right ascension and declination in units of degrees. + Right ascension in degrees. + Declination in degrees. """ c = SkyCoord(l=np.float(l) * u.deg, b=np.float(b) * u.deg, frame="galactic") ra = c.icrs.ra.deg @@ -273,7 +277,7 @@ def parse_coord(coord_string: str, coord_frame: str = "icrs") -> SkyCoord: coord_frame (str, optional): The frame of `coord_string`. Defaults to "icrs". Returns: - SkyCoord + The SkyCoord object. """ # if both coord components are decimals, assume they're in degrees, otherwise assume # hourangles and degrees. Note that the unit parameter is ignored if the units are @@ -304,8 +308,7 @@ def optimize_floats(df: pd.DataFrame) -> pd.DataFrame: input dataframe, no specific columns. Returns: - The input dataframe with the `float64` type - columns downcasted. + The input dataframe with the `float64` type columns downcasted. """ floats = df.select_dtypes(include=["float64"]).columns.tolist() df[floats] = df[floats].apply(pd.to_numeric, downcast="float") @@ -325,8 +328,7 @@ def optimize_ints(df: pd.DataFrame) -> pd.DataFrame: Input dataframe, no specific columns. Returns: - The input dataframe with the `int64` type - columns downcasted. + The input dataframe with the `int64` type columns downcasted. """ ints = df.select_dtypes(include=["int64"]).columns.tolist() df[ints] = df[ints].apply(pd.to_numeric, downcast="integer") @@ -355,7 +357,7 @@ def dict_merge( add_keys (bool): whether to add new keys Returns: - dict: updated dict + Updated dict. """ dct = dct.copy() if not add_keys: diff --git a/vast_pipeline/utils/view.py b/vast_pipeline/utils/view.py index 862d849a9..ec25a26ed 100644 --- a/vast_pipeline/utils/view.py +++ b/vast_pipeline/utils/view.py @@ -201,7 +201,7 @@ def generate_colsfields( be not searchable in the final table. Returns: - colsfields (list): List containing JSON object. + List containing the JSON object. """ colsfields = [] @@ -276,8 +276,7 @@ def get_skyregions_collection(run_id: Optional[int]=None) -> Dict[str, Any]: run_id (int, optional): Run ID to filter on if not None. Returns: - skyregions_collection (dict): Dictionary representing a JSON obejct - containing the sky regions. + Dictionary representing a JSON object containing the sky regions. """ skyregions = SkyRegion.objects.all() if run_id is not None: diff --git a/vast_pipeline/views.py b/vast_pipeline/views.py index 75b933b0d..546cf2994 100644 --- a/vast_pipeline/views.py +++ b/vast_pipeline/views.py @@ -7,7 +7,7 @@ import dask.bag as db import pandas as pd -from typing import Dict, Any, Tuple, List +from typing import Dict, Any, Tuple, List, Optional from glob import glob from itertools import tee from pathlib import Path @@ -72,19 +72,20 @@ def _process_comment_form_get_comments( - request, instance: CommentableModel + request: Request, instance: CommentableModel ) -> Tuple[CommentForm, "QuerySet[Comment]"]: - """Process the comment form and return the form and comment objects. If the `request` - method was POST, create a `Comment` object attached to `instance`. + """Process the comment form and return the form and comment objects. + If the `request` method was POST, create a `Comment` object attached to + `instance`. Args: request: Django HTTP request object. - instance (CommentableModel): Django object that is a subclass of `CommentableModel`. + instance: Django object that is a subclass of `CommentableModel`. This is the object the comment will be attached to. Returns: - Tuple[CommentForm, QuerySet[Comment]]: a new, unbound `CommentForm` instance; and - the `QuerySet` of `Comment` objects attached to `instance`. + A new, unbound `CommentForm` instance. + The `QuerySet` of `Comment` objects attached to `instance`. """ if request.method == "POST": comment_form = CommentForm(request.POST) @@ -250,22 +251,24 @@ def measurements(self, request, pk=None): return Response(serializer.data) @rest_framework.decorators.action(detail=True, methods=['post']) - def run(self, request, pk=None): + def run( + self, request: Request, pk: Optional[int] = None + ) -> HttpResponseRedirect: """ Launches a pipeline run using a Django Q cluster. Includes a check on ownership or admin stataus of the user to make sure processing is allowed. Args: - request (Request): Django REST Framework request object. - pk (int, optional): Run object primary key. Defaults to None. + request: Django REST Framework request object. + pk: Run object primary key. Defaults to None. Raises: Http404: if a Source with the given `pk` cannot be found. Returns: Response: Returns to the orignal request page (the pipeline run - detail). + detail). """ if not pk: messages.error( @@ -347,22 +350,24 @@ def run(self, request, pk=None): ) @rest_framework.decorators.action(detail=True, methods=['post']) - def restore(self, request, pk=None): + def restore( + self, request: Request, pk: Optional[int] = None + ) -> HttpResponseRedirect: """ Launches a restore pipeline run using a Django Q cluster. Includes a check on ownership or admin status of the user to make sure processing is allowed. Args: - request (Request): Django REST Framework request object. - pk (int, optional): Run object primary key. Defaults to None. + request: Django REST Framework request object. + pk: Run object primary key. Defaults to None. Raises: Http404: if a Source with the given `pk` cannot be found. Returns: Response: Returns to the orignal request page (the pipeline run - detail). + detail). """ if not pk: messages.error( @@ -440,7 +445,9 @@ def restore(self, request, pk=None): ) @rest_framework.decorators.action(detail=True, methods=['post']) - def delete(self, request, pk=None): + def delete( + self, request: Request, pk: Optional[int] = None + ) -> HttpResponseRedirect: """ Launches the remove pipeline run using a Django Q cluster. Includes a check on ownership or admin status of the user to make sure @@ -516,22 +523,24 @@ def delete(self, request, pk=None): ) @rest_framework.decorators.action(detail=True, methods=['post']) - def genarrow(self, request, pk=None): + def genarrow( + self, request: Request, pk: Optional[int] = None + ) ->HttpResponseRedirect: """ Launches the create arrow files process for a pipeline run using a Django Q cluster. Includes a check on ownership or admin status of the user to make sure the creation is allowed. Args: - request (Request): Django REST Framework request object. - pk (int, optional): Run object primary key. Defaults to None. + request: Django REST Framework request object. + pk: Run object primary key. Defaults to None. Raises: Http404: if a Source with the given `pk` cannot be found. Returns: Response: Returns to the orignal request page (the pipeline run - detail). + detail). """ if not pk: messages.error( @@ -1977,7 +1986,7 @@ class MeasurementQuery(APIView): def get( self, - request, + request: Request, ra_deg: float, dec_deg: float, image_id: int, @@ -2493,7 +2502,7 @@ def sesame_search(self, request: Request) -> Response: Defaults to "all". Returns: - Response: a Django REST framework Response. Will return JSON with status code: + A Django REST framework Response. Will return JSON with status code: - 400 if the query params fail validation (i.e. if an invalid Sesame service or no object name is provided) or if the name resolution fails. Error messages are returned as an array of strings under the relevant query @@ -2520,7 +2529,7 @@ def coordinate_validator(self, request: Request) -> Response: - frame (str): the frame for the given coordinate string e.g. icrs, galactic. Returns: - Response: a Django REST framework Response. Will return JSON with status code: + A Django REST framework Response. Will return JSON with status code: - 400 if the query params fail validation, i.e. if a frame unknown to Astropy is given, or the coordinate string fails to parse. Error messages are returned as an array of strings under the relevant query parameter key. @@ -2552,7 +2561,7 @@ def external_search(self, request: Request) -> Response: `astropy.coordinates.Angle`, respectively. Returns: - Response: a Django REST framework Response containing result records as a list + A Django REST framework Response containing result records as a list under the data object key. Each record contains the properties: - object_name: the name of the astronomical object. - database: the source of the result, e.g. SIMBAD or NED. From 45ee6cc2f927c68473177ddd802d0a29a0dcd434 Mon Sep 17 00:00:00 2001 From: Adam Stewart Date: Fri, 18 Feb 2022 19:00:58 +0000 Subject: [PATCH 02/10] Updated docs workflows - Publish dev on every dev commit. - Publish new latests version on release (using release name as tag). --- .../{ci-docs.yml => ci-docs-dev.yml} | 6 ++-- .github/workflows/ci-docs-release.yml | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) rename .github/workflows/{ci-docs.yml => ci-docs-dev.yml} (89%) create mode 100644 .github/workflows/ci-docs-release.yml diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs-dev.yml similarity index 89% rename from .github/workflows/ci-docs.yml rename to .github/workflows/ci-docs-dev.yml index 421d9b59f..eba2399c3 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs-dev.yml @@ -1,11 +1,11 @@ -name: ci docs +name: ci docs dev # a merged PR will trigger a push on master so merging to master is # included in the event below on: push: branches: - - master + - dev jobs: deploy: runs-on: ubuntu-latest @@ -28,4 +28,4 @@ jobs: - name: Install python dependencies run: poetry install - name: Build and deploy docs - run: poetry run mkdocs gh-deploy -m 'github action deployment' --force + run: poetry run mike deploy --push dev diff --git a/.github/workflows/ci-docs-release.yml b/.github/workflows/ci-docs-release.yml new file mode 100644 index 000000000..023f37c22 --- /dev/null +++ b/.github/workflows/ci-docs-release.yml @@ -0,0 +1,34 @@ +name: ci docs release + +# a merged PR will trigger a push on master so merging to master is +# included in the event below +on: + release: + types: [published] + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-python@v2 + with: + python-version: 3.x + - uses: Gr1N/setup-poetry@v4 + - name: Set release notes tag + run: | + export RELEASE_TAG_VERSION=${{ github.event.release.tag_name }} + echo "RELEASE_TAG_VERSION=${RELEASE_TAG_VERSION:1}" >> $GITHUB_ENV + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y build-essential libpq-dev libgraphviz-dev + - name: Cache Python dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pypoetry/virtualenvs + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + - name: Install python dependencies + run: poetry install + - name: Build and deploy docs + run: poetry run mike deploy --push --update-aliases ${RELEASE_TAG_VERSION} latest From ef925f4bc3bd281c28119721490b56bef0601f22 Mon Sep 17 00:00:00 2001 From: Adam Stewart Date: Fri, 18 Feb 2022 19:33:08 +0000 Subject: [PATCH 03/10] Test fixes & update CHANGELOG.md --- CHANGELOG.md | 6 ++++++ pyproject.toml | 1 + vast_pipeline/pipeline/config.py | 2 +- vast_pipeline/plots.py | 2 +- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22ea4fd4f..b57baf5a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Added +- Added documentation versioning [#627](https://github.com/askap-vast/vast-pipeline/pull/627). - Added a ZTF cone search button on the source detail page [#626](https://github.com/askap-vast/vast-pipeline/pull/626). - Added the 0-based index of each measurement to the image cutout card headers [#625](https://github.com/askap-vast/vast-pipeline/pull/625). - Added Bokeh hover tooltip to measurement pair graph to display pair metrics [#625](https://github.com/askap-vast/vast-pipeline/pull/625). @@ -35,6 +36,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Changed +- Fixed markupsafe to v2.0.1 for Jinja2 compatibility [#627](https://github.com/askap-vast/vast-pipeline/pull/627). +- Bumped all documentation dependancies to latest versions [#627](https://github.com/askap-vast/vast-pipeline/pull/627). +- Changed GitHub workflows for new documentation versioning [#627](https://github.com/askap-vast/vast-pipeline/pull/627). - Bumped Jinja2 to 3.0.3 to fix a Markupsafe error caused by a removed function [#634](https://github.com/askap-vast/vast-pipeline/pull/634). - Dependancies updated using npm audit fix (non-breaking) [#620](https://github.com/askap-vast/vast-pipeline/pull/620). - Refactored adding source to favourites button to use ajax to avoid page reload [#614](https://github.com/askap-vast/vast-pipeline/pull/614). @@ -54,6 +58,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Fixed +- Fixed docs issue that stopped serializers and views being shown in the code reference [#627](https://github.com/askap-vast/vast-pipeline/pull/627). - Fixed broken links to external search results from NED by URI encoding source names [#633](https://github.com/askap-vast/vast-pipeline/pull/633). - Fixed a regression from pandas==1.4.0 that caused empty groups to be passed to an apply function [#632](https://github.com/askap-vast/vast-pipeline/pull/632). - Fixed source names to be IAU compliant [#618](https://github.com/askap-vast/vast-pipeline/pull/618). @@ -80,6 +85,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### List of PRs +- [#627](https://github.com/askap-vast/vast-pipeline/pull/627): dep, docs: Documentation update and versioning. - [#633](https://github.com/askap-vast/vast-pipeline/pull/633): fix: URI encode NED object names. - [#632](https://github.com/askap-vast/vast-pipeline/pull/632): fix: skip empty groups in new sources groupby-apply. - [#634](https://github.com/askap-vast/vast-pipeline/pull/634): dep: bump Jinja2 to v3. diff --git a/pyproject.toml b/pyproject.toml index 4df7d6ec0..e6cc16235 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ matplotlib = "^3.5.0" holoviews = "^1.14.7" datashader = "^0.13.0" channels = "^3.0.4" +MarkupSafe = "2.0.1" [tool.poetry.dev-dependencies] mkdocs-material = "^8.0.0" diff --git a/vast_pipeline/pipeline/config.py b/vast_pipeline/pipeline/config.py index 6c964ce57..71b20620f 100644 --- a/vast_pipeline/pipeline/config.py +++ b/vast_pipeline/pipeline/config.py @@ -514,7 +514,7 @@ def check_prev_config_diff(self) -> bool: settings are the same (the requirement for add mode). Otherwise False is returned. Returns: - `True` if images are different but general settings are the same, + `True` if images are different but general settings are the same, otherwise `False` is returned. """ prev_config = PipelineConfig.from_file( diff --git a/vast_pipeline/plots.py b/vast_pipeline/plots.py index ec4f5cb59..05c93347a 100644 --- a/vast_pipeline/plots.py +++ b/vast_pipeline/plots.py @@ -328,7 +328,7 @@ def fit_eta_v( integrated fluxes, defaults to 'False'. Returns: - The mean of the eta fit. + The mean of the eta fit. The sigma of the eta fit. The mean of the v fit. The sigma of the v fit. From 283d801949f015fdbac2301ae5d42a26e76a3d0d Mon Sep 17 00:00:00 2001 From: Adam Stewart Date: Fri, 18 Feb 2022 20:00:54 +0000 Subject: [PATCH 04/10] Updated docs docsdev page [skip ci] --- docs/developing/docsdev.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/developing/docsdev.md b/docs/developing/docsdev.md index 81eda98a8..da7a362a4 100644 --- a/docs/developing/docsdev.md +++ b/docs/developing/docsdev.md @@ -211,6 +211,14 @@ For example, using the `vast_pipeline` as the base for the python files and `ref Please refer to the [`mkdocstrings` documentation ](https://mkdocstrings.github.io){:target="_blank"} for full details of the options available to declare in `mkdocs.yml`. +## Versioning + +Versioning is enabled on the documentation by using the [`mike`](https://github.com/jimporter/mike){:target="_blank"} package. +The documentation is published for each release version, along with a development version that is updated with every commit to the default `dev` branch. +Setup of this has been completed on the initial deployment, and the GitHub workflows will automatically take care of the continual deployment. +Vist the `mike` package page linked above for details on general management commands such as how to delete a version. + ## Deployment to GitHub pages -Automatic deployment to GitHub pages is set up using GitHub actions and workflows. See source code under the `.github` folder. +Automatic deployment to GitHub pages is set up using GitHub actions and workflows. +See the workflows `ci-docs-dev.yml` and `ci-docs-release.yml`. From 2aaed2abd24db086c6acb10bc39d16501a3bfc3a Mon Sep 17 00:00:00 2001 From: Adam Stewart Date: Fri, 18 Feb 2022 20:14:30 +0000 Subject: [PATCH 05/10] Tidy ci-docs workflow comments [skip ci] --- .github/workflows/ci-docs-dev.yml | 2 +- .github/workflows/ci-docs-release.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci-docs-dev.yml b/.github/workflows/ci-docs-dev.yml index eba2399c3..876c64c41 100644 --- a/.github/workflows/ci-docs-dev.yml +++ b/.github/workflows/ci-docs-dev.yml @@ -1,6 +1,6 @@ name: ci docs dev -# a merged PR will trigger a push on master so merging to master is +# a merged PR will trigger a push on dev so merging to dev is # included in the event below on: push: diff --git a/.github/workflows/ci-docs-release.yml b/.github/workflows/ci-docs-release.yml index 023f37c22..cf0ec7501 100644 --- a/.github/workflows/ci-docs-release.yml +++ b/.github/workflows/ci-docs-release.yml @@ -1,7 +1,5 @@ name: ci docs release -# a merged PR will trigger a push on master so merging to master is -# included in the event below on: release: types: [published] From 2cc6a4f9b68420caaf3efc7caf948d8a8332769e Mon Sep 17 00:00:00 2001 From: Adam Stewart Date: Tue, 22 Feb 2022 23:55:02 +0000 Subject: [PATCH 06/10] Remove vaex from docs dependencies - [skip ci] --- docs/architecture/intro.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/architecture/intro.md b/docs/architecture/intro.md index 76c4fe9e0..00c9912d4 100644 --- a/docs/architecture/intro.md +++ b/docs/architecture/intro.md @@ -35,7 +35,6 @@ The front-end is built in simple HTML, CSS and Javascript using a freely availab * [Pyarrow 0.17+](https://arrow.apache.org/docs/python/install.html){:target="_blank"} * [Postgres 10+](https://www.postgresql.org/){:target="_blank"} * [Q3C](https://github.com/segasai/q3c){:target="_blank"} -* [Vaex 3+](https://vaex.readthedocs.io/en/latest/){:target="_blank"} ### Front-End From 1e71c43a7b6363beb4476b6434f100a9b2ef476f Mon Sep 17 00:00:00 2001 From: Adam Stewart Date: Tue, 1 Mar 2022 12:28:21 +0000 Subject: [PATCH 07/10] Removed MarkupSafe dependency - [skip ci] --- poetry.lock | 9 +++------ pyproject.toml | 1 - 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0cd806207..85b62f39e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1877,9 +1877,8 @@ category = "main" optional = false python-versions = ">=3.7" -[package.dependencies] -astropy = ">=4.0" -requests = "*" +[package.extras] +docs = ["sphinx-astropy"] [[package]] name = "pywin32-ctypes" @@ -2458,7 +2457,7 @@ prod = ["gevent", "gunicorn"] [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "fdec0ba9ca437d08246bc3251de4be222a78fc6b50742d98066e88191760a611" +content-hash = "2f2045c83c783144483c93662957d56e9910dda0e9f80c00c4ffb81dd5c544eb" [metadata.files] ansicon = [ @@ -2862,7 +2861,6 @@ griffe = [ {file = "griffe-0.12.5.tar.gz", hash = "sha256:9ba7ce1742e8069eb9a67d3cce9b1967af0b46130d04e41c4d7e0c654c930e5c"}, ] gunicorn = [ - {file = "gunicorn-20.1.0-py3-none-any.whl", hash = "sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e"}, {file = "gunicorn-20.1.0.tar.gz", hash = "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"}, ] heapdict = [ @@ -3559,7 +3557,6 @@ pyviz-comms = [ {file = "pyviz_comms-2.1.0.tar.gz", hash = "sha256:f4a7126f318fb6b964fef3f92fa55bc46b9218f62a8464a8b18e968b3087dbc0"}, ] pyvo = [ - {file = "pyvo-1.2.1-py3-none-any.whl", hash = "sha256:f90eb8e687fc950df9b2df9b63b837d33c4db52a68bebc8f4970ec3d1fbba9d2"}, {file = "pyvo-1.2.1.tar.gz", hash = "sha256:3630ad7c62acf9ce435a9b2d997376fa9fae6865ada6632735680c90ccf525e6"}, ] pywin32-ctypes = [ diff --git a/pyproject.toml b/pyproject.toml index e6cc16235..4df7d6ec0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,6 @@ matplotlib = "^3.5.0" holoviews = "^1.14.7" datashader = "^0.13.0" channels = "^3.0.4" -MarkupSafe = "2.0.1" [tool.poetry.dev-dependencies] mkdocs-material = "^8.0.0" From 041e762bee21a7f3fe675921cde1c124485ffd03 Mon Sep 17 00:00:00 2001 From: Adam Stewart Date: Wed, 2 Mar 2022 12:56:26 +0000 Subject: [PATCH 08/10] Add index.md to date exclude list - [skip ci] --- mkdocs.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index ce8230585..d5c97823e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -56,7 +56,7 @@ markdown_extensions: - pymdownx.details - pymdownx.inlinehilite - pymdownx.tabbed: - alternate_style: true + alternate_style: true - pymdownx.superfences - pymdownx.highlight - pymdownx.tabbed @@ -181,6 +181,8 @@ plugins: - git-revision-date-localized: fallback_to_build_date: true enable_creation_date: true + exclude: + - index.md - gen-files: scripts: # - docs/gen_credits.py From 8a86bcc056f3a0f8bfc7c15ef71ef90197adc05e Mon Sep 17 00:00:00 2001 From: Adam Stewart Date: Wed, 2 Mar 2022 15:03:41 +0000 Subject: [PATCH 09/10] Bumped minimum mkdocs-material to 8.2.4 --- poetry.lock | 14 +++++++------- pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/poetry.lock b/poetry.lock index 85b62f39e..3c4f12600 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1288,7 +1288,7 @@ mkdocs = ">=1.0" [[package]] name = "mkdocs-material" -version = "8.2.1" +version = "8.2.4" description = "A Material Design theme for MkDocs" category = "dev" optional = false @@ -1326,7 +1326,7 @@ mkdocs = ">=1.0.4" [[package]] name = "mkdocstrings" -version = "0.18.0" +version = "0.18.1" description = "Automatic documentation from sources, for MkDocs." category = "dev" optional = false @@ -2457,7 +2457,7 @@ prod = ["gevent", "gunicorn"] [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "2f2045c83c783144483c93662957d56e9910dda0e9f80c00c4ffb81dd5c544eb" +content-hash = "a6807671965298b65bd200ec70574a531c09b531765cc7e5fa9a0cd2faf5ffaa" [metadata.files] ansicon = [ @@ -3112,8 +3112,8 @@ mkdocs-git-revision-date-localized-plugin = [ {file = "mkdocs_git_revision_date_localized_plugin-0.12.1-py3-none-any.whl", hash = "sha256:2b05dcaa0700119d92462ada141be1353b1bf90e5e11331ea32ef61a3009c958"}, ] mkdocs-material = [ - {file = "mkdocs-material-8.2.1.tar.gz", hash = "sha256:efcb3a24e1ef374268306cb8b4d3ba0db5d2c4ee4aa1fc4bcbd2087571b1d975"}, - {file = "mkdocs_material-8.2.1-py2.py3-none-any.whl", hash = "sha256:9496b26593c8a0f9bb18bda7f1b919c0bc16838217f38212dca47cc9006d7fe0"}, + {file = "mkdocs-material-8.2.4.tar.gz", hash = "sha256:3c350a03e2f7288874f0a9cf5aacfbbb8721cc7d4e389b53169180ee2187223e"}, + {file = "mkdocs_material-8.2.4-py2.py3-none-any.whl", hash = "sha256:7aa9ed562f68c8592dad3f568378c98a2350398445dbf46de373585b3cd9080e"}, ] mkdocs-material-extensions = [ {file = "mkdocs-material-extensions-1.0.3.tar.gz", hash = "sha256:bfd24dfdef7b41c312ede42648f9eb83476ea168ec163b613f9abd12bbfddba2"}, @@ -3124,8 +3124,8 @@ mkdocs-minify-plugin = [ {file = "mkdocs_minify_plugin-0.5.0-py2-none-any.whl", hash = "sha256:487c31ae6b8b3230f56910ce6bcf5c7e6ad9a8c4f51c720a4b989f30c2b0233f"}, ] mkdocstrings = [ - {file = "mkdocstrings-0.18.0-py3-none-any.whl", hash = "sha256:75e277f6a56a894a727efcf6d418d36cd43d4db7da9614c2dc23300e257d95ad"}, - {file = "mkdocstrings-0.18.0.tar.gz", hash = "sha256:01d8ab962fc1f388c9b15cbf8c078b8738f92adf983b626d74135aaee2bce33a"}, + {file = "mkdocstrings-0.18.1-py3-none-any.whl", hash = "sha256:4053929356df8cd69ed32eef71d8f676a472ef72980c9ffd4f933ead1debcdad"}, + {file = "mkdocstrings-0.18.1.tar.gz", hash = "sha256:fb7c91ce7e3ab70488d3fa6c073a4f827cdc319042f682ef8ea95459790d64fc"}, ] mkdocstrings-python = [ {file = "mkdocstrings-python-0.6.2.tar.gz", hash = "sha256:74a7c54dfd9ea7c62bfb4311087b717145887d7a4ece120a587e428dd7689517"}, diff --git a/pyproject.toml b/pyproject.toml index 4df7d6ec0..8b983f4af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ datashader = "^0.13.0" channels = "^3.0.4" [tool.poetry.dev-dependencies] -mkdocs-material = "^8.0.0" +mkdocs-material = "^8.2.4" mkdocstrings = {extras = ["python"], version = "^0.18.0"} mkdocs-minify-plugin = "^0" mkdocs-material-extensions = "^1.0.1" From cee0b6c90067ae0c535bd371d48a5322fe1e32ec Mon Sep 17 00:00:00 2001 From: Adam Stewart Date: Wed, 2 Mar 2022 15:29:36 +0000 Subject: [PATCH 10/10] Updated announcement bar and CHANGELOG.md - [skip ci] --- CHANGELOG.md | 3 +-- docs/theme/main.html | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b57baf5a9..5ed5c2f37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,8 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Changed -- Fixed markupsafe to v2.0.1 for Jinja2 compatibility [#627](https://github.com/askap-vast/vast-pipeline/pull/627). -- Bumped all documentation dependancies to latest versions [#627](https://github.com/askap-vast/vast-pipeline/pull/627). +- Bumped all documentation dependancies to latest versions (incl. mkdocs-material minimum 8.2.4) [#627](https://github.com/askap-vast/vast-pipeline/pull/627). - Changed GitHub workflows for new documentation versioning [#627](https://github.com/askap-vast/vast-pipeline/pull/627). - Bumped Jinja2 to 3.0.3 to fix a Markupsafe error caused by a removed function [#634](https://github.com/askap-vast/vast-pipeline/pull/634). - Dependancies updated using npm audit fix (non-breaking) [#620](https://github.com/askap-vast/vast-pipeline/pull/620). diff --git a/docs/theme/main.html b/docs/theme/main.html index 4631d9c48..5375ef178 100644 --- a/docs/theme/main.html +++ b/docs/theme/main.html @@ -36,8 +36,8 @@ - New in version 1.0.0: Documentation, add epoch mode and more 🙌. Check the release notes! - + Development Version 🧑‍💻: Check the release notes + for the latest changes! {% endblock %}