diff --git a/.cookiecutter.yaml b/.cookiecutter.yaml new file mode 100644 index 00000000..457ebb9e --- /dev/null +++ b/.cookiecutter.yaml @@ -0,0 +1,10 @@ +default_context: + author_name: "Eva Maxfield Brown, To Huynh, Isaac Na, Council Data Project Contributors" + author_email: "evamaxfieldbrown@gmail.com" + hosting_github_username_or_org: "CouncilDataProject" + project_name: "cdp-backend" + project_slug: "cdp-backend" + python_slug: "cdp_backend" + project_short_description: "Data storage utilities and processing pipelines used by CDP instances." + hosting_pypi_username: "CouncilDataProject" + license: "MIT License" \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 9081bbc1..2e688de5 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -13,19 +13,19 @@ labels: bug ### Describe the Bug -_A clear and concise description of the bug._ + ### Expected Behavior -_What did you expect to happen instead?_ + ### Reproduction -_Steps to reproduce the behavior and/or a minimal example that exhibits the behavior._ + ### Environment -_Any additional information about your environment._ + - OS Version: _[e.g. macOS 11.3.1]_ -- CDP Backend Version: _[e.g. 0.5.0]_ +- cdp-backend Version: _[e.g. 0.5.0]_ diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index d02b9435..9a8d17eb 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -13,16 +13,16 @@ labels: enhancement ### Feature Description -_A clear and concise description of the feature you're requesting._ + ### Use Case -_Please provide a use case to help us understand your request in context._ + ### Solution -_Please describe your ideal solution._ + ### Alternatives -_Please describe any alternatives you've considered, even if you've dismissed them._ + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9954c2f7..4d540e5d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -20,4 +20,4 @@ This pull request resolves # ### Description of Changes -_Include a description of the proposed changes._ + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..ca9bb9cc --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + commit-message: + prefix: "ci(dependabot):" \ No newline at end of file diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml deleted file mode 100644 index bdbfda29..00000000 --- a/.github/workflows/build-docs.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Documentation - -on: - push: - branches: - - main - -jobs: - docs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2.3.1 - with: - persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' - cache: 'pip' - - name: Install Packages - run: | - sudo apt update - sudo apt-get install graphviz --fix-missing - - name: Install Dependencies - run: | - pip install --upgrade pip - pip install .[dev] - - name: Generate Docs - run: | - make gen-docs - touch docs/_build/html/.nojekyll - - name: Publish Docs - uses: JamesIves/github-pages-deploy-action@3.7.1 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BASE_BRANCH: main # The branch the action should deploy from. - BRANCH: gh-pages # The branch the action should deploy to. - FOLDER: docs/_build/html/ # The folder the action should deploy. diff --git a/.github/workflows/check-pr.yml b/.github/workflows/check-pr.yml deleted file mode 100644 index 97842300..00000000 --- a/.github/workflows/check-pr.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Check Pull Request - -on: pull_request - -jobs: - test: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - python-version: [3.8, 3.9, '3.10'] - os: [ubuntu-latest, macOS-latest, windows-latest] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - name: Install Packages (Ubuntu) - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt update - sudo apt-get install graphviz --fix-missing - sudo apt-get install ffmpeg --fix-missing - - name: Install Packages (Mac) - if: matrix.os == 'macOS-latest' - run: | - brew install graphviz - brew install ffmpeg - - name: Install Packages (Windows) - if: matrix.os == 'windows-latest' - run: | - choco install graphviz - choco install ffmpeg - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install .[test] - - name: Run tests with Tox - run: tox -e py - - name: Upload Codecov - uses: codecov/codecov-action@v1 - - lint: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' - cache: 'pip' - - name: Install graphviz - run: | - sudo apt update - sudo apt-get install graphviz --fix-missing - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install pre-commit - - name: Lint - run: pre-commit run --all-files --show-diff-on-failure - - docs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' - cache: 'pip' - - name: Install graphviz - run: | - sudo apt update - sudo apt-get install graphviz --fix-missing - - name: Install Dependencies - run: | - pip install --upgrade pip - pip install .[dev] - - name: Generate Docs - run: | - make gen-docs diff --git a/.github/workflows/build-main.yml b/.github/workflows/ci.yml similarity index 55% rename from .github/workflows/build-main.yml rename to .github/workflows/ci.yml index f469d390..2660bf31 100644 --- a/.github/workflows/build-main.yml +++ b/.github/workflows/ci.yml @@ -1,9 +1,14 @@ -name: Build +name: CI on: push: branches: - main + tags: + - "v*" + pull_request: + branches: + - main schedule: # # @@ -11,24 +16,39 @@ on: # Run every Monday at 10:24:00 PST # (Since these CRONs are used by a lot of people - # let's be nice to the servers and schedule it _not_ on the hour) - - cron: '24 18 * * 1' + - cron: "24 18 * * 1" + workflow_dispatch: jobs: + # Check that all files listed in manifest make it into build + check-manifest: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + - run: pip install check-manifest && check-manifest + + # Check tests pass on multiple Python and OS combinations test: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: [3.8, 3.9, '3.10'] + python-version: [3.8, 3.9, "3.10"] os: [ubuntu-latest, macOS-latest, windows-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - cache: 'pip' + - uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Install Packages (Ubuntu) if: matrix.os == 'ubuntu-latest' run: | @@ -48,54 +68,53 @@ jobs: - name: Install Dependencies run: | python -m pip install --upgrade pip - pip install .[test] - - name: Run tests with Tox - run: tox -e py + pip install .[infrastructure,pipeline,test] + - name: Run Tests + run: just test - name: Upload Codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v3 + # Check linting, formating, types, etc. lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: '3.10' - cache: 'pip' - - name: Install graphviz - run: | - sudo apt update - sudo apt-get install graphviz --fix-missing + python-version: "3.10" + - uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Install Dependencies run: | python -m pip install --upgrade pip - pip install pre-commit + pip install .[lint] - name: Lint - run: pre-commit run --all-files --show-diff-on-failure + run: just lint + # Publish to PyPI if test, lint, and manifest checks passed publish: - if: "contains(github.event.head_commit.message, 'Bump version')" - needs: [test, lint] + if: "success() && startsWith(github.ref, 'refs/tags/')" + needs: [check-manifest, test, lint] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: '3.10' - cache: 'pip' + python-version: "3.10" - name: Install Dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel + pip install build wheel - name: Build Package run: | - python setup.py sdist bdist_wheel + python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: CouncilDataProject password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..22fc38df --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,36 @@ +name: Documentation + +on: + push: + branches: + - main + +jobs: + docs: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install Packages + run: | + sudo apt update + sudo apt-get install graphviz --fix-missing + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + just install + - name: Generate Docs + run: | + just generate-docs + touch docs/_build/.nojekyll + - name: Publish Docs + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: docs/_build/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c189180f..57e43824 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,28 +1,34 @@ files: cdp_backend repos: - - repo: https://github.com/PyCQA/flake8 - rev: 3.9.2 - hooks: - - id: flake8 - additional_dependencies: [flake8-typing-imports, flake8-debugger] - args: [--count, --show-source, --statistics, --min-python-version=3.8.0] - - repo: https://github.com/myint/autoflake - rev: v1.4 - hooks: - - id: autoflake - args: ["--in-place", "--remove-all-unused-imports"] - - repo: https://github.com/PyCQA/isort - rev: 5.9.3 - hooks: - - id: isort - - repo: https://github.com/psf/black - rev: 22.3.0 - hooks: - - id: black - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.910 - hooks: - - id: mypy - additional_dependencies: + + - repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort + + - repo: https://github.com/myint/autoflake + rev: v1.4 + hooks: + - id: autoflake + args: ["--in-place", "--remove-all-unused-imports"] + + - repo: https://github.com/psf/black + rev: 22.6.0 + hooks: + - id: black + + - repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + additional_dependencies: + - flake8-typing-imports>=1.9.0 + - flake8-pyprojecttoml + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.971 + hooks: + - id: mypy + additional_dependencies: - "types-pytz" - "types-requests" diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index c2b3dea1..8ccaae03 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -14,22 +14,22 @@ appearance, race, religion, or sexual identity and orientation. Examples of behavior that contributes to creating a positive environment include: -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members Examples of unacceptable behavior by participants include: -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting +- The use of sexualized language or imagery and unwelcome sexual attention or + advances +- Trolling, insulting/derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or electronic + address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting ## Our Responsibilities diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 79f2c8fa..cd085f0e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,7 +29,7 @@ Ready to contribute? Here's how to set up `cdp-backend` for local development. ```bash cd cdp-backend/ - pip install -e .[dev] + just install ``` 6. Create a branch for local development: @@ -42,32 +42,56 @@ Ready to contribute? Here's how to set up `cdp-backend` for local development. Now you can make your changes locally. 7. When you're done making changes, check that your changes pass linting and - tests, including testing other Python versions with make: + tests with [just](https://github.com/casey/just): ```bash - make build + just build ``` 8. Commit your changes and push your branch to GitHub: ```bash git add . - git commit -m "Resolves gh-###. Your detailed description of your changes." + git commit -m "Your detailed description of your changes." git push origin {your_development_type}/short-description ``` 9. Submit a pull request through the GitHub website. +## Just Commands + +For development commands we use [just](https://github.com/casey/just). + +```bash +just +``` +``` +Available recipes: + build # run lint and then run tests + clean # clean all build, python, and lint files + default # list all available commands + generate-docs # generate Sphinx HTML documentation + install # install with all deps + lint # lint, format, and check all files + release # release a new version + serve-docs # generate Sphinx HTML documentation and serve to browser + tag-for-release version # tag a new version + test # run tests + update-from-cookiecutter # update this repo using latest cookiecutter-py-package +``` + ## Deploying A reminder for the maintainers on how to deploy. -Make sure all your changes are committed. -Then run: +Make sure the main branch is checked out and all desired changes +are merged. Then run: ```bash -$ bump2version patch # possible: major / minor / patch -$ git push -$ git push --tags +just tag-for-release "vX.Y.Z" +just release ``` -This will release a new package version on Git + GitHub and publish to PyPI. +The presence of a tag starting with "v" will trigger the `publish` step in the +main github workflow, which will build the package and upload it to PyPI. The +version will be injected into the package metadata by +[`setuptools-scm`](https://github.com/pypa/setuptools_scm) diff --git a/Justfile b/Justfile new file mode 100644 index 00000000..57da129c --- /dev/null +++ b/Justfile @@ -0,0 +1,94 @@ +# list all available commands +default: + just --list + +# clean all build, python, and lint files +clean: + rm -fr build + rm -fr docs/_build + rm -fr dist + rm -fr .eggs + find . -name '*.egg-info' -exec rm -fr {} + + find . -name '*.egg' -exec rm -f {} + + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + find . -name '__pycache__' -exec rm -fr {} + + rm -fr .coverage* + rm -fr coverage.xml + rm -fr htmlcov + rm -fr .pytest_cache + rm -fr .mypy_cache + rm -fr index + rm -fr abc123-cdp_*-transcript.json + rm -fr test.err + rm -fr test.out + + +# install with all deps +install: + pip install -e .[pipeline,infrastructure,lint,test,docs,dev] + +# lint, format, and check all files +lint: + pre-commit run --all-files + +# run tests +test: + pytest --cov-report xml --cov-report html --cov=cdp_backend cdp_backend/tests + +# run lint and then run tests +build: + just lint + just test + +# generate Sphinx HTML documentation +generate-docs: + rm -f docs/cdp_backend*.rst + rm -f docs/modules.rst + rm -f docs/_static/cdp_database_diagram.* + create_cdp_database_uml \ + -o docs/_static/cdp_database_diagram.dot + dot \ + -T jpg \ + -o docs/_static/cdp_database_diagram.jpg docs/_static/cdp_database_diagram.dot + create_cdp_ingestion_models_doc \ + -t docs/ingestion_models.template \ + -o docs/ingestion_models.md + create_cdp_transcript_model_doc \ + -t docs/transcript_model.template \ + -o docs/transcript_model.md + create_cdp_event_gather_flow_viz \ + -o docs/_static/cdp_event_gather_flow_{ftype}.png + sphinx-apidoc -o docs cdp_backend **/tests + python -msphinx "docs" "docs/_build" + + +# Generate project URI for browser opening +# We replace here to handle windows paths +# Windows paths are normally `\` separated but even in the browser they use `/` +# https://stackoverflow.com/a/61991869 +project_uri := if "os_family()" == "unix" { + justfile_directory() +} else { + replace(justfile_directory(), "\\", "/") +} + +# generate Sphinx HTML documentation and serve to browser +serve-docs: + just generate-docs + python -mwebbrowser -t "file://{{project_uri}}/docs/_build/index.html" + +# tag a new version +tag-for-release version: + git tag -a "{{version}}" -m "{{version}}" + echo "Tagged: $(git tag --sort=-version:refname| head -n 1)" + +# release a new version +release: + git push --follow-tags + +# update this repo using latest cookiecutter-py-package +update-from-cookiecutter: + pip install cookiecutter + cookiecutter gh:evamaxfield/cookiecutter-py-package --config-file .cookiecutter.yaml --no-input --overwrite-if-exists --output-dir .. \ No newline at end of file diff --git a/LICENSE b/LICENSE index c9f5e4f2..9c16d960 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022, Council Data Project Contributors +Copyright (c) 2022, Eva Maxfield Brown, To Huynh, Isaac Na, Council Data Project Contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 53b31eee..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,11 +0,0 @@ -include CONTRIBUTING.md -include LICENSE -include README.md -include cdp_backend/py.typed - -recursive-include tests * -recursive-exclude * __pycache__ -recursive-exclude * *.py[co] - -recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif -graft cdp_backend/utils/resources/ diff --git a/Makefile b/Makefile deleted file mode 100644 index 08a0550e..00000000 --- a/Makefile +++ /dev/null @@ -1,93 +0,0 @@ -.PHONY: clean build gen-docs docs help -.DEFAULT_GOAL := help - -define BROWSER_PYSCRIPT -import os, webbrowser, sys - -try: - from urllib import pathname2url -except: - from urllib.request import pathname2url - -webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) -endef -export BROWSER_PYSCRIPT - -define PRINT_HELP_PYSCRIPT -import re, sys - -for line in sys.stdin: - match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) - if match: - target, help = match.groups() - print("%-20s %s" % (target, help)) -endef -export PRINT_HELP_PYSCRIPT - -BROWSER := python -c "$$BROWSER_PYSCRIPT" - -help: - @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) - -clean: ## Clean all build, python, and testing files - rm -fr build/ - rm -fr dist/ - rm -fr .eggs/ - find . -name '*.egg-info' -exec rm -fr {} + - find . -name '*.egg' -exec rm -f {} + - find . -name '*.pyc' -exec rm -f {} + - find . -name '*.pyo' -exec rm -f {} + - find . -name '*~' -exec rm -f {} + - find . -name '__pycache__' -exec rm -fr {} + - rm -fr .tox/ - rm -fr .coverage - rm -fr coverage.xml - rm -fr htmlcov/ - rm -fr .pytest_cache - rm -fr .mypy_cache - -build: ## Run tox / run tests and lint - tox - -gen-docs: ## Generate Sphinx HTML documentation, including API docs - rm -f docs/cdp_backend*.rst - rm -f docs/modules.rst - rm -f docs/_static/cdp_database_diagram.* - create_cdp_database_uml \ - -o docs/_static/cdp_database_diagram.dot - dot \ - -T jpg \ - -o docs/_static/cdp_database_diagram.jpg docs/_static/cdp_database_diagram.dot - create_cdp_ingestion_models_doc \ - -t docs/ingestion_models.template \ - -o docs/ingestion_models.md - create_cdp_transcript_model_doc \ - -t docs/transcript_model.template \ - -o docs/transcript_model.md - create_cdp_event_gather_flow_viz \ - -o docs/_static/cdp_event_gather_flow_{ftype}.png - sphinx-apidoc -o docs/ cdp_backend **/tests/ - $(MAKE) -C docs html - -docs: ## Generate Sphinx HTML documentation, including API docs, and serve to browser - make gen-docs - $(BROWSER) docs/_build/html/index.html - -run-rand-event-pipeline: ## Run event pipeline using random event gen - run_cdp_event_gather \ - example-configs/random-event.json \ - --parallel - -run-min-event-pipeline: ## Run event pipeline using minimal event def - run_cdp_event_gather \ - example-configs/min-event.json - -run-filled-event-pipeline: ## Run event pipeline using the filled event - run_cdp_event_gather \ - example-configs/filled-event.json \ - --parallel - -run-many-event-pipeline: ## Run event pipeline using multiple events - run_cdp_event_gather \ - example-configs/many-event.json \ - --parallel \ No newline at end of file diff --git a/README.md b/README.md index ce8eee98..4468eaab 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,10 @@ # cdp-backend -[![Build Status](https://github.com/CouncilDataProject/cdp-backend/workflows/Build/badge.svg)](https://github.com/CouncilDataProject/cdp-backend/actions) +[![Build Status](https://github.com/CouncilDataProject/cdp-backend/workflows/CI/badge.svg)](https://github.com/CouncilDataProject/cdp-backend/actions) [![Documentation](https://github.com/CouncilDataProject/cdp-backend/workflows/Documentation/badge.svg)](https://CouncilDataProject.github.io/cdp-backend) -[![Code Coverage](https://codecov.io/gh/CouncilDataProject/cdp-backend/branch/main/graph/badge.svg)](https://codecov.io/gh/CouncilDataProject/cdp-backend) [![DOI](https://joss.theoj.org/papers/10.21105/joss.03904/status.svg)](https://doi.org/10.21105/joss.03904) -Data storage utilities and processing pipelines to run on Council Data Project server deployments. +Data storage utilities and processing pipelines used by CDP instances. --- @@ -65,4 +64,4 @@ Brown et al., (2021). Council Data Project: Software for Municipal Data Collecti ## License -[MIT](./LICENSE) +**MIT License** diff --git a/cdp_backend/__init__.py b/cdp_backend/__init__.py index 81f4a150..aed605c6 100644 --- a/cdp_backend/__init__.py +++ b/cdp_backend/__init__.py @@ -1,8 +1,13 @@ # -*- coding: utf-8 -*- -"""Top-level package for cdp-backend.""" +"""Top-level package for cdp_backend.""" -__author__ = "Council Data Project Contributors" -__email__ = "evamaxfieldbrown@gmail.com" +from importlib.metadata import PackageNotFoundError, version + +try: + __version__ = version("cdp-backend") +except PackageNotFoundError: + __version__ = "uninstalled" -from .version import __version__ # noqa: F401 +__author__ = "Eva Maxfield Brown, To Huynh, Isaac Na, Council Data Project Contributors" +__email__ = "evamaxfieldbrown@gmail.com" diff --git a/cdp_backend/annotation/speaker_labels.py b/cdp_backend/annotation/speaker_labels.py index cc3e9d33..40a26bd8 100644 --- a/cdp_backend/annotation/speaker_labels.py +++ b/cdp_backend/annotation/speaker_labels.py @@ -101,7 +101,7 @@ def annotate( # Load transcript if isinstance(transcript, (str, Path)): with open(transcript, "r") as open_f: - loaded_transcript = Transcript.from_json(open_f.read()) # type: ignore + loaded_transcript = Transcript.from_json(open_f.read()) else: loaded_transcript = transcript @@ -177,9 +177,13 @@ def annotate( for speaker, scores in chunk_scores.items(): mean_scores[speaker] = sum(scores) / len(scores) - # Get highest score speaker - highest_mean_speaker = max(mean_scores, key=mean_scores.get) # type: ignore - highest_mean_score = mean_scores[highest_mean_speaker] + # Get highest scoring speaker and their score + highest_mean_speaker = "" + highest_mean_score = 0.0 + for speaker, score in mean_scores.items(): + if score > highest_mean_score: + highest_mean_speaker = speaker + highest_mean_score = score # Threshold holdout if highest_mean_score >= min_sentence_mean_confidence: diff --git a/cdp_backend/bin/create_cdp_transcript_model_doc.py b/cdp_backend/bin/create_cdp_transcript_model_doc.py index c081c03b..9aa8a1fe 100644 --- a/cdp_backend/bin/create_cdp_transcript_model_doc.py +++ b/cdp_backend/bin/create_cdp_transcript_model_doc.py @@ -59,7 +59,7 @@ def __parse(self) -> None: def _construct_transcript_model_doc(template_file: Path, output_file: Path) -> Path: - example_transcript_jsons = EXAMPLE_TRANSCRIPT.to_json() # type: ignore + example_transcript_jsons = EXAMPLE_TRANSCRIPT.to_json() example_transcript_dict = json.loads(example_transcript_jsons) example_transcript_str = json.dumps(example_transcript_dict, indent=4) diff --git a/cdp_backend/bin/process_cdp_event_index_chunk.py b/cdp_backend/bin/process_cdp_event_index_chunk.py index 0f471146..33183dda 100644 --- a/cdp_backend/bin/process_cdp_event_index_chunk.py +++ b/cdp_backend/bin/process_cdp_event_index_chunk.py @@ -74,9 +74,7 @@ def main() -> None: try: args = Args() with open(args.config_file, "r") as open_resource: - config = EventIndexPipelineConfig.from_json( # type: ignore - open_resource.read() - ) + config = EventIndexPipelineConfig.from_json(open_resource.read()) # Get flow definition flow = pipeline.create_event_index_upload_pipeline( diff --git a/cdp_backend/bin/process_special_event.py b/cdp_backend/bin/process_special_event.py index f2057791..1698825a 100644 --- a/cdp_backend/bin/process_special_event.py +++ b/cdp_backend/bin/process_special_event.py @@ -57,16 +57,12 @@ def main() -> None: # Read pipeline config with open(args.event_gather_config_file, "r") as open_resource: - config = EventGatherPipelineConfig.from_json( # type: ignore - open_resource.read() - ) + config = EventGatherPipelineConfig.from_json(open_resource.read()) log.info("Parsing event details...") # Convert event details file to EventIngestionModel with open(args.event_details_file, "r") as open_resource: - ingestion_model = EventIngestionModel.from_json( # type: ignore - open_resource.read() - ) + ingestion_model = EventIngestionModel.from_json(open_resource.read()) for session in ingestion_model.sessions: # Copy if remote resource, otherwise use local file uri diff --git a/cdp_backend/bin/run_cdp_event_gather.py b/cdp_backend/bin/run_cdp_event_gather.py index bbc3a28e..43c3369f 100644 --- a/cdp_backend/bin/run_cdp_event_gather.py +++ b/cdp_backend/bin/run_cdp_event_gather.py @@ -82,9 +82,7 @@ def main() -> None: try: args = Args() with open(args.config_file, "r") as open_resource: - config = EventGatherPipelineConfig.from_json( # type: ignore - open_resource.read() - ) + config = EventGatherPipelineConfig.from_json(open_resource.read()) # Get flow definition flow = pipeline.create_event_gather_flow( diff --git a/cdp_backend/bin/run_cdp_event_index_generation.py b/cdp_backend/bin/run_cdp_event_index_generation.py index 79e5cb48..9c95267a 100644 --- a/cdp_backend/bin/run_cdp_event_index_generation.py +++ b/cdp_backend/bin/run_cdp_event_index_generation.py @@ -85,9 +85,7 @@ def main() -> None: try: args = Args() with open(args.config_file, "r") as open_resource: - config = EventIndexPipelineConfig.from_json( # type: ignore - open_resource.read() - ) + config = EventIndexPipelineConfig.from_json(open_resource.read()) # Get flow definition flow = pipeline.create_event_index_generation_pipeline( diff --git a/cdp_backend/database/functions.py b/cdp_backend/database/functions.py index 4c8b2e9b..3d87f49d 100644 --- a/cdp_backend/database/functions.py +++ b/cdp_backend/database/functions.py @@ -332,9 +332,13 @@ def create_minimal_person( db_person.name = _strip_field(person.name) db_person.is_active = person.is_active - db_person.router_string = db_models.Person.generate_router_string( - _strip_field(person.name) # type: ignore - ) + stripped_name = _strip_field(person.name) + if stripped_name: + db_person.router_string = db_models.Person.generate_router_string(stripped_name) + else: + raise ValueError( + f"Something went very wrong. Person doesn't have a name: {person}" + ) return db_person @@ -368,11 +372,7 @@ def create_person( kwargs={"google_credentials_file": credentials_file} ) - if person.router_string is None: - db_person.router_string = db_models.Person.generate_router_string( - _strip_field(person.name) # type: ignore - ) - else: + if person.router_string is not None: db_person.router_string = _strip_field(person.router_string) # Optional diff --git a/cdp_backend/pipeline/event_gather_pipeline.py b/cdp_backend/pipeline/event_gather_pipeline.py index 73f862c7..c44835a9 100644 --- a/cdp_backend/pipeline/event_gather_pipeline.py +++ b/cdp_backend/pipeline/event_gather_pipeline.py @@ -171,8 +171,8 @@ def create_event_gather_flow( # Generate transcript transcript_uri, transcript = generate_transcript( - session_content_hash=session_content_hash, # type: ignore - audio_uri=audio_uri, # type: ignore + session_content_hash=session_content_hash, + audio_uri=audio_uri, session=session, event=event, bucket=config.validated_gcs_bucket_name, @@ -199,7 +199,7 @@ def create_event_gather_flow( # Store all processed and provided data session_processing_results.append( - compile_session_processing_result( # type: ignore + compile_session_processing_result( session=session, session_video_hosted_url=session_video_hosted_url, session_content_hash=session_content_hash, @@ -691,7 +691,7 @@ def finalize_and_archive_transcript( # Dump to JSON with open(transcript_save_path, "w") as open_resource: - open_resource.write(transcript.to_json()) # type: ignore + open_resource.write(transcript.to_json()) # Store to file store transcript_file_uri = fs_functions.upload_file( @@ -753,7 +753,7 @@ def check_for_existing_transcript( if transcript_exists: fs = GCSFileSystem(token=credentials_file) with fs.open(transcript_uri, "r") as open_resource: - transcript = Transcript.from_json(open_resource.read()) # type: ignore + transcript = Transcript.from_json(open_resource.read()) else: transcript = None @@ -865,7 +865,7 @@ def generate_transcript( ) result_transcript.name = "merge_in_memory_transcript" - return (result_transcript_uri, result_transcript) # type: ignore + return (result_transcript_uri, result_transcript) @task(nout=2) @@ -976,7 +976,7 @@ def _process_person_ingestion( # So, if the same person is referenced multiple times in the ingestion model # but most of those references have the same data and only a few have different data # the produced JSON string will note the differences and run when it needs to. - person_cache_key = person.to_json() # type: ignore + person_cache_key = person.to_json() if person_cache_key not in upload_cache: # Store person picture file @@ -1353,7 +1353,7 @@ def store_event_processing_results( ) else: - matter_db_model = None # type: ignore + matter_db_model = None # Create minutes item minutes_item_db_model = db_functions.create_minutes_item( diff --git a/cdp_backend/pipeline/generate_event_index_pipeline.py b/cdp_backend/pipeline/generate_event_index_pipeline.py index 881004cd..dfebc4a4 100644 --- a/cdp_backend/pipeline/generate_event_index_pipeline.py +++ b/cdp_backend/pipeline/generate_event_index_pipeline.py @@ -13,7 +13,7 @@ import pandas as pd import pytz import rapidfuzz -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from gcsfs import GCSFileSystem from nltk import ngrams from nltk.stem import SnowballStemmer @@ -141,17 +141,15 @@ def get_transcripts_per_event( return list(event_transcripts.values()) -@dataclass_json @dataclass -class SentenceManager: +class SentenceManager(DataClassJsonMixin): original_details: Sentence cleaned_text: str n_grams: List[Tuple[str]] -@dataclass_json @dataclass -class ContextualizedGram: +class ContextualizedGram(DataClassJsonMixin): # We attach the id for simpler gram grouping # We attach the datetime for simpler datetime weighting event_id: str @@ -201,7 +199,7 @@ def read_transcripts_and_generate_grams( # Init transcript with open(local_transcript_filepath, "r") as open_f: - transcript = Transcript.from_json(open_f.read()) # type: ignore + transcript = Transcript.from_json(open_f.read()) # Get cleaned sentences by removing stop words cleaned_sentences: List[SentenceManager] = [ @@ -291,7 +289,7 @@ def convert_all_n_grams_to_dataframe( """ return pd.DataFrame( [ - n_gram.to_dict() # type: ignore + n_gram.to_dict() for single_event_n_grams in all_events_n_grams for n_gram in single_event_n_grams ] diff --git a/cdp_backend/pipeline/ingestion_models.py b/cdp_backend/pipeline/ingestion_models.py index b8829fc0..e261a45f 100644 --- a/cdp_backend/pipeline/ingestion_models.py +++ b/cdp_backend/pipeline/ingestion_models.py @@ -7,7 +7,7 @@ from datetime import datetime from typing import List, Optional -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin ############################################################################### @@ -21,9 +21,8 @@ class IngestionModel: ############################################################################### -@dataclass_json @dataclass -class Person(IngestionModel): +class Person(IngestionModel, DataClassJsonMixin): """ Primarily the council members, this could technically include the mayor or city manager, or any other "normal" presenters and attendees of meetings. @@ -51,9 +50,8 @@ class Person(IngestionModel): external_source_id: Optional[str] = None -@dataclass_json @dataclass -class Vote(IngestionModel): +class Vote(IngestionModel, DataClassJsonMixin): """ A reference tying a specific person and an event minutes item together. @@ -68,9 +66,8 @@ class Vote(IngestionModel): external_source_id: Optional[str] = None -@dataclass_json @dataclass -class SupportingFile(IngestionModel): +class SupportingFile(IngestionModel, DataClassJsonMixin): """ A file related tied to a matter or minutes item. @@ -84,9 +81,8 @@ class SupportingFile(IngestionModel): external_source_id: Optional[str] = None -@dataclass_json @dataclass -class Matter(IngestionModel): +class Matter(IngestionModel, DataClassJsonMixin): """ A matter is a specific legislative document. A bill, resolution, initiative, etc. """ @@ -99,9 +95,8 @@ class Matter(IngestionModel): external_source_id: Optional[str] = None -@dataclass_json @dataclass -class MinutesItem(IngestionModel): +class MinutesItem(IngestionModel, DataClassJsonMixin): """ An item referenced during a meeting. This can be a matter but it can be a presentation or budget file, etc. @@ -112,9 +107,8 @@ class MinutesItem(IngestionModel): external_source_id: Optional[str] = None -@dataclass_json @dataclass -class EventMinutesItem(IngestionModel): +class EventMinutesItem(IngestionModel, DataClassJsonMixin): """ Details about a specific item during an event. @@ -135,9 +129,8 @@ class EventMinutesItem(IngestionModel): votes: Optional[List[Vote]] = None -@dataclass_json @dataclass -class Session(IngestionModel): +class Session(IngestionModel, DataClassJsonMixin): """ A session is a working period for an event. For example, an event could have a morning and afternoon session. @@ -150,9 +143,8 @@ class Session(IngestionModel): external_source_id: Optional[str] = None -@dataclass_json @dataclass -class Body(IngestionModel): +class Body(IngestionModel, DataClassJsonMixin): """ A meeting body. This can be full council, a subcommittee, or "off-council" matters such as election debates. @@ -172,9 +164,8 @@ class Body(IngestionModel): external_source_id: Optional[str] = None -@dataclass_json @dataclass -class Role(IngestionModel): +class Role(IngestionModel, DataClassJsonMixin): """ A role is a person's job for a period of time in the city council. A person can (and should) have multiple roles. For example: a person has two terms as city @@ -197,9 +188,8 @@ class Role(IngestionModel): external_source_id: Optional[str] = None -@dataclass_json @dataclass -class Seat(IngestionModel): +class Seat(IngestionModel, DataClassJsonMixin): """ An electable office on the City Council. I.E. "Position 9". @@ -217,9 +207,8 @@ class Seat(IngestionModel): roles: Optional[List[Role]] = None -@dataclass_json @dataclass -class EventIngestionModel(IngestionModel): +class EventIngestionModel(IngestionModel, DataClassJsonMixin): """ An event can be a normally scheduled meeting, a special event such as a press conference or election debate, and, can be upcoming or historical. diff --git a/cdp_backend/pipeline/pipeline_config.py b/cdp_backend/pipeline/pipeline_config.py index 3eebe2ee..4a3b4ee8 100644 --- a/cdp_backend/pipeline/pipeline_config.py +++ b/cdp_backend/pipeline/pipeline_config.py @@ -6,15 +6,14 @@ from pathlib import Path from typing import Optional, Union -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from gcsfs import GCSFileSystem ############################################################################### -@dataclass_json @dataclass -class EventGatherPipelineConfig: +class EventGatherPipelineConfig(DataClassJsonMixin): """ Configuration options for the CDP event gather pipeline. @@ -79,9 +78,8 @@ def validated_gcs_bucket_name(self) -> str: return self._validated_gcs_bucket_name -@dataclass_json @dataclass -class EventIndexPipelineConfig: +class EventIndexPipelineConfig(DataClassJsonMixin): """ Configuration options for the CDP event index pipeline. diff --git a/cdp_backend/pipeline/transcript_model.py b/cdp_backend/pipeline/transcript_model.py index 8746a047..815fe852 100644 --- a/cdp_backend/pipeline/transcript_model.py +++ b/cdp_backend/pipeline/transcript_model.py @@ -5,7 +5,7 @@ from datetime import datetime from typing import List, Optional -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin ############################################################################### # Annotation Definitions @@ -14,25 +14,22 @@ # Docstrings for constants go _after_ the constant. -@dataclass_json @dataclass -class WordAnnotations: +class WordAnnotations(DataClassJsonMixin): """ Annotations that can appear on an individual word level. """ -@dataclass_json @dataclass -class SentenceAnnotations: +class SentenceAnnotations(DataClassJsonMixin): """ Annotations that can appear on an individual sentence level. """ -@dataclass_json @dataclass -class SectionAnnotation: +class SectionAnnotation(DataClassJsonMixin): """ A section annotation used for topic segmentation and minutes item alignment. @@ -88,9 +85,8 @@ class SectionAnnotation: description: Optional[str] = None -@dataclass_json @dataclass -class TranscriptAnnotations: +class TranscriptAnnotations(DataClassJsonMixin): """ Annotations that can appear (but are not guaranteed) for the whole transcript. """ @@ -101,9 +97,8 @@ class TranscriptAnnotations: ############################################################################### -@dataclass_json @dataclass -class Word: +class Word(DataClassJsonMixin): """ Data for a word in a transcript. @@ -129,9 +124,8 @@ class Word: annotations: Optional[WordAnnotations] = None -@dataclass_json @dataclass -class Sentence: +class Sentence(DataClassJsonMixin): """ Data for a sentence in a transcript. @@ -169,9 +163,8 @@ class Sentence: annotations: Optional[SentenceAnnotations] = None -@dataclass_json @dataclass -class Transcript: +class Transcript(DataClassJsonMixin): """ Transcript model for all transcripts in CDP databases / filestores. diff --git a/cdp_backend/tests/pipeline/test_event_gather_pipeline.py b/cdp_backend/tests/pipeline/test_event_gather_pipeline.py index e95410a0..acc37106 100644 --- a/cdp_backend/tests/pipeline/test_event_gather_pipeline.py +++ b/cdp_backend/tests/pipeline/test_event_gather_pipeline.py @@ -90,7 +90,7 @@ def test_split_audio( mock_get_file_uri.return_value = get_file_uri_value mock_upload_file.return_value = audio_upload_file_return - audio_uri = pipeline.split_audio.run( # type: ignore + audio_uri = pipeline.split_audio.run( session_content_hash=VIDEO_CONTENT_HASH, tmp_video_filepath=str(example_video), bucket="bucket", @@ -139,10 +139,7 @@ def test_generate_thumbnails( example_static_thumbnail_url, example_hover_thumbnail_url, ] - ( - static_thumbnail_url, - hover_thumbnail_url, - ) = pipeline.generate_thumbnails.run( # type: ignore + (static_thumbnail_url, hover_thumbnail_url,) = pipeline.generate_thumbnails.run( session_content_hash=example_session_content_hash, tmp_video_path=str(example_video), event=event, @@ -190,7 +187,7 @@ def test_generate_thumbnails( def test_construct_speech_to_text_phrases_context( event: EventIngestionModel, expected_phrases: List[str] ) -> None: - phrases = pipeline.construct_speech_to_text_phrases_context.run( # type: ignore + phrases = pipeline.construct_speech_to_text_phrases_context.run( event, ) @@ -261,7 +258,7 @@ def test_generate_transcript( state = flow.run() # Check state and results - assert state.is_successful() # type: ignore + assert state.is_successful() example_person = ingestion_models.Person(name="Bob Boberson") @@ -554,7 +551,7 @@ def test_store_event_processing_results( if fail_file_uploads: mock_upload_file.side_effect = FileNotFoundError() - pipeline.store_event_processing_results.run( # type: ignore + pipeline.store_event_processing_results.run( event=event, session_processing_results=session_processing_results, credentials_file="fake/credentials.json", @@ -636,7 +633,7 @@ def test_convert_video_and_handle_host( ( mp4_filepath, session_video_hosted_url, - ) = pipeline.convert_video_and_handle_host.run( # type: ignore + ) = pipeline.convert_video_and_handle_host.run( session_content_hash="abc123", video_filepath=video_filepath, session=session, diff --git a/cdp_backend/tests/pipeline/test_event_index_pipeline.py b/cdp_backend/tests/pipeline/test_event_index_pipeline.py index c23ad996..e70707b8 100644 --- a/cdp_backend/tests/pipeline/test_event_index_pipeline.py +++ b/cdp_backend/tests/pipeline/test_event_index_pipeline.py @@ -157,10 +157,8 @@ def test_get_highest_confidence_transcript_for_each_session( """ All we are really testing here is that we are reducing the set properly. """ - result_selections = ( - pipeline.get_highest_confidence_transcript_for_each_session.run( # type: ignore - transcripts - ) + result_selections = pipeline.get_highest_confidence_transcript_for_each_session.run( + transcripts ) assert set(result_selections) == set(expected_selections) @@ -211,9 +209,7 @@ def test_get_transcripts_per_event( transcripts: List[db_models.Transcript], expected_selections: List[pipeline.EventTranscripts], ) -> None: - result_selections = pipeline.get_transcripts_per_event.run( # type: ignore - transcripts - ) + result_selections = pipeline.get_transcripts_per_event.run(transcripts) for result_et, expected_et in zip(result_selections, expected_selections): assert result_et.event_id == expected_et.event_id assert set( diff --git a/cdp_backend/tests/pipeline/test_transcript_model.py b/cdp_backend/tests/pipeline/test_transcript_model.py index e5b8db23..2d034b52 100644 --- a/cdp_backend/tests/pipeline/test_transcript_model.py +++ b/cdp_backend/tests/pipeline/test_transcript_model.py @@ -7,9 +7,7 @@ def test_model_serdes() -> None: - example_transcript_str = EXAMPLE_TRANSCRIPT.to_json() # type: ignore - example_transcript_des = Transcript.from_json( # type: ignore - example_transcript_str - ) + example_transcript_str = EXAMPLE_TRANSCRIPT.to_json() + example_transcript_des = Transcript.from_json(example_transcript_str) assert EXAMPLE_TRANSCRIPT == example_transcript_des diff --git a/cdp_backend/tests/resources/example_video.mp4 b/cdp_backend/tests/resources/example_video.mp4 index 3ac71112..5dd374c1 100644 Binary files a/cdp_backend/tests/resources/example_video.mp4 and b/cdp_backend/tests/resources/example_video.mp4 differ diff --git a/cdp_backend/tests/sr_models/test_webvtt_sr_model.py b/cdp_backend/tests/sr_models/test_webvtt_sr_model.py index b0df01e8..43b47a6f 100644 --- a/cdp_backend/tests/sr_models/test_webvtt_sr_model.py +++ b/cdp_backend/tests/sr_models/test_webvtt_sr_model.py @@ -26,7 +26,7 @@ def test_transcribe( # Get URIs and read file_uri = str((resources_dir / webvtt_filename).absolute()) with open(resources_dir / expected_filename) as open_resource: - expected = Transcript.from_json(open_resource.read()) # type: ignore + expected = Transcript.from_json(open_resource.read()) # Generate transcript model = WebVTTSRModel() diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index a50b4dc1..00000000 --- a/codecov.yml +++ /dev/null @@ -1,2 +0,0 @@ -ignore: - - "cdp_backend/bin/.*" diff --git a/dev-infrastructure/Justfile b/dev-infrastructure/Justfile new file mode 100644 index 00000000..5074f829 --- /dev/null +++ b/dev-infrastructure/Justfile @@ -0,0 +1,75 @@ +# list all available commands +default: + just --list + +# get and store user +USER := env_var("USER") + +# run gcloud and pulumi logins +login: + pulumi logout + pulumi login + gcloud auth login + gcloud auth application-default login + +# create a new gcloud project and pulumi stack +init project: + gcloud projects create {{project}} --set-as-default + pulumi stack init {{project}} + echo "----------------------------------------------------------------------------" + echo "Follow the link to setup billing for the created GCloud account." + echo "https://console.cloud.google.com/billing/linkedaccount?project={{project}}" + +# set a CORS policy for the bucket +set-cors project: + gsutil cors set cors.json gs://{{project}}.appspot.com/ + +# run pulumi up for infra deploy +build: + pulumi up -p 4 + +# switch active gcloud project +switch-project project: + gcloud config set project {{project}} + +# remove all database documents (except search index) and filestore objects +clean key: + clean_cdp_database {{key}} + clean_cdp_filestore {{key}} + +# remove all database documents (including search index) and filestore objects +clean-full key: + clean_cdp_database {{key}} --clean-index + clean_cdp_filestore {{key}} + +# destroy just the stack (not the gcloud project) and rebuild +reset: + pulumi destroy -p 4 + echo "----------------------------------------------------------------------------" + echo "Sleeping for three minutes while resources clean up" + sleep 180 + make build + +# fully teardown project +destroy project: + pulumi stack rm {{project}} --force + gcloud projects delete {{project}} + rm -f ../.keys/{{project}}.json + +# generate a service account JSON +gen-key project: + mkdir ../.keys/ -p + rm -rf ../.keys/{{project}}.json + gcloud iam service-accounts create {{project}} \ + --description="CDP Dev Service Account for {{USER}}" \ + --display-name="{{project}}" + gcloud projects add-iam-policy-binding {{project}} \ + --member="serviceAccount:{{project}}@{{project}}.iam.gserviceaccount.com" \ + --role="roles/owner" + gcloud iam service-accounts keys create ../.keys/{{project}}.json \ + --iam-account "{{project}}@{{project}}.iam.gserviceaccount.com" + echo "----------------------------------------------------------------------------" + echo "Sleeping for one minute while resources set up" + sleep 60 + cp -rf ../.keys/{{project}}.json ../.keys/cdp-dev.json + echo "Key generation complete, updated ../.keys/cdp-dev.json to {{project}} key. Be sure to update GOOGLE_CREDENTIALS." \ No newline at end of file diff --git a/dev-infrastructure/Makefile b/dev-infrastructure/Makefile deleted file mode 100644 index 12e311eb..00000000 --- a/dev-infrastructure/Makefile +++ /dev/null @@ -1,75 +0,0 @@ -.PHONY: help login init build clean reset destroy gen-key -.DEFAULT_GOAL := help - -define PRINT_HELP_PYSCRIPT -import re, sys - -for line in sys.stdin: - match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) - if match: - target, help = match.groups() - print("%-20s %s" % (target, help)) -endef -export PRINT_HELP_PYSCRIPT - -help: - @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) - -login: ## Run GCloud and Pulumi logins - pulumi logout - pulumi login - gcloud auth login - gcloud auth application-default login - -init: ## See README, must provide "project" - gcloud projects create $(project) --set-as-default - pulumi stack init $(project) - echo "----------------------------------------------------------------------------" - echo "Follow the link to setup billing for the created GCloud account." - echo "https://console.cloud.google.com/billing/linkedaccount?project=${project}" - -set-cors: ## Set cors for the bucket, must provide "project" - gsutil cors set cors.json gs://${project}.appspot.com/ - -build: ## Run pulumi up for infra setup - pulumi up -p 4 - -switch-project: ## Switch active gcloud project, must provide "project" - gcloud config set project ${project} - -clean: ## Remove all database documents (except search index) and filestore objects, must provide "key" - clean_cdp_database ${key} - clean_cdp_filestore ${key} - -clean-full: ## Remove all database documents (including search index) and filestore objects, must provide "key" - clean_cdp_database ${key} --clean-index - clean_cdp_filestore ${key} - -reset: ## Run pulumi destroy and make build, must provide "project" - pulumi destroy -p 4 - echo "----------------------------------------------------------------------------" - echo "Sleeping for three minutes while resources clean up" - sleep 180 - make build - -destroy: ## Fully teardown Pulumi and GCloud, must provide "project" - pulumi stack rm ${project} --force - gcloud projects delete ${project} - rm -f ../.keys/$(project).json - -gen-key: ## Generate a service account JSON, must provide "project" - mkdir ../.keys/ -p - rm -rf ../.keys/$(project).json - gcloud iam service-accounts create $(project) \ - --description="CDP Dev Service Account for $(USER)" \ - --display-name="$(project)" - gcloud projects add-iam-policy-binding $(project) \ - --member="serviceAccount:$(project)@$(project).iam.gserviceaccount.com" \ - --role="roles/owner" - gcloud iam service-accounts keys create ../.keys/$(project).json \ - --iam-account "$(project)@$(project).iam.gserviceaccount.com" - echo "----------------------------------------------------------------------------" - echo "Sleeping for one minute while resources set up" - sleep 60 - cp -rf ../.keys/$(project).json ../.keys/cdp-dev.json - echo "Key generation complete, updated ../.keys/cdp-dev.json to $(project) key. Be sure to update GOOGLE_CREDENTIALS." diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 7975ebfb..00000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = python -msphinx -SPHINXPROJ = cdp_backend -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py index 477b6947..327d48bf 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# cdp_backend documentation build configuration file, created by +# cdp-backend documentation build configuration file, created by # sphinx-quickstart on Fri Jun 9 13:47:02 2017. # # This file is execfile()d with the current directory set to its @@ -26,6 +26,7 @@ sys.path.insert(0, os.path.abspath("..")) + # -- General configuration --------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. @@ -35,18 +36,24 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named "sphinx.ext.*") or your custom ones. extensions = [ + # Sphinx lib ext "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", "sphinx.ext.viewcode", - "sphinx.ext.napoleon", - "sphinx.ext.mathjax", + # Installed Sphinx ext + "sphinx_copybutton", + # Doc installs "m2r2", + "numpydoc", ] -# Control napoleon -napoleon_google_docstring = False -napolean_include_init_with_doc = True -napoleon_use_ivar = True -napoleon_use_param = False +copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: " +copybutton_prompt_is_regexp = True + +numpydoc_show_class_members = False + +sphinx_tabs_disable_tab_closing = True # Control autodoc autoclass_content = "both" # include init doc with class @@ -63,13 +70,13 @@ ".md": "markdown", } -# The master toctree document. -master_doc = "index" +# The main toctree document. +main_doc = "index" # General information about the project. -project = u"cdp-backend" -copyright = u"2022, Council Data Project Contributors" -author = u"Council Data Project Contributors" +project = "cdp-backend" +copyright = "2022" +author = "Eva Maxfield Brown, To Huynh, Isaac Na, Council Data Project Contributors" # The version info for the project you"re documenting, acts as replacement # for |version| and |release|, also used in various other places throughout @@ -90,7 +97,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ".template"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" @@ -146,10 +153,10 @@ # [howto, manual, or own class]). latex_documents = [ ( - master_doc, + main_doc, "cdp_backend.tex", - u"cdp-backend Documentation", - u"Council Data Project Contributors", + "cdp-backend Documentation", + "Eva Maxfield Brown, To Huynh, Isaac Na, Council Data Project Contributors", "manual", ), ] @@ -159,7 +166,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [(master_doc, "cdp_backend", u"cdp-backend Documentation", [author], 1)] +man_pages = [(main_doc, "cdp_backend", "cdp-backend Documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------- @@ -169,17 +176,14 @@ # dir menu entry, description, category) texinfo_documents = [ ( - master_doc, + main_doc, "cdp_backend", - u"cdp-backend Documentation", + "cdp-backend Documentation", author, "cdp_backend", - "One line description of project.", - "Miscellaneous", ), ] - # -- Extra docstring configurations ------------------------------------ def no_namedtuple_attrib_docstring(app, what, name, obj, options, lines): is_namedtuple_docstring = len(lines) == 1 and lines[0].startswith( diff --git a/docs/index.rst b/docs/index.rst index 5f4eb9d3..313c739c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,9 +16,3 @@ Welcome to cdp-backend's documentation! contributing .. mdinclude:: ../README.md - -Indices and tables -================== -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 0882dd9c..00000000 --- a/docs/make.bat +++ /dev/null @@ -1,36 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=python -msphinx -) -set SOURCEDIR=. -set BUILDDIR=_build -set SPHINXPROJ=cdp_backend - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The Sphinx module was not found. Make sure you have Sphinx installed, - echo.then set the SPHINXBUILD environment variable to point to the full - echo.path of the 'sphinx-build' executable. Alternatively you may add the - echo.Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% - -:end -popd diff --git a/example-configs/filled-event.json b/example-configs/filled-event.json deleted file mode 100644 index 7a41efa2..00000000 --- a/example-configs/filled-event.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "google_credentials_file": ".keys/cdp-dev.json", - "get_events_function_path": "cdp_backend.pipeline.mock_get_events.filled_get_events" -} diff --git a/example-configs/many-event.json b/example-configs/many-event.json deleted file mode 100644 index 85da03cf..00000000 --- a/example-configs/many-event.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "google_credentials_file": ".keys/cdp-dev.json", - "get_events_function_path": "cdp_backend.pipeline.mock_get_events.many_get_events" -} diff --git a/example-configs/min-event.json b/example-configs/min-event.json deleted file mode 100644 index 0e70f0cc..00000000 --- a/example-configs/min-event.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "google_credentials_file": ".keys/cdp-dev.json", - "get_events_function_path": "cdp_backend.pipeline.mock_get_events.min_get_events" -} diff --git a/example-configs/random-event.json b/example-configs/random-event.json deleted file mode 100644 index f4a48f0f..00000000 --- a/example-configs/random-event.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "google_credentials_file": ".keys/cdp-dev.json", - "get_events_function_path": "cdp_backend.pipeline.mock_get_events.get_events" -} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..46e7512e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,204 @@ +# package build +# https://peps.python.org/pep-0517/ +[build-system] +requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] + +# package basics +# https://peps.python.org/pep-0621/ +[project] +name = "cdp-backend" +description = "Data storage utilities and processing pipelines used by CDP instances." +keywords = [ + "civic technology", + "open government", +] +readme = "README.md" +requires-python = ">=3.8" +license = { text = "MIT License" } +authors = [ + { name = "Eva Maxfield Brown", email = "evamaxfieldbrown@gmail.com" }, + { name = "To Huynh" }, + { name = "Isaac Na" }, + { name = "Council Data Project Contributors" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", +] +dynamic = ["version"] +dependencies = [ + "aiohttp>=3.8.1", + "dataclasses-json>=0.5", + "fireo>=1.5", + "fsspec", # Version pin set by gcsfs + "gcsfs>=2022.7.1", + "requests>=2.26.0", +] + +[project.urls] +Homepage = "https://github.com/CouncilDataProject/cdp-backend" +"Bug Tracker" = "https://github.com/CouncilDataProject/cdp-backend/issues" +Documentation = "https://CouncilDataProject.github.io/cdp-backend" +"User Support" = "https://github.com/CouncilDataProject/cdp-backend/issues" + +# extra dependencies +# https://peps.python.org/pep-0621/#dependencies-optional-dependencies +[project.optional-dependencies] +infrastructure = [ + "pulumi~=3.31", + "pulumi-google-native~=0.18", + "pulumi-gcp~=6.0", +] +pipeline = [ + "dask[distributed]>=2021.7.0", + "ffmpeg-python==0.2.0", + "google-cloud-speech~=2.13", + "graphviz~=0.16", + "imageio~=2.18", + "imageio-ffmpeg~=0.4", + "m3u8-To-MP4==0.1.10", + "nltk~=3.6", + "numpy~=1.0", + "pandas~=1.2", + "prefect~=1.2", + "pyarrow~=8.0", + "pydub~=0.25.1", + "rapidfuzz~=2.0", + "spacy~=3.0", + "torch~=1.10", + "tqdm~=4.62", + "transformers~=4.16", + "truecase~=0.0.14", + "webvtt-py~=0.4.6", + "yt-dlp>=2022.2.4", + "vimeo_downloader~=0.4" +] +lint = [ + "black>=22.3.0", + "check-manifest>=0.48", + "flake8>=3.8.3", + "flake8-debugger>=3.2.1", + "flake8-pyprojecttoml", + "flake8-typing-imports>=1.9.0", + "isort>=5.7.0", + "mypy>=0.790", + "pre-commit>=2.20.0", + "types-pytz>=2022.1.2", + "types-requests>=2.28.5", +] +test = [ + # Pytest + "coverage>=5.1", + "pytest>=5.4.3", + "pytest-cov>=2.9.0", + "pytest-raises>=0.11", + # Extras + "networkx>=2.5", + "pydot>=1.4", +] +docs = [ + # Sphinx + Doc Gen + Styling + "m2r2>=0.2.7", + "Sphinx>=4.0.0", + "furo>=2022.4.7", + # Extensions + "numpydoc", + "sphinx-copybutton", + # TODO: Pins + "docutils>=0.18,<0.19", + # Pipeline viz + "prefect[viz]", +] +dev = [ + "bokeh>=2.3.2", + "ipython>=8.4.0", +] + +# entry points +# https://peps.python.org/pep-0621/#entry-points +[project.entry-points."console_scripts"] +clean_cdp_database = "cdp_backend.bin.clean_cdp_database:main" +clean_cdp_filestore = "cdp_backend.bin.clean_cdp_filestore:main" +create_cdp_database_uml = "cdp_backend.bin.create_cdp_database_uml:main" +create_cdp_ingestion_models_doc = "cdp_backend.bin.create_cdp_ingestion_models_doc:main" +create_cdp_transcript_model_doc = "cdp_backend.bin.create_cdp_transcript_model_doc:main" +create_cdp_event_gather_flow_viz = "cdp_backend.bin.create_cdp_event_gather_flow_viz:main" +run_cdp_event_gather = "cdp_backend.bin.run_cdp_event_gather:main" +run_cdp_event_index_generation = "cdp_backend.bin.run_cdp_event_index_generation:main" +process_cdp_event_index_chunk = "cdp_backend.bin.process_cdp_event_index_chunk:main" +search_cdp_events = "cdp_backend.bin.search_cdp_events:main" +process_special_event = "cdp_backend.bin.process_special_event:main" +add_content_hash_to_sessions = "cdp_backend.bin.add_content_hash_to_sessions:main" + +# build settings +# https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html +[tool.setuptools] +zip-safe = false +include-package-data = true + +[tool.setuptools.packages.find] +exclude = ["*docs/*", "*tests/*"] + +[tool.setuptools.package-data] +"*" = ["*.yaml", "py.typed", "*.csv"] + +# tools +[tool.black] +line-length = 88 + +[tool.isort] +ensure_newline_before_comments = true +force_grid_wrap = 0 +include_trailing_comma = true +line_length = 88 +multi_line_output = 3 +profile = "black" +use_parentheses = true + +# https://github.com/mgedmin/check-manifest#configuration +[tool.check-manifest] +ignore = [ + ".editorconfig", + ".pre-commit-config.yaml", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "Justfile", + ".cookiecutter.yaml", + "*docs/*", + "codecov.yml", + "dev-infrastructure/*", + "example-configs/*", + "*_captions.vtt", + "*_transcript.json", + "*.vtt", + "*example_*.mp4", + "*example_*.mkv", + "*.parquet", + "*fake_*", + "*generated_*", +] + +[tool.mypy] +files = "cdp_backend/*.py" +ignore_missing_imports = true +disallow_untyped_defs = true +check_untyped_defs = true +show_error_codes = true + +# https://flake8.pycqa.org/en/latest/user/options.html +# https://gitlab.com/durko/flake8-pyprojecttoml +[tool.flake8] +max-line-length = 88 +ignore = "E203,E402,W291,W503" +min-python-version = "3.8.0" +per-file-ignores = [ + "__init__.py:F401", +] \ No newline at end of file diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index ea36c2c8..00000000 --- a/setup.cfg +++ /dev/null @@ -1,52 +0,0 @@ -[bumpversion] -current_version = 3.1.3 -parse = (?P\d+)\.(?P\d+)\.(?P\d+) -serialize = {major}.{minor}.{patch} -commit = True -tag = True - -[bumpversion:file:setup.py] -search = version="{current_version}" -replace = version="{new_version}" - -[bumpversion:file:cdp_backend/version.py] -search = {current_version} -replace = {new_version} - -[bdist_wheel] -universal = 1 - -[aliases] -test = pytest - -[tool:pytest] -collect_ignore = ['setup.py'] -xfail_strict = true -filterwarnings = - ignore::UserWarning - ignore::FutureWarning - -[flake8] -exclude = - docs/ -ignore = - E203 - E402 - W291 - W503 -max-line-length = 88 - -[mypy] -ignore_missing_imports = True -disallow_untyped_defs = True -check_untyped_defs = True -exclude = infrastructure/ - -[isort] -profile = black -multi_line_output = 3 -include_trailing_comma = True -force_grid_wrap = 0 -use_parentheses = True -ensure_newline_before_comments = True -line_length = 88 diff --git a/setup.py b/setup.py deleted file mode 100644 index bd1fecb6..00000000 --- a/setup.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -"""The setup script.""" - -from setuptools import find_packages, setup - -with open("README.md") as readme_file: - readme = readme_file.read() - -setup_requirements = [ - "pytest-runner>=5.2", -] - -infrastructure_requirements = [ - "pulumi~=3.31", - "pulumi-google-native~=0.18", - "pulumi-gcp~=6.0", -] - -pipeline_requirements = [ - "dask[distributed]>=2021.7.0", - "ffmpeg-python==0.2.0", - "google-cloud-speech~=2.13", - "graphviz~=0.16", - "imageio~=2.18", - "imageio-ffmpeg~=0.4", - "m3u8-To-MP4==0.1.10", - "nltk~=3.6", - "numpy~=1.0", - "pandas~=1.2", - "prefect~=1.2", - "pyarrow~=8.0", - "pydub~=0.25.1", - "rapidfuzz~=2.0", - "spacy~=3.0", - "torch~=1.10", - "tqdm~=4.62", - "transformers~=4.16", - "truecase~=0.0.14", - "webvtt-py~=0.4.6", - "yt-dlp>=2022.2.4", - "vimeo_downloader~=0.4" -] - -test_requirements = [ - *infrastructure_requirements, - *pipeline_requirements, - "black>=22.3.0", - "codecov>=2.1.12", - "flake8>=3.8.3", - "flake8-debugger>=3.2.1", - "isort>=5.7.0", - "mypy>=0.790", - "networkx>=2.5", - "pydot>=1.4", - "pytest>=5.4.3", - "pytest-cov>=2.9.0", - "pytest-raises>=0.11", - "tox>=3.15.2", - "types-pytz>=2021.1.2", - "types-requests", -] - -dev_requirements = [ - *setup_requirements, - *test_requirements, - "bokeh>=2.3.2", - "bump2version>=1.0.1", - "coverage>=5.4", - "ipython>=7.15.0", - "jinja2>=2.11.2", - "m2r2>=0.2.7", - "prefect[viz]", - "Sphinx>=3.4.3", - "furo>=2022.4.7", - "twine>=3.1.1", - "wheel>=0.34.2", - # TODO: fix pin - "docutils>=0.18,<0.19", -] - -requirements = [ - "aiohttp>=3.7.4.post0", - "dataclasses-json>=0.5", - "fireo>=1.4", - "fsspec", # Version pin set by gcsfs - "gcsfs>=2021.7.0", - "requests>=2.26.0", -] - -extra_requirements = { - "setup": setup_requirements, - "infrastructure": infrastructure_requirements, - "pipeline": pipeline_requirements, - "test": test_requirements, - "dev": dev_requirements, - "all": [ - *requirements, - *dev_requirements, - ], -} - -setup( - author=( - "Eva Maxfield Brown, To Huynh, Isaac Na, Council Data Project Contributors" - ), - author_email="evamaxfieldbrown@gmail.com", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Natural Language :: English", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], - description=( - "Data storage utilities and processing pipelines to run on CDP server " - "deployments." - ), - entry_points={ - "console_scripts": [ - "clean_cdp_database=cdp_backend.bin.clean_cdp_database:main", - "clean_cdp_filestore=cdp_backend.bin.clean_cdp_filestore:main", - "create_cdp_database_uml=cdp_backend.bin.create_cdp_database_uml:main", - ( - "create_cdp_ingestion_models_doc=" - "cdp_backend.bin.create_cdp_ingestion_models_doc:main" - ), - ( - "create_cdp_transcript_model_doc=" - "cdp_backend.bin.create_cdp_transcript_model_doc:main" - ), - ( - "create_cdp_event_gather_flow_viz=" - "cdp_backend.bin.create_cdp_event_gather_flow_viz:main" - ), - "run_cdp_event_gather=cdp_backend.bin.run_cdp_event_gather:main", - ( - "run_cdp_event_index_generation=" - "cdp_backend.bin.run_cdp_event_index_generation:main" - ), - ( - "process_cdp_event_index_chunk=" - "cdp_backend.bin.process_cdp_event_index_chunk:main" - ), - "search_cdp_events=cdp_backend.bin.search_cdp_events:main", - "process_special_event=cdp_backend.bin.process_special_event:main", - ( - "add_content_hash_to_sessions=" - "cdp_backend.bin.add_content_hash_to_sessions:main" - ), - ], - }, - install_requires=requirements, - license="MIT license", - long_description=readme, - long_description_content_type="text/markdown", - include_package_data=True, - keywords="civic technology, open government", - name="cdp-backend", - packages=find_packages(exclude=["tests", "*.tests", "*.tests.*"]), - python_requires=">=3.8", - setup_requires=setup_requirements, - test_suite="cdp_backend/tests", - tests_require=test_requirements, - extras_require=extra_requirements, - url="https://github.com/CouncilDataProject/cdp-backend", - # Do not edit this string manually, always use bump2version - # Details in CONTRIBUTING.rst - version="3.1.3", - zip_safe=False, -) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 68f11aaa..00000000 --- a/tox.ini +++ /dev/null @@ -1,19 +0,0 @@ -[tox] -envlist = py38, py39, py310, lint -skip_missing_interpreters = true -toxworkdir={env:TOX_WORK_DIR:.tox} - -[testenv:lint] -skip_install = true -deps = pre-commit -commands = pre-commit run --all-files --show-diff-on-failure - -[testenv] -passenv = - CI -setenv = - PYTHONPATH = {toxinidir} -extras = - test -commands = - pytest --basetemp={envtmpdir} --cov-report xml --cov-report html --cov=cdp_backend cdp_backend/tests/ {posargs}