maintenance & updates 2022 (#3)

* replace yapf with black & flake8

* fix tests with newer valhalla:latest image

* update dependencies

* jeez.. weird bug in flask-restx, needed to install stuff from git

* edit the action workflow

.github/workflows/test-ubuntu.yml

@@ -10,11 +10,7 @@ on:
 
 jobs:
   build:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-20.04, ubuntu-18.04]
-        python_version: [3.7, 3.8, 3.9]
+    runs-on: ubuntu-20.04
     services:
       postgres:
         image: kartoza/postgis:12.1
@@ -43,16 +39,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Python ${{ matrix.python_version }}
+    - name: Set up Python 3.10
       uses: actions/setup-python@v2
       with:
-        python-version: ${{ matrix.python_version }}
-
-    # From https://github.com/python-poetry/poetry/actions
-    - name: Get full python version
-      id: full-python-version
-      run: |
-        echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info[:3]))")
+        python-version: '3.10'
 
     - name: Install and set up Poetry
       run: |
@@ -64,7 +54,7 @@ jobs:
       uses: actions/cache@v2
       with:
         path: .venv
-        key: venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('**/poetry.lock') }}
+        key: venv-3.10-${{ hashFiles('**/poetry.lock') }}
 
     - name: Install dependencies
       run: |
@@ -77,29 +67,25 @@ jobs:
 
     - name: Install osmium
       run: |
-        sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 648ACFD622F3D138
-        sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 04EE7237B7D453EC
-        sudo add-apt-repository 'deb http://ftp.debian.org/debian sid main'
         sudo apt-get update
         sudo apt-get install -y -qq osmium-tool osmctools
         echo $(osmium --version)
 
-    - name: yapf linting
+    - name: linting
       run: |
         source .venv/bin/activate
-        yapf -r -vv --diff -p tests routing_packager_app config.py http_app.py gunicorn.py
+        pre-commit run --all-files
 
-    - name: pytest
+    - name: pytest and coverage
       run: |
         source .venv/bin/activate
         sudo python -m smtpd -n -c DebuggingServer localhost:1025 &
         sudo docker volume create routing-packager_packages --driver local --opt type=none --opt device=$PWD --opt o=bind
         pytest --cov=routing_packager_app --ignore=tests/test_tasks.py
+        coverage lcov --include "routing_packager_app/*"
 
     - name: coveralls
-      run: |
-        source .venv/bin/activate
-        coveralls
-
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      uses: coverallsapp/github-action@master
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        path-to-lcov: ./coverage.lcov

.pre-commit-config.yaml

@@ -1,5 +1,11 @@
 repos:
-- repo: https://github.com/pre-commit/mirrors-yapf
-  rev: v0.30.0  # Use the sha / tag you want to point at
+- repo: https://github.com/psf/black
+  rev: 22.3.0
   hooks:
-    - id: yapf
+    - id: black
+      language_version: python3
+      args: [routing_packager_app, tests]
+- repo: https://github.com/pycqa/flake8
+  rev: 4.0.1  # pick a git hash / tag to point to
+  hooks:
+    - id: flake8

Dockerfile

@@ -1,5 +1,5 @@
 #--- BEGIN Usual Python stuff ---
-FROM python:3.9-slim-buster
+FROM python:3.10-slim-bullseye
 LABEL maintainer=nils@gis-ops.com
 
 # Install poetry
@@ -21,14 +21,17 @@ RUN apt-get update -y > /dev/null && \
 RUN apt-get update -y > /dev/null && \
     apt-get install -y --fix-missing \
         software-properties-common \
-        gnupg-agent \
+        apt-transport-https \
+        ca-certificates \
+        curl \
+        gnupg \
+        lsb-release \
         nano \
         jq \
         cron -o APT::Immediate-Configure=0 > /dev/null && \
     # install docker & osmium
     curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - && \
     add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \
-    add-apt-repository 'deb http://ftp.debian.org/debian sid main' && \
     apt-get update -y > /dev/null && \
     apt-get install -y docker-ce docker-ce-cli containerd.io osmium-tool osmctools > /dev/null && \
     systemctl enable docker

config.py

@@ -3,60 +3,62 @@
 from distutils.util import strtobool
 
 basedir = os.path.abspath(os.path.dirname(__file__))
-load_dotenv(os.path.join(basedir, '.env'))
+load_dotenv(os.path.join(basedir, ".env"))
 
 
 def _get_list_var(var):
     out = []
     if var:
-        out.extend(var.split(','))
+        out.extend(var.split(","))
 
     return out
 
 
 class BaseConfig(object):
     ### FLASK ###
-    SECRET_KEY = os.getenv(
-        'SECRET_KEY'
-    ) or '<MMs8?u_;rTt>;LarIGI&FjWhKNSe=%3|W;=DFDqOdx+~-rBS+K=p8#t#9E+;{e$'
+    SECRET_KEY = (
+        os.getenv("SECRET_KEY") or "<MMs8?u_;rTt>;LarIGI&FjWhKNSe=%3|W;=DFDqOdx+~-rBS+K=p8#t#9E+;{e$"
+    )
     ERROR_INCLUDE_MESSAGE = False  # No default "message" field in error responses
     RESTX_MASK_SWAGGER = False  # No default MASK header in Swagger
     SQLALCHEMY_TRACK_MODIFICATIONS = False
 
     ### APP ###
-    ADMIN_EMAIL = os.getenv('ADMIN_EMAIL') or 'admin@example.org'
-    ADMIN_PASS = os.getenv('ADMIN_PASSWORD') or 'admin'
+    ADMIN_EMAIL = os.getenv("ADMIN_EMAIL") or "admin@example.org"
+    ADMIN_PASS = os.getenv("ADMIN_PASSWORD") or "admin"
 
-    DATA_DIR = os.getenv('DATA_DIR') or os.path.join(basedir, 'data')
+    DATA_DIR = os.getenv("DATA_DIR") or os.path.join(basedir, "data")
     # if we're inside a docker container, we need to reference the fixed directory instead
     # Watch out for CI, also runs within docker
-    if os.path.isdir('/app/data') and not os.getenv('CI', None):
-        DATA_DIR = '/app/data'
+    if os.path.isdir("/app/data") and not os.getenv("CI", None):
+        DATA_DIR = "/app/data"
 
-    ENABLED_PROVIDERS = _get_list_var(os.getenv('ENABLED_PROVIDERS')) or ['osm']
-    ENABLED_ROUTERS = _get_list_var(os.getenv('ENABLED_ROUTERS')) or ['valhalla']
-    VALHALLA_IMAGE = os.getenv('VALHALLA_IMAGE') or 'gisops/valhalla:latest'
-    OSRM_IMAGE = os.getenv('OSRM_IMAGE') or 'osrm/osrm-backend:latest'
-    ORS_IMAGE = os.getenv('ORS_IMAGE') or 'openrouteservice/openrouteservice:latest'
-    GRAPHHOPPER_IMAGE = os.getenv('GRAPHHOPPER_IMAGE') or 'graphhopper/graphhopper:latest'
+    ENABLED_PROVIDERS = _get_list_var(os.getenv("ENABLED_PROVIDERS")) or ["osm"]
+    ENABLED_ROUTERS = _get_list_var(os.getenv("ENABLED_ROUTERS")) or ["valhalla"]
+    VALHALLA_IMAGE = os.getenv("VALHALLA_IMAGE") or "gisops/valhalla:latest"
+    OSRM_IMAGE = os.getenv("OSRM_IMAGE") or "osrm/osrm-backend:latest"
+    ORS_IMAGE = os.getenv("ORS_IMAGE") or "openrouteservice/openrouteservice:latest"
+    GRAPHHOPPER_IMAGE = os.getenv("GRAPHHOPPER_IMAGE") or "graphhopper/graphhopper:latest"
 
     ### DATABASES ###
-    POSTGRES_HOST = os.getenv('POSTGRES_HOST') or 'localhost'
-    POSTGRES_PORT = int(os.getenv('POSTGRES_PORT', int())) or 5432
-    POSTGRES_DB = os.getenv('POSTGRES_DB') or 'gis'
-    POSTGRES_USER = os.getenv('POSTGRES_USER') or 'docker'
-    POSTGRES_PASS = os.getenv('POSTGRES_PASS') or 'docker'
-    SQLALCHEMY_DATABASE_URI = os.getenv('POSTGRES_URL') or \
-        f'postgresql://{POSTGRES_USER}:{POSTGRES_PASS}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}'
-    REDIS_URL = os.getenv('REDIS_URL') or f'redis://localhost:6379/0'
+    POSTGRES_HOST = os.getenv("POSTGRES_HOST") or "localhost"
+    POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", int())) or 5432
+    POSTGRES_DB = os.getenv("POSTGRES_DB") or "gis"
+    POSTGRES_USER = os.getenv("POSTGRES_USER") or "docker"
+    POSTGRES_PASS = os.getenv("POSTGRES_PASS") or "docker"
+    SQLALCHEMY_DATABASE_URI = (
+        os.getenv("POSTGRES_URL")
+        or f"postgresql://{POSTGRES_USER}:{POSTGRES_PASS}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}"
+    )
+    REDIS_URL = os.getenv("REDIS_URL") or "redis://localhost:6379/0"
 
     ### SMTP ###
-    SMTP_HOST = os.getenv('SMTP_HOST') or 'localhost'
-    SMTP_PORT = int(os.getenv('SMTP_PORT', int())) or 1025
-    SMTP_FROM = os.getenv('SMTP_FROM') or 'valhalla@kadas.org'
-    SMTP_USER = os.getenv('SMTP_USER')
-    SMTP_PASS = os.getenv('SMTP_PASS')
-    SMTP_SECURE = bool(strtobool(os.getenv('SMTP_SECURE', 'False'))) or False
+    SMTP_HOST = os.getenv("SMTP_HOST") or "localhost"
+    SMTP_PORT = int(os.getenv("SMTP_PORT", int())) or 1025
+    SMTP_FROM = os.getenv("SMTP_FROM") or "valhalla@kadas.org"
+    SMTP_USER = os.getenv("SMTP_USER")
+    SMTP_PASS = os.getenv("SMTP_PASS")
+    SMTP_SECURE = bool(strtobool(os.getenv("SMTP_SECURE", "False"))) or False
 
 
 class DevConfig(BaseConfig):
@@ -71,22 +73,24 @@ class TestingConfig(BaseConfig):
     TESTING = True
     FLASK_DEBUG = 0
 
-    POSTGRES_HOST = os.getenv('POSTGRES_HOST') or 'localhost'
-    POSTGRES_PORT = os.getenv('POSTGRES_PORT') or '5432'
-    POSTGRES_DB_TEST = os.getenv('POSTGRES_DB_TEST') or 'gis_test'
-    POSTGRES_USER = os.getenv('POSTGRES_USER') or 'admin'
-    POSTGRES_PASS = os.getenv('POSTGRES_PASS') or 'admin'
+    POSTGRES_HOST = os.getenv("POSTGRES_HOST") or "localhost"
+    POSTGRES_PORT = os.getenv("POSTGRES_PORT") or "5432"
+    POSTGRES_DB_TEST = os.getenv("POSTGRES_DB_TEST") or "gis_test"
+    POSTGRES_USER = os.getenv("POSTGRES_USER") or "admin"
+    POSTGRES_PASS = os.getenv("POSTGRES_PASS") or "admin"
 
-    SQLALCHEMY_DATABASE_URI = os.getenv('POSTGRES_TEST_URL') or \
-        f'postgresql://{POSTGRES_USER}:{POSTGRES_PASS}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB_TEST}'
+    SQLALCHEMY_DATABASE_URI = (
+        os.getenv("POSTGRES_TEST_URL")
+        or f"postgresql://{POSTGRES_USER}:{POSTGRES_PASS}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB_TEST}"
+    )
 
-    DATA_DIR = os.path.join(basedir, 'tests', 'data')
-    OSM_PBF_PATH = os.path.join(basedir, 'tests', 'data', 'andorra-200827.osm.pbf')
-    TOMTOM_PBF_PATH = os.path.join(basedir, 'tests', 'data', 'liechtenstein-201109.tomtom.pbf')
-    HERE_PBF_PATH = os.path.join(basedir, 'tests', 'data', 'liechtenstein-201109.here.pbf')
+    DATA_DIR = os.path.join(basedir, "tests", "data")
+    OSM_PBF_PATH = os.path.join(basedir, "tests", "data", "andorra-200827.osm.pbf")
+    TOMTOM_PBF_PATH = os.path.join(basedir, "tests", "data", "liechtenstein-201109.tomtom.pbf")
+    HERE_PBF_PATH = os.path.join(basedir, "tests", "data", "liechtenstein-201109.here.pbf")
 
-    ENABLED_ROUTERS = _get_list_var('valhalla')
-    ENABLED_PROVIDERS = _get_list_var('osm,tomtom,here')
+    ENABLED_ROUTERS = _get_list_var("valhalla")
+    ENABLED_PROVIDERS = _get_list_var("osm,tomtom,here")
 
-    ADMIN_EMAIL = 'admin@example.org'
-    ADMIN_PASS = 'admin'
+    ADMIN_EMAIL = "admin@example.org"
+    ADMIN_PASS = "admin"

gunicorn.py

@@ -1,7 +1,7 @@
 bind = "0.0.0.0:5000"
 # Generally we recommend (2 x $num_cores) + 1 as the number of workers to start off with
 workers = 5
-worker_class = 'gevent'
+worker_class = "gevent"
 worker_connections = 10
 timeout = 30
 keepalive = 2

http_app.py

@@ -7,4 +7,4 @@
 
 @app.shell_context_processor
 def make_shell_context():
-    return {'db': db, 'User': User, 'Job': Job}
+    return {"db": db, "User": User, "Job": Job}