From 22cbedf7b61dc63c66a968b1c8544d2fe1157d7e Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Thu, 7 Mar 2019 10:44:16 -0500
Subject: [PATCH 01/13] use pyyaml beta with security fix

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 3e978f9d1..f3cee3d03 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,7 +15,7 @@ pyramid_chameleon==0.3
 pyramid_mako>=1.0.2
 pyramid_tm==2.2.1
 python-dotenv
-PyYAML<=3.13
+pyyaml>=4.2b4
 requests
 requests_file
 simplejson

From 559119257c7262a1afbf2deed1bccbb7ccb922ce Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Tue, 26 Mar 2019 17:54:45 -0400
Subject: [PATCH 02/13] minor naming changes

---
 magpie/security.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/magpie/security.py b/magpie/security.py
index dbaccb64a..2251f0807 100644
--- a/magpie/security.py
+++ b/magpie/security.py
@@ -1,11 +1,16 @@
-from magpie.definitions.pyramid_definitions import *
-from magpie.definitions.ziggurat_definitions import *
 from magpie.api.login import esgfopenid, wso2
-from magpie.constants import get_constant
 from magpie.common import get_logger
+from magpie.constants import get_constant
+from magpie.definitions.pyramid_definitions import (
+    AuthTktAuthenticationPolicy, ACLAuthorizationPolicy, Configurator, asbool
+)
+from magpie.definitions.ziggurat_definitions import groupfinder
 from authomatic import Authomatic, provider_id
 from authomatic.providers import oauth2, openid
+from typing import TYPE_CHECKING
 import logging
+if TYPE_CHECKING:
+    from magpie.definitions.typedefs import JsonBody
 AUTHOMATIC_LOGGER = get_logger('magpie.authomatic', level=logging.DEBUG)
 
 
@@ -52,18 +57,18 @@ def authomatic_setup(request):
 
 def authomatic_config(request=None):
 
-    DEFAULTS = {
+    defaults_config = {
         'popup': True,
     }
 
-    OPENID = {
+    openid_config = {
         'openid': {
             'class_': openid.OpenID,
             'display_name': 'OpenID',
         },
     }
 
-    ESGF = {
+    esgf_config = {
         'dkrz': {
             'class_': esgfopenid.ESGFOpenID,
             'hostname': 'esgf-data.dkrz.de',
@@ -94,7 +99,7 @@ def authomatic_config(request=None):
     }
 
     _get_const_info = dict(raise_missing=False, raise_not_set=False, print_missing=True)
-    OAUTH2 = {
+    oauth2_config = {
         'github': {
             'class_': oauth2.GitHub,
             'display_name': 'GitHub',
@@ -123,11 +128,11 @@ def authomatic_config(request=None):
     }
 
     # Concatenate the configs.
-    config = {}
-    config.update(OAUTH2)
-    config.update(OPENID)
-    config.update(ESGF)
-    config['__defaults__'] = DEFAULTS
+    config = {}  # type: JsonBody
+    config.update(oauth2_config)
+    config.update(openid_config)
+    config.update(esgf_config)
+    config['__defaults__'] = defaults_config
     return config
 
 

From c698ff6a4c5a35895e51b951f1f02ee3f38a1cac Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Tue, 26 Mar 2019 18:26:25 -0400
Subject: [PATCH 03/13] magpiectl -> app + test include 'magpie'

---
 magpie/{magpiectl.py => app.py} | 0
 setup.py                        | 2 +-
 tests/utils.py                  | 7 ++++---
 3 files changed, 5 insertions(+), 4 deletions(-)
 rename magpie/{magpiectl.py => app.py} (100%)

diff --git a/magpie/magpiectl.py b/magpie/app.py
similarity index 100%
rename from magpie/magpiectl.py
rename to magpie/app.py
diff --git a/setup.py b/setup.py
index 4cbce3cb8..56cf4f6f2 100644
--- a/setup.py
+++ b/setup.py
@@ -99,7 +99,7 @@ def _parse_requirements(file_path, requirements, links):
     # -- script entry points -----------------------------------------------
     entry_points="""\
           [paste.app_factory]
-          main = magpie.magpiectl:main
+          main = magpie.app:main
           [console_scripts]
           """,
 )
diff --git a/tests/utils.py b/tests/utils.py
index 0cea0f31c..084435e66 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -12,7 +12,7 @@
 from webtest import TestApp
 # noinspection PyPackageRequirements
 from webtest.response import TestResponse
-from magpie import __meta__, services, magpiectl
+from magpie import __meta__, services, app
 from magpie.common import get_settings_from_config_ini, get_header, JSON_TYPE, HTML_TYPE
 from magpie.constants import get_constant
 from magpie.common import str2bool
@@ -133,8 +133,9 @@ def get_test_magpie_app(settings=None):
     if settings:
         config.registry.settings.update(settings)
     # create the test application
-    app = TestApp(magpiectl.main({}, **config.registry.settings))
-    return app
+    config.include('magpie')
+    magpie_app = TestApp(app.main({}, **config.registry.settings))
+    return magpie_app
 
 
 def get_hostname(app_or_url):

From e8ddff0b007dcf4f5cb1422fe7b2e41adc3bbaf4 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Tue, 26 Mar 2019 19:09:14 -0400
Subject: [PATCH 04/13] use yaml>=5.1

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index f3cee3d03..8a6ea7081 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,7 +15,7 @@ pyramid_chameleon==0.3
 pyramid_mako>=1.0.2
 pyramid_tm==2.2.1
 python-dotenv
-pyyaml>=4.2b4
+pyyaml>=5.1
 requests
 requests_file
 simplejson

From 2c21d4c16abfff115df6b5555ae79b47ff8b39bb Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Tue, 26 Mar 2019 19:29:06 -0400
Subject: [PATCH 05/13] autoresolve setup.py test

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 56cf4f6f2..036401372 100644
--- a/setup.py
+++ b/setup.py
@@ -93,7 +93,7 @@ def _parse_requirements(file_path, requirements, links):
     # test_suite='nose.collector',
     # test_suite='tests.test_runner',
     # test_loader='tests.test_runner:run_suite',
-    test_suite='tests',
+    # test_suite='tests',
     tests_require=TEST_REQUIREMENTS,
 
     # -- script entry points -----------------------------------------------

From 238c04309112f4e06b77e2b0d3e398555696eaf3 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Tue, 26 Mar 2019 19:37:24 -0400
Subject: [PATCH 06/13] revert setup-tests + pip freeze travis

---
 .travis.yml | 1 +
 setup.py    | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 6cbb4ee4c..091a6a9c5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -56,6 +56,7 @@ install:
     fi
   - make install install-dev
   - make version
+  - ${CONDA_PREFIX}/bin/pip freeze
 before_script:
   - psql -c 'create database magpie;' -U postgres
   - echo $CONDA_PREFIX
diff --git a/setup.py b/setup.py
index 036401372..56cf4f6f2 100644
--- a/setup.py
+++ b/setup.py
@@ -93,7 +93,7 @@ def _parse_requirements(file_path, requirements, links):
     # test_suite='nose.collector',
     # test_suite='tests.test_runner',
     # test_loader='tests.test_runner:run_suite',
-    # test_suite='tests',
+    test_suite='tests',
     tests_require=TEST_REQUIREMENTS,
 
     # -- script entry points -----------------------------------------------

From c2a0d40855da797883353d9e0800249148baa4b4 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Wed, 27 Mar 2019 16:27:29 -0400
Subject: [PATCH 07/13] attempt using pytest runner

---
 .gitignore |  3 ++-
 Makefile   | 12 +++++++++---
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index 8c11471cf..5fc2dccd4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,7 @@ docs/magpie.login.rst
 docs/magpie.management.rst
 docs/magpie.tests.rst
 docs/magpie.rst
+downloads/
 magpieenv/
 magpie/api_docs/
 magpie/api_test/
@@ -29,6 +30,6 @@ magpie/share/
 magpie.egg-info/
 src/
 share/
+tests/results.xml
 /gunicorn.app.wsgiapp
-downloads/
 /sys
diff --git a/Makefile b/Makefile
index 1c84303d9..ca32dbc1a 100644
--- a/Makefile
+++ b/Makefile
@@ -115,17 +115,23 @@ lint: install-dev
 .PHONY: test
 test: install-dev install
 	@echo "Running tests..."
-	@bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; python setup.py test'
+	# @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; python setup.py test'
+	bash -c "source $(CONDA_HOME)/bin/activate $(CONDA_ENV); \
+		pytest tests -vv --junitxml $(CURDIR)/tests/results.xml"
 
 .PHONY: test-local
 test-local: install-dev install
 	@echo "Running local tests..."
-	@bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; MAGPIE_TEST_REMOTE=false python setup.py test'
+	# @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; MAGPIE_TEST_REMOTE=false python setup.py test'
+	bash -c "source $(CONDA_HOME)/bin/activate $(CONDA_ENV); \
+		pytest tests -vv -m 'not remote' --junitxml $(CURDIR)/tests/results.xml"
 
 .PHONY: test-remote
 test-remote: install-dev install
 	@echo "Running remote tests..."
-	@bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; MAGPIE_TEST_LOCAL=false python setup.py test'
+	# @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; MAGPIE_TEST_LOCAL=false python setup.py test'
+	bash -c "source $(CONDA_HOME)/bin/activate $(CONDA_ENV); \
+		pytest tests -vv -m 'not local' --junitxml $(CURDIR)/tests/results.xml"
 
 .PHONY: test-tox
 test-tox: install-dev install

From 8c1cb0ba13bc5ad285acc86e080cd713d2ae027c Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Wed, 27 Mar 2019 18:14:44 -0400
Subject: [PATCH 08/13] testapp getter

---
 magpie/definitions/typedefs.py |   2 +
 tests/interfaces.py            | 160 ++++++++++++++++-----------------
 tests/test_magpie_api.py       |   6 +-
 tests/utils.py                 |  94 ++++++++++++-------
 4 files changed, 147 insertions(+), 115 deletions(-)

diff --git a/magpie/definitions/typedefs.py b/magpie/definitions/typedefs.py
index c1cefb352..e1ca8ddb0 100644
--- a/magpie/definitions/typedefs.py
+++ b/magpie/definitions/typedefs.py
@@ -17,6 +17,7 @@
     from requests.structures import CaseInsensitiveDict
     # noinspection PyUnresolvedReferences, PyProtectedMember
     from logging import _loggerClass as LoggerType  # noqa: F401
+    from tests.interfaces import Base_Magpie_TestCase
     import six
 
     if six.PY2:
@@ -43,3 +44,4 @@
     UserServicesTypes = Union[Dict[Str, Dict[Str, Any]], List[Dict[Str, Any]]]
 
     TestAppOrUrlType = Union[Str, TestApp]
+    AnyMagpieTestType = Union[Type[Base_Magpie_TestCase], Base_Magpie_TestCase, TestAppOrUrlType]
diff --git a/tests/interfaces.py b/tests/interfaces.py
index cfbf8f128..3a9abc2d3 100644
--- a/tests/interfaces.py
+++ b/tests/interfaces.py
@@ -8,7 +8,6 @@
 import yaml
 import six
 # noinspection PyPackageRequirements
-from webtest import TestApp
 from six.moves.urllib.parse import urlparse
 from distutils.version import LooseVersion
 from magpie.api.api_rest_schemas import SwaggerGenerator
@@ -50,7 +49,7 @@ class Interface_MagpieAPI_NoAuth(Base_Magpie_TestCase):
 
     @runner.MAGPIE_TEST_LOGIN
     def test_GetSession_Anonymous(self):
-        resp = utils.test_request(self.url, 'GET', '/session', headers=self.json_headers)
+        resp = utils.test_request(self, 'GET', '/session', headers=self.json_headers)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_equal(json_body['authenticated'], False)
         if LooseVersion(self.version) >= LooseVersion('0.6.3'):
@@ -61,7 +60,7 @@ def test_GetSession_Anonymous(self):
             utils.check_val_not_in('group_names', json_body)
 
     def test_GetVersion(self):
-        resp = utils.test_request(self.url, 'GET', '/version', headers=self.json_headers)
+        resp = utils.test_request(self, 'GET', '/version', headers=self.json_headers)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_is_in('db_version', json_body)
         utils.check_val_is_in('version', json_body)
@@ -74,7 +73,7 @@ def test_GetVersion(self):
     @runner.MAGPIE_TEST_USERS
     def test_GetCurrentUser(self):
         logged_user = get_constant('MAGPIE_LOGGED_USER')
-        resp = utils.test_request(self.url, 'GET', '/users/{}'.format(logged_user), headers=self.json_headers)
+        resp = utils.test_request(self, 'GET', '/users/{}'.format(logged_user), headers=self.json_headers)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         if LooseVersion(self.version) >= LooseVersion('0.6.3'):
             utils.check_val_equal(json_body['user']['user_name'], self.usr)
@@ -114,7 +113,7 @@ def tearDown(self):
 
     @classmethod
     def check_requirements(cls):
-        headers, cookies = utils.check_or_try_login_user(cls.url, cls.usr, cls.pwd,
+        headers, cookies = utils.check_or_try_login_user(cls, cls.usr, cls.pwd,
                                                          use_ui_form_submit=True, version=cls.version)
         assert headers and cookies, cls.require
         assert cls.headers and cls.cookies, cls.require
@@ -154,7 +153,7 @@ def setUp(self):
         utils.TestSetup.delete_TestUser(self)
 
     def test_GetAPI(self):
-        resp = utils.test_request(self.url, 'GET', SwaggerGenerator.path, headers=self.json_headers)
+        resp = utils.test_request(self, 'GET', SwaggerGenerator.path, headers=self.json_headers)
         json_body = utils.get_json_body(resp)
         content_types = utils.get_response_content_types_list(resp)
         utils.check_val_is_in(JSON_TYPE, content_types)
@@ -179,25 +178,26 @@ def test_unauthorized_forbidden_responses(self):
         """
         utils.warn_version(self, "check for response (401/403) statuses", '0.9.1', skip=True)
 
-        if isinstance(self.url, six.string_types):
+        app_or_url = utils.get_app_or_url(self)
+        if isinstance(app_or_url, six.string_types):
             warnings.warn("cannot validate 403 status with remote server (no mock possible, test with local)",
                           RuntimeWarning)
         else:
             # call a route that will make a forbidden access to db
             with mock.patch('magpie.models.User.all', side_effect=Exception('Test')):
-                resp = utils.test_request(self.url, 'GET', '/users', headers=self.json_headers, expect_errors=True)
+                resp = utils.test_request(self, 'GET', '/users', headers=self.json_headers, expect_errors=True)
                 body = utils.check_response_basic_info(resp, 403, expected_method='GET')
                 utils.check_val_equal(body['code'], 403)
 
         # call a route that is admin-only
-        utils.check_or_try_logout_user(self.url)
-        resp = utils.test_request(self.url, 'GET', '/services', headers=self.json_headers, expect_errors=True)
+        utils.check_or_try_logout_user(app_or_url)
+        resp = utils.test_request(self, 'GET', '/services', headers=self.json_headers, expect_errors=True)
         body = utils.check_response_basic_info(resp, 401, expected_method='GET')
         utils.check_val_equal(body['code'], 401)
 
     @runner.MAGPIE_TEST_LOGIN
     def test_GetSession_Administrator(self):
-        resp = utils.test_request(self.url, 'GET', '/session', headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', '/session', headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_equal(json_body['authenticated'], True)
         if LooseVersion(self.version) >= LooseVersion('0.6.3'):
@@ -214,7 +214,7 @@ def test_GetSession_Administrator(self):
 
     @runner.MAGPIE_TEST_USERS
     def test_GetUsers(self):
-        resp = utils.test_request(self.url, 'GET', '/users', headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', '/users', headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_is_in('user_names', json_body)
         utils.check_val_type(json_body['user_names'], list)
@@ -225,7 +225,7 @@ def test_GetUsers(self):
     @runner.MAGPIE_TEST_USERS
     @runner.MAGPIE_TEST_DEFAULTS
     def test_ValidateDefaultUsers(self):
-        resp = utils.test_request(self.url, 'GET', '/users', headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', '/users', headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         users = json_body['user_names']
         utils.check_val_is_in(get_constant('MAGPIE_ANONYMOUS_USER'), users)
@@ -235,7 +235,7 @@ def test_ValidateDefaultUsers(self):
     def check_GetUserResourcesPermissions(cls, user_name, resource_id, query=None):
         query = '?{}'.format(query) if query else ''
         route = '/users/{usr}/resources/{res_id}/permissions{q}'.format(res_id=resource_id, usr=user_name, q=query)
-        resp = utils.test_request(cls.url, 'GET', route, headers=cls.json_headers, cookies=cls.cookies)
+        resp = utils.test_request(cls, 'GET', route, headers=cls.json_headers, cookies=cls.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_is_in('permission_names', json_body)
         utils.check_val_type(json_body['permission_names'], list)
@@ -245,7 +245,7 @@ def check_GetUserResourcesPermissions(cls, user_name, resource_id, query=None):
     def test_GetCurrentUser(self):
         logged_user = get_constant('MAGPIE_LOGGED_USER')
         path = '/users/{}'.format(logged_user)
-        resp = utils.test_request(self.url, 'GET', path, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', path, headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         if LooseVersion(self.version) >= LooseVersion('0.6.3'):
             utils.check_val_equal(json_body['user']['user_name'], self.usr)
@@ -289,15 +289,15 @@ def test_GetCurrentUserResourcesPermissions_Queries(self):
         data_recur = {u'permission_name': perm_recur}
         data_match = {u'permission_name': perm_match}
         path = '/users/{usr}/resources/{res_id}/permissions'.format(res_id=test_svc_res_id, usr=self.usr)
-        utils.test_request(self.url, 'POST', path, data=data_recur, headers=self.json_headers, cookies=self.cookies)
+        utils.test_request(self, 'POST', path, data=data_recur, headers=self.json_headers, cookies=self.cookies)
         path = '/groups/{grp}/resources/{res_id}/permissions'.format(res_id=test_svc_res_id, grp=self.grp)
-        utils.test_request(self.url, 'POST', path, data=data_match, headers=self.json_headers, cookies=self.cookies)
+        utils.test_request(self, 'POST', path, data=data_match, headers=self.json_headers, cookies=self.cookies)
         path = '/groups/{grp}/resources/{res_id}/permissions'.format(res_id=test_parent_res_id, grp=self.grp)
-        utils.test_request(self.url, 'POST', path, data=data_match, headers=self.json_headers, cookies=self.cookies)
+        utils.test_request(self, 'POST', path, data=data_match, headers=self.json_headers, cookies=self.cookies)
         path = '/users/{usr}/resources/{res_id}/permissions'.format(res_id=test_child_res_id, usr=anonym_usr)
-        utils.test_request(self.url, 'POST', path, data=data_match, headers=self.json_headers, cookies=self.cookies)
+        utils.test_request(self, 'POST', path, data=data_match, headers=self.json_headers, cookies=self.cookies)
         path = '/groups/{grp}/resources/{res_id}/permissions'.format(res_id=test_svc_res_id, grp=anonym_grp)
-        utils.test_request(self.url, 'POST', path, data=data_recur, headers=self.json_headers, cookies=self.cookies)
+        utils.test_request(self, 'POST', path, data=data_recur, headers=self.json_headers, cookies=self.cookies)
 
         # tests
         q_groups = "inherit=true"
@@ -358,7 +358,7 @@ def test_PostUserResourcesPermissions_Created(self):
         # test permission creation
         path = '/users/{usr}/resources/{res_id}/permissions'.format(res_id=test_res_id, usr=self.usr)
         data = {u'permission_name': self.test_resource_perm}
-        resp = utils.test_request(self.url, 'POST', path, data=data, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'POST', path, data=data, headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 201, expected_method='POST')
         utils.check_val_is_in('permission_name', json_body)
         utils.check_val_is_in('resource_id', json_body)
@@ -381,12 +381,12 @@ def test_PostUserResourcesPermissions_Conflict(self):
 
         path = '/users/{usr}/resources/{res_id}/permissions'.format(res_id=test_res_id, usr=self.usr)
         data = {u'permission_name': self.test_resource_perm}
-        utils.test_request(self.url, 'POST', path, data=data, headers=self.json_headers, cookies=self.cookies)
+        utils.test_request(self, 'POST', path, data=data, headers=self.json_headers, cookies=self.cookies)
         json_body = self.check_GetUserResourcesPermissions(self.usr, resource_id=test_res_id)
         utils.check_val_is_in(self.test_resource_perm, json_body['permission_names'],
                               msg="Can't test for conflicting permissions if it doesn't exist first.")
 
-        resp = utils.test_request(self.url, 'POST', path, data=data, headers=self.json_headers, cookies=self.cookies,
+        resp = utils.test_request(self, 'POST', path, data=data, headers=self.json_headers, cookies=self.cookies,
                                   expect_errors=True)
         json_body = utils.check_response_basic_info(resp, 409, expected_method='POST')
         utils.check_val_is_in('permission_name', json_body)
@@ -401,7 +401,7 @@ def test_PostUserResourcesPermissions_Conflict(self):
 
     @runner.MAGPIE_TEST_USERS
     def test_GetCurrentUserGroups(self):
-        resp = utils.test_request(self.url, 'GET', '/users/current/groups',
+        resp = utils.test_request(self, 'GET', '/users/current/groups',
                                   headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_is_in('group_names', json_body)
@@ -416,7 +416,7 @@ def test_GetUserInheritedResources(self):
             route = '/users/{usr}/inherited_resources'.format(usr=self.usr)
         else:
             route = '/users/{usr}/resources?inherit=true'.format(usr=self.usr)
-        resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', route, headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_is_in('resources', json_body)
         utils.check_val_type(json_body['resources'], dict)
@@ -450,7 +450,7 @@ def test_GetUserInheritedResources(self):
     @runner.MAGPIE_TEST_USERS
     def test_GetUserServices(self):
         route = '/users/{usr}/services'.format(usr=self.usr)
-        resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', route, headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_is_in('services', json_body)
         services = json_body['services']
@@ -489,7 +489,7 @@ def test_GetUserServiceResources(self):
         utils.TestSetup.create_TestService(self)
         utils.TestSetup.create_TestServiceResource(self)
         route = '/users/{usr}/services/{svc}/resources'.format(usr=self.usr, svc=self.test_service_name)
-        resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', route, headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_is_in('service', json_body)
         svc_dict = json_body['service']
@@ -537,7 +537,7 @@ def test_PostUsers_ReservedKeyword_Current(self):
             'email': 'email@mail.com',
             'group_name': 'users',
         }
-        resp = utils.test_request(self.url, 'POST', '/users', data=data,
+        resp = utils.test_request(self, 'POST', '/users', data=data,
                                   headers=self.json_headers, cookies=self.cookies, expect_errors=True)
         utils.check_response_basic_info(resp, 400, expected_method='POST')
 
@@ -546,7 +546,7 @@ def test_PutUser_ReservedKeyword_Current(self):
         utils.TestSetup.create_TestUser(self)
         route = '/users/{usr}'.format(usr=get_constant('MAGPIE_LOGGED_USER'))
         data = {'user_name': self.test_user_name + '-new-put-over-current'}
-        resp = utils.test_request(self.url, 'PUT', route, data=data,
+        resp = utils.test_request(self, 'PUT', route, data=data,
                                   headers=self.json_headers, cookies=self.cookies, expect_errors=True)
         utils.check_response_basic_info(resp, 400, expected_method='PUT')
 
@@ -554,7 +554,7 @@ def test_PutUser_ReservedKeyword_Current(self):
     def test_PutUsers_nothing(self):
         utils.TestSetup.create_TestUser(self)
         route = '/users/{usr}'.format(usr=self.test_user_name)
-        resp = utils.test_request(self.url, 'PUT', route, data={},
+        resp = utils.test_request(self, 'PUT', route, data={},
                                   headers=self.json_headers, cookies=self.cookies, expect_errors=True)
         utils.check_response_basic_info(resp, 400, expected_method='PUT')
 
@@ -569,37 +569,37 @@ def test_PutUsers_username(self):
         # update existing user name
         data = {'user_name': new_name}
         route = '/users/{usr}'.format(usr=self.test_user_name)
-        resp = utils.test_request(self.url, 'PUT', route, headers=self.json_headers, cookies=self.cookies, data=data)
+        resp = utils.test_request(self, 'PUT', route, headers=self.json_headers, cookies=self.cookies, data=data)
         utils.check_response_basic_info(resp, 200, expected_method='PUT')
 
         # validate change of user name
         route = '/users/{usr}'.format(usr=new_name)
-        resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', route, headers=self.json_headers, cookies=self.cookies)
         body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_equal(body['user']['user_name'], new_name)
 
         # validate removed previous user name
         route = '/users/{usr}'.format(usr=self.test_user_name)
-        resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies,
+        resp = utils.test_request(self, 'GET', route, headers=self.json_headers, cookies=self.cookies,
                                   expect_errors=True)
         utils.check_response_basic_info(resp, 404, expected_method='GET')
 
         # validate effective new user name
-        utils.check_or_try_logout_user(self.url)
-        headers, cookies = utils.check_or_try_login_user(self.url, username=new_name, password=self.test_user_name,
+        utils.check_or_try_logout_user(self)
+        headers, cookies = utils.check_or_try_login_user(self, username=new_name, password=self.test_user_name,
                                                          use_ui_form_submit=True, version=self.version)
-        resp = utils.test_request(self.url, 'GET', '/session', headers=headers, cookies=cookies)
+        resp = utils.test_request(self, 'GET', '/session', headers=headers, cookies=cookies)
         body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_equal(body['authenticated'], True)
         utils.check_val_equal(body['user']['user_name'], new_name)
 
         # validate ineffective previous user name
-        utils.check_or_try_logout_user(self.url)
+        utils.check_or_try_logout_user(self)
         headers, cookies = utils.check_or_try_login_user(
-            self.url, username=self.test_user_name, password=self.test_user_name, version=self.version,
+            self, username=self.test_user_name, password=self.test_user_name, version=self.version,
             use_ui_form_submit=True, expect_errors=True)
         utils.check_val_equal(cookies, {}, msg="CookiesType should be empty from login failure.")
-        resp = utils.test_request(self.url, 'GET', '/session', headers=headers, cookies=cookies)
+        resp = utils.test_request(self, 'GET', '/session', headers=headers, cookies=cookies)
         body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_equal(body['authenticated'], False)
 
@@ -609,10 +609,10 @@ def test_PutUsers_email(self):
         new_email = 'toto@new-email.lol'
         data = {'email': new_email}
         route = '/users/{usr}'.format(usr=self.test_user_name)
-        resp = utils.test_request(self.url, 'PUT', route, headers=self.json_headers, cookies=self.cookies, data=data)
+        resp = utils.test_request(self, 'PUT', route, headers=self.json_headers, cookies=self.cookies, data=data)
         utils.check_response_basic_info(resp, 200, expected_method='PUT')
 
-        resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', route, headers=self.json_headers, cookies=self.cookies)
         body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_equal(body['user']['email'], new_email)
 
@@ -623,24 +623,25 @@ def test_PutUsers_password(self):
         new_password = 'n0t-SO-ez-2-Cr4cK'
         data = {'password': new_password}
         route = '/users/{usr}'.format(usr=self.test_user_name)
-        resp = utils.test_request(self.url, 'PUT', route, headers=self.json_headers, cookies=self.cookies, data=data)
+        resp = utils.test_request(self, 'PUT', route, headers=self.json_headers, cookies=self.cookies, data=data)
         utils.check_response_basic_info(resp, 200, expected_method='PUT')
-        utils.check_or_try_logout_user(self.url)
+        utils.check_or_try_logout_user(self)
 
         # validate that the new password is effective
-        headers, cookies = utils.check_or_try_login_user(self.url, username=self.test_user_name, password=new_password,
-                                                         use_ui_form_submit=True, version=self.version)
-        resp = utils.test_request(self.url, 'GET', '/session', headers=headers, cookies=cookies)
+        headers, cookies = utils.check_or_try_login_user(
+            self, username=self.test_user_name, password=new_password,
+            use_ui_form_submit=True, version=self.version)
+        resp = utils.test_request(self, 'GET', '/session', headers=headers, cookies=cookies)
         body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_equal(body['authenticated'], True)
         utils.check_val_equal(body['user']['user_name'], self.test_user_name)
-        utils.check_or_try_logout_user(self.url)
+        utils.check_or_try_logout_user(self)
 
         # validate that previous password is ineffective
         headers, cookies = utils.check_or_try_login_user(
-            self.url, username=self.test_user_name, password=old_password, version=self.version,
+            self, username=self.test_user_name, password=old_password, version=self.version,
             use_ui_form_submit=True, expect_errors=True)
-        resp = utils.test_request(self.url, 'GET', '/session', headers=headers, cookies=cookies)
+        resp = utils.test_request(self, 'GET', '/session', headers=headers, cookies=cookies)
         body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_equal(body['authenticated'], False)
 
@@ -649,7 +650,7 @@ def test_GetUser_existing(self):
         utils.TestSetup.create_TestUser(self)
 
         route = '/users/{usr}'.format(usr=self.test_user_name)
-        resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', route, headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         if LooseVersion(self.version) >= LooseVersion('0.6.3'):
             utils.check_val_is_in('user', json_body)
@@ -671,14 +672,14 @@ def test_GetUser_existing(self):
     def test_GetUser_missing(self):
         utils.TestSetup.check_NonExistingTestUser(self)
         route = '/users/{usr}'.format(usr=self.test_user_name)
-        resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers,
+        resp = utils.test_request(self, 'GET', route, headers=self.json_headers,
                                   cookies=self.cookies, expect_errors=True)
         utils.check_response_basic_info(resp, 404, expected_method='GET')
 
     @runner.MAGPIE_TEST_GROUPS
     @runner.MAGPIE_TEST_DEFAULTS
     def test_ValidateDefaultGroups(self):
-        resp = utils.test_request(self.url, 'GET', '/groups', headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', '/groups', headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         groups = json_body['group_names']
         utils.check_val_is_in(get_constant('MAGPIE_ANONYMOUS_GROUP'), groups)
@@ -695,7 +696,7 @@ def test_PostUserGroup_assign(self):
     def test_PostUserGroup_not_found(self):
         route = '/users/{usr}/groups'.format(usr=get_constant('MAGPIE_ADMIN_USER'))
         data = {'group_name': 'not_found'}
-        resp = utils.test_request(self.url, 'POST', route, expect_errors=True,
+        resp = utils.test_request(self, 'POST', route, expect_errors=True,
                                   headers=self.json_headers, cookies=self.cookies, data=data)
         utils.check_response_basic_info(resp, 404, expected_method='POST')
 
@@ -703,14 +704,14 @@ def test_PostUserGroup_not_found(self):
     def test_PostUserGroup_conflict(self):
         route = '/users/{usr}/groups'.format(usr=get_constant('MAGPIE_ADMIN_USER'))
         data = {'group_name': get_constant('MAGPIE_ADMIN_GROUP')}
-        resp = utils.test_request(self.url, 'POST', route, expect_errors=True,
+        resp = utils.test_request(self, 'POST', route, expect_errors=True,
                                   headers=self.json_headers, cookies=self.cookies, data=data)
         utils.check_response_basic_info(resp, 409, expected_method='POST')
 
     @runner.MAGPIE_TEST_GROUPS
     def test_GetGroupUsers(self):
         route = '/groups/{grp}/users'.format(grp=get_constant('MAGPIE_ADMIN_GROUP'))
-        resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', route, headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_is_in('user_names', json_body)
         utils.check_val_type(json_body['user_names'], list)
@@ -720,7 +721,7 @@ def test_GetGroupUsers(self):
     @runner.MAGPIE_TEST_GROUPS
     def test_GetGroupServices(self):
         route = '/groups/{grp}/services'.format(grp=self.grp)
-        resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', route, headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_is_in('services', json_body)
         services = json_body['services']
@@ -758,7 +759,7 @@ def test_GetGroupServiceResources(self):
         utils.TestSetup.create_TestService(self)
         utils.TestSetup.create_TestServiceResource(self)
         route = '/groups/{grp}/services/{svc}/resources'.format(grp=self.grp, svc=self.test_service_name)
-        resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', route, headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_is_in('service', json_body)
         svc_dict = json_body['service']
@@ -813,7 +814,7 @@ def test_PutService_UpdateSuccess(self):
         utils.TestSetup.delete_TestService(self, override_service_name=new_svc_name)
         path = '/services/{svc}'.format(svc=service['service_name'])
         data = {'service_name': new_svc_name, 'service_url': new_svc_url}
-        resp = utils.test_request(self.url, 'PUT', path, data=data, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'PUT', path, data=data, headers=self.json_headers, cookies=self.cookies)
         body = utils.check_response_basic_info(resp, expected_method='PUT')
         utils.check_val_is_in('service', body)
         utils.check_val_type(body['service'], dict)
@@ -845,7 +846,7 @@ def test_PutService_UpdateConflict(self):
             utils.TestSetup.create_TestService(self, override_service_name=new_svc_name)
             path = '/services/{svc}'.format(svc=service['service_name'])
             data = {'service_name': new_svc_name, 'service_url': new_svc_url}
-            resp = utils.test_request(self.url, 'PUT', path, data=data, expect_errors=True,
+            resp = utils.test_request(self, 'PUT', path, data=data, expect_errors=True,
                                       headers=self.json_headers, cookies=self.cookies)
             utils.check_response_basic_info(resp, 409, expected_method='PUT')
         finally:
@@ -855,7 +856,7 @@ def test_PutService_UpdateConflict(self):
     def test_PutService_NoUpdateInfo(self):
         # no route PUT on '/services/types' (not equivalent to '/services/{service_name}')
         # so not even a forbidden case to handle
-        resp = utils.test_request(self.url, 'PUT', '/services/types', data={}, expect_errors=True,
+        resp = utils.test_request(self, 'PUT', '/services/types', data={}, expect_errors=True,
                                   headers=self.json_headers, cookies=self.cookies)
         if LooseVersion(self.version) >= LooseVersion('0.9.5'):
             # directly interpreted as expected route `/services/types` behaviour, so method PUT not allowed
@@ -868,7 +869,7 @@ def test_PutService_NoUpdateInfo(self):
     def test_PutService_ReservedKeyword_Types(self):
         # try to PUT on 'types' route should raise the error
         data = {'service_name': 'dummy', 'service_url': 'dummy'}
-        resp = utils.test_request(self.url, 'PUT', '/services/types', data=data, expect_errors=True,
+        resp = utils.test_request(self, 'PUT', '/services/types', data=data, expect_errors=True,
                                   headers=self.json_headers, cookies=self.cookies)
         if LooseVersion(self.version) >= LooseVersion('0.9.5'):
             # directly interpreted as expected route `/services/types` behaviour, so method PUT not allowed
@@ -882,7 +883,7 @@ def test_PutService_ReservedKeyword_Types(self):
         utils.TestSetup.create_TestService(self)
         path = '/services/{}'.format(self.test_service_name)
         data = {'service_name': 'types'}
-        resp = utils.test_request(self.url, 'PUT', path, data=data, expect_errors=True,
+        resp = utils.test_request(self, 'PUT', path, data=data, expect_errors=True,
                                   headers=self.json_headers, cookies=self.cookies)
         utils.check_response_basic_info(resp, 400, expected_method='PUT')   # don't allow naming to 'types'
 
@@ -890,7 +891,7 @@ def test_PutService_ReservedKeyword_Types(self):
     def test_GetService_ResponseFormat(self):
         utils.TestSetup.create_TestService(self)
         path = '/services/{svc}'.format(svc=self.test_service_name)
-        resp = utils.test_request(self.url, 'GET', path, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', path, headers=self.json_headers, cookies=self.cookies)
         body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         if LooseVersion(self.version) < LooseVersion('0.9.1'):
             utils.check_val_is_in(self.test_service_name, body)
@@ -927,7 +928,7 @@ def test_GetService_ResponseFormat(self):
     def test_GetServiceTypes_ResponseFormat(self):
         utils.warn_version(self, "get service types", '0.9.1', skip=True)
 
-        resp = utils.test_request(self.url, 'GET', '/services/types', headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', '/services/types', headers=self.json_headers, cookies=self.cookies)
         body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_is_in('service_types', body)
         utils.check_val_type(body['service_types'], list)
@@ -939,7 +940,7 @@ def test_GetServiceTypeResources_ResponseFormat(self):
 
         utils.TestSetup.create_TestService(self)
         path = '/services/types/{svc_type}/resources'.format(svc_type=self.test_service_type)
-        resp = utils.test_request(self.url, 'GET', path, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', path, headers=self.json_headers, cookies=self.cookies)
         body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         utils.check_val_is_in('resource_types', body)
         utils.check_val_type(body['resource_types'], list)
@@ -980,7 +981,7 @@ def test_GetServiceTypeResources_CheckValues(self):
         ]:
             # test response details
             path = '/services/types/{}/resources'.format(svc_type)
-            resp = utils.test_request(self.url, 'GET', path, headers=self.json_headers, cookies=self.cookies)
+            resp = utils.test_request(self, 'GET', path, headers=self.json_headers, cookies=self.cookies)
             body = utils.check_response_basic_info(resp, 200, expected_method='GET')
             utils.check_val_type(body['resource_types'], list)
             utils.check_val_equal(len(body['resource_types']), len(svc_res_info))
@@ -995,7 +996,7 @@ def test_GetServiceResources(self):
         utils.TestSetup.create_TestService(self)
         utils.TestSetup.create_TestServiceResource(self)
         route = '/services/{svc}/resources'.format(svc=self.test_service_name)
-        resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', route, headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         svc_dict = json_body[self.test_service_name]
         utils.check_val_is_in(self.test_service_name, json_body)
@@ -1026,7 +1027,7 @@ def test_GetServicePermissions(self):
             svc_name = svc['service_name']
             service_perms = service_type_dict[svc['service_type']].permission_names
             route = '/services/{svc}/permissions'.format(svc=svc_name)
-            resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies)
+            resp = utils.test_request(self, 'GET', route, headers=self.json_headers, cookies=self.cookies)
             json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
             utils.check_val_is_in('permission_names', json_body)
             utils.check_val_type(json_body['permission_names'], list)
@@ -1102,7 +1103,7 @@ def test_PostServiceResources_ChildrenResource_ParentID(self):
         service_root_id = utils.TestSetup.get_ExistingTestServiceInfo(self)['resource_id']
         child_resource_id = json_body['resource_id']
         route = '/resources/{res_id}'.format(res_id=child_resource_id)
-        resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', route, headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         if LooseVersion(self.version) >= LooseVersion('0.9.2'):
             utils.check_val_is_in('resource', json_body)
@@ -1123,7 +1124,7 @@ def test_PostServiceResources_DirectResource_Conflict(self):
         utils.TestSetup.create_TestServiceResource(self)
         route = '/services/{svc}/resources'.format(svc=self.test_service_name)
         data = {"resource_name": self.test_resource_name, "resource_type": self.test_resource_type}
-        resp = utils.test_request(self.url, 'POST', route, headers=self.json_headers,
+        resp = utils.test_request(self, 'POST', route, headers=self.json_headers,
                                   cookies=self.cookies, json=data, expect_errors=True)
         json_body = utils.check_response_basic_info(resp, 409, expected_method='POST')
         utils.check_error_param_structure(json_body, version=self.version,
@@ -1141,7 +1142,7 @@ def test_ValidateDefaultServiceProviders(self):
             svc_name = svc['service_name']
             if svc_name in self.test_services_info:
                 utils.check_val_equal(svc['service_type'], self.test_services_info[svc_name]['type'])
-                hostname = utils.get_hostname(self.url)
+                hostname = utils.get_hostname(self)
                 # private service URL should match format of Magpie (schema/host)
                 svc_url = self.test_services_info[svc_name]['url'].replace('${HOSTNAME}', hostname)
                 utils.check_val_equal(svc['service_url'], svc_url)
@@ -1162,7 +1163,7 @@ def test_ValidateDefaultServiceProviders(self):
         anonymous = get_constant('MAGPIE_ANONYMOUS_USER')
         services_list_getcap = [svc for svc in services_list if 'getcapabilities' in svc['permission_names']]
         route = '/users/{usr}/services'.format(usr=anonymous)
-        resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'GET', route, headers=self.json_headers, cookies=self.cookies)
         json_body = utils.check_response_basic_info(resp, 200, expected_method='GET')
         services_body = json_body['services']
         for svc in services_list_getcap:
@@ -1185,7 +1186,7 @@ def test_PostResources_DirectServiceResource(self):
             "resource_type": self.test_resource_type,
             "parent_id": service_resource_id
         }
-        resp = utils.test_request(self.url, 'POST', '/resources',
+        resp = utils.test_request(self, 'POST', '/resources',
                                   headers=self.json_headers, cookies=self.cookies, data=data)
         json_body = utils.check_response_basic_info(resp, 201, expected_method='POST')
         utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type,
@@ -1203,7 +1204,7 @@ def test_PostResources_DirectServiceResourceOptional(self):
             "resource_type": self.test_resource_type,
             "parent_id": service_resource_id
         }
-        resp = utils.test_request(self.url, 'POST', '/resources',
+        resp = utils.test_request(self, 'POST', '/resources',
                                   headers=self.json_headers, cookies=self.cookies, data=data)
         json_body = utils.check_response_basic_info(resp, 201, expected_method='POST')
         utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type,
@@ -1223,7 +1224,7 @@ def test_PostResources_ChildrenResource(self):
             "resource_type": self.test_resource_type,
             "parent_id": direct_resource_id
         }
-        resp = utils.test_request(self.url, 'POST', '/resources',
+        resp = utils.test_request(self, 'POST', '/resources',
                                   headers=self.json_headers, cookies=self.cookies, data=data)
         json_body = utils.check_response_basic_info(resp, 201, expected_method='POST')
         utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type,
@@ -1235,7 +1236,7 @@ def test_PostResources_MissingParentID(self):
             "resource_name": self.test_resource_name,
             "resource_type": self.test_resource_type,
         }
-        resp = utils.test_request(self.url, 'POST', '/resources',
+        resp = utils.test_request(self, 'POST', '/resources',
                                   headers=self.json_headers, cookies=self.cookies, data=data, expect_errors=True)
         json_body = utils.check_response_basic_info(resp, 422, expected_method='POST')
         utils.check_error_param_structure(json_body, version=self.version,
@@ -1250,7 +1251,7 @@ def test_DeleteResource(self):
             resource_id = json_body['resource_id']
 
         route = '/resources/{res_id}'.format(res_id=resource_id)
-        resp = utils.test_request(self.url, 'DELETE', route, headers=self.json_headers, cookies=self.cookies)
+        resp = utils.test_request(self, 'DELETE', route, headers=self.json_headers, cookies=self.cookies)
         utils.check_response_basic_info(resp, 200, expected_method='DELETE')
         utils.TestSetup.check_NonExistingTestServiceResource(self)
 
@@ -1336,7 +1337,7 @@ class Interface_MagpieUI_AdminAuth(Base_Magpie_TestCase):
 
     @classmethod
     def check_requirements(cls):
-        headers, cookies = utils.check_or_try_login_user(cls.url, cls.usr, cls.pwd)
+        headers, cookies = utils.check_or_try_login_user(cls, cls.usr, cls.pwd)
         assert headers and cookies, cls.require
         assert cls.headers and cls.cookies, cls.require
 
@@ -1410,11 +1411,10 @@ def test_EditService(self):
         path = '/ui/services/{type}/{name}'.format(type=self.test_service_type, name=self.test_service_name)
         utils.TestSetup.check_UpStatus(self, method='GET', path=path)
 
+    @runner.MAGPIE_TEST_LOCAL   # not implemented for remote URL
     @runner.MAGPIE_TEST_STATUS
     @runner.MAGPIE_TEST_FUNCTIONAL
     def test_EditService_GotoAddChild_BackToEditService(self):
-        if not isinstance(self.url, TestApp):
-            self.skipTest("test not implemented with remote URL")
         try:
             # make sure any sub-resource are all deleted to avoid conflict, then recreate service to add sub-resource
             utils.TestSetup.delete_TestService(self, override_service_name=self.test_service_parent_resource_name)
diff --git a/tests/test_magpie_api.py b/tests/test_magpie_api.py
index 41ab49c21..cf051e22e 100644
--- a/tests/test_magpie_api.py
+++ b/tests/test_magpie_api.py
@@ -31,8 +31,7 @@ class TestCase_MagpieAPI_NoAuth_Local(ti.Interface_MagpieAPI_NoAuth, unittest.Te
     @classmethod
     def setUpClass(cls):
         cls.app = utils.get_test_magpie_app()
-        cls.url = cls.app  # to simplify calls of TestSetup (all use .url)
-        cls.json_headers = utils.get_headers(cls.url, {'Accept': JSON_TYPE, 'Content-Type': JSON_TYPE})
+        cls.json_headers = utils.get_headers(cls.app, {'Accept': JSON_TYPE, 'Content-Type': JSON_TYPE})
         cls.version = __meta__.__version__
         cls.cookies = None
         cls.usr = get_constant('MAGPIE_ANONYMOUS_USER')
@@ -67,11 +66,10 @@ class TestCase_MagpieAPI_AdminAuth_Local(ti.Interface_MagpieAPI_AdminAuth, unitt
     @classmethod
     def setUpClass(cls):
         cls.app = utils.get_test_magpie_app()
-        cls.url = cls.app  # to simplify calls of TestSetup (all use .url)
         cls.grp = get_constant('MAGPIE_ADMIN_GROUP')
         cls.usr = get_constant('MAGPIE_TEST_ADMIN_USERNAME')
         cls.pwd = get_constant('MAGPIE_TEST_ADMIN_PASSWORD')
-        cls.json_headers = utils.get_headers(cls.url, {'Accept': JSON_TYPE, 'Content-Type': JSON_TYPE})
+        cls.json_headers = utils.get_headers(cls.app, {'Accept': JSON_TYPE, 'Content-Type': JSON_TYPE})
         cls.cookies = None
         cls.version = utils.TestSetup.get_Version(cls)
         # TODO: fix UI views so that they can be 'found' directly in the WebTest.TestApp
diff --git a/tests/utils.py b/tests/utils.py
index 084435e66..13866be71 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -21,7 +21,8 @@
 if TYPE_CHECKING:
     from tests.interfaces import Base_Magpie_TestCase    # noqa: F401
     from magpie.definitions.typedefs import (  # noqa: F401
-        Str, Callable, Dict, HeadersType, OptionalHeaderCookiesType, Optional, Type, TestAppOrUrlType, AnyResponseType
+        Str, Callable, Dict, HeadersType, OptionalHeaderCookiesType, Optional, Type,
+        AnyMagpieTestType, AnyResponseType, TestAppOrUrlType
     )
 
 OptionalStringType = six.string_types + tuple([type(None)])
@@ -138,7 +139,19 @@ def get_test_magpie_app(settings=None):
     return magpie_app
 
 
-def get_hostname(app_or_url):
+def get_app_or_url(test_item):
+    # type: (AnyMagpieTestType) -> TestAppOrUrlType
+    if isinstance(test_item, TestApp) or isinstance(test_item, six.string_types):
+        return test_item
+    app_or_url = getattr(test_item, 'app', None) or getattr(test_item, 'url', None)
+    if not app_or_url:
+        raise ValueError("Invalid test class, application or URL could not be found.")
+    return app_or_url
+
+
+def get_hostname(test_item):
+    # type: (AnyMagpieTestType) -> Str
+    app_or_url = get_app_or_url(test_item)
     if isinstance(app_or_url, TestApp):
         app_or_url = get_magpie_url(app_or_url.app.registry)
     return urlparse(app_or_url).hostname
@@ -183,10 +196,10 @@ def warn_version(test, functionality, version, skip=True):
             test.skipTest(reason=msg)
 
 
-def test_request(app_or_url, method, path, timeout=5, allow_redirects=True, **kwargs):
+def test_request(test_item, method, path, timeout=5, allow_redirects=True, **kwargs):
     """
     Calls the request using either a :class:`webtest.TestApp` instance or :class:`requests.Request` from a string URL.
-    :param app_or_url: `webtest.TestApp` instance of the test application or remote server URL to call with `requests`
+    :param test_item: one of `Base_Magpie_TestCase`, `webtest.TestApp` or remote server URL to call with `requests`
     :param method: request method (GET, POST, PUT, DELETE)
     :param path: test path starting at base path
     :param timeout: `timeout` to pass down to `request`
@@ -205,6 +218,7 @@ def test_request(app_or_url, method, path, timeout=5, allow_redirects=True, **kw
             kwargs.pop(kw)
     json_body = json_body or {}
 
+    app_or_url = get_app_or_url(test_item)
     if isinstance(app_or_url, TestApp):
         # remove any 'cookies' keyword handled by the 'TestApp' instance
         if 'cookies' in kwargs:
@@ -254,7 +268,7 @@ def get_session_user(app_or_url, headers=None):
     return resp
 
 
-def check_or_try_login_user(app_or_url,                     # type: TestAppOrUrlType
+def check_or_try_login_user(test_item,                      # type: AnyMagpieTestType
                             username=None,                  # type: Optional[Str]
                             password=None,                  # type: Optional[Str]
                             provider='ziggurat',            # type: Optional[Str]
@@ -267,7 +281,7 @@ def check_or_try_login_user(app_or_url,                     # type: TestAppOrUrl
     Verifies that the required user is already logged in (or none is if username=None), or tries to login him otherwise.
     Validates that the logged user (if any), matched the one specified with `username`.
 
-    :param app_or_url: instance of the test application or remote server URL to call
+    :param test_item: instance of the test application or remote server URL to call
     :param username: name of the user to login or None otherwise
     :param password: password to use for login if the user was not already logged in
     :param provider: provider string to use for login (default: ziggurat, ie: magpie's local signin)
@@ -279,7 +293,7 @@ def check_or_try_login_user(app_or_url,                     # type: TestAppOrUrl
     :return: headers and cookies of the user session or (None, None)
     :raise: Exception on any login failure as required by the caller's specifications (username/password)
     """
-
+    app_or_url = get_app_or_url(test_item)
     headers = headers or {}
     resp = get_session_user(app_or_url, headers)
     body = get_json_body(resp)
@@ -328,15 +342,16 @@ def check_or_try_login_user(app_or_url,                     # type: TestAppOrUrl
     return resp.headers, resp_cookies
 
 
-def check_or_try_logout_user(app_or_url):
-    # type: (TestAppOrUrlType) -> None
+def check_or_try_logout_user(test_item):
+    # type: (AnyMagpieTestType) -> None
     """
     Verifies that any user is logged out, or tries to logout him otherwise.
 
-    :param app_or_url: instance of the test application or remote server URL to call
     :raise: Exception on any logout failure or incapability to validate logout
     """
 
+    app_or_url = get_app_or_url(test_item)
+
     def _is_logged_out():
         resp = get_session_user(app_or_url)
         body = get_json_body(resp)
@@ -589,8 +604,8 @@ def check_resource_children(resource_dict, parent_resource_id, root_service_id):
 class TestSetup(object):
     @staticmethod
     def get_Version(test_class):
-        resp = test_request(test_class.url, 'GET', '/version',
-                            headers=test_class.json_headers, cookies=test_class.cookies)
+        app_or_url = get_app_or_url(test_class)
+        resp = test_request(app_or_url, 'GET', '/version', headers=test_class.json_headers, cookies=test_class.cookies)
         json_body = check_response_basic_info(resp, 200)
         return json_body['version']
 
@@ -602,7 +617,8 @@ def check_UpStatus(test_class, method, path, timeout=20):
 
         :returns: response from the rendered page for further tests
         """
-        resp = test_request(test_class.url, method, path, cookies=test_class.cookies, timeout=timeout)
+        app_or_url = get_app_or_url(test_class)
+        resp = test_request(app_or_url, method, path, cookies=test_class.cookies, timeout=timeout)
         check_ui_response_basic_info(resp)
         return resp
 
@@ -634,12 +650,13 @@ def check_FormSubmit(test_class, form_match, form_data=None, form_submit='submit
 
         :returns: response from the rendered page for further tests
         """
-        if not isinstance(test_class.url, TestApp):
+        app_or_url = get_app_or_url(test_class)
+        if not isinstance(app_or_url, TestApp):
             test_class.skipTest(reason='test form submit with remote URL not implemented')
         if isinstance(previous_response, TestResponse):
             resp = previous_response
         else:
-            resp = test_request(test_class.url, method, path, cookies=test_class.cookies, timeout=timeout)
+            resp = test_request(app_or_url, method, path, cookies=test_class.cookies, timeout=timeout)
         check_val_equal(resp.status_code, 200, msg="Cannot test form submission, initial page returned an error.")
         form = None
         if isinstance(form_match, int) or isinstance(form_match, six.string_types):
@@ -668,13 +685,15 @@ def check_Unauthorized(test_class, method, path, content_type=JSON_TYPE):
         Verifies that Magpie returned an Unauthorized response.
         Validates that at the bare minimum, no underlying internal error occurred from the API or UI calls.
         """
-        resp = test_request(test_class.url, method, path, cookies=test_class.cookies, expect_errors=True)
+        app_or_url = get_app_or_url(test_class)
+        resp = test_request(app_or_url, method, path, cookies=test_class.cookies, expect_errors=True)
         check_response_basic_info(resp, expected_code=401, expected_type=content_type, expected_method=method)
 
     @staticmethod
     def get_AnyServiceOfTestServiceType(test_class):
+        app_or_url = get_app_or_url(test_class)
         route = '/services/types/{}'.format(test_class.test_service_type)
-        resp = test_request(test_class.url, 'GET', route, headers=test_class.json_headers, cookies=test_class.cookies)
+        resp = test_request(app_or_url, 'GET', route, headers=test_class.json_headers, cookies=test_class.cookies)
         json_body = check_response_basic_info(resp, 200, expected_method='GET')
         check_val_is_in('services', json_body)
         check_val_is_in(test_class.test_service_type, json_body['services'])
@@ -685,6 +704,7 @@ def get_AnyServiceOfTestServiceType(test_class):
 
     @staticmethod
     def create_TestServiceResource(test_class, data_override=None):
+        app_or_url = get_app_or_url(test_class)
         TestSetup.create_TestService(test_class)
         route = '/services/{svc}/resources'.format(svc=test_class.test_service_name)
         data = {
@@ -693,15 +713,16 @@ def create_TestServiceResource(test_class, data_override=None):
         }
         if data_override:
             data.update(data_override)
-        resp = test_request(test_class.url, 'POST', route,
+        resp = test_request(app_or_url, 'POST', route,
                             headers=test_class.json_headers,
                             cookies=test_class.cookies, json=data)
         return check_response_basic_info(resp, 201, expected_method='POST')
 
     @staticmethod
     def get_ExistingTestServiceInfo(test_class):
+        app_or_url = get_app_or_url(test_class)
         route = '/services/{svc}'.format(svc=test_class.test_service_name)
-        resp = test_request(test_class.url, 'GET', route,
+        resp = test_request(app_or_url, 'GET', route,
                             headers=test_class.json_headers, cookies=test_class.cookies)
         json_body = get_json_body(resp)
         svc_getter = 'service'
@@ -711,8 +732,9 @@ def get_ExistingTestServiceInfo(test_class):
 
     @staticmethod
     def get_TestServiceDirectResources(test_class, ignore_missing_service=False):
+        app_or_url = get_app_or_url(test_class)
         route = '/services/{svc}/resources'.format(svc=test_class.test_service_name)
-        resp = test_request(test_class.url, 'GET', route,
+        resp = test_request(app_or_url, 'GET', route,
                             headers=test_class.json_headers, cookies=test_class.cookies,
                             expect_errors=ignore_missing_service)
         if ignore_missing_service and resp.status_code == 404:
@@ -729,6 +751,7 @@ def check_NonExistingTestServiceResource(test_class):
 
     @staticmethod
     def delete_TestServiceResource(test_class, override_resource_name=None):
+        app_or_url = get_app_or_url(test_class)
         resource_name = override_resource_name or test_class.test_resource_name
         resources = TestSetup.get_TestServiceDirectResources(test_class, ignore_missing_service=True)
         test_resource = list(filter(lambda r: r['resource_name'] == resource_name, resources))
@@ -736,7 +759,7 @@ def delete_TestServiceResource(test_class, override_resource_name=None):
         if len(test_resource) > 0:
             resource_id = test_resource[0]['resource_id']
             route = '/services/{svc}/resources/{res_id}'.format(svc=test_class.test_service_name, res_id=resource_id)
-            resp = test_request(test_class.url, 'DELETE', route,
+            resp = test_request(app_or_url, 'DELETE', route,
                                 headers=test_class.json_headers,
                                 cookies=test_class.cookies)
             check_val_equal(resp.status_code, 200)
@@ -744,6 +767,7 @@ def delete_TestServiceResource(test_class, override_resource_name=None):
 
     @staticmethod
     def create_TestService(test_class, override_service_name=None, override_service_type=None):
+        app_or_url = get_app_or_url(test_class)
         svc_name = override_service_name or test_class.test_service_name
         svc_type = override_service_type or test_class.test_service_type
         data = {
@@ -751,12 +775,12 @@ def create_TestService(test_class, override_service_name=None, override_service_
             u'service_type': svc_type,
             u'service_url': u'http://localhost:9000/{}'.format(svc_name)
         }
-        resp = test_request(test_class.url, 'POST', '/services', json=data,
+        resp = test_request(app_or_url, 'POST', '/services', json=data,
                             headers=test_class.json_headers, cookies=test_class.cookies,
                             expect_errors=True)
         if resp.status_code == 409:
             path = '/services/{svc}'.format(svc=svc_name)
-            resp = test_request(test_class.url, 'GET', path,
+            resp = test_request(app_or_url, 'GET', path,
                                 headers=test_class.json_headers,
                                 cookies=test_class.cookies)
             body = check_response_basic_info(resp, 200, expected_method='GET')
@@ -775,13 +799,14 @@ def check_NonExistingTestService(test_class, override_service_name=None):
 
     @staticmethod
     def delete_TestService(test_class, override_service_name=None):
+        app_or_url = get_app_or_url(test_class)
         service_name = override_service_name or test_class.test_service_name
         services_info = TestSetup.get_RegisteredServicesList(test_class)
         test_service = list(filter(lambda r: r['service_name'] == service_name, services_info))
         # delete as required, skip if non-existing
         if len(test_service) > 0:
             route = '/services/{svc_name}'.format(svc_name=service_name)
-            resp = test_request(test_class.url, 'DELETE', route,
+            resp = test_request(app_or_url, 'DELETE', route,
                                 headers=test_class.json_headers,
                                 cookies=test_class.cookies)
             check_val_equal(resp.status_code, 200)
@@ -789,7 +814,8 @@ def delete_TestService(test_class, override_service_name=None):
 
     @staticmethod
     def get_RegisteredServicesList(test_class):
-        resp = test_request(test_class.url, 'GET', '/services',
+        app_or_url = get_app_or_url(test_class)
+        resp = test_request(app_or_url, 'GET', '/services',
                             headers=test_class.json_headers,
                             cookies=test_class.cookies)
         json_body = check_response_basic_info(resp, 200, expected_method='GET')
@@ -803,7 +829,8 @@ def get_RegisteredServicesList(test_class):
 
     @staticmethod
     def get_RegisteredUsersList(test_class):
-        resp = test_request(test_class.url, 'GET', '/users',
+        app_or_url = get_app_or_url(test_class)
+        resp = test_request(app_or_url, 'GET', '/users',
                             headers=test_class.json_headers,
                             cookies=test_class.cookies)
         json_body = check_response_basic_info(resp, 200, expected_method='GET')
@@ -817,6 +844,7 @@ def check_NonExistingTestUser(test_class, override_user_name=None):
 
     @staticmethod
     def create_TestUser(test_class, override_data=None):
+        app_or_url = get_app_or_url(test_class)
         data = {
             "user_name": test_class.test_user_name,
             "email": '{}@mail.com'.format(test_class.test_user_name),
@@ -825,19 +853,20 @@ def create_TestUser(test_class, override_data=None):
         }
         if override_data:
             data.update(override_data)
-        resp = test_request(test_class.url, 'POST', '/users',
+        resp = test_request(app_or_url, 'POST', '/users',
                             headers=test_class.json_headers,
                             cookies=test_class.cookies, json=data)
         return check_response_basic_info(resp, 201, expected_method='POST')
 
     @staticmethod
     def delete_TestUser(test_class, override_user_name=None):
+        app_or_url = get_app_or_url(test_class)
         users = TestSetup.get_RegisteredUsersList(test_class)
         user_name = override_user_name or test_class.test_user_name
         # delete as required, skip if non-existing
         if user_name in users:
             route = '/users/{usr}'.format(usr=user_name)
-            resp = test_request(test_class.url, 'DELETE', route,
+            resp = test_request(app_or_url, 'DELETE', route,
                                 headers=test_class.json_headers,
                                 cookies=test_class.cookies)
             check_response_basic_info(resp, 200, expected_method='DELETE')
@@ -845,7 +874,8 @@ def delete_TestUser(test_class, override_user_name=None):
 
     @staticmethod
     def get_RegisteredGroupsList(test_class):
-        resp = test_request(test_class.url, 'GET', '/groups',
+        app_or_url = get_app_or_url(test_class)
+        resp = test_request(app_or_url, 'GET', '/groups',
                             headers=test_class.json_headers,
                             cookies=test_class.cookies)
         json_body = check_response_basic_info(resp, 200, expected_method='GET')
@@ -859,20 +889,22 @@ def check_NonExistingTestGroup(test_class, override_group_name=None):
 
     @staticmethod
     def create_TestGroup(test_class, override_group_name=None):
+        app_or_url = get_app_or_url(test_class)
         data = {"group_name": override_group_name or test_class.test_group_name}
-        resp = test_request(test_class.url, 'POST', '/groups',
+        resp = test_request(app_or_url, 'POST', '/groups',
                             headers=test_class.json_headers,
                             cookies=test_class.cookies, json=data)
         return check_response_basic_info(resp, 201, expected_method='POST')
 
     @staticmethod
     def delete_TestGroup(test_class, override_group_name=None):
+        app_or_url = get_app_or_url(test_class)
         groups = TestSetup.get_RegisteredGroupsList(test_class)
         group_name = override_group_name or test_class.test_group_name
         # delete as required, skip if non-existing
         if group_name in groups:
             route = '/groups/{grp}'.format(grp=group_name)
-            resp = test_request(test_class.url, 'DELETE', route,
+            resp = test_request(app_or_url, 'DELETE', route,
                                 headers=test_class.json_headers,
                                 cookies=test_class.cookies)
             check_response_basic_info(resp, 200, expected_method='DELETE')

From 43ee382095b4c4e652a0810331e0e69a08b7922b Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Wed, 27 Mar 2019 19:15:06 -0400
Subject: [PATCH 09/13] [DEBUG] avoid get_version call to test failure cause

---
 tests/test_magpie_api.py | 10 ++++++++--
 tests/test_magpie_ui.py  |  7 +++++--
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/tests/test_magpie_api.py b/tests/test_magpie_api.py
index cf051e22e..834f11351 100644
--- a/tests/test_magpie_api.py
+++ b/tests/test_magpie_api.py
@@ -32,7 +32,9 @@ class TestCase_MagpieAPI_NoAuth_Local(ti.Interface_MagpieAPI_NoAuth, unittest.Te
     def setUpClass(cls):
         cls.app = utils.get_test_magpie_app()
         cls.json_headers = utils.get_headers(cls.app, {'Accept': JSON_TYPE, 'Content-Type': JSON_TYPE})
+        # FIXME: use get_version()
         cls.version = __meta__.__version__
+        #cls.version = utils.TestSetup.get_Version(cls)
         cls.cookies = None
         cls.usr = get_constant('MAGPIE_ANONYMOUS_USER')
         cls.grp = get_constant('MAGPIE_ANONYMOUS_GROUP')
@@ -71,7 +73,9 @@ def setUpClass(cls):
         cls.pwd = get_constant('MAGPIE_TEST_ADMIN_PASSWORD')
         cls.json_headers = utils.get_headers(cls.app, {'Accept': JSON_TYPE, 'Content-Type': JSON_TYPE})
         cls.cookies = None
-        cls.version = utils.TestSetup.get_Version(cls)
+        # FIXME: use get_version()
+        cls.version = __meta__.__version__
+        #cls.version = utils.TestSetup.get_Version(cls)
         # TODO: fix UI views so that they can be 'found' directly in the WebTest.TestApp
         # NOTE: localhost magpie has to be running for following login call to work
         cls.headers, cls.cookies = utils.check_or_try_login_user(cls.app, cls.usr, cls.pwd,
@@ -98,7 +102,9 @@ def setUpClass(cls):
         cls.cookies = None
         cls.usr = get_constant('MAGPIE_ANONYMOUS_USER')
         cls.grp = get_constant('MAGPIE_ANONYMOUS_GROUP')
-        cls.version = utils.TestSetup.get_Version(cls)
+        # FIXME: use get_version()
+        cls.version = __meta__.__version__
+        #cls.version = utils.TestSetup.get_Version(cls)
 
 
 @runner.MAGPIE_TEST_API
diff --git a/tests/test_magpie_ui.py b/tests/test_magpie_ui.py
index 2717177cc..dae5af818 100644
--- a/tests/test_magpie_ui.py
+++ b/tests/test_magpie_ui.py
@@ -8,13 +8,14 @@
 Tests for `magpie.ui` module.
 """
 
-import unittest
+from magpie import __meta__
 from magpie.common import JSON_TYPE
 from magpie.constants import get_constant
 from tests import utils, runner
 
 # NOTE: must be imported without 'from', otherwise the interface's test cases are also executed
 import tests.interfaces as ti
+import unittest
 
 
 @runner.MAGPIE_TEST_UI
@@ -58,7 +59,9 @@ def setUpClass(cls):
         cls.url = cls.app  # to simplify calls of TestSetup (all use .url)
         cls.json_headers = utils.get_headers(cls.app, {'Accept': JSON_TYPE, 'Content-Type': JSON_TYPE})
         cls.cookies = None
-        cls.version = utils.TestSetup.get_Version(cls)
+        # FIXME: use get_version()
+        cls.version = __meta__.__version__
+        # cls.version = utils.TestSetup.get_Version(cls)
         cls.headers, cls.cookies = utils.check_or_try_login_user(cls.url, cls.usr, cls.pwd, use_ui_form_submit=True)
         cls.require = "cannot run tests without logged in user with '{}' permissions".format(cls.grp)
         cls.check_requirements()

From 436ca5b30635325751f2c38ed4b77d59b94cad18 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Thu, 28 Mar 2019 14:27:26 -0400
Subject: [PATCH 10/13] [DEBUG] investigate get_version error

---
 magpie/api/home/home.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/magpie/api/home/home.py b/magpie/api/home/home.py
index bc5310a73..98cca6f15 100644
--- a/magpie/api/home/home.py
+++ b/magpie/api/home/home.py
@@ -12,7 +12,7 @@ def get_version(request):
     """
     version = {
         u'version': __meta__.__version__,
-        u'db_version': db.get_database_revision(request.db)
+        # u'db_version': db.get_database_revision(request.db)
     }
     return ax.valid_http(httpSuccess=HTTPOk, content=version, contentType=JSON_TYPE,
                          detail=s.Version_GET_OkResponseSchema.description)

From 8a8373e34270a3c85ffae757056f8d1723d33bb6 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Thu, 28 Mar 2019 15:02:04 -0400
Subject: [PATCH 11/13] reapply get_version + dont skip migration in tests

---
 .travis.yml                                   |  1 -
 magpie/api/home/home.py                       |  2 +-
 .../api/management/resource/resource_utils.py |  4 ++--
 .../api/management/resource/resource_views.py |  4 ++--
 .../api/management/service/service_views.py   |  9 +++++----
 magpie/api/management/user/user_views.py      | 19 ++++++++++---------
 magpie/app.py                                 |  7 ++++---
 magpie/common.py                              | 11 ++++-------
 magpie/constants.py                           | 11 +++++++----
 magpie/db.py                                  |  9 +++++----
 magpie/definitions/pyramid_definitions.py     |  4 ++--
 magpie/ui/management/views.py                 |  7 ++++---
 tests/test_magpie_api.py                      | 12 +++---------
 tests/test_magpie_ui.py                       |  5 +----
 tests/utils.py                                |  7 +++----
 15 files changed, 53 insertions(+), 59 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 091a6a9c5..5e3ed6732 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,7 +16,6 @@ env:
     - CONDA_HOME=$HOME/conda
     - DOWNLOAD_CACHE=$HOME/downloads
   matrix:
-    # FIXME: local login not functional, cannot run local tests on travis
     - TEST_TARGET=test-local    START_TARGET=
     - TEST_TARGET=test-remote   START_TARGET=start
     - TEST_TARGET=coverage      START_TARGET=
diff --git a/magpie/api/home/home.py b/magpie/api/home/home.py
index 98cca6f15..bc5310a73 100644
--- a/magpie/api/home/home.py
+++ b/magpie/api/home/home.py
@@ -12,7 +12,7 @@ def get_version(request):
     """
     version = {
         u'version': __meta__.__version__,
-        # u'db_version': db.get_database_revision(request.db)
+        u'db_version': db.get_database_revision(request.db)
     }
     return ax.valid_http(httpSuccess=HTTPOk, content=version, contentType=JSON_TYPE,
                          detail=s.Version_GET_OkResponseSchema.description)
diff --git a/magpie/api/management/resource/resource_utils.py b/magpie/api/management/resource/resource_utils.py
index c582ca186..4f9ff63d6 100644
--- a/magpie/api/management/resource/resource_utils.py
+++ b/magpie/api/management/resource/resource_utils.py
@@ -1,8 +1,8 @@
-from magpie.common import str2bool
 from magpie.services import service_type_dict
 from magpie.register import sync_services_phoenix
 from magpie.definitions.ziggurat_definitions import ResourceService
 from magpie.definitions.pyramid_definitions import (
+    asbool,
     HTTPOk,
     HTTPCreated,
     HTTPBadRequest,
@@ -182,7 +182,7 @@ def add_resource_in_tree(new_res, db):
 
 def delete_resource(request):
     resource = ar.get_resource_matchdict_checked(request)
-    service_push = str2bool(ar.get_multiformat_post(request, 'service_push'))
+    service_push = asbool(ar.get_multiformat_post(request, 'service_push'))
     res_content = {u'resource': format_resource(resource, basic_info=True)}
     ax.evaluate_call(lambda: models.resource_tree_service.delete_branch(
                         resource_id=resource.resource_id, db_session=request.db),
diff --git a/magpie/api/management/resource/resource_views.py b/magpie/api/management/resource/resource_views.py
index 1b3178271..102844404 100644
--- a/magpie/api/management/resource/resource_views.py
+++ b/magpie/api/management/resource/resource_views.py
@@ -3,13 +3,13 @@
 from magpie.api.management.service.service_formats import format_service_resources
 from magpie.api.management.resource import resource_utils as ru, resource_formats as rf
 from magpie.definitions.pyramid_definitions import (
+    asbool,
     view_config,
     HTTPOk,
     HTTPForbidden,
     HTTPNotAcceptable,
     HTTPInternalServerError,
 )
-from magpie.common import str2bool
 from magpie.register import sync_services_phoenix
 from magpie.services import service_type_dict
 from magpie import models
@@ -69,7 +69,7 @@ def delete_resource_view(request):
 def update_resource(request):
     """Update a resource information."""
     resource = ar.get_resource_matchdict_checked(request, 'resource_id')
-    service_push = str2bool(ar.get_multiformat_post(request, 'service_push'))
+    service_push = asbool(ar.get_multiformat_post(request, 'service_push'))
     res_old_name = resource.resource_name
     res_new_name = ar.get_value_multiformat_post_checked(request, 'resource_name')
 
diff --git a/magpie/api/management/service/service_views.py b/magpie/api/management/service/service_views.py
index 29f78e933..1c5e133cc 100644
--- a/magpie/api/management/service/service_views.py
+++ b/magpie/api/management/service/service_views.py
@@ -2,6 +2,7 @@
 from magpie.api.management.service import service_formats as sf, service_utils as su
 from magpie.api import api_requests as ar, api_except as ax, api_rest_schemas as s
 from magpie.definitions.pyramid_definitions import (
+    asbool,
     view_config,
     HTTPOk,
     HTTPBadRequest,
@@ -10,7 +11,7 @@
     HTTPNotAcceptable,
     HTTPConflict,
 )
-from magpie.common import str2bool, JSON_TYPE
+from magpie.common import JSON_TYPE
 from magpie.register import sync_services_phoenix, SERVICES_PHOENIX_ALLOWED
 from magpie.services import service_type_dict
 from magpie import models
@@ -68,7 +69,7 @@ def register_service_view(request):
     service_name = ar.get_value_multiformat_post_checked(request, 'service_name')
     service_url = ar.get_value_multiformat_post_checked(request, 'service_url')
     service_type = ar.get_value_multiformat_post_checked(request, 'service_type')
-    service_push = str2bool(ar.get_multiformat_post(request, 'service_push'))
+    service_push = asbool(ar.get_multiformat_post(request, 'service_push'))
     ax.verify_param(service_type, isIn=True, paramCompare=service_type_dict.keys(),
                     httpError=HTTPBadRequest, msgOnFail=s.Services_POST_BadRequestResponseSchema.description)
     ax.verify_param(models.Service.by_service_name(service_name, db_session=request.db), isNone=True,
@@ -83,7 +84,7 @@ def register_service_view(request):
 def update_service_view(request):
     """Update a service information."""
     service = ar.get_service_matchdict_checked(request)
-    service_push = str2bool(ar.get_multiformat_post(request, 'service_push', default=False))
+    service_push = asbool(ar.get_multiformat_post(request, 'service_push', default=False))
 
     def select_update(new_value, old_value):
         return new_value if new_value is not None and not new_value == '' else old_value
@@ -142,7 +143,7 @@ def get_service_view(request):
 def unregister_service_view(request):
     """Unregister a service."""
     service = ar.get_service_matchdict_checked(request)
-    service_push = str2bool(ar.get_multiformat_delete(request, 'service_push', default=False))
+    service_push = asbool(ar.get_multiformat_delete(request, 'service_push', default=False))
     svc_content = sf.format_service(service, show_private_url=True)
     svc_res_id = service.resource_id
     ax.evaluate_call(lambda: models.resource_tree_service.delete_branch(resource_id=svc_res_id, db_session=request.db),
diff --git a/magpie/api/management/user/user_views.py b/magpie/api/management/user/user_views.py
index e0fa2bb47..cb567e3a2 100644
--- a/magpie/api/management/user/user_views.py
+++ b/magpie/api/management/user/user_views.py
@@ -2,6 +2,7 @@
 from magpie.api.management.user import user_utils as uu, user_formats as uf
 from magpie.api.management.service.service_formats import format_service_resources
 from magpie.definitions.pyramid_definitions import (
+    asbool,
     view_config,
     HTTPOk,
     HTTPCreated,
@@ -14,7 +15,7 @@
 )
 from magpie.definitions.ziggurat_definitions import UserService, GroupService
 from magpie.constants import get_constant
-from magpie.common import str2bool, get_logger
+from magpie.common import get_logger
 from magpie import models
 LOGGER = get_logger(__name__)
 
@@ -177,7 +178,7 @@ def del_usr_grp(usr, grp):
 @view_config(route_name=s.UserResourcesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
 def get_user_resources_view(request):
     """List all resources a user has permissions on."""
-    inherit_groups_perms = str2bool(ar.get_query_param(request, 'inherit'))
+    inherit_groups_perms = asbool(ar.get_query_param(request, 'inherit'))
     user = ar.get_user_matchdict_checked_or_logged(request)
     db = request.db
 
@@ -234,8 +235,8 @@ def get_user_resource_permissions_view(request):
     """List all permissions a user has on a specific resource."""
     user = ar.get_user_matchdict_checked_or_logged(request)
     resource = ar.get_resource_matchdict_checked(request, 'resource_id')
-    inherit_groups_perms = str2bool(ar.get_query_param(request, 'inherit'))
-    effective_perms = str2bool(ar.get_query_param(request, 'effective'))
+    inherit_groups_perms = asbool(ar.get_query_param(request, 'inherit'))
+    effective_perms = asbool(ar.get_query_param(request, 'effective'))
     perm_names = uu.get_user_resource_permissions(resource=resource, user=user, request=request,
                                                   inherit_groups_permissions=inherit_groups_perms,
                                                   effective_permissions=effective_perms)
@@ -293,9 +294,9 @@ def delete_user_resource_permission_view(request):
 def get_user_services_view(request):
     """List all services a user has permissions on."""
     user = ar.get_user_matchdict_checked_or_logged(request)
-    cascade_resources = str2bool(ar.get_query_param(request, 'cascade'))
-    inherit_groups_perms = str2bool(ar.get_query_param(request, 'inherit'))
-    format_as_list = str2bool(ar.get_query_param(request, 'list'))
+    cascade_resources = asbool(ar.get_query_param(request, 'cascade'))
+    inherit_groups_perms = asbool(ar.get_query_param(request, 'inherit'))
+    format_as_list = asbool(ar.get_query_param(request, 'list'))
 
     svc_json = uu.get_user_services(user, request=request,
                                     cascade_resources=cascade_resources,
@@ -346,7 +347,7 @@ def get_user_service_permissions_view(request):
     """List all permissions a user has on a service."""
     user = ar.get_user_matchdict_checked_or_logged(request)
     service = ar.get_service_matchdict_checked(request)
-    inherit_groups_perms = str2bool(ar.get_query_param(request, 'inherit'))
+    inherit_groups_perms = asbool(ar.get_query_param(request, 'inherit'))
     perms = ax.evaluate_call(lambda: uu.get_user_service_permissions(service=service, user=user, request=request,
                                                                      inherit_groups_permissions=inherit_groups_perms),
                              fallback=lambda: request.db.rollback(), httpError=HTTPNotFound,
@@ -391,7 +392,7 @@ def delete_user_service_permission_view(request):
 @view_config(route_name=s.UserServiceResourcesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
 def get_user_service_resources_view(request):
     """List all resources under a service a user has permission on."""
-    inherit_groups_perms = str2bool(ar.get_query_param(request, 'inherit'))
+    inherit_groups_perms = asbool(ar.get_query_param(request, 'inherit'))
     user = ar.get_user_matchdict_checked_or_logged(request)
     service = ar.get_service_matchdict_checked(request)
     service_perms = uu.get_user_service_permissions(
diff --git a/magpie/app.py b/magpie/app.py
index e27eeee39..8080b8354 100644
--- a/magpie/app.py
+++ b/magpie/app.py
@@ -5,8 +5,9 @@
 Magpie is a service for AuthN and AuthZ based on Ziggurat-Foundations
 """
 
-from magpie.common import print_log, str2bool, get_logger
+from magpie.common import print_log, get_logger
 from magpie.constants import get_constant
+from magpie.definitions.pyramid_definitions import asbool
 from magpie.helpers.register_default_users import register_default_users
 from magpie.register import (
     magpie_register_services_from_config,
@@ -58,8 +59,8 @@ def main(global_config=None, **settings):
     register_default_users(db_session=db_session)
 
     print_log('Register configuration providers...', logger=LOGGER)
-    push_phoenix = str2bool(get_constant('PHOENIX_PUSH', settings=settings, settings_name='magpie.phoenix_push',
-                                         raise_missing=False, raise_not_set=False, print_missing=True))
+    push_phoenix = asbool(get_constant('PHOENIX_PUSH', settings=settings, settings_name='magpie.phoenix_push',
+                                       raise_missing=False, raise_not_set=False, print_missing=True))
     prov_cfg = get_constant('MAGPIE_PROVIDERS_CONFIG_PATH', default_value='',
                             raise_missing=False, raise_not_set=False, print_missing=True)
     if os.path.isfile(prov_cfg):
diff --git a/magpie/common.py b/magpie/common.py
index 6eb7b7c9a..84f93d917 100644
--- a/magpie/common.py
+++ b/magpie/common.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-from magpie.definitions.pyramid_definitions import Response, HTTPException
+from magpie.definitions.pyramid_definitions import Response, HTTPException, truthy
 from webob.headers import ResponseHeaders, EnvironHeaders
 from requests.structures import CaseInsensitiveDict
 from distutils.dir_util import mkpath
@@ -13,7 +13,7 @@
 import os
 if TYPE_CHECKING:
     from magpie.definitions.typedefs import (  # noqa: F401
-        AnyResponseType, AnyHeadersType, LoggerType, Str, List, Optional, Type, Union
+        AnyResponseType, AnyHeadersType, LoggerType, Any, Str, List, Optional, Type, Union
     )
 
 JSON_TYPE = 'application/json'
@@ -55,11 +55,8 @@ def raise_log(msg, exception=Exception, logger=None, level=logging.ERROR):
 
 
 def bool2str(value):
-    return 'true' if value in ['on', 'true', 'True', True] else 'false'
-
-
-def str2bool(value):
-    return True if value in ['on', 'true', 'True', True] else False
+    # type: (Any) -> Str
+    return 'true' if str(value).lower() in truthy else 'false'
 
 
 def islambda(func):
diff --git a/magpie/constants.py b/magpie/constants.py
index 390ca8cee..d86fac25d 100644
--- a/magpie/constants.py
+++ b/magpie/constants.py
@@ -6,7 +6,8 @@
 import dotenv
 import logging
 import warnings
-from magpie.common import str2bool, raise_log, print_log, get_settings_from_config_ini
+from magpie.common import raise_log, print_log, get_settings_from_config_ini
+from magpie.definitions.pyramid_definitions import asbool
 
 # ===========================
 # path variables
@@ -75,13 +76,15 @@
 MAGPIE_EDITOR_GROUP = os.getenv('MAGPIE_EDITOR_GROUP', 'editors')
 MAGPIE_USERS_GROUP = os.getenv('MAGPIE_USERS_GROUP', 'users')
 MAGPIE_CRON_LOG = os.getenv('MAGPIE_CRON_LOG', '~/magpie-cron.log')
+MAGPIE_DB_MIGRATION = asbool(os.getenv('MAGPIE_DB_MIGRATION', True))
+MAGPIE_DB_MIGRATION_ATTEMPTS = int(os.getenv('MAGPIE_DB_MIGRATION_ATTEMPTS', 5))
 MAGPIE_LOG_LEVEL = os.getenv('MAGPIE_LOG_LEVEL', _default_log_lvl)
-MAGPIE_LOG_REQUEST = str2bool(os.getenv('MAGPIE_LOG_REQUEST', True))
-MAGPIE_LOG_EXCEPTION = str2bool(os.getenv('MAGPIE_LOG_EXCEPTION', True))
+MAGPIE_LOG_REQUEST = asbool(os.getenv('MAGPIE_LOG_REQUEST', True))
+MAGPIE_LOG_EXCEPTION = asbool(os.getenv('MAGPIE_LOG_EXCEPTION', True))
 PHOENIX_USER = os.getenv('PHOENIX_USER', 'phoenix')
 PHOENIX_PASSWORD = os.getenv('PHOENIX_PASSWORD', 'qwerty')
 PHOENIX_PORT = int(os.getenv('PHOENIX_PORT', 8443))
-PHOENIX_PUSH = str2bool(os.getenv('PHOENIX_PUSH', True))
+PHOENIX_PUSH = asbool(os.getenv('PHOENIX_PUSH', True))
 TWITCHER_PROTECTED_PATH = os.getenv('TWITCHER_PROTECTED_PATH', '/ows/proxy')
 TWITCHER_PROTECTED_URL = os.getenv('TWITCHER_PROTECTED_URL', None)
 
diff --git a/magpie/db.py b/magpie/db.py
index 1f512b01a..3d83d727a 100644
--- a/magpie/db.py
+++ b/magpie/db.py
@@ -7,6 +7,7 @@
     register, sessionmaker, engine_from_config, ZopeTransactionExtension,
     configure_mappers, select, Inspector, Session, sa_exc
 )
+from magpie.definitions.pyramid_definitions import asbool
 from typing import TYPE_CHECKING
 import transaction
 import inspect
@@ -142,10 +143,10 @@ def run_database_migration_when_ready(settings, db_session=None):
     """
 
     db_ready = False
-    if get_constant('MAGPIE_DB_MIGRATION', settings, 'magpie.db_migration', True,
-                    raise_missing=False, raise_not_set=False, print_missing=True):
-        attempts = get_constant('MAGPIE_DB_MIGRATION_ATTEMPTS', settings, 'magpie.db_migration_attempts',
-                                default_value=5, raise_missing=False, raise_not_set=False, print_missing=True)
+    if asbool(get_constant('MAGPIE_DB_MIGRATION', settings, 'magpie.db_migration', True,
+                           raise_missing=False, raise_not_set=False, print_missing=True)):
+        attempts = int(get_constant('MAGPIE_DB_MIGRATION_ATTEMPTS', settings, 'magpie.db_migration_attempts',
+                                    default_value=5, raise_missing=False, raise_not_set=False, print_missing=True))
 
         print_log('Running database migration (as required) ...')
         attempts = max(attempts, 2)     # enforce at least 2 attempts, 1 for db creation and one for actual migration
diff --git a/magpie/definitions/pyramid_definitions.py b/magpie/definitions/pyramid_definitions.py
index 8543777db..cb8fae9fb 100644
--- a/magpie/definitions/pyramid_definitions.py
+++ b/magpie/definitions/pyramid_definitions.py
@@ -34,7 +34,7 @@
     exception_response,
 )
 # noinspection PyUnresolvedReferences
-from pyramid.settings import asbool                                                 # noqa: F401
+from pyramid.settings import asbool, truthy                                         # noqa: F401
 # noinspection PyUnresolvedReferences
 from pyramid.registry import Registry                                               # noqa: F401
 # noinspection PyUnresolvedReferences
@@ -52,7 +52,7 @@
     exception_view_config,
     forbidden_view_config
 )
-# noinspection PyUnresolvedReferences
+# noinspection PyUnresolvedReferences, PyPep8Naming
 from pyramid.security import (  # noqa: F401
     Authenticated,
     Allow as ALLOW,
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index ec38d274a..13da3ddcf 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -1,5 +1,6 @@
 from magpie.api import api_rest_schemas as schemas
 from magpie.definitions.pyramid_definitions import (
+    asbool,
     view_config,
     HTTPFound,
     HTTPMovedPermanently,
@@ -8,7 +9,7 @@
     HTTPConflict,
 )
 from magpie.constants import get_constant
-from magpie.common import str2bool, get_json, get_logger, JSON_TYPE
+from magpie.common import get_json, get_logger, JSON_TYPE
 from magpie.helpers.sync_resources import OUT_OF_SYNC
 from magpie.helpers import sync_resources
 from magpie.models import resource_type_dict, remote_resource_tree_service  # TODO: remove, implement getters via API
@@ -260,7 +261,7 @@ def edit_user(self):
             requires_update_name = False
 
             if u'inherit_groups_permissions' in self.request.POST:
-                inherit_grp_perms = str2bool(self.request.POST[u'inherit_groups_permissions'])
+                inherit_grp_perms = asbool(self.request.POST[u'inherit_groups_permissions'])
                 user_info[u'inherit_groups_permissions'] = inherit_grp_perms
 
             if u'delete' in self.request.POST:
@@ -804,7 +805,7 @@ def edit_service(self):
         # apply default state if arriving on the page for the first time
         # future editions on the page will transfer the last saved state
         service_push_show = cur_svc_type in register.SERVICES_PHOENIX_ALLOWED
-        service_push = str2bool(self.request.POST.get('service_push', service_push_show))
+        service_push = asbool(self.request.POST.get('service_push', service_push_show))
 
         service_info = {u'edit_mode': u'no_edit', u'service_name': service_name, u'service_url': service_url,
                         u'public_url': register.get_twitcher_protected_service_url(service_name),
diff --git a/tests/test_magpie_api.py b/tests/test_magpie_api.py
index 834f11351..08bba8761 100644
--- a/tests/test_magpie_api.py
+++ b/tests/test_magpie_api.py
@@ -32,9 +32,7 @@ class TestCase_MagpieAPI_NoAuth_Local(ti.Interface_MagpieAPI_NoAuth, unittest.Te
     def setUpClass(cls):
         cls.app = utils.get_test_magpie_app()
         cls.json_headers = utils.get_headers(cls.app, {'Accept': JSON_TYPE, 'Content-Type': JSON_TYPE})
-        # FIXME: use get_version()
-        cls.version = __meta__.__version__
-        #cls.version = utils.TestSetup.get_Version(cls)
+        cls.version = utils.TestSetup.get_Version(cls)
         cls.cookies = None
         cls.usr = get_constant('MAGPIE_ANONYMOUS_USER')
         cls.grp = get_constant('MAGPIE_ANONYMOUS_GROUP')
@@ -73,9 +71,7 @@ def setUpClass(cls):
         cls.pwd = get_constant('MAGPIE_TEST_ADMIN_PASSWORD')
         cls.json_headers = utils.get_headers(cls.app, {'Accept': JSON_TYPE, 'Content-Type': JSON_TYPE})
         cls.cookies = None
-        # FIXME: use get_version()
-        cls.version = __meta__.__version__
-        #cls.version = utils.TestSetup.get_Version(cls)
+        cls.version = utils.TestSetup.get_Version(cls)
         # TODO: fix UI views so that they can be 'found' directly in the WebTest.TestApp
         # NOTE: localhost magpie has to be running for following login call to work
         cls.headers, cls.cookies = utils.check_or_try_login_user(cls.app, cls.usr, cls.pwd,
@@ -102,9 +98,7 @@ def setUpClass(cls):
         cls.cookies = None
         cls.usr = get_constant('MAGPIE_ANONYMOUS_USER')
         cls.grp = get_constant('MAGPIE_ANONYMOUS_GROUP')
-        # FIXME: use get_version()
-        cls.version = __meta__.__version__
-        #cls.version = utils.TestSetup.get_Version(cls)
+        cls.version = utils.TestSetup.get_Version(cls)
 
 
 @runner.MAGPIE_TEST_API
diff --git a/tests/test_magpie_ui.py b/tests/test_magpie_ui.py
index dae5af818..549abbdd6 100644
--- a/tests/test_magpie_ui.py
+++ b/tests/test_magpie_ui.py
@@ -8,7 +8,6 @@
 Tests for `magpie.ui` module.
 """
 
-from magpie import __meta__
 from magpie.common import JSON_TYPE
 from magpie.constants import get_constant
 from tests import utils, runner
@@ -59,9 +58,7 @@ def setUpClass(cls):
         cls.url = cls.app  # to simplify calls of TestSetup (all use .url)
         cls.json_headers = utils.get_headers(cls.app, {'Accept': JSON_TYPE, 'Content-Type': JSON_TYPE})
         cls.cookies = None
-        # FIXME: use get_version()
-        cls.version = __meta__.__version__
-        # cls.version = utils.TestSetup.get_Version(cls)
+        cls.version = utils.TestSetup.get_Version(cls)
         cls.headers, cls.cookies = utils.check_or_try_login_user(cls.url, cls.usr, cls.pwd, use_ui_form_submit=True)
         cls.require = "cannot run tests without logged in user with '{}' permissions".format(cls.grp)
         cls.check_requirements()
diff --git a/tests/utils.py b/tests/utils.py
index 13866be71..2a764e910 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -15,7 +15,7 @@
 from magpie import __meta__, services, app
 from magpie.common import get_settings_from_config_ini, get_header, JSON_TYPE, HTML_TYPE
 from magpie.constants import get_constant
-from magpie.common import str2bool
+from magpie.definitions.pyramid_definitions import asbool
 from magpie.utils import get_magpie_url
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
@@ -47,8 +47,8 @@ def __repr__(self):
         return '{}[{}]'.format(type(self).__name__, self.message)
 
     def _default_run(self):
-        option_value = str2bool(get_constant(self._name, default_value=True,
-                                             raise_missing=False, raise_not_set=False, print_missing=True))
+        option_value = asbool(get_constant(self._name, default_value=True,
+                                           raise_missing=False, raise_not_set=False, print_missing=True))
         return True if option_value is None else option_value
 
     @property
@@ -129,7 +129,6 @@ def get_test_magpie_app(settings=None):
     config = config_setup_from_ini(get_constant('MAGPIE_INI_FILE_PATH'))
     config.include('ziggurat_foundations.ext.pyramid.sign_in')
     config.include('ziggurat_foundations.ext.pyramid.get_user')
-    config.registry.settings['magpie.db_migration'] = False
     config.registry.settings['magpie.url'] = 'http://localhost:80'
     if settings:
         config.registry.settings.update(settings)

From 94b9aa66247bc938fa74f073af9e2e959fc4708e Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Thu, 28 Mar 2019 17:41:33 -0400
Subject: [PATCH 12/13] pep8 fixes + enable pep8 travis test + mock
 get_db_revision

---
 .travis.yml                                   |   5 +
 Makefile                                      |   5 +-
 magpie/adapter/magpieprocess.py               |  16 +-
 .../20671b28c538_change_all_linking_k.py      |   6 +-
 ...c1c9_normalize_constraint_and_key_names.py |   6 +-
 magpie/api/api_except.py                      |  11 +-
 magpie/api/api_requests.py                    |  50 ++++--
 magpie/api/api_rest_schemas.py                |  18 +-
 magpie/api/home/home.py                       |   5 +-
 magpie/api/login/esgfopenid.py                |   2 +-
 magpie/api/login/login.py                     | 148 ++++++++-------
 magpie/api/management/group/group_formats.py  |   3 +-
 magpie/api/management/group/group_utils.py    |  14 +-
 .../management/resource/resource_formats.py   |   2 +-
 .../api/management/resource/resource_utils.py |  15 +-
 .../api/management/service/service_formats.py |  11 +-
 .../api/management/service/service_utils.py   |  12 +-
 magpie/api/management/user/user_utils.py      |  10 +-
 magpie/db.py                                  |   2 +-
 magpie/definitions/typedefs.py                |  13 +-
 magpie/helpers/register_default_users.py      |   9 +-
 magpie/models.py                              |  83 +++++----
 magpie/owsrequest.py                          |   6 +-
 magpie/register.py                            |   9 +-
 magpie/security.py                            |   2 +-
 magpie/services.py                            | 169 +++++++++---------
 magpie/ui/management/views.py                 |   2 +-
 magpie/ui/utils.py                            |   5 +-
 magpie/utils.py                               |   6 +-
 setup.cfg                                     |  14 ++
 tests/test_magpie_api.py                      |   3 +-
 tests/utils.py                                |  33 ++--
 32 files changed, 409 insertions(+), 286 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 5e3ed6732..efa23b81a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -71,6 +71,11 @@ before_script:
 script:
   - export MAGPIE_LOG_LEVEL=INFO
   - make $START_TARGET $TEST_TARGET
+  # run static code checks only once
+  - |
+    if [ "${TRAVIS_PYTHON_VERSION}" -eq "3.6" ]; then
+      make lint
+    fi
 notifications:
   email: false
 after_success:
diff --git a/Makefile b/Makefile
index ca32dbc1a..74494f1ec 100644
--- a/Makefile
+++ b/Makefile
@@ -110,26 +110,23 @@ clean-test:
 .PHONY: lint
 lint: install-dev
 	@echo "Checking code style with flake8..."
-	@bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; flake8 magpie tests --ignore=E501,W291 || true'
+	@bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; flake8'
 
 .PHONY: test
 test: install-dev install
 	@echo "Running tests..."
-	# @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; python setup.py test'
 	bash -c "source $(CONDA_HOME)/bin/activate $(CONDA_ENV); \
 		pytest tests -vv --junitxml $(CURDIR)/tests/results.xml"
 
 .PHONY: test-local
 test-local: install-dev install
 	@echo "Running local tests..."
-	# @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; MAGPIE_TEST_REMOTE=false python setup.py test'
 	bash -c "source $(CONDA_HOME)/bin/activate $(CONDA_ENV); \
 		pytest tests -vv -m 'not remote' --junitxml $(CURDIR)/tests/results.xml"
 
 .PHONY: test-remote
 test-remote: install-dev install
 	@echo "Running remote tests..."
-	# @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; MAGPIE_TEST_LOCAL=false python setup.py test'
 	bash -c "source $(CONDA_HOME)/bin/activate $(CONDA_ENV); \
 		pytest tests -vv -m 'not local' --junitxml $(CURDIR)/tests/results.xml"
 
diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index 5307e9edf..311f81ae2 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -12,20 +12,29 @@
     HTTPConflict,
     HTTPNotImplemented,
     asbool,
-    Registry,
 )
 
 # import 'process' elements separately than 'twitcher_definitions' because not defined in master
+# noinspection PyUnresolvedReferences
 from twitcher.utils import get_twitcher_url
+# noinspection PyUnresolvedReferences
 from twitcher.config import get_twitcher_configuration, TWITCHER_CONFIGURATION_EMS
-from twitcher.datatype import Process
+# noinspection PyUnresolvedReferences
 from twitcher.exceptions import ProcessNotFound, ProcessRegistrationError
+# noinspection PyUnresolvedReferences
 from twitcher.adapter.default import DefaultAdapter
+# noinspection PyUnresolvedReferences
 from twitcher.store.base import ProcessStore
+# noinspection PyUnresolvedReferences
 from twitcher.visibility import VISIBILITY_PUBLIC, VISIBILITY_PRIVATE, visibility_values
-from typing import List, Optional, Iterable, Union, AnyStr
+# noinspection PyUnresolvedReferences
+from twitcher.datatype import Process  # noqa: F401
+from typing import TYPE_CHECKING
 import six
 import requests
+if TYPE_CHECKING:
+    from typing import List, Optional, Iterable, Union, AnyStr  # noqa: F401
+    from magpie.definitions.pyramid_definitions import Registry  # noqa: F401
 LOGGER = get_logger("TWITCHER")
 
 
@@ -480,5 +489,6 @@ def set_visibility(self, process_id, visibility, request=None):
         # update visibility of process, which will also reflect changes to route permissions during 'list_processes'
         self.default_process_store.set_visibility(process_id, visibility=visibility, request=request)
 
+    # noinspection PyMethodMayBeStatic, PyUnusedLocal
     def clear_processes(self, request=None):
         raise_http(httpError=HTTPNotImplemented, detail="Clear processes not supported via MagpieAdapter.")
diff --git a/magpie/alembic/versions/20671b28c538_change_all_linking_k.py b/magpie/alembic/versions/20671b28c538_change_all_linking_k.py
index 5cc2522eb..c34dded82 100644
--- a/magpie/alembic/versions/20671b28c538_change_all_linking_k.py
+++ b/magpie/alembic/versions/20671b28c538_change_all_linking_k.py
@@ -33,12 +33,10 @@ def upgrade():
     # inspected keys
     groups_permissions_pkey = insp.get_pk_constraint('groups_permissions')['name']
     groups_pkey = insp.get_pk_constraint('groups')['name']
-    groups_resources_permissions_pkey = \
-    insp.get_pk_constraint('groups_resources_permissions')['name']
+    groups_resources_permissions_pkey = insp.get_pk_constraint('groups_resources_permissions')['name']
     users_groups_pkey = insp.get_pk_constraint('users_groups')['name']
     users_permissions_pkey = insp.get_pk_constraint('users_permissions')['name']
-    users_resources_permissions_pkey = \
-    insp.get_pk_constraint('users_resources_permissions')['name']
+    users_resources_permissions_pkey = insp.get_pk_constraint('users_resources_permissions')['name']
 
     op.drop_constraint('groups_pkey', 'groups', type_='primary')
 
diff --git a/magpie/alembic/versions/438c27ec1c9_normalize_constraint_and_key_names.py b/magpie/alembic/versions/438c27ec1c9_normalize_constraint_and_key_names.py
index 8f9dec6da..04de0d66c 100644
--- a/magpie/alembic/versions/438c27ec1c9_normalize_constraint_and_key_names.py
+++ b/magpie/alembic/versions/438c27ec1c9_normalize_constraint_and_key_names.py
@@ -29,8 +29,8 @@
 
 
 def upgrade():
-    c = get_context()
-    insp = Inspector.from_engine(c.connection.engine)
+    context = get_context()
+    insp = Inspector.from_engine(context.connection.engine)
     # existing migration
     # pre naming convention keys
     groups_permissions_pkey = 'groups_permissions_pkey'
@@ -40,7 +40,7 @@ def upgrade():
     users_permissions_pkey = 'users_permissions_pkey'
     users_resources_permissions_pkey = 'users_resources_permissions_pkey'
 
-    if isinstance(c.connection.engine.dialect, PGDialect):
+    if isinstance(context.connection.engine.dialect, PGDialect):
         op.execute(
             'ALTER INDEX groups_unique_group_name_key RENAME to ix_groups_uq_group_name_key')
 
diff --git a/magpie/api/api_except.py b/magpie/api/api_except.py
index 0a24aeaf9..72d1f79e4 100644
--- a/magpie/api/api_except.py
+++ b/magpie/api/api_except.py
@@ -24,7 +24,7 @@
 
 
 # noinspection PyPep8Naming
-def verify_param(
+def verify_param(   # noqa: E126
                  # --- verification values ---
                  param,                             # type: Any
                  paramCompare=None,                 # type: Optional[Union[Any, List[Any]]]
@@ -48,7 +48,7 @@ def verify_param(
                  isIn=False,                        # type: Optional[bool]
                  isEqual=False,                     # type: Optional[bool]
                  ofType=False,                      # type: Optional[bool]
-                 ):                                 # type: (...) -> None
+                ):                                  # type: (...) -> None
     """
     Evaluate various parameter combinations given the requested verification flags.
     Given a failing verification, directly raises the specified ``httpError``.
@@ -415,9 +415,10 @@ def generate_response_http_format(httpClass, httpKWArgs, jsonContent, outputType
         # otherwise json is contained within the html <body> section
         elif outputType == HTML_TYPE:
             # add preformat <pre> section to output as is within the <body> section
-            htmlBody = httpClass.explanation + "<br><h2>Exception Details</h2>" + \
-                       "<pre style='word-wrap: break-word; white-space: pre-wrap;'>" + \
-                       jsonContent + "</pre>"
+            htmlBody = "{}<br><h2>Exception Details</h2>" \
+                       "<br><h2>Exception Details</h2>" \
+                       "<pre style='word-wrap: break-word; white-space: pre-wrap;'>{}</pre>" \
+                .format(httpClass.explanation, jsonContent)
             httpResponse = httpClass(body_template=htmlBody, content_type=HTML_TYPE, **httpKWArgs)
 
         # default back to plain text
diff --git a/magpie/api/api_requests.py b/magpie/api/api_requests.py
index 96cfca683..1ff91a7f5 100644
--- a/magpie/api/api_requests.py
+++ b/magpie/api/api_requests.py
@@ -1,8 +1,21 @@
-from magpie.definitions import ziggurat_definitions as zig
-from magpie.definitions.typedefs import Any, AnyStr, Optional
 from magpie.api.api_except import evaluate_call, verify_param
-from magpie.api.api_rest_schemas import *
+from magpie.api import api_rest_schemas as s
+from magpie.common import JSON_TYPE
+from magpie.constants import get_constant
+from magpie.definitions import ziggurat_definitions as zig
+from magpie.definitions.pyramid_definitions import (
+    IAuthenticationPolicy,
+    HTTPForbidden,
+    HTTPNotFound,
+    HTTPNotAcceptable,
+    HTTPUnprocessableEntity,
+    HTTPInternalServerError,
+)
 from magpie import models
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from magpie.definitions.pyramid_definitions import Request  # noqa: F401
+    from magpie.definitions.typedefs import Any, AnyStr, Optional  # noqa: F401
 
 
 def get_request_method_content(request):
@@ -47,7 +60,7 @@ def get_permission_multiformat_post_checked(request, service_resource, permissio
 def get_value_multiformat_post_checked(request, key, default=None):
     val = get_multiformat_any(request, key, default=default)
     verify_param(val, notNone=True, notEmpty=True, httpError=HTTPUnprocessableEntity,
-                 paramName=key, msgOnFail=UnprocessableEntityResponseSchema.description)
+                 paramName=key, msgOnFail=s.UnprocessableEntityResponseSchema.description)
     return val
 
 
@@ -76,9 +89,9 @@ def get_user(request, user_name_or_token):
             anonymous_user = get_constant('MAGPIE_ANONYMOUS_USER')
             anonymous = evaluate_call(lambda: zig.UserService.by_user_name(anonymous_user, db_session=request.db),
                                       fallback=lambda: request.db.rollback(), httpError=HTTPForbidden,
-                                      msgOnFail=User_CheckAnonymous_ForbiddenResponseSchema.description)
+                                      msgOnFail=s.User_CheckAnonymous_ForbiddenResponseSchema.description)
             verify_param(anonymous, notNone=True, httpError=HTTPNotFound,
-                         msgOnFail=User_CheckAnonymous_NotFoundResponseSchema.description)
+                         msgOnFail=s.User_CheckAnonymous_NotFoundResponseSchema.description)
             return anonymous
     else:
         authn_policy = request.registry.queryUtility(IAuthenticationPolicy)
@@ -89,14 +102,15 @@ def get_user(request, user_name_or_token):
             raise HTTPForbidden()
         user = evaluate_call(lambda: zig.UserService.by_user_name(user_name_or_token, db_session=request.db),
                              fallback=lambda: request.db.rollback(),
-                             httpError=HTTPForbidden, msgOnFail=User_GET_ForbiddenResponseSchema.description)
-        verify_param(user, notNone=True, httpError=HTTPNotFound, msgOnFail=User_GET_NotFoundResponseSchema.description)
+                             httpError=HTTPForbidden, msgOnFail=s.User_GET_ForbiddenResponseSchema.description)
+        verify_param(user, notNone=True, httpError=HTTPNotFound,
+                     msgOnFail=s.User_GET_NotFoundResponseSchema.description)
         return user
 
 
 def get_user_matchdict_checked_or_logged(request, user_name_key='user_name'):
     logged_user_name = get_constant('MAGPIE_LOGGED_USER')
-    logged_user_path = UserAPI.path.replace('{' + user_name_key + '}', logged_user_name)
+    logged_user_path = s.UserAPI.path.replace('{' + user_name_key + '}', logged_user_name)
     if user_name_key not in request.matchdict and request.path_info.startswith(logged_user_path):
         return get_user(request, logged_user_name)
     return get_user_matchdict_checked(request, user_name_key)
@@ -110,22 +124,22 @@ def get_user_matchdict_checked(request, user_name_key='user_name'):
 def get_group_matchdict_checked(request, group_name_key='group_name'):
     group_name = get_value_matchdict_checked(request, group_name_key)
     group = evaluate_call(lambda: zig.GroupService.by_group_name(group_name, db_session=request.db),
-                          fallback=lambda: request.db.rollback(),
-                          httpError=HTTPForbidden, msgOnFail=Group_MatchDictCheck_ForbiddenResponseSchema.description)
+                          fallback=lambda: request.db.rollback(), httpError=HTTPForbidden,
+                          msgOnFail=s.Group_MatchDictCheck_ForbiddenResponseSchema.description)
     verify_param(group, notNone=True, httpError=HTTPNotFound,
-                 msgOnFail=Group_MatchDictCheck_NotFoundResponseSchema.description)
+                 msgOnFail=s.Group_MatchDictCheck_NotFoundResponseSchema.description)
     return group
 
 
 def get_resource_matchdict_checked(request, resource_name_key='resource_id'):
     resource_id = get_value_matchdict_checked(request, resource_name_key)
     resource_id = evaluate_call(lambda: int(resource_id), httpError=HTTPNotAcceptable,
-                                msgOnFail=Resource_MatchDictCheck_NotAcceptableResponseSchema.description)
+                                msgOnFail=s.Resource_MatchDictCheck_NotAcceptableResponseSchema.description)
     resource = evaluate_call(lambda: zig.ResourceService.by_resource_id(resource_id, db_session=request.db),
                              fallback=lambda: request.db.rollback(), httpError=HTTPForbidden,
-                             msgOnFail=Resource_MatchDictCheck_ForbiddenResponseSchema.description)
+                             msgOnFail=s.Resource_MatchDictCheck_ForbiddenResponseSchema.description)
     verify_param(resource, notNone=True, httpError=HTTPNotFound,
-                 msgOnFail=Resource_MatchDictCheck_NotFoundResponseSchema.description)
+                 msgOnFail=s.Resource_MatchDictCheck_NotFoundResponseSchema.description)
     return resource
 
 
@@ -133,9 +147,9 @@ def get_service_matchdict_checked(request, service_name_key='service_name'):
     service_name = get_value_matchdict_checked(request, service_name_key)
     service = evaluate_call(lambda: models.Service.by_service_name(service_name, db_session=request.db),
                             fallback=lambda: request.db.rollback(), httpError=HTTPForbidden,
-                            msgOnFail=Service_MatchDictCheck_ForbiddenResponseSchema.description)
+                            msgOnFail=s.Service_MatchDictCheck_ForbiddenResponseSchema.description)
     verify_param(service, notNone=True, httpError=HTTPNotFound, content={u'service_name': service_name},
-                 msgOnFail=Service_MatchDictCheck_NotFoundResponseSchema.description)
+                 msgOnFail=s.Service_MatchDictCheck_NotFoundResponseSchema.description)
     return service
 
 
@@ -150,7 +164,7 @@ def get_permission_matchdict_checked(request, service_resource, permission_name_
 def get_value_matchdict_checked(request, key):
     val = request.matchdict.get(key)
     verify_param(val, notNone=True, notEmpty=True, httpError=HTTPUnprocessableEntity,
-                 paramName=key, msgOnFail=UnprocessableEntityResponseSchema.description)
+                 paramName=key, msgOnFail=s.UnprocessableEntityResponseSchema.description)
     return val
 
 
diff --git a/magpie/api/api_rest_schemas.py b/magpie/api/api_rest_schemas.py
index 63d828da9..236cdb944 100644
--- a/magpie/api/api_rest_schemas.py
+++ b/magpie/api/api_rest_schemas.py
@@ -1,5 +1,19 @@
-from magpie.definitions.cornice_definitions import *
-from magpie.definitions.pyramid_definitions import *
+from magpie.definitions.cornice_definitions import colander, Service, CorniceSwagger, get_services
+from magpie.definitions.pyramid_definitions import (
+    HTTPOk,
+    HTTPCreated,
+    HTTPFound,
+    HTTPBadRequest,
+    HTTPUnauthorized,
+    HTTPForbidden,
+    HTTPNotFound,
+    HTTPMethodNotAllowed,
+    HTTPNotAcceptable,
+    HTTPConflict,
+    HTTPUnprocessableEntity,
+    HTTPInternalServerError,
+    NO_PERMISSION_REQUIRED,
+)
 from magpie.common import JSON_TYPE
 from magpie.constants import get_constant
 from magpie.utils import get_magpie_url
diff --git a/magpie/api/home/home.py b/magpie/api/home/home.py
index bc5310a73..cf3564ce7 100644
--- a/magpie/api/home/home.py
+++ b/magpie/api/home/home.py
@@ -1,7 +1,8 @@
 from magpie.api import api_except as ax, api_rest_schemas as s
 from magpie.definitions.pyramid_definitions import NO_PERMISSION_REQUIRED, HTTPOk, view_config
 from magpie.common import JSON_TYPE
-from magpie import db, __meta__
+from magpie.db import get_database_revision
+from magpie import __meta__
 
 
 @s.VersionAPI.get(tags=[s.APITag], api_security=s.SecurityEveryoneAPI, response_schemas=s.Version_GET_responses)
@@ -12,7 +13,7 @@ def get_version(request):
     """
     version = {
         u'version': __meta__.__version__,
-        u'db_version': db.get_database_revision(request.db)
+        u'db_version': get_database_revision(request.db)
     }
     return ax.valid_http(httpSuccess=HTTPOk, content=version, contentType=JSON_TYPE,
                          detail=s.Version_GET_OkResponseSchema.description)
diff --git a/magpie/api/login/esgfopenid.py b/magpie/api/login/esgfopenid.py
index 67789e1ab..c2c97972a 100644
--- a/magpie/api/login/esgfopenid.py
+++ b/magpie/api/login/esgfopenid.py
@@ -11,7 +11,7 @@
 from six.moves.urllib.request import urlopen
 from authomatic.providers.openid import OpenID
 # noinspection PyProtectedMember, PyUnresolvedReferences
-from openid.fetchers import setDefaultFetcher, Urllib2Fetcher
+from openid.fetchers import Urllib2Fetcher  # , setDefaultFetcher
 from magpie.common import get_logger
 LOGGER = get_logger(__name__)
 
diff --git a/magpie/api/login/login.py b/magpie/api/login/login.py
index 43bb73848..2fc8ca39c 100644
--- a/magpie/api/login/login.py
+++ b/magpie/api/login/login.py
@@ -1,16 +1,42 @@
-from authomatic.adapters import WebObAdapter
-from authomatic.core import LoginResult, Credentials, resolve_provider_class
-from authomatic.exceptions import OAuth2Error
 from magpie.security import authomatic_setup, get_provider_names
-from magpie.definitions.ziggurat_definitions import *
-from magpie.api.api_generic import get_request_info
-from magpie.api.api_except import *
-from magpie.api.api_requests import *
-from magpie.api.api_rest_schemas import *
-from magpie.api.management.user.user_formats import *
+from magpie.definitions.pyramid_definitions import (
+    view_config,
+    forget,
+    remember,
+    Authenticated,
+    IAuthenticationPolicy,
+    Request,
+    Response,
+    HTTPOk,
+    HTTPFound,
+    HTTPTemporaryRedirect,
+    HTTPBadRequest,
+    HTTPUnauthorized,
+    HTTPForbidden,
+    HTTPNotFound,
+    HTTPConflict,
+    HTTPNotAcceptable,
+    HTTPInternalServerError,
+    NO_PERMISSION_REQUIRED,
+)
+from magpie.definitions.ziggurat_definitions import (
+    ExternalIdentityService,
+    UserService,
+    ZigguratSignInSuccess,
+    ZigguratSignInBadAuth,
+    ZigguratSignOut,
+)
+from magpie.api import api_generic as ag, api_except as ax, api_rest_schemas as s
+from magpie.api.api_requests import get_multiformat_post, get_value_multiformat_post_checked
+from magpie.api.management.user.user_formats import format_user
 from magpie.api.management.user.user_utils import create_user
-from magpie.common import convert_response, get_logger
+from magpie.common import convert_response, get_logger, JSON_TYPE
 from magpie.constants import get_constant
+from magpie.models import User, ExternalIdentity
+from magpie.utils import get_magpie_url
+from authomatic.adapters import WebObAdapter
+from authomatic.core import LoginResult, Credentials, resolve_provider_class
+from authomatic.exceptions import OAuth2Error
 from six.moves.urllib.parse import urlparse
 LOGGER = get_logger(__name__)
 
@@ -36,17 +62,17 @@ def process_sign_in_external(request, username, provider):
 
     came_from = request.POST.get('came_from', '/')
     request.response.set_cookie('homepage_route', came_from)
-    external_login_route = request.route_url(ProviderSigninAPI.name, provider_name=provider_name, _query=query_field)
+    external_login_route = request.route_url(s.ProviderSigninAPI.name, provider_name=provider_name, _query=query_field)
     return HTTPTemporaryRedirect(location=external_login_route, headers=request.response.headers)
 
 
 def verify_provider(provider_name):
-    verify_param(provider_name, paramName=u'provider_name', paramCompare=MAGPIE_PROVIDER_KEYS, isIn=True,
-                 httpError=HTTPNotFound, msgOnFail=ProviderSignin_GET_NotFoundResponseSchema.description)
+    ax.verify_param(provider_name, paramName=u'provider_name', paramCompare=MAGPIE_PROVIDER_KEYS, isIn=True,
+                    httpError=HTTPNotFound, msgOnFail=s.ProviderSignin_GET_NotFoundResponseSchema.description)
 
 
-@SigninAPI.post(schema=Signin_POST_RequestSchema(), tags=[LoginTag], response_schemas=Signin_POST_responses)
-@view_config(route_name=SigninAPI.name, request_method='POST', permission=NO_PERMISSION_REQUIRED)
+@s.SigninAPI.post(schema=s.Signin_POST_RequestSchema(), tags=[s.LoginTag], response_schemas=s.Signin_POST_responses)
+@view_config(route_name=s.SigninAPI.name, request_method='POST', permission=NO_PERMISSION_REQUIRED)
 def sign_in(request):
     """Signs in a user session."""
     provider_name = get_value_multiformat_post_checked(request, 'provider_name', default=default_provider).lower()
@@ -66,18 +92,18 @@ def sign_in(request):
         login_failure(request, "Incorrect credentials.")
 
     elif provider_name in MAGPIE_EXTERNAL_PROVIDERS.keys():
-        return evaluate_call(lambda: process_sign_in_external(request, user_name, provider_name),
-                             httpError=HTTPInternalServerError,
-                             content={u'user_name': user_name,  u'provider_name': provider_name},
-                             msgOnFail=Signin_POST_InternalServerErrorResponseSchema.description)
+        return ax.evaluate_call(lambda: process_sign_in_external(request, user_name, provider_name),
+                                httpError=HTTPInternalServerError,
+                                content={u'user_name': user_name, u'provider_name': provider_name},
+                                msgOnFail=s.Signin_POST_InternalServerErrorResponseSchema.description)
 
 
 # swagger responses referred in `sign_in`
 @view_config(context=ZigguratSignInSuccess, permission=NO_PERMISSION_REQUIRED)
 def login_success_ziggurat(request):
     # headers contains login authorization cookie
-    return valid_http(httpSuccess=HTTPOk, httpKWArgs={'headers': request.context.headers},
-                      detail=Signin_POST_OkResponseSchema.description)
+    return ax.valid_http(httpSuccess=HTTPOk, httpKWArgs={'headers': request.context.headers},
+                         detail=s.Signin_POST_OkResponseSchema.description)
 
 
 # swagger responses referred in `sign_in`
@@ -86,21 +112,21 @@ def login_failure(request, reason=None):
     http_err = HTTPUnauthorized
     if reason is None:
         http_err = HTTPNotAcceptable
-        reason = Signin_POST_NotAcceptableResponseSchema.description
+        reason = s.Signin_POST_NotAcceptableResponseSchema.description
         user_name = get_multiformat_post(request, 'user_name', default=None)
         if user_name is None:
             http_err = HTTPBadRequest
-            reason = Signin_POST_BadRequestResponseSchema.description
+            reason = s.Signin_POST_BadRequestResponseSchema.description
         else:
-            user_name_list = evaluate_call(lambda: [user.user_name for user in models.User.all(db_session=request.db)],
-                                           fallback=lambda: request.db.rollback(), httpError=HTTPForbidden,
-                                           msgOnFail=Signin_POST_ForbiddenResponseSchema.description)
+            user_name_list = ax.evaluate_call(lambda: [user.user_name for user in User.all(db_session=request.db)],
+                                              fallback=lambda: request.db.rollback(), httpError=HTTPForbidden,
+                                              msgOnFail=s.Signin_POST_ForbiddenResponseSchema.description)
             if user_name in user_name_list:
                 http_err = HTTPUnauthorized
                 reason = "Incorrect credentials."
-    content = get_request_info(request, default_message=Signin_POST_UnauthorizedResponseSchema.description)
+    content = ag.get_request_info(request, default_message=s.Signin_POST_UnauthorizedResponseSchema.description)
     content.update({u'reason': str(reason)})
-    raise_http(httpError=http_err, content=content, detail=Signin_POST_UnauthorizedResponseSchema.description)
+    ax.raise_http(httpError=http_err, content=content, detail=s.Signin_POST_UnauthorizedResponseSchema.description)
 
 
 def new_user_external(external_user_name, external_id, email, provider_name, db_session):
@@ -112,14 +138,14 @@ def new_user_external(external_user_name, external_id, email, provider_name, db_
 
     user = UserService.by_user_name(internal_user_name, db_session=db_session)
     # noinspection PyArgumentList
-    ex_identity = models.ExternalIdentity(external_user_name=external_user_name, external_id=external_id,
-                                          local_user_id=user.id, provider_name=provider_name)
-    evaluate_call(lambda: db_session.add(ex_identity), fallback=lambda: db_session.rollback(),
-                  httpError=HTTPConflict, msgOnFail=Signin_POST_ConflictResponseSchema.description,
-                  content={u'provider_name': str(provider_name),
-                           u'internal_user_name': str(internal_user_name),
-                           u'external_user_name': str(external_user_name),
-                           u'external_id': str(external_id)})
+    ex_identity = ExternalIdentity(external_user_name=external_user_name, external_id=external_id,
+                                   local_user_id=user.id, provider_name=provider_name)
+    ax.evaluate_call(lambda: db_session.add(ex_identity), fallback=lambda: db_session.rollback(),
+                     httpError=HTTPConflict, msgOnFail=s.Signin_POST_ConflictResponseSchema.description,
+                     content={u'provider_name': str(provider_name),
+                              u'internal_user_name': str(internal_user_name),
+                              u'external_user_name': str(external_user_name),
+                              u'external_id': str(external_id)})
     user.external_identities.append(ex_identity)
     return user
 
@@ -144,17 +170,17 @@ def login_success_external(request, external_user_name, external_id, email, prov
     header_host = urlparse(homepage_route).hostname
     magpie_host = get_magpie_url(request)
     if header_host and header_host != magpie_host:
-        raise_http(httpError=HTTPForbidden, detail=ProviderSignin_GET_ForbiddenResponseSchema.description)
+        ax.raise_http(httpError=HTTPForbidden, detail=s.ProviderSignin_GET_ForbiddenResponseSchema.description)
     if not header_host:
         homepage_route = magpie_host + ('/' if not homepage_route.startswith('/') else '') + homepage_route
-    return valid_http(httpSuccess=HTTPFound, detail=ProviderSignin_GET_FoundResponseSchema.description,
-                      content={u'homepage_route': homepage_route},
-                      httpKWArgs={'location': homepage_route, 'headers': headers})
+    return ax.valid_http(httpSuccess=HTTPFound, detail=s.ProviderSignin_GET_FoundResponseSchema.description,
+                         content={u'homepage_route': homepage_route},
+                         httpKWArgs={'location': homepage_route, 'headers': headers})
 
 
-@ProviderSigninAPI.get(schema=ProviderSignin_GET_RequestSchema, tags=[LoginTag],
-                       response_schemas=ProviderSignin_GET_responses)
-@view_config(route_name=ProviderSigninAPI.name, permission=NO_PERMISSION_REQUIRED)
+@s.ProviderSigninAPI.get(schema=s.ProviderSignin_GET_RequestSchema, tags=[s.LoginTag],
+                         response_schemas=s.ProviderSignin_GET_responses)
+@view_config(route_name=s.ProviderSigninAPI.name, permission=NO_PERMISSION_REQUIRED)
 def authomatic_login(request):
     """Signs in a user session using an external provider."""
 
@@ -201,13 +227,13 @@ def authomatic_login(request):
                     # this error can happen if providing incorrectly formed authorization header
                     except OAuth2Error as exc:
                         LOGGER.debug("Login failure with Authorization header.")
-                        raise_http(httpError=HTTPBadRequest, content={u'reason': str(exc.message)},
-                                   detail=ProviderSignin_GET_BadRequestResponseSchema.description)
+                        ax.raise_http(httpError=HTTPBadRequest, content={u'reason': str(exc.message)},
+                                      detail=s.ProviderSignin_GET_BadRequestResponseSchema.description)
                     # verify that the update procedure succeeded with provided token
                     if 400 <= response.status < 500:
                         LOGGER.debug("Login failure with invalid token.")
-                        raise_http(httpError=HTTPUnauthorized,
-                                   detail=ProviderSignin_GET_UnauthorizedResponseSchema.description)
+                        ax.raise_http(httpError=HTTPUnauthorized,
+                                      detail=s.ProviderSignin_GET_UnauthorizedResponseSchema.description)
                 # create/retrieve the user using found details from login provider
                 return login_success_external(request,
                                               external_id=result.user.username or result.user.id,
@@ -217,22 +243,22 @@ def authomatic_login(request):
     except Exception as exc:
         exc_msg = "Unhandled error during external provider `{}` login. [{!s}]".format(provider_name, exc)
         LOGGER.exception(exc_msg, exc_info=True)
-        raise_http(httpError=HTTPInternalServerError, detail=exc_msg)
+        ax.raise_http(httpError=HTTPInternalServerError, detail=exc_msg)
 
     LOGGER.debug('Reached end of login function. Response: {!r}'.format(response))
     return response
 
 
-@SignoutAPI.get(tags=[LoginTag], response_schemas=Signout_GET_responses)
+@s.SignoutAPI.get(tags=[s.LoginTag], response_schemas=s.Signout_GET_responses)
 @view_config(context=ZigguratSignOut, permission=NO_PERMISSION_REQUIRED)
 def sign_out(request):
     """Signs out the current user session."""
-    return valid_http(httpSuccess=HTTPOk, httpKWArgs={'headers': forget(request)},
-                      detail=Signout_GET_OkResponseSchema.description)
+    return ax.valid_http(httpSuccess=HTTPOk, httpKWArgs={'headers': forget(request)},
+                         detail=s.Signout_GET_OkResponseSchema.description)
 
 
-@SessionAPI.get(tags=[LoginTag], response_schemas=Session_GET_responses)
-@view_config(route_name=SessionAPI.name, permission=NO_PERMISSION_REQUIRED)
+@s.SessionAPI.get(tags=[s.LoginTag], response_schemas=s.Session_GET_responses)
+@view_config(route_name=s.SessionAPI.name, permission=NO_PERMISSION_REQUIRED)
 def get_session(request):
     """Get information about current session."""
     def _get_session(req):
@@ -245,16 +271,16 @@ def _get_session(req):
             json_resp = {u'authenticated': False}
         return json_resp
 
-    session_json = evaluate_call(lambda: _get_session(request), httpError=HTTPInternalServerError,
-                                 msgOnFail=Session_GET_InternalServerErrorResponseSchema.description)
-    return valid_http(httpSuccess=HTTPOk, detail=Session_GET_OkResponseSchema.description, content=session_json)
+    session_json = ax.evaluate_call(lambda: _get_session(request), httpError=HTTPInternalServerError,
+                                    msgOnFail=s.Session_GET_InternalServerErrorResponseSchema.description)
+    return ax.valid_http(httpSuccess=HTTPOk, detail=s.Session_GET_OkResponseSchema.description, content=session_json)
 
 
 # noinspection PyUnusedLocal
-@ProvidersAPI.get(tags=[LoginTag], response_schemas=Providers_GET_responses)
-@view_config(route_name=ProvidersAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
+@s.ProvidersAPI.get(tags=[s.LoginTag], response_schemas=s.Providers_GET_responses)
+@view_config(route_name=s.ProvidersAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
 def get_providers(request):
     """Get list of login providers."""
-    return valid_http(httpSuccess=HTTPOk, detail=Providers_GET_OkResponseSchema.description,
-                      content={u'providers': {u'internal': sorted(MAGPIE_INTERNAL_PROVIDERS.values()),
-                                              u'external': sorted(MAGPIE_EXTERNAL_PROVIDERS.values()), }})
+    return ax.valid_http(httpSuccess=HTTPOk, detail=s.Providers_GET_OkResponseSchema.description,
+                         content={u'providers': {u'internal': sorted(MAGPIE_INTERNAL_PROVIDERS.values()),
+                                                 u'external': sorted(MAGPIE_EXTERNAL_PROVIDERS.values()), }})
diff --git a/magpie/api/management/group/group_formats.py b/magpie/api/management/group/group_formats.py
index 3fbeba37b..01a944a90 100644
--- a/magpie/api/management/group/group_formats.py
+++ b/magpie/api/management/group/group_formats.py
@@ -1,4 +1,5 @@
-from magpie.api.api_requests import *
+from magpie.api.api_requests import HTTPInternalServerError
+from magpie.api.api_except import evaluate_call
 
 
 def format_group(group, basic_info=False):
diff --git a/magpie/api/management/group/group_utils.py b/magpie/api/management/group/group_utils.py
index 805494443..a19dd274c 100644
--- a/magpie/api/management/group/group_utils.py
+++ b/magpie/api/management/group/group_utils.py
@@ -4,7 +4,6 @@
 from magpie.api.management.resource.resource_formats import format_resource
 from magpie.api.management.service.service_formats import format_service_resources, format_service
 from magpie.api.management.group.group_formats import format_group
-from magpie.definitions.sqlalchemy_definitions import Session
 from magpie.definitions.ziggurat_definitions import GroupService, GroupResourcePermissionService, ResourceService
 from magpie.definitions.pyramid_definitions import (
     HTTPOk,
@@ -12,11 +11,14 @@
     HTTPForbidden,
     HTTPNotFound,
     HTTPConflict,
-    HTTPException,  # noqa: F401
     HTTPInternalServerError
 )
 from magpie import models
-from typing import AnyStr
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from magpie.definitions.pyramid_definitions import HTTPException  # noqa: F401
+    from magpie.definitions.sqlalchemy_definitions import Session  # noqa: F401
+    from magpie.definitions.typedefs import Str  # noqa: F401
 
 
 def get_all_groups(db_session):
@@ -46,7 +48,7 @@ def get_group_resources(group, db_session):
 
 
 def create_group(group_name, db_session):
-    # type: (AnyStr, Session) -> models.Group
+    # type: (Str, Session) -> HTTPException
     """
     Creates a group if it is permitted and not conflicting.
     :returns: corresponding HTTP response according to the encountered situation.
@@ -67,7 +69,7 @@ def create_group(group_name, db_session):
 
 
 def create_group_resource_permission(permission_name, resource, group, db_session):
-    # type: (AnyStr, models.Resource, models.Group, Session) -> HTTPException
+    # type: (Str, models.Resource, models.Group, Session) -> HTTPException
     """
     Creates a permission on a group/resource combination if it is permitted and not conflicting.
     :returns: corresponding HTTP response according to the encountered situation.
@@ -143,7 +145,7 @@ def delete_group_resource_permission(permission_name, resource, group, db_sessio
         fallback=lambda: db_session.rollback(), httpError=HTTPForbidden,
         msgOnFail=s.GroupServicePermission_DELETE_ForbiddenGetResponseSchema.description, content=perm_content
     )
-    ax.verify_param(del_perm, notNone=True, httpError=HTTPNotFound,  content=perm_content,
+    ax.verify_param(del_perm, notNone=True, httpError=HTTPNotFound, content=perm_content,
                     msgOnFail=s.GroupServicePermission_DELETE_NotFoundResponseSchema.description)
     ax.evaluate_call(lambda: db_session.delete(del_perm), fallback=lambda: db_session.rollback(),
                      httpError=HTTPForbidden, content=perm_content,
diff --git a/magpie/api/management/resource/resource_formats.py b/magpie/api/management/resource/resource_formats.py
index 4a5595d20..56faf6b49 100644
--- a/magpie/api/management/resource/resource_formats.py
+++ b/magpie/api/management/resource/resource_formats.py
@@ -1,4 +1,4 @@
-from magpie.definitions.pyramid_definitions import *
+from magpie.definitions.pyramid_definitions import HTTPInternalServerError
 from magpie.definitions.ziggurat_definitions import ResourceService
 from magpie.models import resource_tree_service
 from magpie.services import service_type_dict
diff --git a/magpie/api/management/resource/resource_utils.py b/magpie/api/management/resource/resource_utils.py
index 4f9ff63d6..bf59329c3 100644
--- a/magpie/api/management/resource/resource_utils.py
+++ b/magpie/api/management/resource/resource_utils.py
@@ -17,9 +17,9 @@
 from magpie import models
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
-    from magpie.definitions.pyramid_definitions import HTTPException
-    from magpie.definitions.sqlalchemy_definitions import Session
-    from magpie.definitions.typedefs import Str, Union
+    from magpie.definitions.pyramid_definitions import HTTPException  # noqa: F401
+    from magpie.definitions.sqlalchemy_definitions import Session  # noqa: F401
+    from magpie.definitions.typedefs import Str, Union  # noqa: F401
 
 
 def check_valid_service_resource_permission(permission_name, service_resource, db_session):
@@ -184,10 +184,11 @@ def delete_resource(request):
     resource = ar.get_resource_matchdict_checked(request)
     service_push = asbool(ar.get_multiformat_post(request, 'service_push'))
     res_content = {u'resource': format_resource(resource, basic_info=True)}
-    ax.evaluate_call(lambda: models.resource_tree_service.delete_branch(
-                        resource_id=resource.resource_id, db_session=request.db),
-                     fallback=lambda: request.db.rollback(), httpError=HTTPForbidden,
-                     msgOnFail="Delete resource branch from tree service failed.", content=res_content)
+    ax.evaluate_call(
+        lambda: models.resource_tree_service.delete_branch(resource_id=resource.resource_id, db_session=request.db),
+        fallback=lambda: request.db.rollback(), httpError=HTTPForbidden,
+        msgOnFail="Delete resource branch from tree service failed.", content=res_content
+    )
 
     def remove_service_magpie_and_phoenix(res, svc_push, db):
         if res.resource_type != 'service':
diff --git a/magpie/api/management/service/service_formats.py b/magpie/api/management/service/service_formats.py
index fde6afafc..e5dffdbac 100644
--- a/magpie/api/management/service/service_formats.py
+++ b/magpie/api/management/service/service_formats.py
@@ -1,14 +1,15 @@
 from magpie.api.api_except import evaluate_call
 from magpie.api.management.resource.resource_utils import crop_tree_with_permission
 from magpie.api.management.resource.resource_formats import get_resource_children, format_resource_tree
-from magpie.definitions.pyramid_definitions import *
-from magpie.definitions.sqlalchemy_definitions import Session
+from magpie.definitions.pyramid_definitions import HTTPInternalServerError
 from magpie.utils import get_twitcher_protected_service_url
-from magpie.models import Resource, Service
-from magpie.services import service_type_dict, ServiceI
+from magpie.services import service_type_dict
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
-    from magpie.definitions.typedefs import Optional, JsonBody, AnyStr, Dict, List
+    from magpie.definitions.typedefs import Optional, JsonBody, AnyStr, Dict, List  # noqa: F401
+    from magpie.definitions.sqlalchemy_definitions import Session  # noqa: F401
+    from magpie.models import Resource, Service  # noqa: F401
+    from magpie.services import ServiceI  # noqa: F401
 
 
 def format_service(service, permissions=None, show_private_url=False, show_resources_allowed=False):
diff --git a/magpie/api/management/service/service_utils.py b/magpie/api/management/service/service_utils.py
index af91417eb..b74ab63a1 100644
--- a/magpie/api/management/service/service_utils.py
+++ b/magpie/api/management/service/service_utils.py
@@ -3,24 +3,26 @@
 from magpie.api.management.service.service_formats import format_service
 from magpie.constants import get_constant
 from magpie.common import get_logger
-from magpie.definitions.sqlalchemy_definitions import Session
 from magpie.definitions.pyramid_definitions import (
     HTTPCreated,
     HTTPForbidden,
     HTTPNotAcceptable,
     HTTPInternalServerError,
-    HTTPException,  # noqa: F401
 )
 from magpie.definitions.ziggurat_definitions import GroupService, ResourceService
 from magpie.register import sync_services_phoenix, SERVICES_PHOENIX_ALLOWED
 from magpie.services import service_type_dict
 from magpie import models
-from typing import AnyStr
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from magpie.definitions.typedefs import Str  # noqa: F401
+    from magpie.definitions.sqlalchemy_definitions import Session  # noqa: F401
+    from magpie.definitions.pyramid_definitions import HTTPException  # noqa: F401
 LOGGER = get_logger(__name__)
 
 
 def create_service(service_name, service_type, service_url, service_push, db_session):
-    # type: (AnyStr, AnyStr, AnyStr, bool, Session) -> HTTPException
+    # type: (Str, Str, Str, bool, Session) -> HTTPException
     """Generates an instance to register a new service."""
 
     def _add_service_magpie_and_phoenix(svc, svc_push, db):
@@ -37,7 +39,7 @@ def _add_service_magpie_and_phoenix(svc, svc_push, db):
             ax.verify_param(svc.resource_id, notNone=True, paramCompare=int, ofType=True,
                             httpError=HTTPInternalServerError,
                             msgOnFail=s.Services_POST_InternalServerErrorResponseSchema.description,
-                            content={u'service_name': str(service_name),  u'resource_id': svc.resource_id},
+                            content={u'service_name': str(service_name), u'resource_id': svc.resource_id},
                             paramName=u'service_name')
         return svc
 
diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py
index 9bdcb6a21..a44cfd988 100644
--- a/magpie/api/management/user/user_utils.py
+++ b/magpie/api/management/user/user_utils.py
@@ -3,7 +3,6 @@
 from magpie.api.management.resource.resource_utils import check_valid_service_resource_permission
 from magpie.api.management.user import user_formats as uf
 from magpie.constants import get_constant
-from magpie.definitions.sqlalchemy_definitions import Session
 from magpie.definitions.ziggurat_definitions import (
     GroupService,
     UserService,
@@ -11,7 +10,6 @@
     UserResourcePermissionService,
 )
 from magpie.definitions.pyramid_definitions import (
-    Request,
     HTTPOk,
     HTTPCreated,
     HTTPBadRequest,
@@ -19,13 +17,15 @@
     HTTPNotFound,
     HTTPNotAcceptable,
     HTTPConflict,
-    HTTPException,  # noqa: F401
 )
-from magpie.services import service_factory, ResourcePermissionType, ServiceI
+from magpie.services import service_factory
 from magpie import models
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
-    from magpie.definitions.typedefs import Any, Str, Dict, List, Optional, Union, UserServicesType
+    from magpie.services import ResourcePermissionType, ServiceI  # noqa: F401
+    from magpie.definitions.pyramid_definitions import Request, HTTPException  # noqa: F401
+    from magpie.definitions.sqlalchemy_definitions import Session  # noqa: F401
+    from magpie.definitions.typedefs import Any, Str, Dict, List, Optional, Union, UserServicesType  # noqa: F401
 
 
 def create_user(user_name, password, email, group_name, db_session):
diff --git a/magpie/db.py b/magpie/db.py
index 3d83d727a..056c16519 100644
--- a/magpie/db.py
+++ b/magpie/db.py
@@ -20,7 +20,7 @@
 from magpie import models
 
 if TYPE_CHECKING:
-    from magpie.definitions.typedefs import Str, SettingsType, Optional, Union
+    from magpie.definitions.typedefs import Str, SettingsType, Optional, Union  # noqa: F401
 
 
 LOGGER = get_logger(__name__)
diff --git a/magpie/definitions/typedefs.py b/magpie/definitions/typedefs.py
index e1ca8ddb0..994d7db00 100644
--- a/magpie/definitions/typedefs.py
+++ b/magpie/definitions/typedefs.py
@@ -3,13 +3,15 @@
 
 # noinspection PyUnresolvedReferences
 from typing import (  # noqa: F401
-    Any, AnyStr, Callable, Dict, List, Iterable, Optional, Tuple, Type, Union, TYPE_CHECKING
+    Any, AnyStr as _AnyStr, Callable, Dict, List, Iterable, Optional, Tuple, Type, Union, TYPE_CHECKING
 )
 if TYPE_CHECKING:
+    from magpie.definitions.sqlalchemy_definitions import Session
     from webob.headers import ResponseHeaders, EnvironHeaders
     from webob.response import Response as WebobResponse
     from webtest.response import TestResponse
     from webtest.app import TestApp
+    from requests.cookies import RequestsCookieJar
     from pyramid.response import Response as PyramidResponse
     from pyramid.registry import Registry
     from pyramid.request import Request
@@ -21,9 +23,10 @@
     import six
 
     if six.PY2:
-        Str = Union[AnyStr, unicode]
+        Str = Union[_AnyStr, unicode]
     else:
-        Str = AnyStr
+        Str = _AnyStr
+    AnyStr = Str
 
     SettingValue = Union[Str, int, float, bool, None]
     SettingsType = Dict[Str, SettingValue]
@@ -36,12 +39,14 @@
     AnyHeadersType = Union[HeadersType, ResponseHeaders, EnvironHeaders, CaseInsensitiveDict]
     AnyResponseType = Union[WebobResponse, PyramidResponse, TestResponse]
 
+    CookiesOrSessionType = Union[RequestsCookieJar, Session]
+
     JsonField = Union[Str, int, float, bool, None]
     JsonBody = Dict[Str, Union[JsonField, Dict[Str, Any], List[Any]]]
 
     ParamKWArgs = Dict[Str, Any]
 
-    UserServicesTypes = Union[Dict[Str, Dict[Str, Any]], List[Dict[Str, Any]]]
+    UserServicesType = Union[Dict[Str, Dict[Str, Any]], List[Dict[Str, Any]]]
 
     TestAppOrUrlType = Union[Str, TestApp]
     AnyMagpieTestType = Union[Type[Base_Magpie_TestCase], Base_Magpie_TestCase, TestAppOrUrlType]
diff --git a/magpie/helpers/register_default_users.py b/magpie/helpers/register_default_users.py
index 6433541e0..872fadfc2 100644
--- a/magpie/helpers/register_default_users.py
+++ b/magpie/helpers/register_default_users.py
@@ -2,10 +2,12 @@
 from magpie.common import print_log, raise_log, get_logger
 from magpie.definitions.sqlalchemy_definitions import Session
 from magpie.definitions.ziggurat_definitions import GroupService, UserService
-from magpie.definitions.typedefs import Optional
+from typing import TYPE_CHECKING
 import transaction
 import logging
 import time
+if TYPE_CHECKING:
+    from magpie.definitions.typedefs import Optional  # noqa: F401
 LOGGER = get_logger(__name__)
 
 
@@ -49,8 +51,8 @@ def init_anonymous(db_session):
 
 
 def init_admin(db_session):
-    if not (UserService.by_user_name(constants.MAGPIE_ADMIN_USER, db_session=db_session)
-            and GroupService.by_group_name(constants.MAGPIE_ADMIN_GROUP, db_session=db_session)):
+    if not (UserService.by_user_name(constants.MAGPIE_ADMIN_USER, db_session=db_session) and
+            GroupService.by_group_name(constants.MAGPIE_ADMIN_GROUP, db_session=db_session)):
         register_user_with_group(user_name=constants.MAGPIE_ADMIN_USER,
                                  group_name=constants.MAGPIE_ADMIN_GROUP,
                                  email=constants.MAGPIE_ADMIN_EMAIL,
@@ -84,7 +86,6 @@ def register_default_users(db_session=None):
     # type: (Optional[Session]) -> None
     if not isinstance(db_session, Session):
         db_session = db.get_db_session_from_config_ini(constants.MAGPIE_INI_FILE_PATH)
-        db_close = False
     if not db.is_database_ready(db_session):
         time.sleep(2)
         raise_log('Database not ready')
diff --git a/magpie/models.py b/magpie/models.py
index 72440f74d..630f54e7f 100644
--- a/magpie/models.py
+++ b/magpie/models.py
@@ -1,10 +1,28 @@
-from magpie.definitions.pyramid_definitions import *
-from magpie.definitions.ziggurat_definitions import *
-from magpie.definitions.sqlalchemy_definitions import *
-from magpie.definitions.typedefs import AnyStr
 from magpie.api.api_except import evaluate_call
-from magpie.permissions import *
-from six import with_metaclass
+from magpie.definitions.pyramid_definitions import ALLOW, ALL_PERMISSIONS, HTTPInternalServerError
+from magpie.definitions.sqlalchemy_definitions import sa, declared_attr, relationship, declarative_base
+from magpie.definitions.ziggurat_definitions import (
+    get_db_session,
+    permission_to_pyramid_acls,
+    ziggurat_model_init,
+    BaseModel,
+    ExternalIdentityMixin,
+    GroupMixin,
+    GroupPermissionMixin,
+    GroupResourcePermissionMixin,
+    ResourceMixin,
+    ResourceTreeService,
+    ResourceTreeServicePostgreSQL,
+    UserGroupMixin,
+    UserMixin,
+    UserPermissionMixin,
+    UserResourcePermissionMixin,
+    UserService,
+)
+from magpie import permissions as p
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from magpie.definitions.typedefs import Str  # noqa: F401
 
 Base = declarative_base()
 
@@ -39,14 +57,13 @@ class GroupResourcePermission(GroupResourcePermissionMixin, Base):
 class ResourceMeta(type):
     @property
     def resource_type_name(cls):
-        # type: (...) -> AnyStr
+        # type: (...) -> Str
         """Generic resource type identifier."""
         raise NotImplemented("Resource implementation must define unique type representation"
                              "and must be added to `resource_type_dict`.")
 
 
 class Resource(ResourceMixin, Base):
-#class Resource(with_metaclass(ResourceMeta, ResourceMixin, Base)):
     permission_names = []
     child_resource_allowed = True
     resource_display_name = sa.Column(sa.Unicode(100), nullable=True)
@@ -156,13 +173,13 @@ def extend_acl(self, resource, user):
 
 class Path(object):
     permission_names = [
-        PERMISSION_READ,
-        PERMISSION_WRITE,
-        PERMISSION_GET_CAPABILITIES,
-        PERMISSION_GET_MAP,
-        PERMISSION_GET_FEATURE_INFO,
-        PERMISSION_GET_LEGEND_GRAPHIC,
-        PERMISSION_GET_METADATA,
+        p.PERMISSION_READ,
+        p.PERMISSION_WRITE,
+        p.PERMISSION_GET_CAPABILITIES,
+        p.PERMISSION_GET_MAP,
+        p.PERMISSION_GET_FEATURE_INFO,
+        p.PERMISSION_GET_LEGEND_GRAPHIC,
+        p.PERMISSION_GET_METADATA,
     ]
 
 
@@ -182,15 +199,15 @@ class Workspace(Resource):
     __mapper_args__ = {u'polymorphic_identity': resource_type_name}
 
     permission_names = [
-        PERMISSION_GET_CAPABILITIES,
-        PERMISSION_GET_MAP,
-        PERMISSION_GET_FEATURE_INFO,
-        PERMISSION_GET_LEGEND_GRAPHIC,
-        PERMISSION_GET_METADATA,
-        PERMISSION_GET_FEATURE,
-        PERMISSION_DESCRIBE_FEATURE_TYPE,
-        PERMISSION_LOCK_FEATURE,
-        PERMISSION_TRANSACTION,
+        p.PERMISSION_GET_CAPABILITIES,
+        p.PERMISSION_GET_MAP,
+        p.PERMISSION_GET_FEATURE_INFO,
+        p.PERMISSION_GET_LEGEND_GRAPHIC,
+        p.PERMISSION_GET_METADATA,
+        p.PERMISSION_GET_FEATURE,
+        p.PERMISSION_DESCRIBE_FEATURE_TYPE,
+        p.PERMISSION_LOCK_FEATURE,
+        p.PERMISSION_TRANSACTION,
     ]
 
 
@@ -199,10 +216,10 @@ class Route(Resource):
     __mapper_args__ = {u'polymorphic_identity': resource_type_name}
 
     permission_names = [
-        PERMISSION_READ,            # access with inheritance (this route and all under it)
-        PERMISSION_WRITE,           # access with inheritance (this route and all under it)
-        PERMISSION_READ_MATCH,      # access without inheritance (only on this specific route)
-        PERMISSION_WRITE_MATCH,     # access without inheritance (only on this specific route)
+        p.PERMISSION_READ,            # access with inheritance (this route and all under it)
+        p.PERMISSION_WRITE,           # access with inheritance (this route and all under it)
+        p.PERMISSION_READ_MATCH,      # access without inheritance (only on this specific route)
+        p.PERMISSION_WRITE_MATCH,     # access without inheritance (only on this specific route)
     ]
 
 
@@ -286,11 +303,11 @@ class RemoteResourceTreeServicePostgresSQL(ResourceTreeServicePostgreSQL):
 remote_resource_tree_service = RemoteResourceTreeService(RemoteResourceTreeServicePostgresSQL)
 
 resource_type_dict = {
-    Service.resource_type_name:     Service,
-    Directory.resource_type_name:   Directory,
-    File.resource_type_name:        File,
-    Workspace.resource_type_name:   Workspace,
-    Route.resource_type_name:       Route,
+    Service.resource_type_name:     Service,    # noqa: E241
+    Directory.resource_type_name:   Directory,  # noqa: E241
+    File.resource_type_name:        File,       # noqa: E241
+    Workspace.resource_type_name:   Workspace,  # noqa: E241
+    Route.resource_type_name:       Route,      # noqa: E241
 }
 
 
diff --git a/magpie/owsrequest.py b/magpie/owsrequest.py
index 97afa4a25..6fd2d4e96 100644
--- a/magpie/owsrequest.py
+++ b/magpie/owsrequest.py
@@ -8,9 +8,12 @@
 from magpie.api.api_except import raise_http
 from magpie.common import get_logger, get_header, JSON_TYPE, PLAIN_TYPE
 from pyramid.httpexceptions import HTTPMethodNotAllowed
-from requests import Request
+from typing import TYPE_CHECKING
+# noinspection PyUnresolvedReferences
 import lxml.etree
 import json
+if TYPE_CHECKING:
+    from requests import Request  # noqa: F401
 LOGGER = get_logger(__name__)
 
 
@@ -93,6 +96,7 @@ class WPSPost(OWSParser):
 
     def __init__(self, request):
         super(WPSPost, self).__init__(request)
+        # noinspection PyUnresolvedReferences
         self.document = lxml.etree.fromstring(self.request.body)
         lxml_strip_ns(self.document)
 
diff --git a/magpie/register.py b/magpie/register.py
index 7d7a53404..2b9737262 100644
--- a/magpie/register.py
+++ b/magpie/register.py
@@ -23,7 +23,6 @@
 from magpie.services import service_type_dict
 from magpie import models
 from magpie.utils import get_twitcher_protected_service_url, get_phoenix_url, get_magpie_url, get_admin_cookies
-from requests.cookies import RequestsCookieJar
 from typing import TYPE_CHECKING
 import os
 import six
@@ -34,7 +33,9 @@
 import transaction
 import logging
 if TYPE_CHECKING:
-    from magpie.definitions.typedefs import Str, Dict, List, JsonBody, Optional, Tuple, Union  # noqa: F401
+    from magpie.definitions.typedefs import (  # noqa: F401
+        Str, Dict, List, JsonBody, Optional, Tuple, Union, CookiesOrSessionType
+    )
 
 LOGGER = get_logger(__name__)
 
@@ -508,7 +509,7 @@ def use_request(cookies_or_session):
 def parse_resource_path(permission_config_entry,    # type: ConfigItem
                         entry_index,                # type: int
                         service_info,               # type: ConfigItem
-                        cookies_or_session=None,    # type: Union[RequestsCookieJar, Session]
+                        cookies_or_session=None,    # type: CookiesOrSessionType
                         magpie_url=None,            # type: Optional[Str]
                         ):                          # type: (...) -> Tuple[Union[int, None], bool]
     """
@@ -589,7 +590,7 @@ def parse_resource_path(permission_config_entry,    # type: ConfigItem
 def apply_permission_entry(permission_config_entry,     # type: ConfigItem
                            entry_index,                 # type: int
                            resource_id,                 # type: int
-                           cookies_or_session,          # type: Union[RequestsCookieJar, Session]
+                           cookies_or_session,          # type: CookiesOrSessionType
                            magpie_url,                  # type: Optional[Str]
                            ):                           # type: (...) -> None
     """
diff --git a/magpie/security.py b/magpie/security.py
index 2251f0807..043e45194 100644
--- a/magpie/security.py
+++ b/magpie/security.py
@@ -10,7 +10,7 @@
 from typing import TYPE_CHECKING
 import logging
 if TYPE_CHECKING:
-    from magpie.definitions.typedefs import JsonBody
+    from magpie.definitions.typedefs import JsonBody  # noqa: F401
 AUTHOMATIC_LOGGER = get_logger('magpie.authomatic', level=logging.DEBUG)
 
 
diff --git a/magpie/services.py b/magpie/services.py
index 28b3b9b0e..da0c59ea3 100644
--- a/magpie/services.py
+++ b/magpie/services.py
@@ -1,27 +1,28 @@
 from magpie.constants import get_constant
-from magpie.definitions.ziggurat_definitions import *
+from magpie.definitions.ziggurat_definitions import UserService, ResourceService, permission_to_pyramid_acls
 from magpie.definitions.pyramid_definitions import (
     EVERYONE,
     ALLOW,
-    Request,
     HTTPNotFound,
     HTTPBadRequest,
     HTTPNotImplemented,
+    HTTPInternalServerError,
 )
-from magpie.api.api_except import *
-from magpie.permissions import *
-from magpie.owsrequest import *
-from magpie import models
-from typing import AnyStr, List, Dict, Union
+from magpie.api import api_except as ax
+from magpie.owsrequest import ows_parser_factory
+from magpie import models, permissions as p
+from typing import TYPE_CHECKING
 from six import with_metaclass
-
-ResourcePermissionType = Union[models.GroupPermission, models.UserPermission]
+if TYPE_CHECKING:
+    from magpie.definitions.typedefs import Str, List, Dict, Union  # noqa: F401
+    from magpie.definitions.pyramid_definitions import Request  # noqa: F401
+    ResourcePermissionType = Union[models.GroupPermission, models.UserPermission]
 
 
 class ServiceMeta(type):
     @property
     def resource_types(cls):
-        # type: (...) -> List[AnyStr]
+        # type: (...) -> List[Str]
         """Allowed resources types under the service."""
         return list(cls.resource_types_permissions.keys())
 
@@ -33,11 +34,11 @@ def child_resource_allowed(cls):
 
 class ServiceI(with_metaclass(ServiceMeta)):
     # required request parameters for the service
-    params_expected = []                # type: List[AnyStr]
+    params_expected = []                # type: List[Str]
     # global permissions allowed for the service (top-level resource)
-    permission_names = []               # type: List[AnyStr]
+    permission_names = []               # type: List[Str]
     # dict of list for each corresponding allowed resource permissions (children resources)
-    resource_types_permissions = {}     # type: Dict[AnyStr, List[AnyStr]]
+    resource_types_permissions = {}     # type: Dict[Str, List[Str]]
 
     def __init__(self, service, request):
         self.service = service
@@ -70,7 +71,7 @@ def expand_acl(self, resource, user):
                         self.acl.append((outcome, EVERYONE, perm_name,))
 
     def permission_requested(self):
-        # type: (...) -> AnyStr
+        # type: (...) -> Str
         try:
             return self.parser.params[u'request']
         except Exception as e:
@@ -97,9 +98,9 @@ def effective_permissions(self, resource, user):
 class ServiceWPS(ServiceI):
 
     permission_names = [
-        PERMISSION_GET_CAPABILITIES,
-        PERMISSION_DESCRIBE_PROCESS,
-        PERMISSION_EXECUTE,
+        p.PERMISSION_GET_CAPABILITIES,
+        p.PERMISSION_DESCRIBE_PROCESS,
+        p.PERMISSION_EXECUTE,
     ]
 
     params_expected = [
@@ -121,11 +122,11 @@ def __acl__(self):
 
 class ServiceWMS(ServiceI):
     permission_names = [
-        PERMISSION_GET_CAPABILITIES,
-        PERMISSION_GET_MAP,
-        PERMISSION_GET_FEATURE_INFO,
-        PERMISSION_GET_LEGEND_GRAPHIC,
-        PERMISSION_GET_METADATA,
+        p.PERMISSION_GET_CAPABILITIES,
+        p.PERMISSION_GET_MAP,
+        p.PERMISSION_GET_FEATURE_INFO,
+        p.PERMISSION_GET_LEGEND_GRAPHIC,
+        p.PERMISSION_GET_METADATA,
     ]
 
     params_expected = [
@@ -139,11 +140,11 @@ class ServiceWMS(ServiceI):
 
     resource_types_permissions = {
         models.Workspace.resource_type_name: [
-            PERMISSION_GET_CAPABILITIES,
-            PERMISSION_GET_MAP,
-            PERMISSION_GET_FEATURE_INFO,
-            PERMISSION_GET_LEGEND_GRAPHIC,
-            PERMISSION_GET_METADATA,
+            p.PERMISSION_GET_CAPABILITIES,
+            p.PERMISSION_GET_MAP,
+            p.PERMISSION_GET_FEATURE_INFO,
+            p.PERMISSION_GET_LEGEND_GRAPHIC,
+            p.PERMISSION_GET_METADATA,
         ]
     }
 
@@ -159,18 +160,18 @@ class ServiceNCWMS2(ServiceWMS):
 
     resource_types_permissions = {
         models.File.resource_type_name: [
-            PERMISSION_GET_CAPABILITIES,
-            PERMISSION_GET_MAP,
-            PERMISSION_GET_FEATURE_INFO,
-            PERMISSION_GET_LEGEND_GRAPHIC,
-            PERMISSION_GET_METADATA,
+            p.PERMISSION_GET_CAPABILITIES,
+            p.PERMISSION_GET_MAP,
+            p.PERMISSION_GET_FEATURE_INFO,
+            p.PERMISSION_GET_LEGEND_GRAPHIC,
+            p.PERMISSION_GET_METADATA,
         ],
         models.Directory.resource_type_name: [
-            PERMISSION_GET_CAPABILITIES,
-            PERMISSION_GET_MAP,
-            PERMISSION_GET_FEATURE_INFO,
-            PERMISSION_GET_LEGEND_GRAPHIC,
-            PERMISSION_GET_METADATA,
+            p.PERMISSION_GET_CAPABILITIES,
+            p.PERMISSION_GET_MAP,
+            p.PERMISSION_GET_FEATURE_INFO,
+            p.PERMISSION_GET_LEGEND_GRAPHIC,
+            p.PERMISSION_GET_METADATA,
         ]
     }
 
@@ -184,13 +185,13 @@ def __acl__(self):
         # According to the permission, the resource we want to authorize is not formatted the same way
         permission_requested = self.permission_requested()
         netcdf_file = None
-        if permission_requested == PERMISSION_GET_CAPABILITIES:
+        if permission_requested == p.PERMISSION_GET_CAPABILITIES:
             # https://colibri.crim.ca/twitcher/ows/proxy/ncWMS2/wms?SERVICE=WMS&REQUEST=GetCapabilities&
             #   VERSION=1.3.0&DATASET=outputs/ouranos/subdaily/aet/pcp/aet_pcp_1961.nc
             if 'dataset' in self.parser.params.keys():
                 netcdf_file = self.parser.params['dataset']
 
-        elif permission_requested == PERMISSION_GET_MAP:
+        elif permission_requested == p.PERMISSION_GET_MAP:
             # https://colibri.crim.ca/ncWMS2/wms?SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&FORMAT=image%2Fpng&
             #   TRANSPARENT=TRUE&ABOVEMAXCOLOR=extend&STYLES=default-scalar%2Fseq-Blues&
             #   LAYERS=outputs/ouranos/subdaily/aet/pcp/aet_pcp_1961.nc/PCP&EPSG=4326
@@ -198,7 +199,7 @@ def __acl__(self):
             if netcdf_file:
                 netcdf_file = netcdf_file.rsplit('/', 1)[0]
 
-        elif permission_requested == PERMISSION_GET_METADATA:
+        elif permission_requested == p.PERMISSION_GET_METADATA:
             # https://colibri.crim.ca/ncWMS2/wms?request=GetMetadata&item=layerDetails&
             #   layerName=outputs/ouranos/subdaily/aet/pcp/aet_pcp_1961.nc/PCP
             netcdf_file = self.parser.params['layername']
@@ -209,8 +210,8 @@ def __acl__(self):
             return [(ALLOW, EVERYONE, permission_requested,)]
 
         if netcdf_file:
-            verify_param('outputs/', paramCompare=netcdf_file, httpError=HTTPNotFound,
-                         msgOnFail='outputs/ is not in path', notIn=True)
+            ax.verify_param('outputs/', paramCompare=netcdf_file, httpError=HTTPNotFound,
+                            msgOnFail='outputs/ is not in path', notIn=True)
             netcdf_file = netcdf_file.replace('outputs/', 'birdhouse/')
 
             db_session = self.request.db
@@ -244,11 +245,11 @@ def __acl__(self):
         # here we need to check the workspace in the path
 
         request_type = self.permission_requested()
-        if request_type == PERMISSION_GET_CAPABILITIES:
+        if request_type == p.PERMISSION_GET_CAPABILITIES:
             path_elem = self.request.path.split('/')
             wms_idx = path_elem.index('wms')
-            if path_elem[wms_idx-1] != 'geoserver':
-                workspace_name = path_elem[wms_idx-1]
+            if path_elem[wms_idx - 1] != 'geoserver':
+                workspace_name = path_elem[wms_idx - 1]
             else:
                 workspace_name = ''
         else:
@@ -265,7 +266,7 @@ def __acl__(self):
 
 
 class ServiceAccess(ServiceI):
-    permission_names = [PERMISSION_ACCESS]
+    permission_names = [p.PERMISSION_ACCESS]
     params_expected = []
     resource_types_permissions = {}
 
@@ -278,7 +279,7 @@ def __acl__(self):
         return self.acl
 
     def permission_requested(self):
-        return PERMISSION_ACCESS
+        return p.PERMISSION_ACCESS
 
 
 class ServiceAPI(ServiceI):
@@ -323,38 +324,38 @@ def route_acl(self, sub_api_route=None):
         # process read/write-match specific permission access
         # (convert exact route 'match' to read/write counterparts only if matching last item's permissions)
         for i in range(match_index, len(self.acl)):
-            if self.acl[i][2] == PERMISSION_READ_MATCH:
-                self.acl[i] = (self.acl[i][0], self.acl[i][1], PERMISSION_READ)
-            if self.acl[i][2] == PERMISSION_WRITE_MATCH:
-                self.acl[i] = (self.acl[i][0], self.acl[i][1], PERMISSION_WRITE)
+            if self.acl[i][2] == p.PERMISSION_READ_MATCH:
+                self.acl[i] = (self.acl[i][0], self.acl[i][1], p.PERMISSION_READ)
+            if self.acl[i][2] == p.PERMISSION_WRITE_MATCH:
+                self.acl[i] = (self.acl[i][0], self.acl[i][1], p.PERMISSION_WRITE)
 
         return self.acl
 
     def permission_requested(self):
         # only read/write are used for 'real' access control, 'match' permissions must be updated accordingly
         if self.request.method.upper() in ['GET', 'HEAD']:
-            return PERMISSION_READ
-        return PERMISSION_WRITE
+            return p.PERMISSION_READ
+        return p.PERMISSION_WRITE
 
     def effective_permissions(self, resource, user):
         # if 'match' permissions are on the specified 'resource', keep them
         # otherwise, keep only the non 'match' variations from inherited parent resources permissions
         resource_effective_perms = super(ServiceAPI, self).effective_permissions(resource, user)
-        return filter(lambda p:
-                      (p.perm_name in [PERMISSION_READ, PERMISSION_WRITE]) or
-                      (p.perm_name in [PERMISSION_READ_MATCH, PERMISSION_WRITE_MATCH]
-                       and p.resource.resource_id == resource.resource_id),
+        return filter(lambda perm:
+                      (perm.perm_name in [p.PERMISSION_READ, p.PERMISSION_WRITE]) or
+                      (perm.perm_name in [p.PERMISSION_READ_MATCH, p.PERMISSION_WRITE_MATCH] and
+                       perm.resource.resource_id == resource.resource_id),
                       resource_effective_perms)
 
 
 class ServiceWFS(ServiceI):
 
     permission_names = [
-        PERMISSION_GET_CAPABILITIES,
-        PERMISSION_DESCRIBE_FEATURE_TYPE,
-        PERMISSION_GET_FEATURE,
-        PERMISSION_LOCK_FEATURE,
-        PERMISSION_TRANSACTION,
+        p.PERMISSION_GET_CAPABILITIES,
+        p.PERMISSION_DESCRIBE_FEATURE_TYPE,
+        p.PERMISSION_GET_FEATURE,
+        p.PERMISSION_LOCK_FEATURE,
+        p.PERMISSION_TRANSACTION,
     ]
 
     params_expected = [
@@ -373,7 +374,7 @@ def __init__(self, service, request):
     def __acl__(self):
         self.expand_acl(self.service, self.request.user)
         request_type = self.permission_requested()
-        if request_type == PERMISSION_GET_CAPABILITIES:
+        if request_type == p.PERMISSION_GET_CAPABILITIES:
             path_elem = self.request.path.split('/')
             wms_idx = path_elem.index('wfs')
             if path_elem[wms_idx - 1] != 'geoserver':
@@ -396,8 +397,8 @@ def __acl__(self):
 class ServiceTHREDDS(ServiceI):
 
     permission_names = [
-        PERMISSION_READ,
-        PERMISSION_WRITE,
+        p.PERMISSION_READ,
+        p.PERMISSION_WRITE,
     ]
 
     params_expected = [
@@ -429,12 +430,12 @@ def __acl__(self):
         else:
             return self.acl
 
-        elems = elems[first_idx+1::]
+        elems = elems[first_idx + 1::]
         new_child = self.service
         while new_child and elems:
             elem_name = elems.pop(0)
             if ".nc" in elem_name:
-                elem_name = elem_name.split(".nc")[0]+".nc"  # in case there is more extension to discard such as .dds
+                elem_name = elem_name.split(".nc")[0] + ".nc"  # in case there is more extension to discard such as .dds
             parent_id = new_child.resource_id
             new_child = models.find_children_by_name(elem_name, parent_id=parent_id, db_session=self.request.db)
             self.expand_acl(new_child, self.request.user)
@@ -446,28 +447,30 @@ def permission_requested(self):
 
 
 service_type_dict = {
-    u'access':          ServiceAccess,
-    u'api':             ServiceAPI,
-    u'geoserverwms':    ServiceGeoserver,
-    u'ncwms':           ServiceNCWMS2,
-    u'thredds':         ServiceTHREDDS,
-    u'wfs':             ServiceWFS,
-    u'wps':             ServiceWPS,
+    u'access':          ServiceAccess,      # noqa: E241
+    u'api':             ServiceAPI,         # noqa: E241
+    u'geoserverwms':    ServiceGeoserver,   # noqa: E241
+    u'ncwms':           ServiceNCWMS2,      # noqa: E241
+    u'thredds':         ServiceTHREDDS,     # noqa: E241
+    u'wfs':             ServiceWFS,         # noqa: E241
+    u'wps':             ServiceWPS,         # noqa: E241
 }
 
 
 def service_factory(service, request):
     # type: (models.Service, Request) -> ServiceI
     """Retrieve the specific service class from the provided database service entry."""
-    verify_param(service, paramCompare=models.Service, ofType=True,
-                 httpError=HTTPBadRequest, content={u'service': repr(service)},
-                 msgOnFail="Cannot process invalid service object")
-    service_type = evaluate_call(lambda: service.type, httpError=HTTPInternalServerError,
-                                 msgOnFail="Cannot retrieve service type from object")
-    verify_param(service_type, isIn=True, paramCompare=service_type_dict.keys(), httpError=HTTPNotImplemented,
-                 msgOnFail="Undefined service type mapping to service object", content={u'service_type': service_type})
-    return evaluate_call(lambda: service_type_dict[service_type](service, request), httpError=HTTPInternalServerError,
-                         msgOnFail="Failed to find requested service type.")
+    ax.verify_param(service, paramCompare=models.Service, ofType=True,
+                    httpError=HTTPBadRequest, content={u'service': repr(service)},
+                    msgOnFail="Cannot process invalid service object")
+    service_type = ax.evaluate_call(lambda: service.type, httpError=HTTPInternalServerError,
+                                    msgOnFail="Cannot retrieve service type from object")
+    ax.verify_param(service_type, isIn=True, paramCompare=service_type_dict.keys(),
+                    httpError=HTTPNotImplemented, content={u'service_type': service_type},
+                    msgOnFail="Undefined service type mapping to service object")
+    return ax.evaluate_call(lambda: service_type_dict[service_type](service, request),
+                            httpError=HTTPInternalServerError,
+                            msgOnFail="Failed to find requested service type.")
 
 
 def get_all_service_permission_names():
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 13da3ddcf..88f14757d 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -758,7 +758,7 @@ def view_services(self):
         if 'edit' in self.request.POST:
             service_name = self.request.POST.get('service_name')
             return HTTPFound(self.request.route_url('edit_service',
-                                       service_name=service_name, cur_svc_type=cur_svc_type))
+                             service_name=service_name, cur_svc_type=cur_svc_type))
 
         return add_template_data(self.request,
                                  {u'cur_svc_type': cur_svc_type,
diff --git a/magpie/ui/utils.py b/magpie/ui/utils.py
index c1e8a5b67..09f3c0c6a 100644
--- a/magpie/ui/utils.py
+++ b/magpie/ui/utils.py
@@ -1,9 +1,10 @@
-from magpie.definitions.pyramid_definitions import exception_response, Request, Response, HTTPBadRequest
+from magpie.definitions.pyramid_definitions import exception_response, Request, HTTPBadRequest
 from magpie.common import get_header, JSON_TYPE
 from typing import TYPE_CHECKING
 import json
 if TYPE_CHECKING:
-    from magpie.definitions.typedefs import Str, JsonBody, CookiesType, HeadersType, Optional
+    from magpie.definitions.typedefs import Str, JsonBody, CookiesType, HeadersType, Optional  # noqa: F401
+    from magpie.definitions.pyramid_definitions import Response  # noqa: F401
 
 
 def check_response(response):
diff --git a/magpie/utils.py b/magpie/utils.py
index 60947ab98..6747507c7 100644
--- a/magpie/utils.py
+++ b/magpie/utils.py
@@ -1,15 +1,15 @@
 from magpie.common import raise_log, get_logger, JSON_TYPE
 from magpie.constants import get_constant
-from magpie.definitions.pyramid_definitions import (  # noqa: F401
+from magpie.definitions.pyramid_definitions import (
     HTTPOk, HTTPClientError, ConfigurationError, Configurator, Registry, Request
 )
 from six.moves.urllib.parse import urlparse
-from typing import Optional, TYPE_CHECKING
+from typing import TYPE_CHECKING
 from requests.cookies import RequestsCookieJar
 import logging
 import requests
 if TYPE_CHECKING:
-    from magpie.definitions.typedefs import Str, CookiesType, SettingsType, SettingsContainer
+    from magpie.definitions.typedefs import Str, CookiesType, SettingsType, SettingsContainer, Optional  # noqa: F401
 LOGGER = get_logger(__name__)
 
 
diff --git a/setup.cfg b/setup.cfg
index 7a9839380..173887783 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -9,3 +9,17 @@ tag = True
 
 [wheel]
 universal = 1
+
+[flake8]
+ignore=E501,W291
+max-line-length=120
+exclude =
+    .git,
+    __pycache__,
+    docs,
+    build,
+    dist,
+    env,
+    eggs,
+    parts,
+    share,
diff --git a/tests/test_magpie_api.py b/tests/test_magpie_api.py
index 08bba8761..87c8da169 100644
--- a/tests/test_magpie_api.py
+++ b/tests/test_magpie_api.py
@@ -11,7 +11,6 @@
 import unittest
 from magpie.constants import get_constant
 from magpie.common import JSON_TYPE
-from magpie import __meta__
 from tests import utils, runner
 
 # NOTE: must be imported without 'from', otherwise the interface's test cases are also executed
@@ -32,8 +31,8 @@ class TestCase_MagpieAPI_NoAuth_Local(ti.Interface_MagpieAPI_NoAuth, unittest.Te
     def setUpClass(cls):
         cls.app = utils.get_test_magpie_app()
         cls.json_headers = utils.get_headers(cls.app, {'Accept': JSON_TYPE, 'Content-Type': JSON_TYPE})
-        cls.version = utils.TestSetup.get_Version(cls)
         cls.cookies = None
+        cls.version = utils.TestSetup.get_Version(cls)
         cls.usr = get_constant('MAGPIE_ANONYMOUS_USER')
         cls.grp = get_constant('MAGPIE_ANONYMOUS_GROUP')
 
diff --git a/tests/utils.py b/tests/utils.py
index 2a764e910..b5398df56 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -1,10 +1,8 @@
-import unittest
-import requests
-import warnings
-import json
-import six
-# noinspection PyPackageRequirements
-import pytest
+from magpie import __meta__, services, app
+from magpie.common import get_settings_from_config_ini, get_header, JSON_TYPE, HTML_TYPE
+from magpie.constants import get_constant
+from magpie.definitions.pyramid_definitions import asbool
+from magpie.utils import get_magpie_url
 from six.moves.urllib.parse import urlparse
 from distutils.version import LooseVersion
 from pyramid.testing import setUp as PyramidSetUp
@@ -12,14 +10,17 @@
 from webtest import TestApp
 # noinspection PyPackageRequirements
 from webtest.response import TestResponse
-from magpie import __meta__, services, app
-from magpie.common import get_settings_from_config_ini, get_header, JSON_TYPE, HTML_TYPE
-from magpie.constants import get_constant
-from magpie.definitions.pyramid_definitions import asbool
-from magpie.utils import get_magpie_url
 from typing import TYPE_CHECKING
+import unittest
+import requests
+import warnings
+import json
+import mock
+import six
+# noinspection PyPackageRequirements
+import pytest
 if TYPE_CHECKING:
-    from tests.interfaces import Base_Magpie_TestCase    # noqa: F401
+    from tests.interfaces import Base_Magpie_TestCase  # noqa: F401
     from magpie.definitions.typedefs import (  # noqa: F401
         Str, Callable, Dict, HeadersType, OptionalHeaderCookiesType, Optional, Type,
         AnyMagpieTestType, AnyResponseType, TestAppOrUrlType
@@ -604,7 +605,11 @@ class TestSetup(object):
     @staticmethod
     def get_Version(test_class):
         app_or_url = get_app_or_url(test_class)
-        resp = test_request(app_or_url, 'GET', '/version', headers=test_class.json_headers, cookies=test_class.cookies)
+        # use mocked call since migration seem to fail on travis-ci
+        with mock.patch('magpie.api.home.home.get_database_revision', return_value='mocked-revision'):
+            resp = test_request(app_or_url, 'GET', '/version',
+                                headers=test_class.json_headers,
+                                cookies=test_class.cookies)
         json_body = check_response_basic_info(resp, 200)
         return json_body['version']
 

From 6435886da103b0ca60870eb29707cd17f8d11893 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Thu, 28 Mar 2019 18:44:51 -0400
Subject: [PATCH 13/13] ignore failure db-alembic-version fetch

---
 magpie/api/home/home.py |  8 +++++++-
 magpie/app.py           |  2 +-
 magpie/constants.py     | 11 +++++++++--
 magpie/db.py            |  4 ++--
 magpie/demo.py          | 30 ------------------------------
 magpie/security.py      |  8 ++++----
 magpie/utils.py         |  2 +-
 tests/interfaces.py     | 20 ++++++++++----------
 tests/utils.py          | 10 ++++------
 9 files changed, 38 insertions(+), 57 deletions(-)
 delete mode 100644 magpie/demo.py

diff --git a/magpie/api/home/home.py b/magpie/api/home/home.py
index cf3564ce7..dbc07bac3 100644
--- a/magpie/api/home/home.py
+++ b/magpie/api/home/home.py
@@ -11,9 +11,15 @@ def get_version(request):
     """
     Version information of the API.
     """
+    version_db = None
+    # noinspection PyBroadException
+    try:
+        version_db = get_database_revision(request.db)
+    except Exception:
+        pass
     version = {
         u'version': __meta__.__version__,
-        u'db_version': get_database_revision(request.db)
+        u'db_version': version_db
     }
     return ax.valid_http(httpSuccess=HTTPOk, content=version, contentType=JSON_TYPE,
                          detail=s.Version_GET_OkResponseSchema.description)
diff --git a/magpie/app.py b/magpie/app.py
index 8080b8354..0951a4ee7 100644
--- a/magpie/app.py
+++ b/magpie/app.py
@@ -59,7 +59,7 @@ def main(global_config=None, **settings):
     register_default_users(db_session=db_session)
 
     print_log('Register configuration providers...', logger=LOGGER)
-    push_phoenix = asbool(get_constant('PHOENIX_PUSH', settings=settings, settings_name='magpie.phoenix_push',
+    push_phoenix = asbool(get_constant('PHOENIX_PUSH', settings, settings_name='magpie.phoenix_push',
                                        raise_missing=False, raise_not_set=False, print_missing=True))
     prov_cfg = get_constant('MAGPIE_PROVIDERS_CONFIG_PATH', default_value='',
                             raise_missing=False, raise_not_set=False, print_missing=True)
diff --git a/magpie/constants.py b/magpie/constants.py
index d86fac25d..639b83176 100644
--- a/magpie/constants.py
+++ b/magpie/constants.py
@@ -8,6 +8,9 @@
 import warnings
 from magpie.common import raise_log, print_log, get_settings_from_config_ini
 from magpie.definitions.pyramid_definitions import asbool
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from magpie.definitions.typedefs import Str, Optional, SettingValue, SettingsContainer
 
 # ===========================
 # path variables
@@ -114,8 +117,9 @@
 # ===========================
 
 
-def get_constant(name, settings=None, settings_name=None, default_value=None,
+def get_constant(name, settings_container=None, settings_name=None, default_value=None,
                  raise_missing=True, print_missing=False, raise_not_set=True):
+    # type: (Str, Optional[SettingsContainer], Optional[Str], Optional[SettingValue], bool, bool, bool) -> SettingValue
     """
     Search in order for matched value of `name` :
       1. search in magpie definitions
@@ -123,7 +127,7 @@ def get_constant(name, settings=None, settings_name=None, default_value=None,
       3. search in settings if specified
 
     :param name: key to search for a value
-    :param settings: wsgi app settings
+    :param settings_container: wsgi app settings container
     :param settings_name: alternative name for `settings` if specified
     :param default_value: default value to be returned if not found anywhere, and exception raises are disabled.
     :param raise_missing: raise exception if key is not found anywhere
@@ -132,9 +136,12 @@ def get_constant(name, settings=None, settings_name=None, default_value=None,
     :returns: found value or `default_value`
     :raises: according message based on options (by default raise missing/`None` value)
     """
+    from magpie.utils import get_settings
+
     magpie_globals = globals()
     missing = True
     magpie_value = None
+    settings = get_settings(settings_container) if settings_container else None
     if name in magpie_globals:
         missing = False
         magpie_value = magpie_globals.get(name)
diff --git a/magpie/db.py b/magpie/db.py
index 056c16519..40bdf08bc 100644
--- a/magpie/db.py
+++ b/magpie/db.py
@@ -143,8 +143,8 @@ def run_database_migration_when_ready(settings, db_session=None):
     """
 
     db_ready = False
-    if asbool(get_constant('MAGPIE_DB_MIGRATION', settings, 'magpie.db_migration', True,
-                           raise_missing=False, raise_not_set=False, print_missing=True)):
+    if asbool(get_constant('MAGPIE_DB_MIGRATION', settings, 'magpie.db_migration',
+                           default_value=True, raise_missing=False, raise_not_set=False, print_missing=True)):
         attempts = int(get_constant('MAGPIE_DB_MIGRATION_ATTEMPTS', settings, 'magpie.db_migration_attempts',
                                     default_value=5, raise_missing=False, raise_not_set=False, print_missing=True))
 
diff --git a/magpie/demo.py b/magpie/demo.py
deleted file mode 100644
index e5e81d558..000000000
--- a/magpie/demo.py
+++ /dev/null
@@ -1,30 +0,0 @@
-from pyramid.config import Configurator
-from pyramid.session import SignedCookieSessionFactory
-
-
-def main(global_settings, **settings):
-    session_factory = SignedCookieSessionFactory(settings['auth.secret'])
-
-    config = Configurator(
-        settings=settings,
-        session_factory=session_factory
-    )
-
-    config.include('pyramid_mako')
-    config.include('magpie.ui')
-
-    config.scan()
-    return config.make_wsgi_app()
-
-
-if __name__ == '__main__':
-    settings = {
-        'auth.secret': 'magpie',
-        'magpie.url': 'http://localhost:2001',
-        'pyramid.reload_templates': True
-    }
-    app = main({}, **settings)
-    from wsgiref.simple_server import make_server
-
-    server = make_server('0.0.0.0', 5003, app)
-    server.serve_forever()
diff --git a/magpie/security.py b/magpie/security.py
index 043e45194..643e7d8d4 100644
--- a/magpie/security.py
+++ b/magpie/security.py
@@ -15,11 +15,11 @@
 
 
 def auth_config_from_settings(settings):
-    magpie_secret = get_constant('MAGPIE_SECRET', settings=settings, settings_name='magpie.secret')
-    magpie_cookie_expire = get_constant('MAGPIE_COOKIE_EXPIRE', settings=settings,
+    magpie_secret = get_constant('MAGPIE_SECRET', settings, settings_name='magpie.secret')
+    magpie_cookie_expire = get_constant('MAGPIE_COOKIE_EXPIRE', settings,
                                         settings_name='magpie.cookie_expire', default_value=None,
                                         raise_missing=False, raise_not_set=False, print_missing=True)
-    magpie_cookie_name = get_constant('MAGPIE_COOKIE_NAME', settings=settings,
+    magpie_cookie_name = get_constant('MAGPIE_COOKIE_NAME', settings,
                                       settings_name='magpie.cookie_name', default_value='auth_tkt',
                                       raise_missing=False, raise_not_set=False, print_missing=True)
     authn_policy = AuthTktAuthenticationPolicy(
@@ -45,7 +45,7 @@ def auth_config_from_settings(settings):
 
 
 def authomatic_setup(request):
-    magpie_secret = get_constant('MAGPIE_SECRET', settings=request.registry.settings, settings_name='magpie.secret')
+    magpie_secret = get_constant('MAGPIE_SECRET', request, settings_name='magpie.secret')
     return Authomatic(
         config=authomatic_config(request),
         secret=magpie_secret,
diff --git a/magpie/utils.py b/magpie/utils.py
index 6747507c7..60ec24441 100644
--- a/magpie/utils.py
+++ b/magpie/utils.py
@@ -53,7 +53,7 @@ def patch_magpie_url(container):
         get_magpie_url(settings)
     except ConfigurationError:
         magpie_url_template = 'http://{hostname}:{port}'
-        port = get_constant('MAGPIE_PORT', settings=settings, settings_name='magpie.port')
+        port = get_constant('MAGPIE_PORT', settings, settings_name='magpie.port')
         if port:
             settings['magpie.port'] = port
         hostname = get_constant('HOSTNAME')
diff --git a/tests/interfaces.py b/tests/interfaces.py
index 3a9abc2d3..797465ca3 100644
--- a/tests/interfaces.py
+++ b/tests/interfaces.py
@@ -1,3 +1,13 @@
+from magpie.api.api_rest_schemas import SwaggerGenerator
+from magpie.common import JSON_TYPE
+from magpie.constants import get_constant
+from magpie.models import resource_type_dict
+from magpie.services import service_type_dict
+from magpie.utils import get_twitcher_protected_service_url
+from tests import utils, runner
+# noinspection PyPackageRequirements
+from six.moves.urllib.parse import urlparse
+from distutils.version import LooseVersion
 import unittest
 import warnings
 # noinspection PyPackageRequirements
@@ -7,16 +17,6 @@
 import mock
 import yaml
 import six
-# noinspection PyPackageRequirements
-from six.moves.urllib.parse import urlparse
-from distutils.version import LooseVersion
-from magpie.api.api_rest_schemas import SwaggerGenerator
-from magpie.common import JSON_TYPE
-from magpie.constants import get_constant
-from magpie.models import resource_type_dict
-from magpie.services import service_type_dict
-from magpie.utils import get_twitcher_protected_service_url
-from tests import utils, runner
 
 
 # don't use 'unittest.TestCase' base
diff --git a/tests/utils.py b/tests/utils.py
index b5398df56..ffa64f7af 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -15,7 +15,6 @@
 import requests
 import warnings
 import json
-import mock
 import six
 # noinspection PyPackageRequirements
 import pytest
@@ -131,6 +130,7 @@ def get_test_magpie_app(settings=None):
     config.include('ziggurat_foundations.ext.pyramid.sign_in')
     config.include('ziggurat_foundations.ext.pyramid.get_user')
     config.registry.settings['magpie.url'] = 'http://localhost:80'
+    config.registry.settings['magpie.db_migration'] = False
     if settings:
         config.registry.settings.update(settings)
     # create the test application
@@ -605,11 +605,9 @@ class TestSetup(object):
     @staticmethod
     def get_Version(test_class):
         app_or_url = get_app_or_url(test_class)
-        # use mocked call since migration seem to fail on travis-ci
-        with mock.patch('magpie.api.home.home.get_database_revision', return_value='mocked-revision'):
-            resp = test_request(app_or_url, 'GET', '/version',
-                                headers=test_class.json_headers,
-                                cookies=test_class.cookies)
+        resp = test_request(app_or_url, 'GET', '/version',
+                            headers=test_class.json_headers,
+                            cookies=test_class.cookies)
         json_body = check_response_basic_info(resp, 200)
         return json_body['version']