Merge pull request #156 from Ouranosinc/yaml-req

use pyyaml beta with security fix

.gitignore

@@ -16,6 +16,7 @@ docs/magpie.login.rst
 docs/magpie.management.rst
 docs/magpie.tests.rst
 docs/magpie.rst
+downloads/
 magpieenv/
 magpie/api_docs/
 magpie/api_test/
@@ -29,6 +30,6 @@ magpie/share/
 magpie.egg-info/
 src/
 share/
+tests/results.xml
 /gunicorn.app.wsgiapp
-downloads/
 /sys

.travis.yml

@@ -16,7 +16,6 @@ env:
     - CONDA_HOME=$HOME/conda
     - DOWNLOAD_CACHE=$HOME/downloads
   matrix:
-    # FIXME: local login not functional, cannot run local tests on travis
     - TEST_TARGET=test-local    START_TARGET=
     - TEST_TARGET=test-remote   START_TARGET=start
     - TEST_TARGET=coverage      START_TARGET=
@@ -56,6 +55,7 @@ install:
     fi
   - make install install-dev
   - make version
+  - ${CONDA_PREFIX}/bin/pip freeze
 before_script:
   - psql -c 'create database magpie;' -U postgres
   - echo $CONDA_PREFIX
@@ -71,6 +71,11 @@ before_script:
 script:
   - export MAGPIE_LOG_LEVEL=INFO
   - make $START_TARGET $TEST_TARGET
+  # run static code checks only once
+  - |
+    if [ "${TRAVIS_PYTHON_VERSION}" -eq "3.6" ]; then
+      make lint
+    fi
 notifications:
   email: false
 after_success:

Makefile

@@ -110,22 +110,25 @@ clean-test:
 .PHONY: lint
 lint: install-dev
 	@echo "Checking code style with flake8..."
-	@bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; flake8 magpie tests --ignore=E501,W291 || true'
+	@bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; flake8'
 
 .PHONY: test
 test: install-dev install
 	@echo "Running tests..."
-	@bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; python setup.py test'
+	bash -c "source $(CONDA_HOME)/bin/activate $(CONDA_ENV); \
+		pytest tests -vv --junitxml $(CURDIR)/tests/results.xml"
 
 .PHONY: test-local
 test-local: install-dev install
 	@echo "Running local tests..."
-	@bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; MAGPIE_TEST_REMOTE=false python setup.py test'
+	bash -c "source $(CONDA_HOME)/bin/activate $(CONDA_ENV); \
+		pytest tests -vv -m 'not remote' --junitxml $(CURDIR)/tests/results.xml"
 
 .PHONY: test-remote
 test-remote: install-dev install
 	@echo "Running remote tests..."
-	@bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; MAGPIE_TEST_LOCAL=false python setup.py test'
+	bash -c "source $(CONDA_HOME)/bin/activate $(CONDA_ENV); \
+		pytest tests -vv -m 'not local' --junitxml $(CURDIR)/tests/results.xml"
 
 .PHONY: test-tox
 test-tox: install-dev install

magpie/adapter/magpieprocess.py

@@ -12,20 +12,29 @@
     HTTPConflict,
     HTTPNotImplemented,
     asbool,
-    Registry,
 )
 
 # import 'process' elements separately than 'twitcher_definitions' because not defined in master
+# noinspection PyUnresolvedReferences
 from twitcher.utils import get_twitcher_url
+# noinspection PyUnresolvedReferences
 from twitcher.config import get_twitcher_configuration, TWITCHER_CONFIGURATION_EMS
-from twitcher.datatype import Process
+# noinspection PyUnresolvedReferences
 from twitcher.exceptions import ProcessNotFound, ProcessRegistrationError
+# noinspection PyUnresolvedReferences
 from twitcher.adapter.default import DefaultAdapter
+# noinspection PyUnresolvedReferences
 from twitcher.store.base import ProcessStore
+# noinspection PyUnresolvedReferences
 from twitcher.visibility import VISIBILITY_PUBLIC, VISIBILITY_PRIVATE, visibility_values
-from typing import List, Optional, Iterable, Union, AnyStr
+# noinspection PyUnresolvedReferences
+from twitcher.datatype import Process  # noqa: F401
+from typing import TYPE_CHECKING
 import six
 import requests
+if TYPE_CHECKING:
+    from typing import List, Optional, Iterable, Union, AnyStr  # noqa: F401
+    from magpie.definitions.pyramid_definitions import Registry  # noqa: F401
 LOGGER = get_logger("TWITCHER")
 
 
@@ -480,5 +489,6 @@ def set_visibility(self, process_id, visibility, request=None):
         # update visibility of process, which will also reflect changes to route permissions during 'list_processes'
         self.default_process_store.set_visibility(process_id, visibility=visibility, request=request)
 
+    # noinspection PyMethodMayBeStatic, PyUnusedLocal
     def clear_processes(self, request=None):
         raise_http(httpError=HTTPNotImplemented, detail="Clear processes not supported via MagpieAdapter.")

magpie/alembic/versions/20671b28c538_change_all_linking_k.py

@@ -33,12 +33,10 @@ def upgrade():
     # inspected keys
     groups_permissions_pkey = insp.get_pk_constraint('groups_permissions')['name']
     groups_pkey = insp.get_pk_constraint('groups')['name']
-    groups_resources_permissions_pkey = \
-    insp.get_pk_constraint('groups_resources_permissions')['name']
+    groups_resources_permissions_pkey = insp.get_pk_constraint('groups_resources_permissions')['name']
     users_groups_pkey = insp.get_pk_constraint('users_groups')['name']
     users_permissions_pkey = insp.get_pk_constraint('users_permissions')['name']
-    users_resources_permissions_pkey = \
-    insp.get_pk_constraint('users_resources_permissions')['name']
+    users_resources_permissions_pkey = insp.get_pk_constraint('users_resources_permissions')['name']
 
     op.drop_constraint('groups_pkey', 'groups', type_='primary')
 

magpie/alembic/versions/438c27ec1c9_normalize_constraint_and_key_names.py

@@ -29,8 +29,8 @@
 
 
 def upgrade():
-    c = get_context()
-    insp = Inspector.from_engine(c.connection.engine)
+    context = get_context()
+    insp = Inspector.from_engine(context.connection.engine)
     # existing migration
     # pre naming convention keys
     groups_permissions_pkey = 'groups_permissions_pkey'
@@ -40,7 +40,7 @@ def upgrade():
     users_permissions_pkey = 'users_permissions_pkey'
     users_resources_permissions_pkey = 'users_resources_permissions_pkey'
 
-    if isinstance(c.connection.engine.dialect, PGDialect):
+    if isinstance(context.connection.engine.dialect, PGDialect):
         op.execute(
             'ALTER INDEX groups_unique_group_name_key RENAME to ix_groups_uq_group_name_key')
 

magpie/api/api_except.py

@@ -24,7 +24,7 @@
 
 
 # noinspection PyPep8Naming
-def verify_param(
+def verify_param(   # noqa: E126
                  # --- verification values ---
                  param,                             # type: Any
                  paramCompare=None,                 # type: Optional[Union[Any, List[Any]]]
@@ -48,7 +48,7 @@ def verify_param(
                  isIn=False,                        # type: Optional[bool]
                  isEqual=False,                     # type: Optional[bool]
                  ofType=False,                      # type: Optional[bool]
-                 ):                                 # type: (...) -> None
+                ):                                  # type: (...) -> None
     """
     Evaluate various parameter combinations given the requested verification flags.
     Given a failing verification, directly raises the specified ``httpError``.
@@ -415,9 +415,10 @@ def generate_response_http_format(httpClass, httpKWArgs, jsonContent, outputType
         # otherwise json is contained within the html <body> section
         elif outputType == HTML_TYPE:
             # add preformat <pre> section to output as is within the <body> section
-            htmlBody = httpClass.explanation + "<br><h2>Exception Details</h2>" + \
-                       "<pre style='word-wrap: break-word; white-space: pre-wrap;'>" + \
-                       jsonContent + "</pre>"
+            htmlBody = "{}<br><h2>Exception Details</h2>" \
+                       "<br><h2>Exception Details</h2>" \
+                       "<pre style='word-wrap: break-word; white-space: pre-wrap;'>{}</pre>" \
+                .format(httpClass.explanation, jsonContent)
             httpResponse = httpClass(body_template=htmlBody, content_type=HTML_TYPE, **httpKWArgs)
 
         # default back to plain text

magpie/api/api_requests.py

@@ -1,8 +1,21 @@
-from magpie.definitions import ziggurat_definitions as zig
-from magpie.definitions.typedefs import Any, AnyStr, Optional
 from magpie.api.api_except import evaluate_call, verify_param
-from magpie.api.api_rest_schemas import *
+from magpie.api import api_rest_schemas as s
+from magpie.common import JSON_TYPE
+from magpie.constants import get_constant
+from magpie.definitions import ziggurat_definitions as zig
+from magpie.definitions.pyramid_definitions import (
+    IAuthenticationPolicy,
+    HTTPForbidden,
+    HTTPNotFound,
+    HTTPNotAcceptable,
+    HTTPUnprocessableEntity,
+    HTTPInternalServerError,
+)
 from magpie import models
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from magpie.definitions.pyramid_definitions import Request  # noqa: F401
+    from magpie.definitions.typedefs import Any, AnyStr, Optional  # noqa: F401
 
 
 def get_request_method_content(request):
@@ -47,7 +60,7 @@ def get_permission_multiformat_post_checked(request, service_resource, permissio
 def get_value_multiformat_post_checked(request, key, default=None):
     val = get_multiformat_any(request, key, default=default)
     verify_param(val, notNone=True, notEmpty=True, httpError=HTTPUnprocessableEntity,
-                 paramName=key, msgOnFail=UnprocessableEntityResponseSchema.description)
+                 paramName=key, msgOnFail=s.UnprocessableEntityResponseSchema.description)
     return val
 
 
@@ -76,9 +89,9 @@ def get_user(request, user_name_or_token):
             anonymous_user = get_constant('MAGPIE_ANONYMOUS_USER')
             anonymous = evaluate_call(lambda: zig.UserService.by_user_name(anonymous_user, db_session=request.db),
                                       fallback=lambda: request.db.rollback(), httpError=HTTPForbidden,
-                                      msgOnFail=User_CheckAnonymous_ForbiddenResponseSchema.description)
+                                      msgOnFail=s.User_CheckAnonymous_ForbiddenResponseSchema.description)
             verify_param(anonymous, notNone=True, httpError=HTTPNotFound,
-                         msgOnFail=User_CheckAnonymous_NotFoundResponseSchema.description)
+                         msgOnFail=s.User_CheckAnonymous_NotFoundResponseSchema.description)
             return anonymous
     else:
         authn_policy = request.registry.queryUtility(IAuthenticationPolicy)
@@ -89,14 +102,15 @@ def get_user(request, user_name_or_token):
             raise HTTPForbidden()
         user = evaluate_call(lambda: zig.UserService.by_user_name(user_name_or_token, db_session=request.db),
                              fallback=lambda: request.db.rollback(),
-                             httpError=HTTPForbidden, msgOnFail=User_GET_ForbiddenResponseSchema.description)
-        verify_param(user, notNone=True, httpError=HTTPNotFound, msgOnFail=User_GET_NotFoundResponseSchema.description)
+                             httpError=HTTPForbidden, msgOnFail=s.User_GET_ForbiddenResponseSchema.description)
+        verify_param(user, notNone=True, httpError=HTTPNotFound,
+                     msgOnFail=s.User_GET_NotFoundResponseSchema.description)
         return user
 
 
 def get_user_matchdict_checked_or_logged(request, user_name_key='user_name'):
     logged_user_name = get_constant('MAGPIE_LOGGED_USER')
-    logged_user_path = UserAPI.path.replace('{' + user_name_key + '}', logged_user_name)
+    logged_user_path = s.UserAPI.path.replace('{' + user_name_key + '}', logged_user_name)
     if user_name_key not in request.matchdict and request.path_info.startswith(logged_user_path):
         return get_user(request, logged_user_name)
     return get_user_matchdict_checked(request, user_name_key)
@@ -110,32 +124,32 @@ def get_user_matchdict_checked(request, user_name_key='user_name'):
 def get_group_matchdict_checked(request, group_name_key='group_name'):
     group_name = get_value_matchdict_checked(request, group_name_key)
     group = evaluate_call(lambda: zig.GroupService.by_group_name(group_name, db_session=request.db),
-                          fallback=lambda: request.db.rollback(),
-                          httpError=HTTPForbidden, msgOnFail=Group_MatchDictCheck_ForbiddenResponseSchema.description)
+                          fallback=lambda: request.db.rollback(), httpError=HTTPForbidden,
+                          msgOnFail=s.Group_MatchDictCheck_ForbiddenResponseSchema.description)
     verify_param(group, notNone=True, httpError=HTTPNotFound,
-                 msgOnFail=Group_MatchDictCheck_NotFoundResponseSchema.description)
+                 msgOnFail=s.Group_MatchDictCheck_NotFoundResponseSchema.description)
     return group
 
 
 def get_resource_matchdict_checked(request, resource_name_key='resource_id'):
     resource_id = get_value_matchdict_checked(request, resource_name_key)
     resource_id = evaluate_call(lambda: int(resource_id), httpError=HTTPNotAcceptable,
-                                msgOnFail=Resource_MatchDictCheck_NotAcceptableResponseSchema.description)
+                                msgOnFail=s.Resource_MatchDictCheck_NotAcceptableResponseSchema.description)
     resource = evaluate_call(lambda: zig.ResourceService.by_resource_id(resource_id, db_session=request.db),
                              fallback=lambda: request.db.rollback(), httpError=HTTPForbidden,
-                             msgOnFail=Resource_MatchDictCheck_ForbiddenResponseSchema.description)
+                             msgOnFail=s.Resource_MatchDictCheck_ForbiddenResponseSchema.description)
     verify_param(resource, notNone=True, httpError=HTTPNotFound,
-                 msgOnFail=Resource_MatchDictCheck_NotFoundResponseSchema.description)
+                 msgOnFail=s.Resource_MatchDictCheck_NotFoundResponseSchema.description)
     return resource
 
 
 def get_service_matchdict_checked(request, service_name_key='service_name'):
     service_name = get_value_matchdict_checked(request, service_name_key)
     service = evaluate_call(lambda: models.Service.by_service_name(service_name, db_session=request.db),
                             fallback=lambda: request.db.rollback(), httpError=HTTPForbidden,
-                            msgOnFail=Service_MatchDictCheck_ForbiddenResponseSchema.description)
+                            msgOnFail=s.Service_MatchDictCheck_ForbiddenResponseSchema.description)
     verify_param(service, notNone=True, httpError=HTTPNotFound, content={u'service_name': service_name},
-                 msgOnFail=Service_MatchDictCheck_NotFoundResponseSchema.description)
+                 msgOnFail=s.Service_MatchDictCheck_NotFoundResponseSchema.description)
     return service
 
 
@@ -150,7 +164,7 @@ def get_permission_matchdict_checked(request, service_resource, permission_name_
 def get_value_matchdict_checked(request, key):
     val = request.matchdict.get(key)
     verify_param(val, notNone=True, notEmpty=True, httpError=HTTPUnprocessableEntity,
-                 paramName=key, msgOnFail=UnprocessableEntityResponseSchema.description)
+                 paramName=key, msgOnFail=s.UnprocessableEntityResponseSchema.description)
     return val
 
 

magpie/api/api_rest_schemas.py

@@ -1,5 +1,19 @@
-from magpie.definitions.cornice_definitions import *
-from magpie.definitions.pyramid_definitions import *
+from magpie.definitions.cornice_definitions import colander, Service, CorniceSwagger, get_services
+from magpie.definitions.pyramid_definitions import (
+    HTTPOk,
+    HTTPCreated,
+    HTTPFound,
+    HTTPBadRequest,
+    HTTPUnauthorized,
+    HTTPForbidden,
+    HTTPNotFound,
+    HTTPMethodNotAllowed,
+    HTTPNotAcceptable,
+    HTTPConflict,
+    HTTPUnprocessableEntity,
+    HTTPInternalServerError,
+    NO_PERMISSION_REQUIRED,
+)
 from magpie.common import JSON_TYPE
 from magpie.constants import get_constant
 from magpie.utils import get_magpie_url

magpie/api/home/home.py

@@ -1,7 +1,8 @@
 from magpie.api import api_except as ax, api_rest_schemas as s
 from magpie.definitions.pyramid_definitions import NO_PERMISSION_REQUIRED, HTTPOk, view_config
 from magpie.common import JSON_TYPE
-from magpie import db, __meta__
+from magpie.db import get_database_revision
+from magpie import __meta__
 
 
 @s.VersionAPI.get(tags=[s.APITag], api_security=s.SecurityEveryoneAPI, response_schemas=s.Version_GET_responses)
@@ -10,9 +11,15 @@ def get_version(request):
     """
     Version information of the API.
     """
+    version_db = None
+    # noinspection PyBroadException
+    try:
+        version_db = get_database_revision(request.db)
+    except Exception:
+        pass
     version = {
         u'version': __meta__.__version__,
-        u'db_version': db.get_database_revision(request.db)
+        u'db_version': version_db
     }
     return ax.valid_http(httpSuccess=HTTPOk, content=version, contentType=JSON_TYPE,
                          detail=s.Version_GET_OkResponseSchema.description)

magpie/api/login/esgfopenid.py

@@ -11,7 +11,7 @@
 from six.moves.urllib.request import urlopen
 from authomatic.providers.openid import OpenID
 # noinspection PyProtectedMember, PyUnresolvedReferences
-from openid.fetchers import setDefaultFetcher, Urllib2Fetcher
+from openid.fetchers import Urllib2Fetcher  # , setDefaultFetcher
 from magpie.common import get_logger
 LOGGER = get_logger(__name__)