diff --git a/.circleci/config.yml b/.circleci/config.yml index 228ad7b..91d3e30 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -67,6 +67,3 @@ workflows: - build_package: requires: - track_web - filters: - branches: - only: master diff --git a/requirements.txt b/requirements.txt index fb8bc52..1ae0ad5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,12 @@ flask==0.12.4 gunicorn==19.6.0 -pyyaml==3.12 +pyyaml==3.13 python-slugify==1.2.1 Flask-PyMongo==0.5.1 -ujson==1.35 flask-compress==1.4.0 click==6.7 Babel==2.6.0 Flask-Caching==1.4.0 pymongo==3.7.0 +azure-keyvault==1.1.0 +msrestazure==0.5.1 diff --git a/setup.py b/setup.py index 7e30aa3..9f8ffb4 100644 --- a/setup.py +++ b/setup.py @@ -20,15 +20,16 @@ install_requires=[ 'flask==0.12.4', 'gunicorn==19.6.0', - 'pyyaml==3.12', + 'pyyaml==3.13', 'python-slugify==1.2.1', 'pymongo==3.7.0', 'Flask-PyMongo==0.5.1', - 'ujson==1.35', 'flask-compress==1.4.0', 'click==6.7', 'Babel==2.6.0', 'Flask-Caching==1.4.0', + 'azure-keyvault==1.1.0', + 'msrestazure==0.5.1' ], extras_require={ 'development': [ diff --git a/tests/test_models.py b/tests/test_models.py index 1144a8b..ca5fffa 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -170,11 +170,10 @@ def test_all(self, clean_model, domain) -> None: # pylint: disable=no-self-use def test_to_csv_en(self, domain) -> None: # pylint: disable=no-self-use csv_string = models.Domain.to_csv([domain], 'https', 'en') + bytes_in = io.BytesIO(csv_string) - with io.StringIO() as sio: - sio.write(csv_string) - sio.seek(0) - reader = csv.DictReader(sio) + with io.TextIOWrapper(bytes_in, encoding='utf-8-sig', newline='') as wrapped_io: + reader = csv.DictReader(wrapped_io) assert sorted(reader.fieldnames) == [ '3DES', 'Approved Certificate', @@ -224,10 +223,10 @@ def test_to_csv_en(self, domain) -> None: # pylint: disable=no-self-use def test_to_csv_fr(self, domain) -> None: # pylint: disable=no-self-use csv_string = models.Domain.to_csv([domain], 'https', 'fr') - with io.StringIO() as sio: - sio.write(csv_string) - sio.seek(0) - reader = csv.DictReader(sio) + bytes_in = io.BytesIO(csv_string) + + with io.TextIOWrapper(bytes_in, encoding='utf-8-sig', newline='') as wrapped_io: + reader = csv.DictReader(wrapped_io) assert sorted(reader.fieldnames) == [ '3DES', 'Absence de protocoles ou de suites de chiffrement ayant des vulnérabilités connues', @@ -350,10 +349,4 @@ def test_all(self, clean_model, organization) -> None: # pylint: disable=no-self class TestFlag(): def test_get_cache_not_set(self, clean_model) -> None: # pylint: disable=no-self-use - assert not clean_model.Flag.get_cache() - - def test_get_cache_set(self, clean_model) -> None: # pylint: disable=no-self-use - clean_model.Flag.set_cache(True) - assert clean_model.Flag.get_cache() - clean_model.Flag.set_cache(False) - assert not clean_model.Flag.get_cache() + assert clean_model.Flag.get_cache() == "1999-12-31 23:59" \ No newline at end of file diff --git a/track/config.py b/track/config.py index 085f96e..d9c3aa6 100644 --- a/track/config.py +++ b/track/config.py @@ -1,6 +1,14 @@ import os +import sys import random +import logging +from azure.keyvault import KeyVaultClient +from msrestazure.azure_active_directory import MSIAuthentication + +LOGGER = logging.getLogger(__name__) + +A_DAY = 60 * 60 * 24 class Config: DEBUG = False @@ -12,16 +20,27 @@ class Config: def init_app(app): pass - -A_DAY = 60 * 60 * 24 class ProductionConfig(Config): - MONGO_URI = os.environ.get("TRACKER_MONGO_URI", None) + CACHE_TYPE = "filesystem" CACHE_DIR = os.environ.get("TRACKER_CACHE_DIR", "./.cache") CACHE_DEFAULT_TIMEOUT = int(os.environ.get("TRACKER_CACHE_TIMEOUT", A_DAY)) + if os.environ.get('TRACKER_ENV', None) == "production": + if os.environ.get("TRACKER_KEYVAULT_URI", None) is None or os.environ.get("SECRET_NAME_RO", None) is None: + # Error and crash hard: Production should be configured as expected. + LOGGER.error("KeyVault uri or secret name missing from local environment.") + sys.exit(4) + + kv_uri = os.environ.get("TRACKER_KEYVAULT_URI") + kv_secret = os.environ.get("SECRET_NAME_RO") + kv_creds = MSIAuthentication(resource='https://vault.azure.net') + kv_client = KeyVaultClient(kv_creds) + MONGO_URI = kv_client.get_secret(kv_uri, kv_secret, "").value + @staticmethod def init_app(app): + Config.init_app(app) import logging @@ -31,13 +50,14 @@ def init_app(app): handler.setLevel(logging.ERROR) app.logger.addHandler(handler) - class DevelopmentConfig(Config): + DEBUG = True CACHE_TYPE = "simple" class TestingConfig(Config): + TESTING = True MONGO_URI = "mongodb://localhost:27017/track_{rand}".format( rand=random.randint(0, 1000) diff --git a/track/data.py b/track/data.py index 5a9ba4f..d544349 100644 --- a/track/data.py +++ b/track/data.py @@ -1,4 +1,3 @@ -import typing # Mapping report/domain/organization field names to display names. class MultiLingualString: diff --git a/track/helpers.py b/track/helpers.py index bac4b29..cc11b89 100644 --- a/track/helpers.py +++ b/track/helpers.py @@ -1,4 +1,3 @@ -import os import pkg_resources import yaml import datetime diff --git a/track/models.py b/track/models.py index a2c30e0..f8f7e95 100644 --- a/track/models.py +++ b/track/models.py @@ -11,7 +11,6 @@ # coordinated here. db = PyMongo() - QueryError = PyMongoError # Data loads should clear the entire database first. @@ -59,7 +58,6 @@ class Domain: # # https: { ... } # - @staticmethod def find(domain_name: str) -> typing.Dict: return db.db.meta.find_one( @@ -73,7 +71,7 @@ def find(domain_name: str) -> typing.Dict: ) @staticmethod - def find_all(query: typing.Dict, projection: typing.Dict={'_id': False, '_collection': False}) -> typing.Dict: + def find_all(query: typing.Dict, projection: typing.Dict = {'_id': False, '_collection': False}) -> typing.Dict: return db.db.meta.find( { '_collection': 'domains', @@ -131,9 +129,14 @@ def all() -> typing.Iterable[typing.Dict]: return db.db.meta.find({'_collection': 'domains'}, {'_id': False, '_collection': False}) @staticmethod - def to_csv(domains: typing.Iterable[typing.Dict], report_type: str, language: str) -> str: - output = io.StringIO() - writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC) + def to_csv(domains: typing.Iterable[typing.Dict], report_type: str, language: str) -> bytes: + if report_type not in track.data.CSV_FIELDS: + return {} + + output = io.BytesIO() + iowrap = io.TextIOWrapper(output, encoding='utf-8-sig', newline='', write_through=True) + + writer = csv.writer(iowrap, quoting=csv.QUOTE_NONNUMERIC) def value_for(value: typing.Union[str, list, bool]) -> str: # if it's a list, convert it to a list of strings and join @@ -226,7 +229,7 @@ def find(slug: str) -> typing.Dict: return db.db.meta.find_one({'_collection': 'organizations', 'slug': slug}, {'_id': False, '_collection': False}) @staticmethod - def find_all(query: typing.Dict, projection: typing.Dict={'_id': False, '_collection': False}) -> typing.Dict: + def find_all(query: typing.Dict, projection: typing.Dict = {'_id': False, '_collection': False}) -> typing.Dict: return db.db.meta.find( { '_collection': 'organizations', @@ -242,11 +245,6 @@ def all() -> typing.Iterable[typing.Dict]: class Flag: @staticmethod - def get_cache() -> bool: + def get_cache() -> str: flags = db.db.meta.find_one({"_collection": "flags"}) - return flags['cache'] if flags else False - - @staticmethod - def set_cache(state: bool) -> None: - db.db.meta.update_one({"_collection": "flags"}, {"$set": {"cache": state}}, upsert=True) - + return flags['cache'] if flags else "1999-12-31 23:59" diff --git a/track/static/js/tables.js b/track/static/js/tables.js index 497c166..325d9f7 100644 --- a/track/static/js/tables.js +++ b/track/static/js/tables.js @@ -63,6 +63,12 @@ var Tables = { Utils.updatePagination(); }); + table.on("page.dt",function(){ + /* scroll page to top of table on page change */ + var top = $(".dataTable").offset().top; + $("html, body").animate({ scrollTop: top }, "slow"); + }); + return table; }, diff --git a/track/templates/en/guidance.html b/track/templates/en/guidance.html index dddd110..4c45ac6 100644 --- a/track/templates/en/guidance.html +++ b/track/templates/en/guidance.html @@ -18,8 +18,8 @@

Read guidance

@@ -27,14 +27,14 @@

Read guidance

-
  • Provide an up-to-date list of all domain and sub-domains of the publicly-accessible websites and web services to the following website: Submit your institution's domains.
    • +
  • Provide an up-to-date list of all domain and sub-domains of the publicly-accessible websites and web services to the following website: Submit your institution's domains.
  • Perform an assessment of the domains and sub-domains to determine the status of the configuration. Tools available to support this activity includes GC HTTPS Dashboard, SSL Labs, Hardenize, etc.
  • Develop a prioritized implementation schedule for each of the affected websites and web services, following the recommended prioritization approach in the ITPIN: @@ -56,7 +56,7 @@

    Read guidance

    • -
  • Based on the assessment, and using the guidance available on GCpedia, the following activities may be required: +
  • Based on the assessment, and using the guidance available on GCpedia, the following activities may be required: