From aaf3509f0c5b44debc204d3d2d50ba0bcdc4ecfb Mon Sep 17 00:00:00 2001 From: Dave Samojlenko Date: Tue, 23 Oct 2018 13:47:47 -0400 Subject: [PATCH 01/12] build all branches (#101) --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 228ad7b..91d3e30 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -67,6 +67,3 @@ workflows: - build_package: requires: - track_web - filters: - branches: - only: master From e499631c143b447103e5a43902a7baefd0740137 Mon Sep 17 00:00:00 2001 From: John O'Brien Date: Fri, 26 Oct 2018 21:22:25 -0400 Subject: [PATCH 02/12] Small tweak to how cache invalidation works to remove necessity for write access in track-web (#100) * Minor changes to cache invalidation to get rid of write access rqmt. * ugh tests. * get_cache should be type-hinting a str return, not bool. Also, I was returning both a datetime, or a str. Whoops. * sigh. tests. remember the tests. --- tests/test_models.py | 8 +------- track/models.py | 9 ++------- track/views.py | 28 ++++++++++++++++++++++++---- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/tests/test_models.py b/tests/test_models.py index 1144a8b..6336fdf 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -350,10 +350,4 @@ def test_all(self, clean_model, organization) -> None: # pylint: disable=no-self class TestFlag(): def test_get_cache_not_set(self, clean_model) -> None: # pylint: disable=no-self-use - assert not clean_model.Flag.get_cache() - - def test_get_cache_set(self, clean_model) -> None: # pylint: disable=no-self-use - clean_model.Flag.set_cache(True) - assert clean_model.Flag.get_cache() - clean_model.Flag.set_cache(False) - assert not clean_model.Flag.get_cache() + assert clean_model.Flag.get_cache() == "1999-12-31 23:59" \ No newline at end of file diff --git a/track/models.py b/track/models.py index a2c30e0..84e8260 100644 --- a/track/models.py +++ b/track/models.py @@ -242,11 +242,6 @@ def all() -> typing.Iterable[typing.Dict]: class Flag: @staticmethod - def get_cache() -> bool: + def get_cache() -> str: flags = db.db.meta.find_one({"_collection": "flags"}) - return flags['cache'] if flags else False - - @staticmethod - def set_cache(state: bool) -> None: - db.db.meta.update_one({"_collection": "flags"}, {"$set": {"cache": state}}, upsert=True) - + return flags['cache'] if flags else "1999-12-31 23:59" diff --git a/track/views.py b/track/views.py index 2c2cbe8..827fcb7 100644 --- a/track/views.py +++ b/track/views.py @@ -4,6 +4,8 @@ from flask import render_template, Response, abort, request, redirect import ujson +from datetime import datetime + from track.data import FIELD_MAPPING from track import models from track.cache import cache @@ -256,7 +258,25 @@ def handle_invalid_usage(error): @app.before_request def verify_cache(): - if not models.Flag.get_cache(): - app.logger.info('Clearing cache...') - cache.clear() - models.Flag.set_cache(True) + cur_time = datetime.now() + + if cache.get('last-cache-bump') is None: + cache.set('last-cache-bump', cur_time) + + if cache.get('cache-timer') is None: + # Set up a 5 minute timer to minimize flailing. + cache.set('cache-timer', cur_time, 5 * 60) + + # Let's check the remote cache flag (time value) + remote_signal = datetime.strptime(models.Flag.get_cache(), "%Y-%m-%d %H:%M") + app.logger.warn("TRACK_CACHE: remote signal @ {}".format(remote_signal)) + + if remote_signal is not None: + if cache.get('last-cache-bump') < remote_signal: + app.logger.warn("TRACK_CACHE: Cache reset @ {} due to signal @ {}".format(cur_time, remote_signal)) + cache.clear() + # We've blown the whole cache, so reset the cache timer, and the remote val. + cache.set('cache-timer', cur_time, 5 * 60) + cache.set('last-cache-bump', remote_signal) + else: + app.logger.error("TRACK_CACHE: remote cache datetime was None. Danger Will Robinson.") From e85d2ee1130a0c05e8c737ff65462169940ac517 Mon Sep 17 00:00:00 2001 From: John O'Brien Date: Fri, 26 Oct 2018 21:25:12 -0400 Subject: [PATCH 03/12] Add the utf-8 byte order marker to simplify issues with loading to Excel (#97) * Add the utf-8 byte order marker to simplify issues with loading to Excel * Brought tests suites inline with models.py, handle the utf-8 BOM, and expect bytes over the wire. * Whoops. params where they should have been. --- tests/test_models.py | 15 +++++++-------- track/models.py | 6 ++++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/test_models.py b/tests/test_models.py index 6336fdf..ca5fffa 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -170,11 +170,10 @@ def test_all(self, clean_model, domain) -> None: # pylint: disable=no-self-use def test_to_csv_en(self, domain) -> None: # pylint: disable=no-self-use csv_string = models.Domain.to_csv([domain], 'https', 'en') + bytes_in = io.BytesIO(csv_string) - with io.StringIO() as sio: - sio.write(csv_string) - sio.seek(0) - reader = csv.DictReader(sio) + with io.TextIOWrapper(bytes_in, encoding='utf-8-sig', newline='') as wrapped_io: + reader = csv.DictReader(wrapped_io) assert sorted(reader.fieldnames) == [ '3DES', 'Approved Certificate', @@ -224,10 +223,10 @@ def test_to_csv_en(self, domain) -> None: # pylint: disable=no-self-use def test_to_csv_fr(self, domain) -> None: # pylint: disable=no-self-use csv_string = models.Domain.to_csv([domain], 'https', 'fr') - with io.StringIO() as sio: - sio.write(csv_string) - sio.seek(0) - reader = csv.DictReader(sio) + bytes_in = io.BytesIO(csv_string) + + with io.TextIOWrapper(bytes_in, encoding='utf-8-sig', newline='') as wrapped_io: + reader = csv.DictReader(wrapped_io) assert sorted(reader.fieldnames) == [ '3DES', 'Absence de protocoles ou de suites de chiffrement ayant des vulnérabilités connues', diff --git a/track/models.py b/track/models.py index 84e8260..46efdc2 100644 --- a/track/models.py +++ b/track/models.py @@ -132,8 +132,10 @@ def all() -> typing.Iterable[typing.Dict]: @staticmethod def to_csv(domains: typing.Iterable[typing.Dict], report_type: str, language: str) -> str: - output = io.StringIO() - writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC) + output = io.BytesIO() + iowrap = io.TextIOWrapper(output, encoding='utf-8-sig', newline='', write_through=True) + + writer = csv.writer(iowrap, quoting=csv.QUOTE_NONNUMERIC) def value_for(value: typing.Union[str, list, bool]) -> str: # if it's a list, convert it to a list of strings and join From 7d160dbfdbf5fd7c040372bb48cf251e3d2b1e3c Mon Sep 17 00:00:00 2001 From: sayaHub Date: Fri, 9 Nov 2018 14:10:59 -0500 Subject: [PATCH 04/12] Hotfix align with canada req (#103) * - removed Beta banner - removed Bold links in some pages - add Terms and Conditions in footers * forgot to remove bold for links for modal (How to read this table?) * removed some unwanted space --- track/templates/en/guidance.html | 12 ++++++------ track/templates/en/help.html | 12 ++++++------ track/templates/en/index.html | 4 ++-- track/templates/fr/guidance.html | 12 ++++++------ track/templates/fr/help.html | 12 ++++++------ track/templates/fr/index.html | 6 +++--- track/templates/includes/en/beta-bar.html | 9 +-------- track/templates/includes/en/footer.html | 3 ++- track/templates/includes/en/header.html | 4 ++-- track/templates/includes/en/modal.html | 4 ++-- track/templates/includes/fr/beta-bar.html | 9 +-------- track/templates/includes/fr/footer.html | 4 +++- track/templates/includes/fr/header.html | 4 ++-- track/templates/includes/fr/modal.html | 4 ++-- 14 files changed, 44 insertions(+), 55 deletions(-) diff --git a/track/templates/en/guidance.html b/track/templates/en/guidance.html index dddd110..4c45ac6 100644 --- a/track/templates/en/guidance.html +++ b/track/templates/en/guidance.html @@ -18,8 +18,8 @@

Read guidance

@@ -27,14 +27,14 @@

Read guidance

    -
  • Internally available HTTPS Dashboard
    • +
  • Internally available HTTPS Dashboard
  • TBS Application Portfolio Management (APM)
  • Departmental business units
-
  • Provide an up-to-date list of all domain and sub-domains of the publicly-accessible websites and web services to the following website: Submit your institution's domains.
    • +
  • Provide an up-to-date list of all domain and sub-domains of the publicly-accessible websites and web services to the following website: Submit your institution's domains.
  • Perform an assessment of the domains and sub-domains to determine the status of the configuration. Tools available to support this activity includes GC HTTPS Dashboard, SSL Labs, Hardenize, etc.
  • Develop a prioritized implementation schedule for each of the affected websites and web services, following the recommended prioritization approach in the ITPIN: @@ -56,7 +56,7 @@

    Read guidance

    • -
  • Based on the assessment, and using the guidance available on GCpedia, the following activities may be required: +
  • Based on the assessment, and using the guidance available on GCpedia, the following activities may be required:
    • Obtain certificates from a GC-approved certificate source as outlined in the Recommendations for TLS Server Certificates for GC Public Facing Web Services
      • @@ -71,7 +71,7 @@

      Read guidance


      -

      For any questions or concerns related to the ITPIN and related implementation guidance, contact TBS Cybersecurity (zzTBSCybers@tbs-sct.gc.ca).

      +

      For any questions or concerns related to the ITPIN and related implementation guidance, contact TBS Cybersecurity (zzTBSCybers@tbs-sct.gc.ca).

      {% endblock %} diff --git a/track/templates/en/help.html b/track/templates/en/help.html index 3c696a9..ce3ad42 100644 --- a/track/templates/en/help.html +++ b/track/templates/en/help.html @@ -9,12 +9,12 @@

      Get help

      -

      General public

      -

      For interpretation of any aspect of Implementing HTTPS for Secure Web Connections: Information Technology Policy Implementation Notice (ITPIN), contact Treasury Board of Canada Secretariat through Public Enquiries.

      -

      Government of Canada employees

      -

      Individuals at departments should contact their departmental information technology group for any questions regarding this ITPIN. Individuals can also join the HTTPS Everywhere group on GCconnex, or the #HTTPSEverywhere channel on GCcollab Message.

      -

      Individuals from a departmental information technology group may contact the TBS Cyber Security mailbox for interpretations of this ITPIN.

      -

      Individuals with questions about the accuracy of their domain’s compliance data may contact the TBS Cyber Security mailbox. Note that compliance data does not automatically refresh. If you modified your domain recently, there may be a delay before your domain’s information updates.

      +

      General public

      +

      For interpretation of any aspect of Implementing HTTPS for Secure Web Connections: Information Technology Policy Implementation Notice (ITPIN), contact Treasury Board of Canada Secretariat through Public Enquiries.

      +

      Government of Canada employees

      +

      Individuals at departments should contact their departmental information technology group for any questions regarding this ITPIN. Individuals can also join the HTTPS Everywhere group on GCconnex, or the #HTTPSEverywhere channel on GCcollab Message.

      +

      Individuals from a departmental information technology group may contact the TBS Cyber Security mailbox for interpretations of this ITPIN.

      +

      Individuals with questions about the accuracy of their domain’s compliance data may contact the TBS Cyber Security mailbox. Note that compliance data does not automatically refresh. If you modified your domain recently, there may be a delay before your domain’s information updates.

      diff --git a/track/templates/en/index.html b/track/templates/en/index.html index a8b2a21..79819fe 100644 --- a/track/templates/en/index.html +++ b/track/templates/en/index.html @@ -14,10 +14,10 @@

      Track web security compliance

      Making government websites more secure

      -

      Canadians rely on the Government of Canada to provide secure digital services. A new policy notice guides government websites to adopt good web security practices. Track how government sites are becoming more secure.

      +

      Canadians rely on the Government of Canada to provide secure digital services. A new policy notice guides government websites to adopt good web security practices. Track how government sites are becoming more secure.

      -

      Check a government website

      +

      Check a government website

      diff --git a/track/templates/fr/guidance.html b/track/templates/fr/guidance.html index efccf9a..99319d8 100644 --- a/track/templates/fr/guidance.html +++ b/track/templates/fr/guidance.html @@ -18,8 +18,8 @@

      Lires les directives

      @@ -27,14 +27,14 @@

      Lires les directives

      -
    • Fournir une liste à jour de tous les domaines et les sous-domaines des sites Web et des services Web publiquement accessibles au site Web suivant : Soumettez vos noms de domaine institutionnels.
      • +
    • Fournir une liste à jour de tous les domaines et les sous-domaines des sites Web et des services Web publiquement accessibles au site Web suivant : Soumettez vos noms de domaine institutionnels.
    • Effectuer une évaluation des domaines et des sous-domaines afin de déterminer l’état de la configuration. Les outils disponibles afin d’appuyer cette activité comprennent le tableau de bord HTTPS du gouvernement du Canada (GC), SSL Labs et Hardenize, entre autres.
    • Élaborer un calendrier de mise en œuvre priorisé pour chacun des sites Web et des services Web touchés en suivant l’approche de priorisation recommandée dans l’Avis de mise en œuvre de la Politique sur la technologie de l’information (AMPTI) :  @@ -55,7 +55,7 @@

      Lires les directives

      • -
    • Selon l’évaluation, et au moyen des conseils disponibles sur GCpédia, les activités suivantes pourraient être requises :  +
    • Selon l’évaluation, et au moyen des conseils disponibles sur GCpédia, les activités suivantes pourraient être requises : 
      • Obtenir les certificats d’une source de certificat approuvée par le GC, comme l’indiquent les recommandations concernant les certificats de serveur TLS pour les services Web publics du GC.
        • @@ -70,7 +70,7 @@

        Lires les directives


        -

        Pour toute question ou préoccupation concernant cet AMPTI et les conseils connexes concernant la mise en œuvre, veuillez communiquer avec l’unité de la Cybersécurité du SCT (zzTBSCybers@tbs-sct.gc.ca).

        +

        Pour toute question ou préoccupation concernant cet AMPTI et les conseils connexes concernant la mise en œuvre, veuillez communiquer avec l’unité de la Cybersécurité du SCT (zzTBSCybers@tbs-sct.gc.ca).

      • {% endblock %} diff --git a/track/templates/fr/help.html b/track/templates/fr/help.html index a3932cc..0165451 100644 --- a/track/templates/fr/help.html +++ b/track/templates/fr/help.html @@ -9,12 +9,12 @@

      Obtenir de l’aide

      -

      Grand public

      -

      Pour des explications concernant tout aspect de l’application de l’Avis de mise en œuvre de la Politique sur la technologie de l’information (AMPTI) : Mise en œuvre de HTTPS pour les connexions Web sécurisées, veuillez communiquer avec le Secrétariat du Conseil du Trésor du Canada par l’entremise des Renseignements au public.

      -

      Employés du gouvernement du Canada

      -

      Les fonctionnaires des ministères devraient communiquer avec leur groupe de technologies de l’information respectif pour toute question relative à cet AMPTI. Ils peuvent aussi se joindre au groupe HTTPS Everywhere sur GCconnex ou à la chaîne #HTTPSEverywhere sur GCcollab.

      -

      Les membres d’un groupe de technologies de l’information d’un ministère peuvent envoyer un courriel à la Cybersécurité du SCT pour obtenir des explications concernant cet AMPTI.

      -

      Les personnes qui ont des questions au sujet de l’exactitude des données sur la conformité de leur domaine peuvent communiquer par courriel avec la Cybersécurité du SCT. Veuillez noter que les données de conformité ne s’actualisent pas automatiquement. Si vous avez modifié votre domaine récemment, il se peut que ses informations ne soient pas encore mises à jour.

      +

      Grand public

      +

      Pour des explications concernant tout aspect de l’application de l’Avis de mise en œuvre de la Politique sur la technologie de l’information (AMPTI) : Mise en œuvre de HTTPS pour les connexions Web sécurisées, veuillez communiquer avec le Secrétariat du Conseil du Trésor du Canada par l’entremise des Renseignements au public.

      +

      Employés du gouvernement du Canada

      +

      Les fonctionnaires des ministères devraient communiquer avec leur groupe de technologies de l’information respectif pour toute question relative à cet AMPTI. Ils peuvent aussi se joindre au groupe HTTPS Everywhere sur GCconnex ou à la chaîne #HTTPSEverywhere sur GCcollab.

      +

      Les membres d’un groupe de technologies de l’information d’un ministère peuvent envoyer un courriel à la Cybersécurité du SCT pour obtenir des explications concernant cet AMPTI.

      +

      Les personnes qui ont des questions au sujet de l’exactitude des données sur la conformité de leur domaine peuvent communiquer par courriel avec la Cybersécurité du SCT. Veuillez noter que les données de conformité ne s’actualisent pas automatiquement. Si vous avez modifié votre domaine récemment, il se peut que ses informations ne soient pas encore mises à jour.

      diff --git a/track/templates/fr/index.html b/track/templates/fr/index.html index 91556eb..b5c83fb 100644 --- a/track/templates/fr/index.html +++ b/track/templates/fr/index.html @@ -14,10 +14,10 @@

      Suivre la conformité en matière de sécurité Web

      Rendre les sites gouvernementaux plus sécuritaires

      -

      Les Canadiens s'attendent à ce que le gouvernement du Canada leur offre des services en ligne sécurisés. Un nouvel avis de politique vise à assurer que les sites gouvernementaux soient conformes aux bonnes pratiques en matière de sécurité Web. Voyez comment les sites gouvernementaux deviennent plus sécuritaires.

      +

      Les Canadiens s'attendent à ce que le gouvernement du Canada leur offre des services en ligne sécurisés. Un nouvel avis de politique vise à assurer que les sites gouvernementaux soient conformes aux bonnes pratiques en matière de sécurité Web. Voyez comment les sites gouvernementaux deviennent plus sécuritaires.

      -

      Vérifier la conformité d’un site gouvernemental

      +

      Vérifier la conformité d’un site gouvernemental

      @@ -25,7 +25,7 @@

      Rendre les sites gouvernementaux plus sécuritaires
      -

      Domaines qui excécutent
      le protocole HTTPS

      +

      Domaines qui excécutent
      le protocole HTTPS

      diff --git a/track/templates/includes/en/beta-bar.html b/track/templates/includes/en/beta-bar.html index e028ff7..661f1d0 100644 --- a/track/templates/includes/en/beta-bar.html +++ b/track/templates/includes/en/beta-bar.html @@ -1,8 +1 @@ -
      -
      - - Beta - - This is a new service, we are constantly improving. -
      -
      \ No newline at end of file +
      \ No newline at end of file diff --git a/track/templates/includes/en/footer.html b/track/templates/includes/en/footer.html index 7f7b424..1903a6c 100644 --- a/track/templates/includes/en/footer.html +++ b/track/templates/includes/en/footer.html @@ -2,7 +2,8 @@
      diff --git a/track/templates/includes/en/header.html b/track/templates/includes/en/header.html index 06c5a2f..edc2dca 100644 --- a/track/templates/includes/en/header.html +++ b/track/templates/includes/en/header.html @@ -1,7 +1,7 @@

      Check compliance

      Last updated {{ scan_date | display_date("en") }}

      -

      This dashboard reports how federal government websites and web services are meeting good web security practices, as outlined in Information Technology Policy Implementation Notice (ITPIN): Implementing HTTPS for Secure Web Connections.

      -

      How do I read this table?

      +

      This dashboard reports how federal government websites and web services are meeting good web security practices, as outlined in Information Technology Policy Implementation Notice (ITPIN): Implementing HTTPS for Secure Web Connections.

      +

      How do I read this table?

      diff --git a/track/templates/includes/en/modal.html b/track/templates/includes/en/modal.html index 6223272..61133a9 100644 --- a/track/templates/includes/en/modal.html +++ b/track/templates/includes/en/modal.html @@ -6,7 +6,7 @@
      ×

      Compliance definitions

      -

      Table columns report compliance with a section of the policy direction.

      +

      Table columns report compliance with a section of the policy direction.

      Each column shows an overall compliance percentage for each organization or domain. Click “show details” or “show subdomains” to view the column values for individual domains.

      ITPIN Compliant

      Whether a domain is compliant with the entire ITPIN 2018-01 direction.

      @@ -51,7 +51,7 @@

      Free of known weak protocols and

      Uses approved certificates

      -

      Whether a domain uses approved certificates as outlined in the Communication Security Establishment’s Guidance on Securely Configuring Network Protocols (ITSP.40.062).

      +

      Whether a domain uses approved certificates as outlined in the Communication Security Establishment’s Guidance on Securely Configuring Network Protocols (ITSP.40.062).

      Values

      • Yes: The domain uses an approved certificate
        • diff --git a/track/templates/includes/fr/beta-bar.html b/track/templates/includes/fr/beta-bar.html index 896e042..661f1d0 100644 --- a/track/templates/includes/fr/beta-bar.html +++ b/track/templates/includes/fr/beta-bar.html @@ -1,8 +1 @@ -
        -
        - - Bêta - - Ceci est un nouveau service, nous l’améliorons constamment. -
        -
        \ No newline at end of file +
        \ No newline at end of file diff --git a/track/templates/includes/fr/footer.html b/track/templates/includes/fr/footer.html index d02f16b..ef00e55 100644 --- a/track/templates/includes/fr/footer.html +++ b/track/templates/includes/fr/footer.html @@ -3,7 +3,9 @@
        +
      • Avis
        • + +
      diff --git a/track/templates/includes/fr/header.html b/track/templates/includes/fr/header.html index 8f2cca8..830ada0 100644 --- a/track/templates/includes/fr/header.html +++ b/track/templates/includes/fr/header.html @@ -1,7 +1,7 @@

      Vérifier la conformité

      Dernière mise à jour {{ scan_date | display_date("fr") }}

      -

      Ce tableau indique dans quelles proportions les sites et services en ligne gouvernementaux respectent les bonnes pratiques en matière de sécurité Web décrites dans l’Avis de mise en œuvre de la Politique sur la technologie de l’information (AMPTI) : Mise en œuvre de HTTPS pour les connexions Web sécurisées.

      -

      Comment interpréter ce tableau?

      +

      Ce tableau indique dans quelles proportions les sites et services en ligne gouvernementaux respectent les bonnes pratiques en matière de sécurité Web décrites dans l’Avis de mise en œuvre de la Politique sur la technologie de l’information (AMPTI) : Mise en œuvre de HTTPS pour les connexions Web sécurisées.

      +

      Comment interpréter ce tableau?

      diff --git a/track/templates/includes/fr/modal.html b/track/templates/includes/fr/modal.html index e47a655..18cdc83 100644 --- a/track/templates/includes/fr/modal.html +++ b/track/templates/includes/fr/modal.html @@ -6,7 +6,7 @@
      ×

      Critères de conformité

      -

      Les colonnes du tableau font état de la conformité aux critères énumérés dans les directives de la Politique.

      +

      Les colonnes du tableau font état de la conformité aux critères énumérés dans les directives de la Politique.

      Chaque colonne indique le pourcentage global de conformité de chaque organisation ou domaine. Cliquez sur « Afficher les détails » ou « Afficher les sous-domaines » pour afficher les valeurs des colonnes pour des domaines spécifiques.

      Conforme à l’AMPTI

      Indique si un domaine est conforme à l’ensemble des directives de l’AMPTI 2018-01.

      @@ -51,7 +51,7 @@

      Absence de protocoles ou de suite

      Utilise des certificats approuvés

      -

      Indique si le domaine utilise des certificats approuvés tel que recommandé dans les Conseils sur la configuration sécurisée des protocoles réseau (ITSP.40.062) du Centre de la sécurité des télécommunications.

      +

      Indique si le domaine utilise des certificats approuvés tel que recommandé dans les Conseils sur la configuration sécurisée des protocoles réseau (ITSP.40.062) du Centre de la sécurité des télécommunications.

      Valeurs

      • Oui : Le domaine utilise un certificat approuvé.
        • From 11f38cc033fb258baa41ad2915aee968a167813c Mon Sep 17 00:00:00 2001 From: John O'Brien Date: Sun, 2 Dec 2018 10:01:16 -0500 Subject: [PATCH 05/12] KeyVault and Managed Service Identities (#104) * Minor tweaks to config to enable usage of Azure Managed Service Identities in combination with Azure KeyVault. * Make Keyvault mandatory for production config, takedown gunicorn (sys.exit(4) when production envvars missing. * Put a production-flag guard around the KV code, since there didn't seem to be a clean way to push it into an __init__ . --- requirements.txt | 2 ++ setup.py | 2 ++ track/config.py | 28 ++++++++++++++++++++++++---- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index fb8bc52..0586a3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,5 @@ click==6.7 Babel==2.6.0 Flask-Caching==1.4.0 pymongo==3.7.0 +azure-keyvault==1.1.0 +msrestazure==0.5.1 diff --git a/setup.py b/setup.py index 7e30aa3..1f832b2 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,8 @@ 'click==6.7', 'Babel==2.6.0', 'Flask-Caching==1.4.0', + 'azure-keyvault==1.1.0', + 'msrestazure==0.5.1' ], extras_require={ 'development': [ diff --git a/track/config.py b/track/config.py index 085f96e..d9c3aa6 100644 --- a/track/config.py +++ b/track/config.py @@ -1,6 +1,14 @@ import os +import sys import random +import logging +from azure.keyvault import KeyVaultClient +from msrestazure.azure_active_directory import MSIAuthentication + +LOGGER = logging.getLogger(__name__) + +A_DAY = 60 * 60 * 24 class Config: DEBUG = False @@ -12,16 +20,27 @@ class Config: def init_app(app): pass - -A_DAY = 60 * 60 * 24 class ProductionConfig(Config): - MONGO_URI = os.environ.get("TRACKER_MONGO_URI", None) + CACHE_TYPE = "filesystem" CACHE_DIR = os.environ.get("TRACKER_CACHE_DIR", "./.cache") CACHE_DEFAULT_TIMEOUT = int(os.environ.get("TRACKER_CACHE_TIMEOUT", A_DAY)) + if os.environ.get('TRACKER_ENV', None) == "production": + if os.environ.get("TRACKER_KEYVAULT_URI", None) is None or os.environ.get("SECRET_NAME_RO", None) is None: + # Error and crash hard: Production should be configured as expected. + LOGGER.error("KeyVault uri or secret name missing from local environment.") + sys.exit(4) + + kv_uri = os.environ.get("TRACKER_KEYVAULT_URI") + kv_secret = os.environ.get("SECRET_NAME_RO") + kv_creds = MSIAuthentication(resource='https://vault.azure.net') + kv_client = KeyVaultClient(kv_creds) + MONGO_URI = kv_client.get_secret(kv_uri, kv_secret, "").value + @staticmethod def init_app(app): + Config.init_app(app) import logging @@ -31,13 +50,14 @@ def init_app(app): handler.setLevel(logging.ERROR) app.logger.addHandler(handler) - class DevelopmentConfig(Config): + DEBUG = True CACHE_TYPE = "simple" class TestingConfig(Config): + TESTING = True MONGO_URI = "mongodb://localhost:27017/track_{rand}".format( rand=random.randint(0, 1000) From 05c6cca0027681fa86c443bb12d3bcc2bb18a56c Mon Sep 17 00:00:00 2001 From: John O'Brien Date: Tue, 11 Dec 2018 11:24:25 -0500 Subject: [PATCH 06/12] Added missing view return (#108) Sidenote: It doesn't look like we're leveraging this api endpoint anymore, so flag for review during housekeeping. --- track/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/track/views.py b/track/views.py index 827fcb7..ac96893 100644 --- a/track/views.py +++ b/track/views.py @@ -109,6 +109,7 @@ def domain_report_csv(report_name, language): response = Response(models.Domain.to_csv(domains, report_name, language)) response.headers["Content-Type"] = "text/csv" + return response @app.route("/data/domains-table.json") @cache.cached() From f1842553904b76ca4000ae2609c6bb2da9fdab7b Mon Sep 17 00:00:00 2001 From: John O'Brien Date: Tue, 11 Dec 2018 14:12:13 -0500 Subject: [PATCH 07/12] Add guard to the 'to_csv' to avoid failing on invalid report name requests. (#109) * Added guard on the 'to_csv' function to protect against keyErrors for invalid report names, and to align with default return empty dict for data/json api's --- track/models.py | 2 ++ track/views.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/track/models.py b/track/models.py index 46efdc2..0b77136 100644 --- a/track/models.py +++ b/track/models.py @@ -132,6 +132,8 @@ def all() -> typing.Iterable[typing.Dict]: @staticmethod def to_csv(domains: typing.Iterable[typing.Dict], report_type: str, language: str) -> str: + if report_type not in track.data.CSV_FIELDS: + return {} output = io.BytesIO() iowrap = io.TextIOWrapper(output, encoding='utf-8-sig', newline='', write_through=True) diff --git a/track/views.py b/track/views.py index ac96893..607e408 100644 --- a/track/views.py +++ b/track/views.py @@ -6,7 +6,6 @@ from datetime import datetime -from track.data import FIELD_MAPPING from track import models from track.cache import cache @@ -162,7 +161,6 @@ def organizations_table(): "crypto.eligible": True, }, ) - # app.logger.debug([o for o in organizations]) response = Response(ujson.dumps({"data": organizations})) response.headers["Content-Type"] = "application/json" return response From 87be207919f150dc268f60a5c5f9dcbe7af0aa70 Mon Sep 17 00:00:00 2001 From: John O'Brien Date: Tue, 11 Dec 2018 19:10:40 -0500 Subject: [PATCH 08/12] Various bits and bobs of cleanup. (#110) 1: Remove use of ujson as it's effectively unsupported and not necessary. 2: Remove unused imports 3: Fix a typing return value issue from str -> bytes --- requirements.txt | 1 - setup.py | 1 - track/data.py | 1 - track/helpers.py | 1 - track/models.py | 11 +++++------ track/views.py | 18 +++++++++--------- track/wsgi.py | 1 - utils/sync_dependencies.py | 1 - 8 files changed, 14 insertions(+), 21 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0586a3d..a4b37a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ gunicorn==19.6.0 pyyaml==3.12 python-slugify==1.2.1 Flask-PyMongo==0.5.1 -ujson==1.35 flask-compress==1.4.0 click==6.7 Babel==2.6.0 diff --git a/setup.py b/setup.py index 1f832b2..1834873 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,6 @@ 'python-slugify==1.2.1', 'pymongo==3.7.0', 'Flask-PyMongo==0.5.1', - 'ujson==1.35', 'flask-compress==1.4.0', 'click==6.7', 'Babel==2.6.0', diff --git a/track/data.py b/track/data.py index 5a9ba4f..d544349 100644 --- a/track/data.py +++ b/track/data.py @@ -1,4 +1,3 @@ -import typing # Mapping report/domain/organization field names to display names. class MultiLingualString: diff --git a/track/helpers.py b/track/helpers.py index bac4b29..cc11b89 100644 --- a/track/helpers.py +++ b/track/helpers.py @@ -1,4 +1,3 @@ -import os import pkg_resources import yaml import datetime diff --git a/track/models.py b/track/models.py index 0b77136..f8f7e95 100644 --- a/track/models.py +++ b/track/models.py @@ -11,7 +11,6 @@ # coordinated here. db = PyMongo() - QueryError = PyMongoError # Data loads should clear the entire database first. @@ -59,7 +58,6 @@ class Domain: # # https: { ... } # - @staticmethod def find(domain_name: str) -> typing.Dict: return db.db.meta.find_one( @@ -73,7 +71,7 @@ def find(domain_name: str) -> typing.Dict: ) @staticmethod - def find_all(query: typing.Dict, projection: typing.Dict={'_id': False, '_collection': False}) -> typing.Dict: + def find_all(query: typing.Dict, projection: typing.Dict = {'_id': False, '_collection': False}) -> typing.Dict: return db.db.meta.find( { '_collection': 'domains', @@ -131,12 +129,13 @@ def all() -> typing.Iterable[typing.Dict]: return db.db.meta.find({'_collection': 'domains'}, {'_id': False, '_collection': False}) @staticmethod - def to_csv(domains: typing.Iterable[typing.Dict], report_type: str, language: str) -> str: + def to_csv(domains: typing.Iterable[typing.Dict], report_type: str, language: str) -> bytes: if report_type not in track.data.CSV_FIELDS: return {} + output = io.BytesIO() iowrap = io.TextIOWrapper(output, encoding='utf-8-sig', newline='', write_through=True) - + writer = csv.writer(iowrap, quoting=csv.QUOTE_NONNUMERIC) def value_for(value: typing.Union[str, list, bool]) -> str: @@ -230,7 +229,7 @@ def find(slug: str) -> typing.Dict: return db.db.meta.find_one({'_collection': 'organizations', 'slug': slug}, {'_id': False, '_collection': False}) @staticmethod - def find_all(query: typing.Dict, projection: typing.Dict={'_id': False, '_collection': False}) -> typing.Dict: + def find_all(query: typing.Dict, projection: typing.Dict = {'_id': False, '_collection': False}) -> typing.Dict: return db.db.meta.find( { '_collection': 'organizations', diff --git a/track/views.py b/track/views.py index 607e408..75746d8 100644 --- a/track/views.py +++ b/track/views.py @@ -2,7 +2,7 @@ import os from flask import render_template, Response, abort, request, redirect -import ujson +import json from datetime import datetime @@ -81,7 +81,7 @@ def cache_bust(): def report(report_name): report_name = "https" if report_name == "compliance" else report_name - response = Response(ujson.dumps(models.Report.latest().get(report_name, {}))) + response = Response(json.dumps(models.Report.latest().get(report_name, {}))) response.headers["Content-Type"] = "application/json" return response @@ -94,7 +94,7 @@ def domain_report(report_name): domains = models.Domain.eligible_parents(report_name) domains = sorted(domains, key=lambda k: k["domain"]) - response = Response(ujson.dumps({"data": domains})) + response = Response(json.dumps({"data": domains})) response.headers["Content-Type"] = "application/json" return response @@ -122,7 +122,6 @@ def domains_table(): "organization_name_fr": True, "is_parent": True, "base_domain": True, - "https.bod_crypto": True, "https.eligible": True, "https.enforces": True, "https.hsts": True, @@ -138,7 +137,7 @@ def domains_table(): "totals.crypto.eligible": True, }, ) - response = Response(ujson.dumps({"data": domains})) + response = Response(json.dumps({"data": list(domains)})) response.headers["Content-Type"] = "application/json" return response @@ -161,7 +160,8 @@ def organizations_table(): "crypto.eligible": True, }, ) - response = Response(ujson.dumps({"data": organizations})) + # app.logger.debug([o for o in organizations]) + response = Response(json.dumps({"data": list(organizations)})) response.headers["Content-Type"] = "application/json" return response @@ -177,7 +177,7 @@ def hostname_report(report_name): domains = sorted(domains, key=lambda k: k["domain"]) domains = sorted(domains, key=lambda k: k["base_domain"]) - response = Response(ujson.dumps({"data": domains})) + response = Response(json.dumps({"data": domains})) response.headers["Content-Type"] = "application/json" return response @@ -208,7 +208,7 @@ def hostname_report_for_domain(domain, report_name): domains = sorted(domains, key=lambda k: k["domain"]) domains = sorted(domains, key=lambda k: k["is_parent"], reverse=True) - response = Response(ujson.dumps({"data": domains})) + response = Response(json.dumps({"data": domains})) response.headers["Content-Type"] = "application/json" return response @@ -234,7 +234,7 @@ def organization_report(report_name): report_name = "https" if report_name == "compliance" else report_name domains = models.Organization.eligible(report_name) - response = Response(ujson.dumps({"data": domains})) + response = Response(json.dumps({"data": list(domains)})) response.headers["Content-Type"] = "application/json" return response diff --git a/track/wsgi.py b/track/wsgi.py index 4c71e54..a88fb86 100644 --- a/track/wsgi.py +++ b/track/wsgi.py @@ -1,6 +1,5 @@ import os from track import create_app -import yaml port = int(os.getenv("PORT", 5000)) environment = os.getenv("TRACKER_ENV", "development") diff --git a/utils/sync_dependencies.py b/utils/sync_dependencies.py index 85d5bd0..fea1f2e 100755 --- a/utils/sync_dependencies.py +++ b/utils/sync_dependencies.py @@ -7,7 +7,6 @@ import sys from unittest import mock -import distutils import setuptools @contextlib.contextmanager From 722b1e8d1bc068dddb75f389f1e5dc87312b547a Mon Sep 17 00:00:00 2001 From: Tim Arney Date: Sat, 5 Jan 2019 20:59:21 -0500 Subject: [PATCH 09/12] Handle request error (#112) * Log error to console when api endpoint fails to return data * update ui to hide text under the donut on api failure --- track/templates/en/index.html | 36 ++++--- track/templates/includes/donut.html | 150 +++++++++++++++------------- 2 files changed, 99 insertions(+), 87 deletions(-) diff --git a/track/templates/en/index.html b/track/templates/en/index.html index 79819fe..a3c6ca7 100644 --- a/track/templates/en/index.html +++ b/track/templates/en/index.html @@ -10,28 +10,32 @@
        -
        -
        -

        Track web security compliance

        -

        Making government websites more secure

        -

        Canadians rely on the Government of Canada to provide secure digital services. A new policy notice guides government websites to adopt good web security practices. Track how government sites are becoming more secure.

        +
        +
        +

        Track web security compliance

        +

        Making government websites more secure

        +

        Canadians rely on the Government of Canada to provide secure digital services. A new + policy notice guides government websites to adopt good web security practices. Track how government sites are + becoming more secure.

        + +
        +

        Check a government + website

        + +
        -
        -

        Check a government website

        - -
        +
        -
        - -
        -
        -

        Domains that
        enforce HTTPS

        -
        +
        +
        +
        +
        {% include 'includes/donut.html' %} -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/track/templates/includes/donut.html b/track/templates/includes/donut.html index e5b613c..74a5a39 100644 --- a/track/templates/includes/donut.html +++ b/track/templates/includes/donut.html @@ -1,79 +1,87 @@ + \ No newline at end of file From 0c49aba9297031c50eb70fb33988d9f41193a6b1 Mon Sep 17 00:00:00 2001 From: John O'Brien Date: Mon, 7 Jan 2019 09:30:10 -0500 Subject: [PATCH 10/12] Removed headers due to duplication.. (#114) The upstream servers are also placing these headers, so removing from here. --- track/views.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/track/views.py b/track/views.py index 75746d8..5290431 100644 --- a/track/views.py +++ b/track/views.py @@ -241,9 +241,6 @@ def organization_report(report_name): # Every response back to the browser will include these web response headers @app.after_request def apply_headers(response): - response.headers["X-Frame-Options"] = "SAMEORIGIN" - response.headers["X-XSS-Protection"] = 1 - response.headers["X-Content-Type-Options"] = "nosniff" return response @app.errorhandler(404) From af235249b593032ffade03d3411b45056098607a Mon Sep 17 00:00:00 2001 From: John O'Brien Date: Tue, 8 Jan 2019 13:34:24 -0500 Subject: [PATCH 11/12] Pyyaml security bump (#116) * Security Update: pyyaml bump to pull in safe_load Fixes this https://github.com/yaml/pyyaml/pull/74. Note we were already using safe_load. * Security Update: pyyaml version bump https://github.com/yaml/pyyaml/pull/74 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index a4b37a7..1ae0ad5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ flask==0.12.4 gunicorn==19.6.0 -pyyaml==3.12 +pyyaml==3.13 python-slugify==1.2.1 Flask-PyMongo==0.5.1 flask-compress==1.4.0 diff --git a/setup.py b/setup.py index 1834873..9f8ffb4 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ install_requires=[ 'flask==0.12.4', 'gunicorn==19.6.0', - 'pyyaml==3.12', + 'pyyaml==3.13', 'python-slugify==1.2.1', 'pymongo==3.7.0', 'Flask-PyMongo==0.5.1', From d00add16e1af735db0b904149cbafcf76a8296c2 Mon Sep 17 00:00:00 2001 From: Tim Arney Date: Wed, 16 Jan 2019 08:07:06 -0500 Subject: [PATCH 12/12] Paginate scroll to top (#118) * Paginate scroll to top --- track/static/js/tables.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/track/static/js/tables.js b/track/static/js/tables.js index 497c166..325d9f7 100644 --- a/track/static/js/tables.js +++ b/track/static/js/tables.js @@ -63,6 +63,12 @@ var Tables = { Utils.updatePagination(); }); + table.on("page.dt",function(){ + /* scroll page to top of table on page change */ + var top = $(".dataTable").offset().top; + $("html, body").animate({ scrollTop: top }, "slow"); + }); + return table; },