Merge pull request #4 from cds-snc/master

pull most recent changes from CDS

.circleci/config.yml

@@ -67,6 +67,3 @@ workflows:
       - build_package:
          requires:
             - track_web
-         filters:
-           branches:
-             only: master

requirements.txt

@@ -1,11 +1,12 @@
 flask==0.12.4
 gunicorn==19.6.0
-pyyaml==3.12
+pyyaml==3.13
 python-slugify==1.2.1
 Flask-PyMongo==0.5.1
-ujson==1.35
 flask-compress==1.4.0
 click==6.7
 Babel==2.6.0
 Flask-Caching==1.4.0
 pymongo==3.7.0
+azure-keyvault==1.1.0
+msrestazure==0.5.1

setup.py

@@ -20,15 +20,16 @@
     install_requires=[
         'flask==0.12.4',
         'gunicorn==19.6.0',
-        'pyyaml==3.12',
+        'pyyaml==3.13',
         'python-slugify==1.2.1',
         'pymongo==3.7.0',
         'Flask-PyMongo==0.5.1',
-        'ujson==1.35',
         'flask-compress==1.4.0',
         'click==6.7',
         'Babel==2.6.0',
         'Flask-Caching==1.4.0',
+        'azure-keyvault==1.1.0',
+        'msrestazure==0.5.1'
     ],
     extras_require={
         'development': [

tests/test_models.py

@@ -170,11 +170,10 @@ def test_all(self, clean_model, domain) -> None: # pylint: disable=no-self-use
 
     def test_to_csv_en(self, domain) -> None: # pylint: disable=no-self-use
         csv_string = models.Domain.to_csv([domain], 'https', 'en')
+        bytes_in = io.BytesIO(csv_string)
 
-        with io.StringIO() as sio:
-            sio.write(csv_string)
-            sio.seek(0)
-            reader = csv.DictReader(sio)
+        with io.TextIOWrapper(bytes_in, encoding='utf-8-sig', newline='') as wrapped_io:
+            reader = csv.DictReader(wrapped_io)
             assert sorted(reader.fieldnames) == [
                 '3DES',
                 'Approved Certificate',
@@ -224,10 +223,10 @@ def test_to_csv_en(self, domain) -> None: # pylint: disable=no-self-use
 
     def test_to_csv_fr(self, domain) -> None: # pylint: disable=no-self-use
         csv_string = models.Domain.to_csv([domain], 'https', 'fr')
-        with io.StringIO() as sio:
-            sio.write(csv_string)
-            sio.seek(0)
-            reader = csv.DictReader(sio)
+        bytes_in = io.BytesIO(csv_string)
+
+        with io.TextIOWrapper(bytes_in, encoding='utf-8-sig', newline='') as wrapped_io:
+            reader = csv.DictReader(wrapped_io)
             assert sorted(reader.fieldnames) == [
                 '3DES',
                 'Absence de protocoles ou de suites de chiffrement ayant des vulnérabilités connues',
@@ -350,10 +349,4 @@ def test_all(self, clean_model, organization) -> None: # pylint: disable=no-self
 class TestFlag():
 
     def test_get_cache_not_set(self, clean_model) -> None: # pylint: disable=no-self-use
-        assert not clean_model.Flag.get_cache()
-
-    def test_get_cache_set(self, clean_model) -> None: # pylint: disable=no-self-use
-        clean_model.Flag.set_cache(True)
-        assert clean_model.Flag.get_cache()
-        clean_model.Flag.set_cache(False)
-        assert not clean_model.Flag.get_cache()
+        assert clean_model.Flag.get_cache() == "1999-12-31 23:59"

track/config.py

@@ -1,6 +1,14 @@
 import os
+import sys
 import random
+import logging
 
+from azure.keyvault import KeyVaultClient
+from msrestazure.azure_active_directory import MSIAuthentication
+
+LOGGER = logging.getLogger(__name__)
+
+A_DAY = 60 * 60 * 24
 
 class Config:
     DEBUG = False
@@ -12,16 +20,27 @@ class Config:
     def init_app(app):
         pass
 
-
-A_DAY = 60 * 60 * 24
 class ProductionConfig(Config):
-    MONGO_URI = os.environ.get("TRACKER_MONGO_URI", None)
+
     CACHE_TYPE = "filesystem"
     CACHE_DIR = os.environ.get("TRACKER_CACHE_DIR", "./.cache")
     CACHE_DEFAULT_TIMEOUT = int(os.environ.get("TRACKER_CACHE_TIMEOUT", A_DAY))
 
+    if os.environ.get('TRACKER_ENV', None) == "production":
+        if os.environ.get("TRACKER_KEYVAULT_URI", None) is None or os.environ.get("SECRET_NAME_RO", None) is None:
+            # Error and crash hard: Production should be configured as expected.
+            LOGGER.error("KeyVault uri or secret name missing from local environment.")
+            sys.exit(4)
+
+        kv_uri = os.environ.get("TRACKER_KEYVAULT_URI")
+        kv_secret = os.environ.get("SECRET_NAME_RO")
+        kv_creds = MSIAuthentication(resource='https://vault.azure.net')
+        kv_client = KeyVaultClient(kv_creds)
+        MONGO_URI = kv_client.get_secret(kv_uri, kv_secret, "").value
+
     @staticmethod
     def init_app(app):
+
         Config.init_app(app)
 
         import logging
@@ -31,13 +50,14 @@ def init_app(app):
         handler.setLevel(logging.ERROR)
         app.logger.addHandler(handler)
 
-
 class DevelopmentConfig(Config):
+
     DEBUG = True
     CACHE_TYPE = "simple"
 
 
 class TestingConfig(Config):
+
     TESTING = True
     MONGO_URI = "mongodb://localhost:27017/track_{rand}".format(
         rand=random.randint(0, 1000)

track/data.py

@@ -1,4 +1,3 @@
-import typing
 # Mapping report/domain/organization field names to display names.
 
 class MultiLingualString:

track/helpers.py

@@ -1,4 +1,3 @@
-import os
 import pkg_resources
 import yaml
 import datetime

track/models.py

@@ -11,7 +11,6 @@
 # coordinated here.
 
 db = PyMongo()
-
 QueryError = PyMongoError
 
 # Data loads should clear the entire database first.
@@ -59,7 +58,6 @@ class Domain:
     #
     # https: { ... }
     #
-
     @staticmethod
     def find(domain_name: str) -> typing.Dict:
         return db.db.meta.find_one(
@@ -73,7 +71,7 @@ def find(domain_name: str) -> typing.Dict:
         )
 
     @staticmethod
-    def find_all(query: typing.Dict, projection: typing.Dict={'_id': False, '_collection': False}) -> typing.Dict:
+    def find_all(query: typing.Dict, projection: typing.Dict = {'_id': False, '_collection': False}) -> typing.Dict:
         return db.db.meta.find(
             {
                 '_collection': 'domains',
@@ -131,9 +129,14 @@ def all() -> typing.Iterable[typing.Dict]:
         return db.db.meta.find({'_collection': 'domains'}, {'_id': False, '_collection': False})
 
     @staticmethod
-    def to_csv(domains: typing.Iterable[typing.Dict], report_type: str, language: str) -> str:
-        output = io.StringIO()
-        writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
+    def to_csv(domains: typing.Iterable[typing.Dict], report_type: str, language: str) -> bytes:
+        if report_type not in track.data.CSV_FIELDS:
+            return {}
+
+        output = io.BytesIO()
+        iowrap = io.TextIOWrapper(output, encoding='utf-8-sig', newline='', write_through=True)
+
+        writer = csv.writer(iowrap, quoting=csv.QUOTE_NONNUMERIC)
 
         def value_for(value: typing.Union[str, list, bool]) -> str:
             # if it's a list, convert it to a list of strings and join
@@ -226,7 +229,7 @@ def find(slug: str) -> typing.Dict:
         return db.db.meta.find_one({'_collection': 'organizations', 'slug': slug}, {'_id': False, '_collection': False})
 
     @staticmethod
-    def find_all(query: typing.Dict, projection: typing.Dict={'_id': False, '_collection': False}) -> typing.Dict:
+    def find_all(query: typing.Dict, projection: typing.Dict = {'_id': False, '_collection': False}) -> typing.Dict:
         return db.db.meta.find(
             {
                 '_collection': 'organizations',
@@ -242,11 +245,6 @@ def all() -> typing.Iterable[typing.Dict]:
 class Flag:
 
     @staticmethod
-    def get_cache() -> bool:
+    def get_cache() -> str:
         flags = db.db.meta.find_one({"_collection": "flags"})
-        return flags['cache'] if flags else False
-
-    @staticmethod
-    def set_cache(state: bool) -> None:
-        db.db.meta.update_one({"_collection": "flags"}, {"$set": {"cache": state}}, upsert=True)
-
+        return flags['cache'] if flags else "1999-12-31 23:59"

track/static/js/tables.js

@@ -63,6 +63,12 @@ var Tables = {
       Utils.updatePagination();
     });
 
+    table.on("page.dt",function(){
+      /* scroll page to top of table on page change */
+      var top = $(".dataTable").offset().top;
+      $("html, body").animate({ scrollTop: top }, "slow");
+    });
+
     return table;
   },
 

track/templates/en/guidance.html

@@ -18,23 +18,23 @@ <h1>Read guidance</h1>
 
                    <ul>
 
-                     <li>GCconnex: <a href="https://gcconnex.gc.ca/groups/profile/35218846/gc-https-everywhere-2018-https-partout-dans-le-gc-2018" class="text-https-blue hover:text-black font-bold">HTTPS Everywhere 2018</a></li>
-                     <li>GCmessage: <a href="https://message.gccollab.ca/channel/httpseverywhere-httpspartout" class="text-https-blue hover:text-black font-bold">#HTTPSEverywhere-HTTPSpartout</a></li>
+                     <li>GCconnex: <a href="https://gcconnex.gc.ca/groups/profile/35218846/gc-https-everywhere-2018-https-partout-dans-le-gc-2018" class="text-https-blue hover:text-black">HTTPS Everywhere 2018</a></li>
+                     <li>GCmessage: <a href="https://message.gccollab.ca/channel/httpseverywhere-httpspartout" class="text-https-blue hover:text-black">#HTTPSEverywhere-HTTPSpartout</a></li>
                    </ul>
            </li>
 
            <li class="mb-6">Perform an inventory of all departmental domains and subdomains. Sources of information include:
 
             <ul>
 
-                <li>Internally available <a href="https://https-everywhere.canada.ca" class="text-https-blue hover:text-black font-bold">HTTPS Dashboard</a></li>
+                <li>Internally available <a href="https://https-everywhere.canada.ca" class="text-https-blue hover:text-black">HTTPS Dashboard</a></li>
                 <li>TBS Application Portfolio Management (APM)</li>
                 <li>Departmental business units </li>
            </ul>
 
            </li>
 
-           <li class="mb-6">Provide an up-to-date list of all domain and sub-domains of the publicly-accessible websites and web services to the following website: <a href="https://canada-ca.github.io/pages/submit-institutional-domains.html" class="text-https-blue hover:text-black font-bold">Submit your institution's domains</a>.</li>
+           <li class="mb-6">Provide an up-to-date list of all domain and sub-domains of the publicly-accessible websites and web services to the following website: <a href="https://canada-ca.github.io/pages/submit-institutional-domains.html" class="text-https-blue hover:text-black">Submit your institution's domains</a>.</li>
            <li class="mb-6">Perform an assessment of the domains and sub-domains to determine the status of the configuration. Tools available to support this activity includes GC HTTPS Dashboard, SSL Labs, Hardenize, etc.</li>
            <li class="mb-6">Develop a prioritized implementation schedule for each of the affected websites and web services, following the recommended prioritization approach in the ITPIN:
 
@@ -56,7 +56,7 @@ <h1>Read guidance</h1>
 
            </li>
 
-           <li class="mb-6">Based on the assessment, and using the <a href="https://www.gcpedia.gc.ca/wiki/HTTPS_Initiative" class="text-https-blue hover:text-black font-bold">guidance available on GCpedia</a>, the following activities may be required:
+           <li class="mb-6">Based on the assessment, and using the <a href="https://www.gcpedia.gc.ca/wiki/HTTPS_Initiative" class="text-https-blue hover:text-black">guidance available on GCpedia</a>, the following activities may be required:
             <ul>
 
                 <li>Obtain certificates from a GC-approved certificate source as outlined in the Recommendations for TLS Server Certificates for GC Public Facing Web Services</li>
@@ -71,7 +71,7 @@ <h1>Read guidance</h1>
            </li>
         </ol>
         <br/>
-        <p>For any questions or concerns related to the ITPIN and related implementation guidance, contact TBS Cybersecurity (<a href="mailto:zzTBSCybers@tbs-sct.gc.ca" class="text-https-blue hover:text-black font-bold">zzTBSCybers@tbs-sct.gc.ca</a>).</p>
+        <p>For any questions or concerns related to the ITPIN and related implementation guidance, contact TBS Cybersecurity (<a href="mailto:zzTBSCybers@tbs-sct.gc.ca" class="text-https-blue hover:text-black">zzTBSCybers@tbs-sct.gc.ca</a>).</p>
     </div>
 </section>
 {% endblock %}

track/templates/en/help.html

@@ -9,12 +9,12 @@
 <section id="main-content" class="flex-1">
 	<div class="container mx-auto items-center sm:w-4/5 xl:w-3/5 mt-6">
 		<h1>Get help</h1>
-		<h2 class="font-bold mb-4">General public</h2>
-		<p>For interpretation of any aspect of <a class="text-https-blue hover:text-black font-bold" href="https://www.canada.ca/en/treasury-board-secretariat/services/information-technology/policy-implementation-notices/implementing-https-secure-web-connections-itpin.html">Implementing HTTPS for Secure Web Connections: Information Technology Policy Implementation Notice (ITPIN)</a>, contact Treasury Board of Canada Secretariat through Public Enquiries.</p>
-		<h2 class="font-bold mt-4 mb-4">Government of Canada employees</h2>
-		<p>Individuals at departments should contact their departmental information technology group for any questions regarding this <abbr title="Information Technology Policy Implementation Notice">ITPIN</abbr>. Individuals can also join the <a href="https://gcconnex.gc.ca/groups/profile/35218846/gc-https-everywhere-2018-https-partout-dans-le-gc-2018?language=en" class="text-https-blue hover:text-black font-bold">HTTPS Everywhere group on GCconnex</a>, or the #HTTPSEverywhere channel on <a href="https://message.gccollab.ca/" class="text-https-blue hover:text-black font-bold">GCcollab Message</a>.</p>
-		<p>Individuals from a departmental information technology group may contact the <a href="mailto:zzTBSCybers@tbs-sct.gc.ca" class="text-https-blue hover:text-black font-bold">TBS Cyber Security mailbox</a> for interpretations of this <abbr title="Information Technology Policy Implementation Notice">ITPIN</abbr>.</p>
-		<p class="mb-8">Individuals with questions about the accuracy of their domain’s compliance data may contact the <a href="mailto:zzTBSCybers@tbs-sct.gc.ca" class="text-https-blue hover:text-black font-bold">TBS Cyber Security mailbox</a>. Note that compliance data does not automatically refresh. If you modified your domain recently, there may be a delay before your domain’s information updates.</p>
+		<h2 class=" mb-4">General public</h2>
+		<p>For interpretation of any aspect of <a class="text-https-blue hover:text-black" href="https://www.canada.ca/en/treasury-board-secretariat/services/information-technology/policy-implementation-notices/implementing-https-secure-web-connections-itpin.html">Implementing HTTPS for Secure Web Connections: Information Technology Policy Implementation Notice (ITPIN)</a>, contact Treasury Board of Canada Secretariat through Public Enquiries.</p>
+		<h2 class=" mt-4 mb-4">Government of Canada employees</h2>
+		<p>Individuals at departments should contact their departmental information technology group for any questions regarding this <abbr title="Information Technology Policy Implementation Notice">ITPIN</abbr>. Individuals can also join the <a href="https://gcconnex.gc.ca/groups/profile/35218846/gc-https-everywhere-2018-https-partout-dans-le-gc-2018?language=en" class="text-https-blue hover:text-black">HTTPS Everywhere group on GCconnex</a>, or the #HTTPSEverywhere channel on <a href="https://message.gccollab.ca/" class="text-https-blue hover:text-black">GCcollab Message</a>.</p>
+		<p>Individuals from a departmental information technology group may contact the <a href="mailto:zzTBSCybers@tbs-sct.gc.ca" class="text-https-blue hover:text-black">TBS Cyber Security mailbox</a> for interpretations of this <abbr title="Information Technology Policy Implementation Notice">ITPIN</abbr>.</p>
+		<p class="mb-8">Individuals with questions about the accuracy of their domain’s compliance data may contact the <a href="mailto:zzTBSCybers@tbs-sct.gc.ca" class="text-https-blue hover:text-black">TBS Cyber Security mailbox</a>. Note that compliance data does not automatically refresh. If you modified your domain recently, there may be a delay before your domain’s information updates.</p>
 	</div>
 
 </section>

track/templates/en/index.html

@@ -10,28 +10,32 @@
 
 <section id="main-content" class="flex-1 bg-https-light-gray py-6">
 
-		<div class="flex flex-col lg:flex-row mx-auto items-center sm:w-4/5 xl:w-3/5">
-			<div class="flex-1 lg:mr-8 lg:pr-8 mb-4 lg:mb-0">
-				<h1 class="text-4xl sm:text-5xl mb-6">Track web security compliance</h1>
-				<h2 class="text-3xl mb-2">Making government websites more secure</h2>
-				<p class="text-xl">Canadians rely on the Government of Canada to provide secure digital services. A <a class="text-https-blue hover:text-black font-bold" href="https://www.canada.ca/en/treasury-board-secretariat/services/information-technology/policy-implementation-notices/implementing-https-secure-web-connections-itpin.html">new policy notice</a> guides government websites to adopt good web security practices. Track how government sites are becoming more secure.</p>
+	<div class="flex flex-col lg:flex-row mx-auto items-center sm:w-4/5 xl:w-3/5">
+		<div class="flex-1 lg:mr-8 lg:pr-8 mb-4 lg:mb-0">
+			<h1 class="text-4xl sm:text-5xl mb-6">Track web security compliance</h1>
+			<h2 class="text-3xl mb-2">Making government websites more secure</h2>
+			<p class="text-xl">Canadians rely on the Government of Canada to provide secure digital services. A <a class="text-https-blue hover:text-black"
+				 href="https://www.canada.ca/en/treasury-board-secretariat/services/information-technology/policy-implementation-notices/implementing-https-secure-web-connections-itpin.html">new
+					policy notice</a> guides government websites to adopt good web security practices. Track how government sites are
+				becoming more secure.</p>
+
+			<div class="flex">
+				<p class="text-xl mt-2 lg:mt-6"><a href="/en/domains/" class="text-https-blue hover:text-black">Check a government
+						website</a></p>
+				<object class="ml-2 lg:mt-4" type="image/svg+xml" tabindex="-1" aria-hidden="true" role="none" data="/static/images/cta-arrow.svg"></object>
+			</div>
 
-				<div class="flex">
-					<p class="text-xl mt-2 lg:mt-6"><a href="/en/domains/" class="text-https-blue hover:text-black font-bold">Check a government website</a></p>
-					<object class="ml-2 lg:mt-4" type="image/svg+xml" tabindex="-1" aria-hidden="true" role="none" data="/static/images/cta-arrow.svg"></object>
-				</div>
+		</div>
 
-			</div>
-
-			<div class="flex-inline flex-col text-center mx-auto">
-				<div class="compliant"></div>
-				<h3 class="font-bold text-xl sm:text-2xl">Domains that <br> enforce HTTPS</h3>
-			</div>
+		<div class="flex-inline flex-col text-center mx-auto">
+			<div class="compliant"></div>
+			<h3 style="display:none" class="domains-text font-bold text-xl sm:text-2xl">Domains that <br> enforce HTTPS</h3>
 		</div>
+	</div>
 
 </section>
 
 {% include 'includes/donut.html' %}
 
 <script src="/static/js/https/organizations.js?{{ now() | date("%Y%m%j%H%M%S") }}"></script>
-{% endblock %}
+{% endblock %}