Pull ethnicity and race coding from filesystem (#4377)

The previously available value sets for ethnicity and race, hosted at
hl7.org, are no longer available for automated download.

Dumped contents to filesystem for a reliable import on deploy.

portal/config/eproms/Coding.json

@@ -5862,12 +5862,6 @@
       "resourceType": "Coding",
       "system": "http://snomed.info/sct"
     },
-    {
-      "code": "",
-      "display": "",
-      "resourceType": "Coding",
-      "system": "http://us.truenth.org/clinical-codes"
-    },
     {
       "code": "111",
       "display": "biopsy",

portal/models/clinical_constants.py

@@ -1,4 +1,6 @@
 """ TrueNTH Clinical Codes """
+import json
+import os
 import requests
 
 from ..database import db
@@ -90,12 +92,23 @@ def parse_concepts(elements, system):
     return ccs
 
 
-def fetch_HL7_V3_Namespace(valueSet):
+def fetch_HL7_V3_Namespace(valueSet, pull_from_hl7=False):
     """Pull and parse the published FHIR ethnicity namespace"""
+    # NB, this used to be pulled on every deploy, but hl7.org now requires human
+    # intervention, to bypass the captcha - now pulling cached version off file
+    # system.
     src_url = 'http://hl7.org/fhir/STU3/v3/{valueSet}/v3-{valueSet}.cs.json'.format(
         valueSet=valueSet)
-    response = requests.get(src_url)
-    return parse_concepts(response.json()['concept'],
+    if pull_from_hl7:
+        response = requests.get(src_url)
+        concept_source = response.json()
+    else:
+        cwd = os.path.dirname(__file__)
+        fp = os.path.join(cwd, f'code_systems/v3-{valueSet}.cs.json')
+        with open(fp, 'r') as jfile:
+            concept_source = json.load(jfile)
+
+    return parse_concepts(concept_source['concept'],
                           system='http://hl7.org/fhir/v3/{}'.format(valueSet))
 
 

portal/models/code_systems/v3-Ethnicity.cs.json

@@ -0,0 +1,274 @@
+{
+  "resourceType": "CodeSystem",
+  "id": "v3-Ethnicity",
+  "meta": {
+    "lastUpdated": "2016-11-11T00:00:00.000+11:00"
+  },
+  "text": {
+    "status": "generated",
+    "div": "<div xmlns=\"http://www.w3.org/1999/xhtml\"><p>Release Date: 2016-11-11</p>\r\n<table class=\"grid\">\r\n <tr><td><b>Level</b></td><td><b>Code</b></td><td><b>Display</b></td><td><b>Definition</b></td></tr>\r\n <tr><td>1</td><td>2135-2<a name=\"2135-2\"> </a></td><td>Hispanic or Latino</td><td/></tr>\r\n <tr><td>2</td><td>  2137-8<a name=\"2137-8\"> </a></td><td>Spaniard</td><td/></tr>\r\n <tr><td>3</td><td>    2138-6<a name=\"2138-6\"> </a></td><td>Andalusian</td><td/></tr>\r\n <tr><td>3</td><td>    2139-4<a name=\"2139-4\"> </a></td><td>Asturian</td><td/></tr>\r\n <tr><td>3</td><td>    2140-2<a name=\"2140-2\"> </a></td><td>Castillian</td><td/></tr>\r\n <tr><td>3</td><td>    2141-0<a name=\"2141-0\"> </a></td><td>Catalonian</td><td/></tr>\r\n <tr><td>3</td><td>    2142-8<a name=\"2142-8\"> </a></td><td>Belearic Islander</td><td/></tr>\r\n <tr><td>3</td><td>    2143-6<a name=\"2143-6\"> </a></td><td>Gallego</td><td/></tr>\r\n <tr><td>3</td><td>    2144-4<a name=\"2144-4\"> </a></td><td>Valencian</td><td/></tr>\r\n <tr><td>3</td><td>    2145-1<a name=\"2145-1\"> </a></td><td>Canarian</td><td/></tr>\r\n <tr><td>3</td><td>    2146-9<a name=\"2146-9\"> </a></td><td>Spanish Basque</td><td/></tr>\r\n <tr><td>2</td><td>  2148-5<a name=\"2148-5\"> </a></td><td>Mexican</td><td/></tr>\r\n <tr><td>3</td><td>    2149-3<a name=\"2149-3\"> </a></td><td>Mexican American</td><td/></tr>\r\n <tr><td>3</td><td>    2150-1<a name=\"2150-1\"> </a></td><td>Mexicano</td><td/></tr>\r\n <tr><td>3</td><td>    2151-9<a name=\"2151-9\"> </a></td><td>Chicano</td><td/></tr>\r\n <tr><td>3</td><td>    2152-7<a name=\"2152-7\"> </a></td><td>La Raza</td><td/></tr>\r\n <tr><td>3</td><td>    2153-5<a name=\"2153-5\"> </a></td><td>Mexican American Indian</td><td/></tr>\r\n <tr><td>2</td><td>  2155-0<a name=\"2155-0\"> </a></td><td>Central American</td><td/></tr>\r\n <tr><td>3</td><td>    2156-8<a name=\"2156-8\"> </a></td><td>Costa Rican</td><td/></tr>\r\n <tr><td>3</td><td>    2157-6<a name=\"2157-6\"> </a></td><td>Guatemalan</td><td/></tr>\r\n <tr><td>3</td><td>    2158-4<a name=\"2158-4\"> </a></td><td>Honduran</td><td/></tr>\r\n <tr><td>3</td><td>    2159-2<a name=\"2159-2\"> </a></td><td>Nicaraguan</td><td/></tr>\r\n <tr><td>3</td><td>    2160-0<a name=\"2160-0\"> </a></td><td>Panamanian</td><td/></tr>\r\n <tr><td>3</td><td>    2161-8<a name=\"2161-8\"> </a></td><td>Salvadoran</td><td/></tr>\r\n <tr><td>3</td><td>    2162-6<a name=\"2162-6\"> </a></td><td>Central American Indian</td><td/></tr>\r\n <tr><td>3</td><td>    2163-4<a name=\"2163-4\"> </a></td><td>Canal Zone</td><td/></tr>\r\n <tr><td>2</td><td>  2165-9<a name=\"2165-9\"> </a></td><td>South American</td><td/></tr>\r\n <tr><td>3</td><td>    2166-7<a name=\"2166-7\"> </a></td><td>Argentinean</td><td/></tr>\r\n <tr><td>3</td><td>    2167-5<a name=\"2167-5\"> </a></td><td>Bolivian</td><td/></tr>\r\n <tr><td>3</td><td>    2168-3<a name=\"2168-3\"> </a></td><td>Chilean</td><td/></tr>\r\n <tr><td>3</td><td>    2169-1<a name=\"2169-1\"> </a></td><td>Colombian</td><td/></tr>\r\n <tr><td>3</td><td>    2170-9<a name=\"2170-9\"> </a></td><td>Ecuadorian</td><td/></tr>\r\n <tr><td>3</td><td>    2171-7<a name=\"2171-7\"> </a></td><td>Paraguayan</td><td/></tr>\r\n <tr><td>3</td><td>    2172-5<a name=\"2172-5\"> </a></td><td>Peruvian</td><td/></tr>\r\n <tr><td>3</td><td>    2173-3<a name=\"2173-3\"> </a></td><td>Uruguayan</td><td/></tr>\r\n <tr><td>3</td><td>    2174-1<a name=\"2174-1\"> </a></td><td>Venezuelan</td><td/></tr>\r\n <tr><td>3</td><td>    2175-8<a name=\"2175-8\"> </a></td><td>South American Indian</td><td/></tr>\r\n <tr><td>3</td><td>    2176-6<a name=\"2176-6\"> </a></td><td>Criollo</td><td/></tr>\r\n <tr><td>2</td><td>  2178-2<a name=\"2178-2\"> </a></td><td>Latin American</td><td/></tr>\r\n <tr><td>2</td><td>  2180-8<a name=\"2180-8\"> </a></td><td>Puerto Rican</td><td/></tr>\r\n <tr><td>2</td><td>  2182-4<a name=\"2182-4\"> </a></td><td>Cuban</td><td/></tr>\r\n <tr><td>2</td><td>  2184-0<a name=\"2184-0\"> </a></td><td>Dominican</td><td/></tr>\r\n <tr><td>1</td><td>2186-5<a name=\"2186-5\"> </a></td><td>Not Hispanic or Latino</td><td>\n                        Note that this term remains in the table for completeness, even though within HL7, the notion of &quot;not otherwise coded&quot; term is deprecated.<br/>\r\n\n                     </td></tr>\r\n</table>\r\n</div>"
+  },
+  "extension": [
+    {
+      "url": "http://hl7.org/fhir/StructureDefinition/structuredefinition-ballot-status",
+      "valueString": "External"
+    },
+    {
+      "url": "http://hl7.org/fhir/StructureDefinition/structuredefinition-fmm",
+      "valueInteger": 0
+    }
+  ],
+  "url": "http://hl7.org/fhir/v3/Ethnicity",
+  "identifier": {
+    "system": "urn:ietf:rfc:3986",
+    "value": "urn:oid:2.16.840.1.113883.5.50"
+  },
+  "version": "2016-11-11",
+  "name": "v3 Code System Ethnicity",
+  "status": "active",
+  "experimental": false,
+  "date": "2016-11-11T00:00:00+11:00",
+  "publisher": "HL7, Inc",
+  "contact": [
+    {
+      "telecom": [
+        {
+          "system": "url",
+          "value": "http://hl7.org"
+        }
+      ]
+    }
+  ],
+  "description": " In the United States, federal standards for classifying data on ethnicity determine the categories used by federal agencies and exert a strong influence on categorization by state and local agencies and private sector organizations. The federal standards do not conceptually define ethnicity, and they recognize the absence of an anthropological or scientific basis for ethnicity classification.  Instead, the federal standards acknowledge that ethnicity is a social-political construct in which an individual's own identification with a particular ethnicity is preferred to observer identification.  The standards specify two minimum ethnicity categories: Hispanic or Latino, and Not Hispanic or Latino.  The standards define a Hispanic or Latino as a person of \"Mexican, Puerto Rican, Cuban, South or Central America, or other Spanish culture or origin, regardless of race.\" The standards stipulate that ethnicity data need not be limited to the two minimum categories, but any expansion must be collapsible to those categories.  In addition, the standards stipulate that an individual can be Hispanic or Latino or can be Not Hispanic or Latino, but cannot be both.",
+  "caseSensitive": true,
+  "valueSet": "http://hl7.org/fhir/ValueSet/v3-Ethnicity",
+  "hierarchyMeaning": "is-a",
+  "content": "complete",
+  "concept": [
+    {
+      "code": "2135-2",
+      "display": "Hispanic or Latino",
+      "definition": "Hispanic or Latino",
+      "concept": [
+        {
+          "code": "2137-8",
+          "display": "Spaniard",
+          "definition": "Spaniard",
+          "concept": [
+            {
+              "code": "2138-6",
+              "display": "Andalusian",
+              "definition": "Andalusian"
+            },
+            {
+              "code": "2139-4",
+              "display": "Asturian",
+              "definition": "Asturian"
+            },
+            {
+              "code": "2140-2",
+              "display": "Castillian",
+              "definition": "Castillian"
+            },
+            {
+              "code": "2141-0",
+              "display": "Catalonian",
+              "definition": "Catalonian"
+            },
+            {
+              "code": "2142-8",
+              "display": "Belearic Islander",
+              "definition": "Belearic Islander"
+            },
+            {
+              "code": "2143-6",
+              "display": "Gallego",
+              "definition": "Gallego"
+            },
+            {
+              "code": "2144-4",
+              "display": "Valencian",
+              "definition": "Valencian"
+            },
+            {
+              "code": "2145-1",
+              "display": "Canarian",
+              "definition": "Canarian"
+            },
+            {
+              "code": "2146-9",
+              "display": "Spanish Basque",
+              "definition": "Spanish Basque"
+            }
+          ]
+        },
+        {
+          "code": "2148-5",
+          "display": "Mexican",
+          "definition": "Mexican",
+          "concept": [
+            {
+              "code": "2149-3",
+              "display": "Mexican American",
+              "definition": "Mexican American"
+            },
+            {
+              "code": "2150-1",
+              "display": "Mexicano",
+              "definition": "Mexicano"
+            },
+            {
+              "code": "2151-9",
+              "display": "Chicano",
+              "definition": "Chicano"
+            },
+            {
+              "code": "2152-7",
+              "display": "La Raza",
+              "definition": "La Raza"
+            },
+            {
+              "code": "2153-5",
+              "display": "Mexican American Indian",
+              "definition": "Mexican American Indian"
+            }
+          ]
+        },
+        {
+          "code": "2155-0",
+          "display": "Central American",
+          "definition": "Central American",
+          "concept": [
+            {
+              "code": "2156-8",
+              "display": "Costa Rican",
+              "definition": "Costa Rican"
+            },
+            {
+              "code": "2157-6",
+              "display": "Guatemalan",
+              "definition": "Guatemalan"
+            },
+            {
+              "code": "2158-4",
+              "display": "Honduran",
+              "definition": "Honduran"
+            },
+            {
+              "code": "2159-2",
+              "display": "Nicaraguan",
+              "definition": "Nicaraguan"
+            },
+            {
+              "code": "2160-0",
+              "display": "Panamanian",
+              "definition": "Panamanian"
+            },
+            {
+              "code": "2161-8",
+              "display": "Salvadoran",
+              "definition": "Salvadoran"
+            },
+            {
+              "code": "2162-6",
+              "display": "Central American Indian",
+              "definition": "Central American Indian"
+            },
+            {
+              "code": "2163-4",
+              "display": "Canal Zone",
+              "definition": "Canal Zone"
+            }
+          ]
+        },
+        {
+          "code": "2165-9",
+          "display": "South American",
+          "definition": "South American",
+          "concept": [
+            {
+              "code": "2166-7",
+              "display": "Argentinean",
+              "definition": "Argentinean"
+            },
+            {
+              "code": "2167-5",
+              "display": "Bolivian",
+              "definition": "Bolivian"
+            },
+            {
+              "code": "2168-3",
+              "display": "Chilean",
+              "definition": "Chilean"
+            },
+            {
+              "code": "2169-1",
+              "display": "Colombian",
+              "definition": "Colombian"
+            },
+            {
+              "code": "2170-9",
+              "display": "Ecuadorian",
+              "definition": "Ecuadorian"
+            },
+            {
+              "code": "2171-7",
+              "display": "Paraguayan",
+              "definition": "Paraguayan"
+            },
+            {
+              "code": "2172-5",
+              "display": "Peruvian",
+              "definition": "Peruvian"
+            },
+            {
+              "code": "2173-3",
+              "display": "Uruguayan",
+              "definition": "Uruguayan"
+            },
+            {
+              "code": "2174-1",
+              "display": "Venezuelan",
+              "definition": "Venezuelan"
+            },
+            {
+              "code": "2175-8",
+              "display": "South American Indian",
+              "definition": "South American Indian"
+            },
+            {
+              "code": "2176-6",
+              "display": "Criollo",
+              "definition": "Criollo"
+            }
+          ]
+        },
+        {
+          "code": "2178-2",
+          "display": "Latin American",
+          "definition": "Latin American"
+        },
+        {
+          "code": "2180-8",
+          "display": "Puerto Rican",
+          "definition": "Puerto Rican"
+        },
+        {
+          "code": "2182-4",
+          "display": "Cuban",
+          "definition": "Cuban"
+        },
+        {
+          "code": "2184-0",
+          "display": "Dominican",
+          "definition": "Dominican"
+        }
+      ]
+    },
+    {
+      "code": "2186-5",
+      "display": "Not Hispanic or Latino",
+      "definition": "Note that this term remains in the table for completeness, even though within HL7, the notion of \"not otherwise coded\" term is deprecated."
+    }
+  ]
+}