Read from new cache (#3216)

* Allow delay in history graph

* Upgrade d3-scale

null in quantitative scales is now considered an “unknown” value.

d3/d3-scale#241

* Fix history mix graph

* Update check as production is always present

* Make area graph handle null values

* Make time slider hourly

* Only run validation when production values are there

* Add preview flags to API calls

* Upgrade build image for d3-scale

* Fix timeslider timescale

Co-authored-by: skovhus <kenneth.skovhus@gmail.com>
Co-authored-by: Kenneth Skovhus <skovhus@users.noreply.github.com>

web/BUILD.yaml

@@ -1,9 +1,8 @@
 steps:
 
   prepare:
-    image: node:10.15.3
+    image: node:12
     inputs:
-      # packages
       - package.json
       - yarn.lock
     commands:

web/package.json

@@ -15,8 +15,7 @@
     "d3-interpolate": "^1.1.6",
     "d3-queue": "^3.0.7",
     "d3-request": "^1.0.6",
-    "d3-scale": "^2.2.0",
-    "d3-scale-chromatic": "^1.2.0",
+    "d3-scale": "^4.0.0",
     "d3-selection": "^1.3.0",
     "d3-shape": "^1.2.0",
     "d3-transition": "^1.1.1",

web/src/components/countryhistorymixgraph.js

@@ -47,9 +47,14 @@ const prepareGraphData = (historyData, co2ColorScale, displayByEmissions, electr
     // Add production
     modeOrder.forEach((k) => {
       const isStorage = k.indexOf('storage') !== -1;
-      const value = isStorage
+      let value = isStorage
         ? -1 * Math.min(0, (d.storage || {})[k.replace(' storage', '')])
         : (d.production || {})[k];
+
+      if (value === null) {
+        value = undefined;
+      }
+
       // in GW or MW
       obj[k] = value / valueFactor;
       if (Number.isFinite(value) && displayByEmissions && obj[k] != null) {

web/src/components/countrytableoverlayifnodata.js

@@ -15,7 +15,7 @@ const CountryTableOverlayIfNoData = ({ zoneTimeIndex }) => {
 
   // TODO: Shouldn't be hardcoded
   const zonesThatCanHaveZeroProduction = ['AX', 'DK-BHM', 'CA-PE', 'ES-IB-FO'];
-  const zoneHasNotProductionDataAtTimestamp = (!zoneData.production || !Object.keys(zoneData.production).length) && !zonesThatCanHaveZeroProduction.includes(zoneId);
+  const zoneHasNotProductionDataAtTimestamp = (!zoneData.production || Object.values(zoneData.production).every(v => v === null)) && !zonesThatCanHaveZeroProduction.includes(zoneId);
   const zoneIsMissingParser = !zoneData.hasParser;
   const zoneHasData = zoneHasNotProductionDataAtTimestamp && !zoneIsMissingParser;
   const isRealtimeData = zoneTimeIndex === null;

web/src/components/graph/areagraph.js

@@ -57,8 +57,10 @@ const getValueScale = (height, totalValues) => scaleLinear()
 
 const getLayers = (data, layerKeys, layerStroke, layerFill, markerFill) => {
   if (!data || !data[0]) return [];
+
   const stackedData = stack()
     .offset(stackOffsetDiverging)
+    .value((d, key) => d[key] === null ? undefined : d[key])
     .keys(layerKeys)(data);
   return layerKeys.map((key, ind) => ({
     key,

web/src/components/timeslider.js

@@ -17,7 +17,8 @@ const getTimeScale = (rangeEnd, datetimes, startTime, endTime) => scaleTime()
     startTime ? moment(startTime).toDate() : first(datetimes),
     endTime ? moment(endTime).toDate() : last(datetimes),
   ])
-  .range([0, rangeEnd]);
+  .range([0, rangeEnd])
+  .nice(25);
 
 const createChangeAndInputHandler = (
   datetimes,

web/src/reducers/dataReducer.js

@@ -121,8 +121,12 @@ module.exports = (state = initialDataState, action) => {
         });
         // Set date
         zone.datetime = action.payload.datetime;
+
+        if (!zone.production || Object.values(zone.production).every(v => v === null)) {
+          return;
+        }
+
         // Validate data
-        if (!zone.production) return;
         modeOrder.forEach((mode) => {
           if (mode === 'other' || mode === 'unknown' || !zone.datetime) { return; }
           // Check missing values

web/src/sagas/index.js

@@ -27,18 +27,17 @@ function* fetchClientVersion() {
 function* fetchZoneHistory(action) {
   const { zoneId } = action.payload;
   try {
-    const payload = yield call(protectedJsonRequest, `/v3/history?countryCode=${zoneId}`);
+    const payload = yield call(protectedJsonRequest, `/v3/history?countryCode=${zoneId}&preview=1`);
     yield put({ type: 'ZONE_HISTORY_FETCH_SUCCEEDED', zoneId, payload });
   } catch (err) {
     yield put({ type: 'ZONE_HISTORY_FETCH_FAILED' });
     handleRequestError(err);
   }
 }
 
-function* fetchGridData(action) {
-  const { datetime } = action.payload;
+function* fetchGridData() {
   try {
-    const payload = yield call(protectedJsonRequest, datetime ? `/v3/state?datetime=${datetime}` : '/v3/state');
+    const payload = yield call(protectedJsonRequest, '/v3/state?preview=1');
     yield put({ type: 'TRACK_EVENT', payload: { eventName: 'pageview' } });
     yield put({ type: 'APPLICATION_STATE_UPDATE', key: 'callerLocation', value: payload.callerLocation });
     yield put({ type: 'APPLICATION_STATE_UPDATE', key: 'callerZone', value: payload.callerZone });

web/yarn.lock

@@ -3036,7 +3036,14 @@ cyclist@~0.2.2:
   version "0.2.2"
   resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-0.2.2.tgz#1b33792e11e914a2fd6d6ed6447464444e5fa640"
 
-d3-array@^1.2.0, d3-array@^1.2.1:
+"d3-array@2 - 3", "d3-array@2.10.0 - 3":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-3.0.1.tgz#ca45c263f5bb780ab5a34a6e1d3d5883fe4a8d14"
+  integrity sha512-l3Bh5o8RSoC3SBm5ix6ogaFW+J6rOUm42yOtZ2sQPCEvCqUMepeX7zgrlLLGIemxgOyo9s2CsWEidnLv5PwwRw==
+  dependencies:
+    internmap "1 - 2"
+
+d3-array@^1.2.1:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-1.2.4.tgz#635ce4d5eea759f6f605863dbcfc30edc737f71f"
 
@@ -3052,6 +3059,11 @@ d3-color@1:
   version "1.2.3"
   resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-1.2.3.tgz#6c67bb2af6df3cc8d79efcc4d3a3e83e28c8048f"
 
+"d3-color@1 - 3":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-3.0.1.tgz#03316e595955d1fcd39d9f3610ad41bb90194d0a"
+  integrity sha512-6/SlHkDOBLyQSJ1j1Ghs82OIUXpKWlR0hCsw0XrLSQhuUPuCSmLQ1QPH98vpnQxMUQM2/gfAkUEWsupVpd9JGw==
+
 d3-dispatch@1:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/d3-dispatch/-/d3-dispatch-1.0.5.tgz#e25c10a186517cd6c82dd19ea018f07e01e39015"
@@ -3068,7 +3080,12 @@ d3-ease@1:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/d3-ease/-/d3-ease-1.0.5.tgz#8ce59276d81241b1b72042d6af2d40e76d936ffb"
 
-d3-format@1, d3-format@^1.2.2:
+"d3-format@1 - 3":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-3.0.1.tgz#e41b81b2ab79277141ec1404aa5d05001da64084"
+  integrity sha512-hdL7+HBIohpgfolhBxr1KX47VMD6+vVD/oEFrxk5yhmzV2prk99EkFKYpXuhVkFpTgHdJ6/4bYcjdLPPXV4tIA==
+
+d3-format@^1.2.2:
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-1.3.2.tgz#6a96b5e31bcb98122a30863f7d92365c00603562"
 
@@ -3078,6 +3095,13 @@ d3-interpolate@1, d3-interpolate@^1.1.6:
   dependencies:
     d3-color "1"
 
+"d3-interpolate@1.2.0 - 3":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/d3-interpolate/-/d3-interpolate-3.0.1.tgz#3c47aa5b32c5b3dfb56ef3fd4342078a632b400d"
+  integrity sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==
+  dependencies:
+    d3-color "1 - 3"
+
 d3-path@1:
   version "1.0.7"
   resolved "https://registry.yarnpkg.com/d3-path/-/d3-path-1.0.7.tgz#8de7cd693a75ac0b5480d3abaccd94793e58aae8"
@@ -3095,23 +3119,16 @@ d3-request@^1.0.6:
     d3-dsv "1"
     xmlhttprequest "1"
 
-d3-scale-chromatic@^1.2.0:
-  version "1.3.3"
-  resolved "https://registry.yarnpkg.com/d3-scale-chromatic/-/d3-scale-chromatic-1.3.3.tgz#dad4366f0edcb288f490128979c3c793583ed3c0"
-  dependencies:
-    d3-color "1"
-    d3-interpolate "1"
-
-d3-scale@^2.2.0:
-  version "2.2.2"
-  resolved "https://registry.yarnpkg.com/d3-scale/-/d3-scale-2.2.2.tgz#4e880e0b2745acaaddd3ede26a9e908a9e17b81f"
+d3-scale@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/d3-scale/-/d3-scale-4.0.0.tgz#294377ea1d7e5a31509ee648b98d7916ac0b34e3"
+  integrity sha512-foHQYKpWQcyndH1CGoHdUC4PECxTxonzwwBXGT8qu+Drb1FIc6ON6dG2P5f4hRRMkLiIKeWK7iFtdznDUrnuPQ==
   dependencies:
-    d3-array "^1.2.0"
-    d3-collection "1"
-    d3-format "1"
-    d3-interpolate "1"
-    d3-time "1"
-    d3-time-format "2"
+    d3-array "2.10.0 - 3"
+    d3-format "1 - 3"
+    d3-interpolate "1.2.0 - 3"
+    d3-time "2.1.1 - 3"
+    d3-time-format "2 - 4"
 
 d3-selection@^1.1.0, d3-selection@^1.3.0:
   version "1.3.2"
@@ -3123,15 +3140,19 @@ d3-shape@^1.2.0:
   dependencies:
     d3-path "1"
 
-d3-time-format@2:
-  version "2.1.3"
-  resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-2.1.3.tgz#ae06f8e0126a9d60d6364eac5b1533ae1bac826b"
+"d3-time-format@2 - 4":
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-4.0.0.tgz#930ded86a9de761702344760d8a25753467f28b7"
+  integrity sha512-nzaCwlj+ZVBIlFuVOT1RmU+6xb/7D5IcnhHzHQcBgS/aTa5K9fWZNN5LCXA27LgF5WxoSNJqKBbLcGMtM6Ca6A==
   dependencies:
-    d3-time "1"
+    d3-time "1 - 3"
 
-d3-time@1:
-  version "1.0.10"
-  resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-1.0.10.tgz#8259dd71288d72eeacfd8de281c4bf5c7393053c"
+"d3-time@1 - 3", "d3-time@2.1.1 - 3":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-3.0.0.tgz#65972cb98ae2d4954ef5c932e8704061335d4975"
+  integrity sha512-zmV3lRnlaLI08y9IMRXSDshQb5Nj77smnfpnd2LrBa/2K281Jijactokeak14QacHs/kKq0AQ121nidNYlarbQ==
+  dependencies:
+    d3-array "2 - 3"
 
 d3-timer@1:
   version "1.0.9"
@@ -4700,6 +4721,11 @@ internal-slot@^1.0.3:
     has "^1.0.3"
     side-channel "^1.0.4"
 
+"internmap@1 - 2":
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/internmap/-/internmap-2.0.1.tgz#33d0fa016185397549fb1a14ea3dbe5a2949d1cd"
+  integrity sha512-Ujwccrj9FkGqjbY3iVoxD1VV+KdZZeENx0rphrtzmRXbFvkFO88L80BL/zeSIguX/7T+y8k04xqtgWgS5vxwxw==
+
 interpret@1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.2.0.tgz#d5061a6224be58e8083985f5014d844359576296"