Add filter for valid fields. Fixes #4098 (#4109)

build/vega-lite-schema.json

@@ -3745,6 +3745,28 @@
       ],
       "type": "object"
     },
+    "FieldValidPredicate": {
+      "additionalProperties": false,
+      "properties": {
+        "field": {
+          "description": "Field to be filtered.",
+          "type": "string"
+        },
+        "timeUnit": {
+          "$ref": "#/definitions/TimeUnit",
+          "description": "Time unit for the field to be filtered."
+        },
+        "valid": {
+          "description": "If set to true the field's value has to be valid, meaning both not `null` and not [`NaN`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/NaN).",
+          "type": "boolean"
+        }
+      },
+      "required": [
+        "field",
+        "valid"
+      ],
+      "type": "object"
+    },
     "FilterTransform": {
       "additionalProperties": false,
       "properties": {
@@ -6986,6 +7008,9 @@
         {
           "$ref": "#/definitions/FieldGTEPredicate"
         },
+        {
+          "$ref": "#/definitions/FieldValidPredicate"
+        },
         {
           "$ref": "#/definitions/SelectionPredicate"
         },

examples/compiled/rect_binned_heatmap.vg.json

@@ -14,6 +14,10 @@
         "parse": {"IMDB_Rating": "number", "Rotten_Tomatoes_Rating": "number"}
       },
       "transform": [
+        {
+          "type": "filter",
+          "expr": "(datum[\"IMDB_Rating\"]!==null&&!isNaN(datum[\"IMDB_Rating\"])) && (datum[\"Rotten_Tomatoes_Rating\"]!==null&&!isNaN(datum[\"Rotten_Tomatoes_Rating\"]))"
+        },
         {
           "type": "extent",
           "field": "IMDB_Rating",

examples/specs/rect_binned_heatmap.vl.json

@@ -1,6 +1,12 @@
 {
   "$schema": "https://vega.github.io/schema/vega-lite/v2.json",
   "data": {"url": "data/movies.json"},
+  "transform": [{
+    "filter": {"and": [
+      {"field": "IMDB_Rating", "valid": true},
+      {"field": "Rotten_Tomatoes_Rating", "valid": true}
+    ]}
+  }],
   "mark": "rect",
   "width": 300,
   "height": 200,

site/docs/transform/filter.md

@@ -112,7 +112,16 @@ For example, to check if the `height` field's value is greater than or equals to
 
 {% include table.html props="oneOf" source="FieldOneOfPredicate" %}
 
-For example, `{"filter": {"field": "car_color", "oneOf":["red", "yellow"]}}` checks if the `car_color` field's value is `"red"` or `"yellow"`.
+For example, `{"filter": {"field": "car_color", "oneOf": ["red", "yellow"]}}` checks if the `car_color` field's value is `"red"` or `"yellow"`.
+
+
+{:#valid-predicate}
+### Field Valid Predicate
+
+{% include table.html props="valid" source="FieldValidPredicate" %}
+
+For example, `{"filter": {"field": "car_color", "valid": true}}` checks if the `car_color` field's value is valid meaning it is both not `null` and not [`NaN`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/NaN).
+
 
 {:#selection-predicate}
 ## Selection Predicate

src/predicate.ts

@@ -18,6 +18,7 @@ export type Predicate =
   | FieldGTPredicate
   | FieldLTEPredicate
   | FieldGTEPredicate
+  | FieldValidPredicate
   // b) Selection Predicate
   | SelectionPredicate
   // c) Vega Expression string
@@ -30,7 +31,8 @@ export type FieldPredicate =
   | FieldLTEPredicate
   | FieldGTEPredicate
   | FieldRangePredicate
-  | FieldOneOfPredicate;
+  | FieldOneOfPredicate
+  | FieldValidPredicate;
 
 export interface SelectionPredicate {
   /**
@@ -139,12 +141,23 @@ export interface FieldOneOfPredicate extends FieldPredicateBase {
   oneOf: string[] | number[] | boolean[] | DateTime[];
 }
 
+export interface FieldValidPredicate extends FieldPredicateBase {
+  /**
+   * If set to true the field's value has to be valid, meaning both not `null` and not [`NaN`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/NaN).
+   */
+  valid: boolean;
+}
+
 export function isFieldOneOfPredicate(predicate: any): predicate is FieldOneOfPredicate {
   return (
     predicate && !!predicate.field && (isArray(predicate.oneOf) || isArray(predicate.in)) // backward compatibility
   );
 }
 
+export function isFieldValidPredicate(predicate: any): predicate is FieldValidPredicate {
+  return predicate && !!predicate.field && predicate.valid !== undefined;
+}
+
 export function isFieldPredicate(
   predicate: Predicate
 ): predicate is
@@ -216,7 +229,9 @@ export function fieldFilterExpression(predicate: FieldPredicate, useInRange = tr
     const lower = predicate.gte;
     return `${fieldExpr}>=${predicateValueExpr(lower, timeUnit)}`;
   } else if (isFieldOneOfPredicate(predicate)) {
-    return 'indexof([' + predicateValuesExpr(predicate.oneOf, timeUnit).join(',') + '], ' + fieldExpr + ') !== -1';
+    return `indexof([${predicateValuesExpr(predicate.oneOf, timeUnit).join(',')}], ${fieldExpr}) !== -1`;
+  } else if (isFieldValidPredicate(predicate)) {
+    return predicate.valid ? `${fieldExpr}!==null&&!isNaN(${fieldExpr})` : `${fieldExpr}===null||isNaN(${fieldExpr})`;
   } else if (isFieldRangePredicate(predicate)) {
     const lower = predicate.range[0];
     const upper = predicate.range[1];

test/predicate.test.ts

@@ -1,29 +1,42 @@
 import {assert} from 'chai';
-
 import {
   expression,
   fieldFilterExpression,
   isFieldEqualPredicate,
   isFieldLTEPredicate,
   isFieldOneOfPredicate,
-  isFieldRangePredicate
+  isFieldRangePredicate,
+  isFieldValidPredicate,
+  Predicate
 } from '../src/predicate';
 import {TimeUnit} from '../src/timeunit';
+import {without} from '../src/util';
+import {FieldValidPredicate} from './../src/predicate';
 
 describe('filter', () => {
   const equalFilter = {field: 'color', equal: 'red'};
   const oneOfFilter = {field: 'color', oneOf: ['red', 'yellow']};
   const rangeFilter = {field: 'x', range: [0, 5]};
   const exprFilter = 'datum["x"]===5';
   const lessThanEqualsFilter = {field: 'x', lte: 'z'};
+  const validFilter: FieldValidPredicate = {field: 'x', valid: true};
+
+  const allFilters: Predicate[] = [
+    equalFilter,
+    lessThanEqualsFilter,
+    oneOfFilter,
+    rangeFilter,
+    validFilter,
+    exprFilter
+  ];
 
   describe('isEqualFilter', () => {
     it('should return true for an equal filter', () => {
       assert.isTrue(isFieldEqualPredicate(equalFilter));
     });
 
     it('should return false for other filters', () => {
-      [oneOfFilter, rangeFilter, exprFilter].forEach(filter => {
+      without(allFilters, [equalFilter]).forEach(filter => {
         assert.isFalse(isFieldEqualPredicate(filter));
       });
     });
@@ -35,7 +48,7 @@ describe('filter', () => {
     });
 
     it('should return false for other filters', () => {
-      [equalFilter, oneOfFilter, rangeFilter, exprFilter].forEach(filter => {
+      without(allFilters, [lessThanEqualsFilter]).forEach(filter => {
         assert.isFalse(isFieldLTEPredicate(filter));
       });
     });
@@ -47,7 +60,7 @@ describe('filter', () => {
     });
 
     it('should return false for other filters', () => {
-      [equalFilter, rangeFilter, exprFilter].forEach(filter => {
+      without(allFilters, [oneOfFilter]).forEach(filter => {
         assert.isFalse(isFieldOneOfPredicate(filter));
       });
     });
@@ -59,12 +72,24 @@ describe('filter', () => {
     });
 
     it('should return false for other filters', () => {
-      [oneOfFilter, equalFilter, exprFilter].forEach(filter => {
+      without(allFilters, [rangeFilter]).forEach(filter => {
         assert.isFalse(isFieldRangePredicate(filter));
       });
     });
   });
 
+  describe('isValidFilter', () => {
+    it('should return true for a valid filter', () => {
+      assert.isTrue(isFieldValidPredicate(validFilter));
+    });
+
+    it('should return false for other filters', () => {
+      without(allFilters, [validFilter]).forEach(filter => {
+        assert.isFalse(isFieldValidPredicate(filter));
+      });
+    });
+  });
+
   describe('expression', () => {
     it('should return a correct expression for an EqualFilter', () => {
       const expr = expression(null, {field: 'color', equal: 'red'});
@@ -91,6 +116,11 @@ describe('filter', () => {
       assert.equal(expr, 'datum["x"]>=1');
     });
 
+    it('should return correct expression for valid', () => {
+      const expr = expression(null, {field: 'x', valid: true});
+      assert.equal(expr, 'datum["x"]!==null&&!isNaN(datum["x"])');
+    });
+
     it('should return a correct expression for an EqualFilter with datetime object', () => {
       const expr = expression(null, {
         field: 'date',