From e28256c01fd298c0207e0253f2a2fd81fd83e875 Mon Sep 17 00:00:00 2001 From: Kanit Wongsuphasawat Date: Thu, 2 May 2024 19:42:03 -0700 Subject: [PATCH] feat: more options for handling invalid --- build/vega-lite-schema.json | 956 ++++++++++++++---- examples/specs/test_invalid_include.vl.json | 91 ++ src/channel.ts | 2 +- src/compile/data/filterinvalid.ts | 35 +- src/compile/data/index.ts | 10 + src/compile/data/parse.ts | 53 +- src/compile/guide.ts | 11 +- src/compile/invalid/ScaleInvalidDataMode.ts | 68 ++ src/compile/invalid/datasources.ts | 63 ++ .../invalid/normalizeInvalidDataMode.ts | 15 + src/compile/mark/encode/aria.ts | 8 +- src/compile/mark/encode/base.ts | 64 +- src/compile/mark/encode/conditional.ts | 59 +- src/compile/mark/encode/defined.ts | 63 +- src/compile/mark/encode/index.ts | 1 - src/compile/mark/encode/invalid.ts | 68 ++ src/compile/mark/encode/nonposition.ts | 28 +- src/compile/mark/encode/position-point.ts | 92 +- src/compile/mark/encode/text.ts | 8 +- src/compile/mark/encode/tooltip.ts | 10 +- src/compile/mark/encode/valueref.ts | 96 +- src/compile/mark/encode/zindex.ts | 8 +- src/compile/model.ts | 7 +- src/compile/scale/component.ts | 9 +- src/compile/scale/domain.ts | 26 +- src/compositemark/boxplot.ts | 3 +- src/data.ts | 4 +- src/invalid.ts | 70 ++ src/mark.ts | 19 +- src/scale.ts | 7 +- test/compile/data/filterinvalid.test.ts | 7 +- .../invalid/ChannelInvalidDataMode.test.ts | 164 +++ test/compile/mark/text.test.ts | 16 +- test/compile/selection/layers.test.ts | 56 +- test/compile/selection/timeunit.test.ts | 2 +- 35 files changed, 1673 insertions(+), 526 deletions(-) create mode 100644 examples/specs/test_invalid_include.vl.json create mode 100644 src/compile/invalid/ScaleInvalidDataMode.ts create mode 100644 src/compile/invalid/datasources.ts create mode 100644 src/compile/invalid/normalizeInvalidDataMode.ts create mode 100644 src/compile/mark/encode/invalid.ts create mode 100644 src/invalid.ts create mode 100644 test/compile/invalid/ChannelInvalidDataMode.test.ts diff --git a/build/vega-lite-schema.json b/build/vega-lite-schema.json index 5b773b2649..625742d840 100644 --- a/build/vega-lite-schema.json +++ b/build/vega-lite-schema.json @@ -513,15 +513,15 @@ ] }, "invalid": { - "description": "Defines how Vega-Lite should handle marks for invalid values (`null` and `NaN`).\n- If set to `\"filter\"` (default), all data items with null values will be skipped (for line, trail, and area marks) or filtered (for other marks).\n- If `null`, all data items are included. In this case, invalid values will be interpreted as zeroes.", - "enum": [ - "filter", - null + "anyOf": [ + { + "$ref": "#/definitions/MarkInvalidDataMode" + }, + { + "type": "null" + } ], - "type": [ - "string", - "null" - ] + "description": "Defines how Vega-Lite should represent invalid values (`null` and `NaN` in continuous scales without defined output for invalid values) in the marks and their scale domains.\n\n- `\"filter\"` = *Exclude* all invalid values from the visualization's *marks* and *scales*. For path marks (for line, area, trail), this option will create paths that connect valid points only.\n\n- `\"break-paths\"` = Break path marks (for line, area, trail) at invalid values. For non-path marks, this is equivalent to `\"filter\"`. All *scale* domains will *exclude* these filtered data points.\n\n- `\"break-paths-keep-domains\"` = Break paths (for line, area, trail) at invalid values. Hide invalid values for non-path marks. All *scale* domains will *include* these filtered data points.\n\n- `\"include\"` | `null`. Include all data points in the marks and scale domains. By default, invalid values will output the same visual values as zeroes if zero is in the scale domain or otherwise the scale's min value.\n\n- `\"break-and-keep-path-domains\"` (default). This is equivalent to `\"break-path-keep-domains` for path-based marks (line/area/trail) and `\"filter\"` for other marks." }, "limit": { "anyOf": [ @@ -3383,15 +3383,15 @@ ] }, "invalid": { - "description": "Defines how Vega-Lite should handle marks for invalid values (`null` and `NaN`).\n- If set to `\"filter\"` (default), all data items with null values will be skipped (for line, trail, and area marks) or filtered (for other marks).\n- If `null`, all data items are included. In this case, invalid values will be interpreted as zeroes.", - "enum": [ - "filter", - null + "anyOf": [ + { + "$ref": "#/definitions/MarkInvalidDataMode" + }, + { + "type": "null" + } ], - "type": [ - "string", - "null" - ] + "description": "Defines how Vega-Lite should represent invalid values (`null` and `NaN` in continuous scales without defined output for invalid values) in the marks and their scale domains.\n\n- `\"filter\"` = *Exclude* all invalid values from the visualization's *marks* and *scales*. For path marks (for line, area, trail), this option will create paths that connect valid points only.\n\n- `\"break-paths\"` = Break path marks (for line, area, trail) at invalid values. For non-path marks, this is equivalent to `\"filter\"`. All *scale* domains will *exclude* these filtered data points.\n\n- `\"break-paths-keep-domains\"` = Break paths (for line, area, trail) at invalid values. Hide invalid values for non-path marks. All *scale* domains will *include* these filtered data points.\n\n- `\"include\"` | `null`. Include all data points in the marks and scale domains. By default, invalid values will output the same visual values as zeroes if zero is in the scale domain or otherwise the scale's min value.\n\n- `\"break-and-keep-path-domains\"` (default). This is equivalent to `\"break-path-keep-domains` for path-based marks (line/area/trail) and `\"filter\"` for other marks." }, "limit": { "anyOf": [ @@ -4613,15 +4613,15 @@ "description": "The extent of the whiskers. Available options include:\n- `\"min-max\"`: min and max are the lower and upper whiskers respectively.\n- A number representing multiple of the interquartile range. This number will be multiplied by the IQR to determine whisker boundary, which spans from the smallest data to the largest data within the range _[Q1 - k * IQR, Q3 + k * IQR]_ where _Q1_ and _Q3_ are the first and third quartiles while _IQR_ is the interquartile range (_Q3-Q1_).\n\n__Default value:__ `1.5`." }, "invalid": { - "description": "Defines how Vega-Lite should handle marks for invalid values (`null` and `NaN`).\n- If set to `\"filter\"` (default), all data items with null values will be skipped (for line, trail, and area marks) or filtered (for other marks).\n- If `null`, all data items are included. In this case, invalid values will be interpreted as zeroes.", - "enum": [ - "filter", - null + "anyOf": [ + { + "$ref": "#/definitions/MarkInvalidDataMode" + }, + { + "type": "null" + } ], - "type": [ - "string", - "null" - ] + "description": "Defines how Vega-Lite should represent invalid values (`null` and `NaN` in continuous scales without defined output for invalid values) in the marks and their scale domains.\n\n- `\"filter\"` = *Exclude* all invalid values from the visualization's *marks* and *scales*. For path marks (for line, area, trail), this option will create paths that connect valid points only.\n\n- `\"break-paths\"` = Break path marks (for line, area, trail) at invalid values. For non-path marks, this is equivalent to `\"filter\"`. All *scale* domains will *exclude* these filtered data points.\n\n- `\"break-paths-keep-domains\"` = Break paths (for line, area, trail) at invalid values. Hide invalid values for non-path marks. All *scale* domains will *include* these filtered data points.\n\n- `\"include\"` | `null`. Include all data points in the marks and scale domains. By default, invalid values will output the same visual values as zeroes if zero is in the scale domain or otherwise the scale's min value.\n\n- `\"break-and-keep-path-domains\"` (default). This is equivalent to `\"break-path-keep-domains` for path-based marks (line/area/trail) and `\"filter\"` for other marks." }, "median": { "anyOf": [ @@ -15059,15 +15059,15 @@ ] }, "invalid": { - "description": "Defines how Vega-Lite should handle marks for invalid values (`null` and `NaN`).\n- If set to `\"filter\"` (default), all data items with null values will be skipped (for line, trail, and area marks) or filtered (for other marks).\n- If `null`, all data items are included. In this case, invalid values will be interpreted as zeroes.", - "enum": [ - "filter", - null + "anyOf": [ + { + "$ref": "#/definitions/MarkInvalidDataMode" + }, + { + "type": "null" + } ], - "type": [ - "string", - "null" - ] + "description": "Defines how Vega-Lite should represent invalid values (`null` and `NaN` in continuous scales without defined output for invalid values) in the marks and their scale domains.\n\n- `\"filter\"` = *Exclude* all invalid values from the visualization's *marks* and *scales*. For path marks (for line, area, trail), this option will create paths that connect valid points only.\n\n- `\"break-paths\"` = Break path marks (for line, area, trail) at invalid values. For non-path marks, this is equivalent to `\"filter\"`. All *scale* domains will *exclude* these filtered data points.\n\n- `\"break-paths-keep-domains\"` = Break paths (for line, area, trail) at invalid values. Hide invalid values for non-path marks. All *scale* domains will *include* these filtered data points.\n\n- `\"include\"` | `null`. Include all data points in the marks and scale domains. By default, invalid values will output the same visual values as zeroes if zero is in the scale domain or otherwise the scale's min value.\n\n- `\"break-and-keep-path-domains\"` (default). This is equivalent to `\"break-path-keep-domains` for path-based marks (line/area/trail) and `\"filter\"` for other marks." }, "limit": { "anyOf": [ @@ -16214,15 +16214,15 @@ ] }, "invalid": { - "description": "Defines how Vega-Lite should handle marks for invalid values (`null` and `NaN`).\n- If set to `\"filter\"` (default), all data items with null values will be skipped (for line, trail, and area marks) or filtered (for other marks).\n- If `null`, all data items are included. In this case, invalid values will be interpreted as zeroes.", - "enum": [ - "filter", - null + "anyOf": [ + { + "$ref": "#/definitions/MarkInvalidDataMode" + }, + { + "type": "null" + } ], - "type": [ - "string", - "null" - ] + "description": "Defines how Vega-Lite should represent invalid values (`null` and `NaN` in continuous scales without defined output for invalid values) in the marks and their scale domains.\n\n- `\"filter\"` = *Exclude* all invalid values from the visualization's *marks* and *scales*. For path marks (for line, area, trail), this option will create paths that connect valid points only.\n\n- `\"break-paths\"` = Break path marks (for line, area, trail) at invalid values. For non-path marks, this is equivalent to `\"filter\"`. All *scale* domains will *exclude* these filtered data points.\n\n- `\"break-paths-keep-domains\"` = Break paths (for line, area, trail) at invalid values. Hide invalid values for non-path marks. All *scale* domains will *include* these filtered data points.\n\n- `\"include\"` | `null`. Include all data points in the marks and scale domains. By default, invalid values will output the same visual values as zeroes if zero is in the scale domain or otherwise the scale's min value.\n\n- `\"break-and-keep-path-domains\"` (default). This is equivalent to `\"break-path-keep-domains` for path-based marks (line/area/trail) and `\"filter\"` for other marks." }, "limit": { "anyOf": [ @@ -17058,15 +17058,15 @@ ] }, "invalid": { - "description": "Defines how Vega-Lite should handle marks for invalid values (`null` and `NaN`).\n- If set to `\"filter\"` (default), all data items with null values will be skipped (for line, trail, and area marks) or filtered (for other marks).\n- If `null`, all data items are included. In this case, invalid values will be interpreted as zeroes.", - "enum": [ - "filter", - null + "anyOf": [ + { + "$ref": "#/definitions/MarkInvalidDataMode" + }, + { + "type": "null" + } ], - "type": [ - "string", - "null" - ] + "description": "Defines how Vega-Lite should represent invalid values (`null` and `NaN` in continuous scales without defined output for invalid values) in the marks and their scale domains.\n\n- `\"filter\"` = *Exclude* all invalid values from the visualization's *marks* and *scales*. For path marks (for line, area, trail), this option will create paths that connect valid points only.\n\n- `\"break-paths\"` = Break path marks (for line, area, trail) at invalid values. For non-path marks, this is equivalent to `\"filter\"`. All *scale* domains will *exclude* these filtered data points.\n\n- `\"break-paths-keep-domains\"` = Break paths (for line, area, trail) at invalid values. Hide invalid values for non-path marks. All *scale* domains will *include* these filtered data points.\n\n- `\"include\"` | `null`. Include all data points in the marks and scale domains. By default, invalid values will output the same visual values as zeroes if zero is in the scale domain or otherwise the scale's min value.\n\n- `\"break-and-keep-path-domains\"` (default). This is equivalent to `\"break-path-keep-domains` for path-based marks (line/area/trail) and `\"filter\"` for other marks." }, "limit": { "anyOf": [ @@ -17640,6 +17640,16 @@ ], "type": "object" }, + "MarkInvalidDataMode": { + "enum": [ + "filter", + "break-paths", + "break-paths-keep-domains", + "break-and-keep-path-domains", + "include" + ], + "type": "string" + }, "MarkPropDef<(Gradient|string|null)>": { "anyOf": [ { @@ -18596,15 +18606,15 @@ ] }, "invalid": { - "description": "Defines how Vega-Lite should handle marks for invalid values (`null` and `NaN`).\n- If set to `\"filter\"` (default), all data items with null values will be skipped (for line, trail, and area marks) or filtered (for other marks).\n- If `null`, all data items are included. In this case, invalid values will be interpreted as zeroes.", - "enum": [ - "filter", - null + "anyOf": [ + { + "$ref": "#/definitions/MarkInvalidDataMode" + }, + { + "type": "null" + } ], - "type": [ - "string", - "null" - ] + "description": "Defines how Vega-Lite should represent invalid values (`null` and `NaN` in continuous scales without defined output for invalid values) in the marks and their scale domains.\n\n- `\"filter\"` = *Exclude* all invalid values from the visualization's *marks* and *scales*. For path marks (for line, area, trail), this option will create paths that connect valid points only.\n\n- `\"break-paths\"` = Break path marks (for line, area, trail) at invalid values. For non-path marks, this is equivalent to `\"filter\"`. All *scale* domains will *exclude* these filtered data points.\n\n- `\"break-paths-keep-domains\"` = Break paths (for line, area, trail) at invalid values. Hide invalid values for non-path marks. All *scale* domains will *include* these filtered data points.\n\n- `\"include\"` | `null`. Include all data points in the marks and scale domains. By default, invalid values will output the same visual values as zeroes if zero is in the scale domain or otherwise the scale's min value.\n\n- `\"break-and-keep-path-domains\"` (default). This is equivalent to `\"break-path-keep-domains` for path-based marks (line/area/trail) and `\"filter\"` for other marks." }, "limit": { "anyOf": [ @@ -20892,15 +20902,15 @@ ] }, "invalid": { - "description": "Defines how Vega-Lite should handle marks for invalid values (`null` and `NaN`).\n- If set to `\"filter\"` (default), all data items with null values will be skipped (for line, trail, and area marks) or filtered (for other marks).\n- If `null`, all data items are included. In this case, invalid values will be interpreted as zeroes.", - "enum": [ - "filter", - null + "anyOf": [ + { + "$ref": "#/definitions/MarkInvalidDataMode" + }, + { + "type": "null" + } ], - "type": [ - "string", - "null" - ] + "description": "Defines how Vega-Lite should represent invalid values (`null` and `NaN` in continuous scales without defined output for invalid values) in the marks and their scale domains.\n\n- `\"filter\"` = *Exclude* all invalid values from the visualization's *marks* and *scales*. For path marks (for line, area, trail), this option will create paths that connect valid points only.\n\n- `\"break-paths\"` = Break path marks (for line, area, trail) at invalid values. For non-path marks, this is equivalent to `\"filter\"`. All *scale* domains will *exclude* these filtered data points.\n\n- `\"break-paths-keep-domains\"` = Break paths (for line, area, trail) at invalid values. Hide invalid values for non-path marks. All *scale* domains will *include* these filtered data points.\n\n- `\"include\"` | `null`. Include all data points in the marks and scale domains. By default, invalid values will output the same visual values as zeroes if zero is in the scale domain or otherwise the scale's min value.\n\n- `\"break-and-keep-path-domains\"` (default). This is equivalent to `\"break-path-keep-domains` for path-based marks (line/area/trail) and `\"filter\"` for other marks." }, "limit": { "anyOf": [ @@ -22102,6 +22112,10 @@ "description": "Default padding for continuous x/y scales.\n\n__Default:__ The bar width for continuous x-scale of a vertical bar and continuous y-scale of a horizontal bar.; `0` otherwise.", "minimum": 0 }, + "invalid": { + "$ref": "#/definitions/ScaleInvalidDataConfig", + "description": "An object that defines scale outputs per channel for invalid values (null, NaN) on a continuous scale.\n- The keys in this object are the scale channels.\n- The values is either `\"zero-or-min\"`or a value definition `{value: ...}`.\n\n_Example:_ Setting this `config.scale.invalid` property to `{color: {value: '#aaa'}}` will make Vega-Lite color all invalid values with '#aaa'." + }, "maxBandSize": { "description": "The default max value for mapping quantitative fields to bar's size/bandSize.\n\nIf undefined (default), we will use the axis's size (width or height) - 1.", "minimum": 0, @@ -22134,7 +22148,7 @@ "type": "number" }, "minFontSize": { - "description": "The default min value for mapping quantitative fields to tick's size/fontSize scale with zero=false\n\n__Default value:__ `8`", + "description": "The default min value for mapping quantitative fields to text's size/fontSize scale with zero=false\n\n__Default value:__ `8`", "minimum": 0, "type": "number" }, @@ -22411,200 +22425,748 @@ ], "type": "object" }, - "ScaleResolveMap": { + "ScaleInvalidDataConfig": { "additionalProperties": false, "properties": { "angle": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"angle\">" }, "color": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"color\">" }, "fill": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"fill\">" }, "fillOpacity": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"fillOpacity\">" }, "opacity": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"opacity\">" }, "radius": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"radius\">" }, "shape": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"shape\">" }, "size": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"size\">" }, "stroke": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"stroke\">" }, "strokeDash": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"strokeDash\">" }, "strokeOpacity": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"strokeOpacity\">" }, "strokeWidth": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"strokeWidth\">" }, "theta": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"theta\">" }, "x": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"x\">" }, "xOffset": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"xOffset\">" }, "y": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"y\">" }, "yOffset": { - "$ref": "#/definitions/ResolveMode" + "$ref": "#/definitions/ScaleInvalidDataIncludeAs<\"yOffset\">" } }, "type": "object" }, - "ScaleType": { - "enum": [ - "linear", - "log", - "pow", - "sqrt", - "symlog", - "identity", - "sequential", - "time", - "utc", - "quantile", - "quantize", - "threshold", - "bin-ordinal", - "ordinal", - "point", - "band" - ], - "type": "string" - }, - "SchemeParams": { - "additionalProperties": false, - "properties": { - "count": { - "description": "The number of colors to use in the scheme. This can be useful for scale types such as `\"quantize\"`, which use the length of the scale range to determine the number of discrete bins for the scale domain.", - "type": "number" + "ScaleInvalidDataIncludeAs<\"angle\">": { + "anyOf": [ + { + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"angle\">" }, - "extent": { - "description": "The extent of the color range to use. For example `[0.2, 1]` will rescale the color scheme such that color values in the range _[0, 0.2)_ are excluded from the scheme.", - "items": { - "type": "number" - }, - "type": "array" + { + "const": "zero-or-min", + "type": "string" }, - "name": { - "$ref": "#/definitions/ColorScheme", - "description": "A color scheme name for ordinal scales (e.g., `\"category10\"` or `\"blues\"`).\n\nFor the full list of supported schemes, please refer to the [Vega Scheme](https://vega.github.io/vega/docs/schemes/#reference) reference." + { + "const": "min", + "type": "string" } - }, - "required": [ - "name" - ], - "type": "object" + ] }, - "SecondaryFieldDef": { - "additionalProperties": false, - "description": "A field definition of a secondary channel that shares a scale with another primary channel. For example, `x2`, `xError` and `xError2` share the same scale with `x`.", - "properties": { - "aggregate": { - "$ref": "#/definitions/Aggregate", - "description": "Aggregation function for the field (e.g., `\"mean\"`, `\"sum\"`, `\"median\"`, `\"min\"`, `\"max\"`, `\"count\"`).\n\n__Default value:__ `undefined` (None)\n\n__See also:__ [`aggregate`](https://vega.github.io/vega-lite/docs/aggregate.html) documentation." - }, - "bandPosition": { - "description": "Relative position on a band of a stacked, binned, time unit, or band scale. For example, the marks will be positioned at the beginning of the band if set to `0`, and at the middle of the band if set to `0.5`.", - "maximum": 1, - "minimum": 0, - "type": "number" + "ScaleInvalidDataIncludeAs<\"color\">": { + "anyOf": [ + { + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"color\">" }, - "bin": { - "description": "A flag for binning a `quantitative` field, [an object defining binning parameters](https://vega.github.io/vega-lite/docs/bin.html#bin-parameters), or indicating that the data for `x` or `y` channel are binned before they are imported into Vega-Lite (`\"binned\"`).\n\n- If `true`, default [binning parameters](https://vega.github.io/vega-lite/docs/bin.html#bin-parameters) will be applied.\n\n- If `\"binned\"`, this indicates that the data for the `x` (or `y`) channel are already binned. You can map the bin-start field to `x` (or `y`) and the bin-end field to `x2` (or `y2`). The scale and axis will be formatted similar to binning in Vega-Lite. To adjust the axis ticks based on the bin step, you can also set the axis's [`tickMinStep`](https://vega.github.io/vega-lite/docs/axis.html#ticks) property.\n\n__Default value:__ `false`\n\n__See also:__ [`bin`](https://vega.github.io/vega-lite/docs/bin.html) documentation.", - "type": "null" + { + "const": "zero-or-min", + "type": "string" }, - "field": { - "$ref": "#/definitions/Field", - "description": "__Required.__ A string defining the name of the field from which to pull a data value or an object defining iterated values from the [`repeat`](https://vega.github.io/vega-lite/docs/repeat.html) operator.\n\n__See also:__ [`field`](https://vega.github.io/vega-lite/docs/field.html) documentation.\n\n__Notes:__ 1) Dots (`.`) and brackets (`[` and `]`) can be used to access nested objects (e.g., `\"field\": \"foo.bar\"` and `\"field\": \"foo['bar']\"`). If field names contain dots or brackets but are not nested, you can use `\\\\` to escape dots and brackets (e.g., `\"a\\\\.b\"` and `\"a\\\\[0\\\\]\"`). See more details about escaping in the [field documentation](https://vega.github.io/vega-lite/docs/field.html). 2) `field` is not required if `aggregate` is `count`." + { + "const": "min", + "type": "string" + } + ] + }, + "ScaleInvalidDataIncludeAs<\"fill\">": { + "anyOf": [ + { + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"fill\">" }, - "timeUnit": { - "anyOf": [ - { - "$ref": "#/definitions/TimeUnit" - }, - { - "$ref": "#/definitions/BinnedTimeUnit" - }, - { - "$ref": "#/definitions/TimeUnitParams" - } - ], - "description": "Time unit (e.g., `year`, `yearmonth`, `month`, `hours`) for a temporal field. or [a temporal field that gets casted as ordinal](https://vega.github.io/vega-lite/docs/type.html#cast).\n\n__Default value:__ `undefined` (None)\n\n__See also:__ [`timeUnit`](https://vega.github.io/vega-lite/docs/timeunit.html) documentation." + { + "const": "zero-or-min", + "type": "string" }, - "title": { - "anyOf": [ - { - "$ref": "#/definitions/Text" - }, - { - "type": "null" - } - ], - "description": "A title for the field. If `null`, the title will be removed.\n\n__Default value:__ derived from the field's name and transformation function (`aggregate`, `bin` and `timeUnit`). If the field has an aggregate function, the function is displayed as part of the title (e.g., `\"Sum of Profit\"`). If the field is binned or has a time unit applied, the applied function is shown in parentheses (e.g., `\"Profit (binned)\"`, `\"Transaction Date (year-month)\"`). Otherwise, the title is simply the field name.\n\n__Notes__:\n\n1) You can customize the default field title format by providing the [`fieldTitle`](https://vega.github.io/vega-lite/docs/config.html#top-level-config) property in the [config](https://vega.github.io/vega-lite/docs/config.html) or [`fieldTitle` function via the `compile` function's options](https://vega.github.io/vega-lite/usage/compile.html#field-title).\n\n2) If both field definition's `title` and axis, header, or legend `title` are defined, axis/header/legend title will be used." + { + "const": "min", + "type": "string" } - }, - "type": "object" + ] }, - "SelectionConfig": { - "additionalProperties": false, - "properties": { - "interval": { - "$ref": "#/definitions/IntervalSelectionConfigWithoutType", - "description": "The default definition for an [`interval`](https://vega.github.io/vega-lite/docs/parameter.html#select) selection. All properties and transformations for an interval selection definition (except `type`) may be specified here.\n\nFor instance, setting `interval` to `{\"translate\": false}` disables the ability to move interval selections by default." + "ScaleInvalidDataIncludeAs<\"fillOpacity\">": { + "anyOf": [ + { + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"fillOpacity\">" }, - "point": { - "$ref": "#/definitions/PointSelectionConfigWithoutType", - "description": "The default definition for a [`point`](https://vega.github.io/vega-lite/docs/parameter.html#select) selection. All properties and transformations for a point selection definition (except `type`) may be specified here.\n\nFor instance, setting `point` to `{\"on\": \"dblclick\"}` populates point selections on double-click by default." + { + "const": "zero-or-min", + "type": "string" + }, + { + "const": "min", + "type": "string" } - }, - "type": "object" + ] }, - "SelectionInit": { + "ScaleInvalidDataIncludeAs<\"opacity\">": { "anyOf": [ { - "$ref": "#/definitions/PrimitiveValue" + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"opacity\">" }, { - "$ref": "#/definitions/DateTime" + "const": "zero-or-min", + "type": "string" + }, + { + "const": "min", + "type": "string" } ] }, - "SelectionInitInterval": { + "ScaleInvalidDataIncludeAs<\"radius\">": { "anyOf": [ { - "$ref": "#/definitions/Vector2" + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"radius\">" }, { - "$ref": "#/definitions/Vector2" + "const": "zero-or-min", + "type": "string" }, { - "$ref": "#/definitions/Vector2" + "const": "min", + "type": "string" + } + ] + }, + "ScaleInvalidDataIncludeAs<\"shape\">": { + "anyOf": [ + { + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"shape\">" }, { - "$ref": "#/definitions/Vector2" + "const": "zero-or-min", + "type": "string" + }, + { + "const": "min", + "type": "string" } ] }, - "SelectionInitIntervalMapping": { - "$ref": "#/definitions/Dict" + "ScaleInvalidDataIncludeAs<\"size\">": { + "anyOf": [ + { + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"size\">" + }, + { + "const": "zero-or-min", + "type": "string" + }, + { + "const": "min", + "type": "string" + } + ] + }, + "ScaleInvalidDataIncludeAs<\"stroke\">": { + "anyOf": [ + { + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"stroke\">" + }, + { + "const": "zero-or-min", + "type": "string" + }, + { + "const": "min", + "type": "string" + } + ] + }, + "ScaleInvalidDataIncludeAs<\"strokeDash\">": { + "anyOf": [ + { + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"strokeDash\">" + }, + { + "const": "zero-or-min", + "type": "string" + }, + { + "const": "min", + "type": "string" + } + ] + }, + "ScaleInvalidDataIncludeAs<\"strokeOpacity\">": { + "anyOf": [ + { + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"strokeOpacity\">" + }, + { + "const": "zero-or-min", + "type": "string" + }, + { + "const": "min", + "type": "string" + } + ] + }, + "ScaleInvalidDataIncludeAs<\"strokeWidth\">": { + "anyOf": [ + { + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"strokeWidth\">" + }, + { + "const": "zero-or-min", + "type": "string" + }, + { + "const": "min", + "type": "string" + } + ] + }, + "ScaleInvalidDataIncludeAs<\"theta\">": { + "anyOf": [ + { + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"theta\">" + }, + { + "const": "zero-or-min", + "type": "string" + }, + { + "const": "min", + "type": "string" + } + ] + }, + "ScaleInvalidDataIncludeAs<\"x\">": { + "anyOf": [ + { + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"x\">" + }, + { + "const": "zero-or-min", + "type": "string" + }, + { + "const": "min", + "type": "string" + } + ] + }, + "ScaleInvalidDataIncludeAs<\"xOffset\">": { + "anyOf": [ + { + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"xOffset\">" + }, + { + "const": "zero-or-min", + "type": "string" + }, + { + "const": "min", + "type": "string" + } + ] + }, + "ScaleInvalidDataIncludeAs<\"y\">": { + "anyOf": [ + { + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"y\">" + }, + { + "const": "zero-or-min", + "type": "string" + }, + { + "const": "min", + "type": "string" + } + ] + }, + "ScaleInvalidDataIncludeAs<\"yOffset\">": { + "anyOf": [ + { + "$ref": "#/definitions/ScaleInvalidDataIncludeAsValue<\"yOffset\">" + }, + { + "const": "zero-or-min", + "type": "string" + }, + { + "const": "min", + "type": "string" + } + ] + }, + "ScaleInvalidDataIncludeAsValue<\"angle\">": { + "additionalProperties": false, + "properties": { + "value": { + "description": "The rotation angle of the text, in degrees.", + "maximum": 360, + "minimum": 0, + "type": "number" + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"color\">": { + "additionalProperties": false, + "properties": { + "value": { + "anyOf": [ + { + "$ref": "#/definitions/Color" + }, + { + "$ref": "#/definitions/Gradient" + } + ], + "description": "Default color.\n\n__Default value:__ ■ `\"#4682b4\"`\n\n__Note:__\n- This property cannot be used in a [style config](https://vega.github.io/vega-lite/docs/mark.html#style-config).\n- The `fill` and `stroke` properties have higher precedence than `color` and will override `color`." + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"fill\">": { + "additionalProperties": false, + "properties": { + "value": { + "anyOf": [ + { + "$ref": "#/definitions/Color" + }, + { + "$ref": "#/definitions/Gradient" + }, + { + "type": "null" + } + ], + "description": "Default fill color. This property has higher precedence than `config.color`. Set to `null` to remove fill.\n\n__Default value:__ (None)" + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"fillOpacity\">": { + "additionalProperties": false, + "properties": { + "value": { + "description": "The fill opacity (value between [0,1]).\n\n__Default value:__ `1`", + "maximum": 1, + "minimum": 0, + "type": "number" + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"opacity\">": { + "additionalProperties": false, + "properties": { + "value": { + "description": "The overall opacity (value between [0,1]).\n\n__Default value:__ `0.7` for non-aggregate plots with `point`, `tick`, `circle`, or `square` marks or layered `bar` charts and `1` otherwise.", + "maximum": 1, + "minimum": 0, + "type": "number" + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"radius\">": { + "additionalProperties": false, + "properties": { + "value": { + "description": "For arc mark, the primary (outer) radius in pixels.\n\nFor text marks, polar coordinate radial offset, in pixels, of the text from the origin determined by the `x` and `y` properties.\n\n__Default value:__ `min(plot_width, plot_height)/2`", + "minimum": 0, + "type": "number" + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"shape\">": { + "additionalProperties": false, + "properties": { + "value": { + "anyOf": [ + { + "$ref": "#/definitions/SymbolShape" + }, + { + "type": "string" + } + ], + "description": "Shape of the point marks. Supported values include:\n- plotting shapes: `\"circle\"`, `\"square\"`, `\"cross\"`, `\"diamond\"`, `\"triangle-up\"`, `\"triangle-down\"`, `\"triangle-right\"`, or `\"triangle-left\"`.\n- the line symbol `\"stroke\"`\n- centered directional shapes `\"arrow\"`, `\"wedge\"`, or `\"triangle\"`\n- a custom [SVG path string](https://developer.mozilla.org/en-US/docs/Web/SVG/Tutorial/Paths) (For correct sizing, custom shape paths should be defined within a square bounding box with coordinates ranging from -1 to 1 along both the x and y dimensions.)\n\n__Default value:__ `\"circle\"`" + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"size\">": { + "additionalProperties": false, + "properties": { + "value": { + "description": "Default size for marks.\n- For `point`/`circle`/`square`, this represents the pixel area of the marks. Note that this value sets the area of the symbol; the side lengths will increase with the square root of this value.\n- For `bar`, this represents the band size of the bar, in pixels.\n- For `text`, this represents the font size, in pixels.\n\n__Default value:__\n- `30` for point, circle, square marks; width/height's `step`\n- `2` for bar marks with discrete dimensions;\n- `5` for bar marks with continuous dimensions;\n- `11` for text marks.", + "minimum": 0, + "type": "number" + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"stroke\">": { + "additionalProperties": false, + "properties": { + "value": { + "anyOf": [ + { + "$ref": "#/definitions/Color" + }, + { + "$ref": "#/definitions/Gradient" + }, + { + "type": "null" + } + ], + "description": "Default stroke color. This property has higher precedence than `config.color`. Set to `null` to remove stroke.\n\n__Default value:__ (None)" + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"strokeDash\">": { + "additionalProperties": false, + "properties": { + "value": { + "description": "An array of alternating stroke, space lengths for creating dashed or dotted lines.", + "items": { + "type": "number" + }, + "type": "array" + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"strokeOpacity\">": { + "additionalProperties": false, + "properties": { + "value": { + "description": "The stroke opacity (value between [0,1]).\n\n__Default value:__ `1`", + "maximum": 1, + "minimum": 0, + "type": "number" + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"strokeWidth\">": { + "additionalProperties": false, + "properties": { + "value": { + "description": "The stroke width, in pixels.", + "minimum": 0, + "type": "number" + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"theta\">": { + "additionalProperties": false, + "properties": { + "value": { + "description": "- For arc marks, the arc length in radians if theta2 is not specified, otherwise the start arc angle. (A value of 0 indicates up or “north”, increasing values proceed clockwise.)\n\n- For text marks, polar coordinate angle in radians.", + "maximum": 360, + "minimum": 0, + "type": "number" + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"x\">": { + "additionalProperties": false, + "properties": { + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "const": "width", + "type": "string" + } + ], + "description": "X coordinates of the marks, or width of horizontal `\"bar\"` and `\"area\"` without specified `x2` or `width`.\n\nThe `value` of this channel can be a number or a string `\"width\"` for the width of the plot." + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"xOffset\">": { + "additionalProperties": false, + "properties": { + "value": { + "description": "Offset for x-position.", + "type": "number" + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"y\">": { + "additionalProperties": false, + "properties": { + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "const": "height", + "type": "string" + } + ], + "description": "Y coordinates of the marks, or height of vertical `\"bar\"` and `\"area\"` without specified `y2` or `height`.\n\nThe `value` of this channel can be a number or a string `\"height\"` for the height of the plot." + } + }, + "type": "object" + }, + "ScaleInvalidDataIncludeAsValue<\"yOffset\">": { + "additionalProperties": false, + "properties": { + "value": { + "description": "Offset for y-position.", + "type": "number" + } + }, + "type": "object" + }, + "ScaleResolveMap": { + "additionalProperties": false, + "properties": { + "angle": { + "$ref": "#/definitions/ResolveMode" + }, + "color": { + "$ref": "#/definitions/ResolveMode" + }, + "fill": { + "$ref": "#/definitions/ResolveMode" + }, + "fillOpacity": { + "$ref": "#/definitions/ResolveMode" + }, + "opacity": { + "$ref": "#/definitions/ResolveMode" + }, + "radius": { + "$ref": "#/definitions/ResolveMode" + }, + "shape": { + "$ref": "#/definitions/ResolveMode" + }, + "size": { + "$ref": "#/definitions/ResolveMode" + }, + "stroke": { + "$ref": "#/definitions/ResolveMode" + }, + "strokeDash": { + "$ref": "#/definitions/ResolveMode" + }, + "strokeOpacity": { + "$ref": "#/definitions/ResolveMode" + }, + "strokeWidth": { + "$ref": "#/definitions/ResolveMode" + }, + "theta": { + "$ref": "#/definitions/ResolveMode" + }, + "x": { + "$ref": "#/definitions/ResolveMode" + }, + "xOffset": { + "$ref": "#/definitions/ResolveMode" + }, + "y": { + "$ref": "#/definitions/ResolveMode" + }, + "yOffset": { + "$ref": "#/definitions/ResolveMode" + } + }, + "type": "object" + }, + "ScaleType": { + "enum": [ + "linear", + "log", + "pow", + "sqrt", + "symlog", + "identity", + "sequential", + "time", + "utc", + "quantile", + "quantize", + "threshold", + "bin-ordinal", + "ordinal", + "point", + "band" + ], + "type": "string" + }, + "SchemeParams": { + "additionalProperties": false, + "properties": { + "count": { + "description": "The number of colors to use in the scheme. This can be useful for scale types such as `\"quantize\"`, which use the length of the scale range to determine the number of discrete bins for the scale domain.", + "type": "number" + }, + "extent": { + "description": "The extent of the color range to use. For example `[0.2, 1]` will rescale the color scheme such that color values in the range _[0, 0.2)_ are excluded from the scheme.", + "items": { + "type": "number" + }, + "type": "array" + }, + "name": { + "$ref": "#/definitions/ColorScheme", + "description": "A color scheme name for ordinal scales (e.g., `\"category10\"` or `\"blues\"`).\n\nFor the full list of supported schemes, please refer to the [Vega Scheme](https://vega.github.io/vega/docs/schemes/#reference) reference." + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "SecondaryFieldDef": { + "additionalProperties": false, + "description": "A field definition of a secondary channel that shares a scale with another primary channel. For example, `x2`, `xError` and `xError2` share the same scale with `x`.", + "properties": { + "aggregate": { + "$ref": "#/definitions/Aggregate", + "description": "Aggregation function for the field (e.g., `\"mean\"`, `\"sum\"`, `\"median\"`, `\"min\"`, `\"max\"`, `\"count\"`).\n\n__Default value:__ `undefined` (None)\n\n__See also:__ [`aggregate`](https://vega.github.io/vega-lite/docs/aggregate.html) documentation." + }, + "bandPosition": { + "description": "Relative position on a band of a stacked, binned, time unit, or band scale. For example, the marks will be positioned at the beginning of the band if set to `0`, and at the middle of the band if set to `0.5`.", + "maximum": 1, + "minimum": 0, + "type": "number" + }, + "bin": { + "description": "A flag for binning a `quantitative` field, [an object defining binning parameters](https://vega.github.io/vega-lite/docs/bin.html#bin-parameters), or indicating that the data for `x` or `y` channel are binned before they are imported into Vega-Lite (`\"binned\"`).\n\n- If `true`, default [binning parameters](https://vega.github.io/vega-lite/docs/bin.html#bin-parameters) will be applied.\n\n- If `\"binned\"`, this indicates that the data for the `x` (or `y`) channel are already binned. You can map the bin-start field to `x` (or `y`) and the bin-end field to `x2` (or `y2`). The scale and axis will be formatted similar to binning in Vega-Lite. To adjust the axis ticks based on the bin step, you can also set the axis's [`tickMinStep`](https://vega.github.io/vega-lite/docs/axis.html#ticks) property.\n\n__Default value:__ `false`\n\n__See also:__ [`bin`](https://vega.github.io/vega-lite/docs/bin.html) documentation.", + "type": "null" + }, + "field": { + "$ref": "#/definitions/Field", + "description": "__Required.__ A string defining the name of the field from which to pull a data value or an object defining iterated values from the [`repeat`](https://vega.github.io/vega-lite/docs/repeat.html) operator.\n\n__See also:__ [`field`](https://vega.github.io/vega-lite/docs/field.html) documentation.\n\n__Notes:__ 1) Dots (`.`) and brackets (`[` and `]`) can be used to access nested objects (e.g., `\"field\": \"foo.bar\"` and `\"field\": \"foo['bar']\"`). If field names contain dots or brackets but are not nested, you can use `\\\\` to escape dots and brackets (e.g., `\"a\\\\.b\"` and `\"a\\\\[0\\\\]\"`). See more details about escaping in the [field documentation](https://vega.github.io/vega-lite/docs/field.html). 2) `field` is not required if `aggregate` is `count`." + }, + "timeUnit": { + "anyOf": [ + { + "$ref": "#/definitions/TimeUnit" + }, + { + "$ref": "#/definitions/BinnedTimeUnit" + }, + { + "$ref": "#/definitions/TimeUnitParams" + } + ], + "description": "Time unit (e.g., `year`, `yearmonth`, `month`, `hours`) for a temporal field. or [a temporal field that gets casted as ordinal](https://vega.github.io/vega-lite/docs/type.html#cast).\n\n__Default value:__ `undefined` (None)\n\n__See also:__ [`timeUnit`](https://vega.github.io/vega-lite/docs/timeunit.html) documentation." + }, + "title": { + "anyOf": [ + { + "$ref": "#/definitions/Text" + }, + { + "type": "null" + } + ], + "description": "A title for the field. If `null`, the title will be removed.\n\n__Default value:__ derived from the field's name and transformation function (`aggregate`, `bin` and `timeUnit`). If the field has an aggregate function, the function is displayed as part of the title (e.g., `\"Sum of Profit\"`). If the field is binned or has a time unit applied, the applied function is shown in parentheses (e.g., `\"Profit (binned)\"`, `\"Transaction Date (year-month)\"`). Otherwise, the title is simply the field name.\n\n__Notes__:\n\n1) You can customize the default field title format by providing the [`fieldTitle`](https://vega.github.io/vega-lite/docs/config.html#top-level-config) property in the [config](https://vega.github.io/vega-lite/docs/config.html) or [`fieldTitle` function via the `compile` function's options](https://vega.github.io/vega-lite/usage/compile.html#field-title).\n\n2) If both field definition's `title` and axis, header, or legend `title` are defined, axis/header/legend title will be used." + } + }, + "type": "object" + }, + "SelectionConfig": { + "additionalProperties": false, + "properties": { + "interval": { + "$ref": "#/definitions/IntervalSelectionConfigWithoutType", + "description": "The default definition for an [`interval`](https://vega.github.io/vega-lite/docs/parameter.html#select) selection. All properties and transformations for an interval selection definition (except `type`) may be specified here.\n\nFor instance, setting `interval` to `{\"translate\": false}` disables the ability to move interval selections by default." + }, + "point": { + "$ref": "#/definitions/PointSelectionConfigWithoutType", + "description": "The default definition for a [`point`](https://vega.github.io/vega-lite/docs/parameter.html#select) selection. All properties and transformations for a point selection definition (except `type`) may be specified here.\n\nFor instance, setting `point` to `{\"on\": \"dblclick\"}` populates point selections on double-click by default." + } + }, + "type": "object" + }, + "SelectionInit": { + "anyOf": [ + { + "$ref": "#/definitions/PrimitiveValue" + }, + { + "$ref": "#/definitions/DateTime" + } + ] + }, + "SelectionInitInterval": { + "anyOf": [ + { + "$ref": "#/definitions/Vector2" + }, + { + "$ref": "#/definitions/Vector2" + }, + { + "$ref": "#/definitions/Vector2" + }, + { + "$ref": "#/definitions/Vector2" + } + ] + }, + "SelectionInitIntervalMapping": { + "$ref": "#/definitions/Dict" }, "SelectionInitMapping": { "$ref": "#/definitions/Dict" @@ -28111,15 +28673,15 @@ ] }, "invalid": { - "description": "Defines how Vega-Lite should handle marks for invalid values (`null` and `NaN`).\n- If set to `\"filter\"` (default), all data items with null values will be skipped (for line, trail, and area marks) or filtered (for other marks).\n- If `null`, all data items are included. In this case, invalid values will be interpreted as zeroes.", - "enum": [ - "filter", - null + "anyOf": [ + { + "$ref": "#/definitions/MarkInvalidDataMode" + }, + { + "type": "null" + } ], - "type": [ - "string", - "null" - ] + "description": "Defines how Vega-Lite should represent invalid values (`null` and `NaN` in continuous scales without defined output for invalid values) in the marks and their scale domains.\n\n- `\"filter\"` = *Exclude* all invalid values from the visualization's *marks* and *scales*. For path marks (for line, area, trail), this option will create paths that connect valid points only.\n\n- `\"break-paths\"` = Break path marks (for line, area, trail) at invalid values. For non-path marks, this is equivalent to `\"filter\"`. All *scale* domains will *exclude* these filtered data points.\n\n- `\"break-paths-keep-domains\"` = Break paths (for line, area, trail) at invalid values. Hide invalid values for non-path marks. All *scale* domains will *include* these filtered data points.\n\n- `\"include\"` | `null`. Include all data points in the marks and scale domains. By default, invalid values will output the same visual values as zeroes if zero is in the scale domain or otherwise the scale's min value.\n\n- `\"break-and-keep-path-domains\"` (default). This is equivalent to `\"break-path-keep-domains` for path-based marks (line/area/trail) and `\"filter\"` for other marks." }, "limit": { "anyOf": [ diff --git a/examples/specs/test_invalid_include.vl.json b/examples/specs/test_invalid_include.vl.json new file mode 100644 index 0000000000..a475bf7830 --- /dev/null +++ b/examples/specs/test_invalid_include.vl.json @@ -0,0 +1,91 @@ +{ + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", + "description": "Testing invalid", + "data": { + "values": [ + {"a": null, "b": 1000}, + {"a": -10, "b": null}, + {"a": -5, "b": 25}, + {"a": -1, "b": 20}, + {"a": 0, "b": null}, + {"a": 1, "b": 30}, + {"a": 5, "b": 40}, + {"a": 10, "b": null} + ] + }, + + "config": { + "mark": {"invalid": "include"} + }, + "hconcat": [{ + "title": "Quantitative X", + "vconcat": [{ + "width": 100, + "height": 100, + "mark": "point", + "encoding": { + "x": {"field": "a", "type": "quantitative"}, + "y": {"field": "b", "type": "quantitative"} + } + }, { + "width": 100, + "height": 100, + "mark": "bar", + "encoding": { + "x": {"field": "a", "type": "quantitative"}, + "y": {"field": "b", "type": "quantitative"} + } + }, { + "width": 100, + "height": 100, + "mark": "line", + "encoding": { + "x": {"field": "a", "type": "quantitative"}, + "y": {"field": "b", "type": "quantitative"} + } + }, { + "width": 100, + "height": 100, + "mark": "area", + "encoding": { + "x": {"field": "a", "type": "quantitative"}, + "y": {"field": "b", "type": "quantitative"} + } + }] + },{ + "title": "Ordinal X", + "vconcat": [{ + "width": 100, + "height": 100, + "mark": "point", + "encoding": { + "x": {"field": "a", "type": "ordinal"}, + "y": {"field": "b", "type": "quantitative"} + } + }, { + "width": 100, + "height": 100, + "mark": "bar", + "encoding": { + "x": {"field": "a", "type": "ordinal"}, + "y": {"field": "b", "type": "quantitative"} + } + }, { + "width": 100, + "height": 100, + "mark": "line", + "encoding": { + "x": {"field": "a", "type": "ordinal"}, + "y": {"field": "b", "type": "quantitative"} + } + }, { + "width": 100, + "height": 100, + "mark": "area", + "encoding": { + "x": {"field": "a", "type": "ordinal"}, + "y": {"field": "b", "type": "quantitative"} + } + }] + }] +} diff --git a/src/channel.ts b/src/channel.ts index a359383603..de1f789261 100644 --- a/src/channel.ts +++ b/src/channel.ts @@ -480,7 +480,7 @@ const SCALE_CHANNEL_INDEX = { export const SCALE_CHANNELS = keys(SCALE_CHANNEL_INDEX); export type ScaleChannel = (typeof SCALE_CHANNELS)[number]; -export function isScaleChannel(channel: Channel): channel is ScaleChannel { +export function isScaleChannel(channel: Channel | ExtendedChannel): channel is ScaleChannel { return !!SCALE_CHANNEL_INDEX[channel]; } diff --git a/src/compile/data/filterinvalid.ts b/src/compile/data/filterinvalid.ts index 5126ea8dd8..531a3bedcb 100644 --- a/src/compile/data/filterinvalid.ts +++ b/src/compile/data/filterinvalid.ts @@ -1,12 +1,12 @@ import {FilterTransform as VgFilterTransform} from 'vega'; import {isScaleChannel} from '../../channel'; import {TypedFieldDef, vgField as fieldRef} from '../../channeldef'; -import {isPathMark} from '../../mark'; -import {hasContinuousDomain} from '../../scale'; import {Dict, hash, keys} from '../../util'; -import {getMarkPropOrConfig} from '../common'; +import {getScaleInvalidDataMode, isScaleInvalidDataInclude} from '../invalid/ScaleInvalidDataMode'; +import {DataSourcesForHandlingInvalidValues} from '../invalid/datasources'; import {UnitModel} from '../unit'; import {DataFlowNode} from './dataflow'; +import {isCountingAggregateOp} from '../../aggregate'; export class FilterInvalidNode extends DataFlowNode { public clone() { @@ -20,27 +20,40 @@ export class FilterInvalidNode extends DataFlowNode { super(parent); } - public static make(parent: DataFlowNode, model: UnitModel): FilterInvalidNode { - const {config, mark, markDef} = model; + public static make( + parent: DataFlowNode, + model: UnitModel, + dataSourcesForHandlingInvalidValues: DataSourcesForHandlingInvalidValues + ): FilterInvalidNode { + const {config, markDef} = model; - const invalid = getMarkPropOrConfig('invalid', markDef, config); - if (invalid !== 'filter') { + const {marks, scales} = dataSourcesForHandlingInvalidValues; + if (marks === 'pre-filter' && scales === 'pre-filter') { + // If neither marks nor scale domains need data source to filter null values, then don't add the filter. return null; } const filter = model.reduceFieldDef( (aggregator: Dict>, fieldDef, channel) => { const scaleComponent = isScaleChannel(channel) && model.getScaleComponent(channel); + if (scaleComponent) { const scaleType = scaleComponent.get('type'); + const {aggregate} = fieldDef; + const invalidDataMode = getScaleInvalidDataMode({ + scaleChannel: channel, + markDef, + config, + scaleType, + isCountAggregate: isCountingAggregateOp(aggregate) + }); - // While discrete domain scales can handle invalid values, continuous scales can't. - // Thus, for non-path marks, we have to filter null for scales with continuous domains. - // (For path marks, we will use "defined" property and skip these values instead.) - if (hasContinuousDomain(scaleType) && fieldDef.aggregate !== 'count' && !isPathMark(mark)) { + // If the invalid data mode is include or always-valid, we don't need to filter invalid values as the scale can handle invalid values. + if (!isScaleInvalidDataInclude(invalidDataMode) && invalidDataMode !== 'always-valid') { aggregator[fieldDef.field] = fieldDef as any; // we know that the fieldDef is a typed field def } } + return aggregator; }, {} as Dict> diff --git a/src/compile/data/index.ts b/src/compile/data/index.ts index 29fefae229..ac7b790cd9 100644 --- a/src/compile/data/index.ts +++ b/src/compile/data/index.ts @@ -27,6 +27,16 @@ export interface DataComponent { */ raw?: OutputNode; + /** + * The output node for scale domain before filter invalid. + */ + preFilterInvalid?: OutputNode; + + /** + * The output node for scale domain after filter invalid. + */ + postFilterInvalid?: OutputNode; + /** * The main output node. */ diff --git a/src/compile/data/parse.ts b/src/compile/data/parse.ts index 6ce7d480d5..60b27ce738 100644 --- a/src/compile/data/parse.ts +++ b/src/compile/data/parse.ts @@ -10,7 +10,9 @@ import { DataSourceType, ParseValue } from '../../data'; +import {getDataSourcesForHandlingInvalidValues, DataSourcesForHandlingInvalidValues} from '../invalid/datasources'; import * as log from '../../log'; +import {isPathMark} from '../../mark'; import { isAggregate, isBin, @@ -33,6 +35,7 @@ import { isWindow } from '../../transform'; import {deepEqual, mergeDeep} from '../../util'; +import {getMarkPropOrConfig} from '../common'; import {isFacetModel, isLayerModel, isUnitModel, Model} from '../model'; import {requiresSelectionId} from '../selection'; import {materializeSelections} from '../selection/parse'; @@ -285,14 +288,26 @@ Formula From Sort Array Stack (in `encoding`) | v - Invalid Filter ++- - - - - - - - - - -+ +| PreFilterInvalid | - - - -> scale domains +|(when scales need it)| ++- - - - - - - - - - -+ + | + v + Invalid Filter (if the main data source needs it) | v +----------+ - | Main | + | Main | - - - -> scale domains +----------+ | v ++- - - - - - - - - - -+ +| PostFilterInvalid | - - - -> scale domains +|(when scales need it)| ++- - - - - - - - - - -+ + | + v +-------+ | Facet |----> "column", "column-layout", and "row" +-------+ @@ -384,13 +399,41 @@ export function parseData(model: Model): DataComponent { head = StackNode.makeFromEncoding(head, model) ?? head; } + let preFilterInvalid: OutputNode | undefined; + let dataSourcesForHandlingInvalidValues: DataSourcesForHandlingInvalidValues | undefined; if (isUnitModel(model)) { - head = FilterInvalidNode.make(head, model) ?? head; + const {markDef, mark, config} = model; + const invalid = getMarkPropOrConfig('invalid', markDef, config); + + const {marks, scales} = (dataSourcesForHandlingInvalidValues = getDataSourcesForHandlingInvalidValues({ + invalid, + isPath: isPathMark(mark) + })); + + if (marks !== scales && scales === 'pre-filter') { + // Create a seperate preFilterInvalid dataSource if scales need pre-filter data but marks needs post-filter. + preFilterInvalid = head = makeOutputNode(DataSourceType.PreFilterInvalid, model, head); + } + + if (marks === 'post-filter') { + head = FilterInvalidNode.make(head, model, dataSourcesForHandlingInvalidValues) ?? head; + } } // output node for marks const main = (head = makeOutputNode(DataSourceType.Main, model, head)); + let postFilterInvalid: OutputNode | undefined; + if (isUnitModel(model) && dataSourcesForHandlingInvalidValues) { + const {marks, scales} = dataSourcesForHandlingInvalidValues; + if (marks === 'pre-filter' && scales === 'post-filter') { + // Create a seperate postFilterInvalid dataSource if scales need post-filter data but marks needs pre-filter. + head = FilterInvalidNode.make(head, model, dataSourcesForHandlingInvalidValues) ?? head; + + postFilterInvalid = head = makeOutputNode(DataSourceType.PostFilterInvalid, model, head); + } + } + if (isUnitModel(model)) { materializeSelections(model, main); } @@ -415,7 +458,9 @@ export function parseData(model: Model): DataComponent { raw, main, facetRoot, - ancestorParse + ancestorParse, + preFilterInvalid, + postFilterInvalid }; } diff --git a/src/compile/guide.ts b/src/compile/guide.ts index 1ef0ed15f0..bdbe0420e5 100644 --- a/src/compile/guide.ts +++ b/src/compile/guide.ts @@ -2,15 +2,20 @@ import {GuideEncodingEntry} from '../guide'; import {keys} from '../util'; import {VgEncodeChannel} from '../vega.schema'; import {signalOrValueRef} from './common'; -import {wrapCondition} from './mark/encode'; +import {wrapCondition} from './mark/encode/conditional'; import {UnitModel} from './unit'; export function guideEncodeEntry(encoding: GuideEncodingEntry, model: UnitModel) { return keys(encoding).reduce((encode, channel: VgEncodeChannel) => { - const valueDef = encoding[channel]; return { ...encode, - ...wrapCondition(model, valueDef, channel, def => signalOrValueRef(def.value)) + ...wrapCondition({ + model, + channelDef: encoding[channel], + vgChannel: channel, + mainRefFn: def => signalOrValueRef(def.value), + invalidValueRef: undefined // guide encoding won't show invalid values for the scale + }) }; }, {}); } diff --git a/src/compile/invalid/ScaleInvalidDataMode.ts b/src/compile/invalid/ScaleInvalidDataMode.ts new file mode 100644 index 0000000000..c21e58f5e7 --- /dev/null +++ b/src/compile/invalid/ScaleInvalidDataMode.ts @@ -0,0 +1,68 @@ +import {SignalRef} from 'vega'; +import {ScaleChannel, isPolarPositionChannel, isXorY} from '../../channel'; +import {Config} from '../../config'; +import {ScaleInvalidDataIncludeAs} from '../../invalid'; +import {MarkDef, isPathMark} from '../../mark'; +import {ScaleType, hasContinuousDomain} from '../../scale'; +import {getMarkPropOrConfig} from '../common'; +import {normalizeInvalidDataMode} from './normalizeInvalidDataMode'; + +export type ScaleInvalidDataMode = + | 'always-valid' + | 'filter' + | 'break-paths' + | 'break-paths-keep-domains' + | ScaleInvalidDataInclude; + +export interface ScaleInvalidDataInclude { + includeAs: ScaleInvalidDataIncludeAs; +} + +export function isScaleInvalidDataInclude( + invalidDataMode: ScaleInvalidDataMode +): invalidDataMode is ScaleInvalidDataInclude { + return (invalidDataMode as ScaleInvalidDataInclude).includeAs !== undefined; +} + +export function getScaleInvalidDataMode({ + markDef, + config, + scaleChannel, + scaleType, + isCountAggregate +}: { + markDef: MarkDef; + config: Config; + scaleChannel: C; + scaleType: ScaleType; + isCountAggregate: boolean; +}): ScaleInvalidDataMode { + if (!scaleType || !hasContinuousDomain(scaleType) || isCountAggregate) { + // - Discrete scales can always display null as another category + // - Count cannot output null values + return 'always-valid'; + } + + const invalidMode = normalizeInvalidDataMode(getMarkPropOrConfig('invalid', markDef, config), { + isPath: isPathMark(markDef.type) + }); + + const scaleOutputForInvalid = config.scale?.invalid?.[scaleChannel]; + if (scaleOutputForInvalid !== undefined) { + // Regardless of the current invalid mode, if the channel has a default value, we consider the field valid. + return {includeAs: scaleOutputForInvalid}; + } + + if (invalidMode === 'include') { + // TODO: it's arguable if we should make the behavior inconsistent between position and non-position. + // But this initial PR, we keep the behavior consistent (no breaking changes). + if (isXorY(scaleChannel) || isPolarPositionChannel(scaleChannel)) { + return {includeAs: 'min'}; + } + return {includeAs: 'zero-or-min'}; + } + return invalidMode; +} +export function shouldBreakPath(mode: ScaleInvalidDataMode): boolean { + return mode === 'break-paths' || mode === 'break-paths-keep-domains'; +} diff --git a/src/compile/invalid/datasources.ts b/src/compile/invalid/datasources.ts new file mode 100644 index 0000000000..d9de873c3a --- /dev/null +++ b/src/compile/invalid/datasources.ts @@ -0,0 +1,63 @@ +import {MarkInvalidDataMode} from '../../invalid'; + +import {DataSourceType} from '../../data'; +import {normalizeInvalidDataMode} from './normalizeInvalidDataMode'; + +type PreOrPostFilteringInvalidValues = 'pre-filter' | 'post-filter'; + +export interface DataSourcesForHandlingInvalidValues { + marks: PreOrPostFilteringInvalidValues; + scales: PreOrPostFilteringInvalidValues; +} +interface GetDataSourcesForHandlingInvalidValuesProps { + invalid: MarkInvalidDataMode | null | undefined; + isPath: boolean; +} + +export function getDataSourcesForHandlingInvalidValues({ + invalid, + isPath +}: GetDataSourcesForHandlingInvalidValuesProps): DataSourcesForHandlingInvalidValues { + const normalizedInvalid = normalizeInvalidDataMode(invalid, {isPath}); + + switch (normalizedInvalid) { + case 'filter': + // Both marks and scales use post-filter data + return { + marks: 'post-filter', + scales: 'post-filter' + }; + case 'break-paths-keep-domains': + return { + // Path-based marks use pre-filter data so we know to skip these invalid points in the path. + // For non-path based marks, we skip by not showing them at all. + marks: isPath ? 'pre-filter' : 'post-filter', + scales: 'pre-filter' + }; + case 'break-paths': + // For path marks, the marks will use unfiltered data (and skip points). But we need a separate data sources to feed the domain. + // For non-path marks, we can use the filtered data for both marks and scales. + return { + marks: isPath ? 'pre-filter' : 'post-filter', + // Unlike 'break-paths-keep-domains', 'break-paths' uses post-filter data to feed scale. + scales: 'post-filter' + }; + case 'include': + return { + marks: 'pre-filter', + scales: 'pre-filter' + }; + } +} + +export function getScaleDataSourceForHandlingInvalidValues( + props: GetDataSourcesForHandlingInvalidValuesProps +): DataSourceType { + const {marks, scales} = getDataSourcesForHandlingInvalidValues(props); + if (marks === scales) { + // If both marks and scales use the same data, there is only the main data source. + return DataSourceType.Main; + } + // If marks and scales use differetnt data, return the pre/post-filter data source accordingly. + return scales === 'pre-filter' ? DataSourceType.PreFilterInvalid : DataSourceType.PostFilterInvalid; +} diff --git a/src/compile/invalid/normalizeInvalidDataMode.ts b/src/compile/invalid/normalizeInvalidDataMode.ts new file mode 100644 index 0000000000..94b716d653 --- /dev/null +++ b/src/compile/invalid/normalizeInvalidDataMode.ts @@ -0,0 +1,15 @@ +import {MarkInvalidDataMode} from '../../invalid'; + +type NormalizedMarkInvalidDataMode = Exclude; + +export function normalizeInvalidDataMode( + mode: MarkInvalidDataMode | null | undefined, + {isPath}: {isPath: boolean} +): NormalizedMarkInvalidDataMode { + if (mode === undefined || mode === 'break-and-keep-path-domains') { + return isPath ? 'break-paths-keep-domains' : 'filter'; + } else if (mode === null) { + return 'include'; + } + return mode; +} diff --git a/src/compile/mark/encode/aria.ts b/src/compile/mark/encode/aria.ts index dac62f19ec..8fb2c8db1e 100644 --- a/src/compile/mark/encode/aria.ts +++ b/src/compile/mark/encode/aria.ts @@ -45,7 +45,13 @@ export function description(model: UnitModel) { const channelDef = encoding.description; if (channelDef) { - return wrapCondition(model, channelDef, 'description', cDef => textRef(cDef, model.config)); + return wrapCondition({ + model, + channelDef, + vgChannel: 'description', + mainRefFn: cDef => textRef(cDef, model.config), + invalidValueRef: undefined // aria encoding doesn't have continuous scales and thus can't have invalid values + }); } // Use default from mark def or config if defined. diff --git a/src/compile/mark/encode/base.ts b/src/compile/mark/encode/base.ts index b739d101e7..55cff371e4 100644 --- a/src/compile/mark/encode/base.ts +++ b/src/compile/mark/encode/base.ts @@ -1,21 +1,15 @@ -import {array} from 'vega-util'; -import {Channel, ScaleChannel, SCALE_CHANNELS} from '../../../channel'; -import {isPathMark, MarkDef} from '../../../mark'; -import {hasContinuousDomain} from '../../../scale'; -import {Dict, keys} from '../../../util'; -import {VgEncodeEntry, VgValueRef, VG_MARK_CONFIGS} from '../../../vega.schema'; -import {getMarkPropOrConfig, signalOrValueRef} from '../../common'; +import {MarkDef} from '../../../mark'; +import {VG_MARK_CONFIGS, VgEncodeEntry, VgValueRef} from '../../../vega.schema'; +import {signalOrValueRef} from '../../common'; import {UnitModel} from '../../unit'; import {aria} from './aria'; import {color} from './color'; import {nonPosition} from './nonposition'; import {text} from './text'; import {tooltip} from './tooltip'; -import {fieldInvalidPredicate} from './valueref'; import {zindex} from './zindex'; export {color} from './color'; -export {wrapCondition} from './conditional'; export {nonPosition} from './nonposition'; export {pointPosition} from './position-point'; export {pointOrRangePosition, rangePosition} from './position-range'; @@ -31,8 +25,8 @@ export function baseEncodeEntry(model: UnitModel, ignore: Ignore) { const {fill = undefined, stroke = undefined} = ignore.color === 'include' ? color(model) : {}; return { ...markDefProperties(model.markDef, ignore), - ...wrapAllFieldsInvalid(model, 'fill', fill), - ...wrapAllFieldsInvalid(model, 'stroke', stroke), + ...colorRef('fill', fill), + ...colorRef('stroke', stroke), ...nonPosition('opacity', model), ...nonPosition('fillOpacity', model), ...nonPosition('strokeOpacity', model), @@ -45,27 +39,7 @@ export function baseEncodeEntry(model: UnitModel, ignore: Ignore) { }; } -// TODO: mark VgValueRef[] as readonly after https://github.com/vega/vega/pull/1987 -function wrapAllFieldsInvalid(model: UnitModel, channel: Channel, valueRef: VgValueRef | VgValueRef[]): VgEncodeEntry { - const {config, mark, markDef} = model; - - const invalid = getMarkPropOrConfig('invalid', markDef, config); - - if (invalid === 'hide' && valueRef && !isPathMark(mark)) { - // For non-path marks, we have to exclude invalid values (null and NaN) for scales with continuous domains. - // For path marks, we will use "defined" property and skip these values instead. - const test = allFieldsInvalidPredicate(model, {invalid: true, channels: SCALE_CHANNELS}); - if (test) { - return { - [channel]: [ - // prepend the invalid case - // TODO: support custom value - {test, value: null}, - ...array(valueRef) - ] - }; - } - } +function colorRef(channel: 'fill' | 'stroke', valueRef: VgValueRef | VgValueRef[]): VgEncodeEntry { return valueRef ? {[channel]: valueRef} : {}; } @@ -77,29 +51,3 @@ function markDefProperties(mark: MarkDef, ignore: Ignore) { return m; }, {}); } - -function allFieldsInvalidPredicate( - model: UnitModel, - {invalid = false, channels}: {invalid?: boolean; channels: ScaleChannel[]} -) { - const filterIndex = channels.reduce((aggregator: Dict, channel) => { - const scaleComponent = model.getScaleComponent(channel); - if (scaleComponent) { - const scaleType = scaleComponent.get('type'); - const field = model.vgField(channel, {expr: 'datum'}); - - // While discrete domain scales can handle invalid values, continuous scales can't. - if (field && hasContinuousDomain(scaleType)) { - aggregator[field] = true; - } - } - return aggregator; - }, {}); - - const fields = keys(filterIndex); - if (fields.length > 0) { - const op = invalid ? '||' : '&&'; - return fields.map(field => fieldInvalidPredicate(field, invalid)).join(` ${op} `); - } - return undefined; -} diff --git a/src/compile/mark/encode/conditional.ts b/src/compile/mark/encode/conditional.ts index 1e4cacb52b..7a9aca8082 100644 --- a/src/compile/mark/encode/conditional.ts +++ b/src/compile/mark/encode/conditional.ts @@ -7,21 +7,37 @@ import {parseSelectionPredicate} from '../../selection/parse'; import {UnitModel} from '../../unit'; /** - * Return a mixin that includes a Vega production rule for a Vega-Lite conditional channel definition - * or a simple mixin if channel def has no condition. + * Return a VgEncodeEntry that includes a Vega production rule for a scale channel's encoding or guide encoding, which includes: + * (1) the conditional rules (if provided as part of channelDef) + * (2) invalidValueRef for handling invalid values (if provided as a parameter of this method) + * (3) main reference for the encoded data. */ -export function wrapCondition( - model: UnitModel, - channelDef: CD, - vgChannel: string, - refFn: (cDef: CD) => VgValueRef -): VgEncodeEntry { +export function wrapCondition({ + model, + channelDef, + vgChannel, + invalidValueRef, + mainRefFn +}: { + model: UnitModel; + channelDef: CD; + vgChannel: string; + + /** + * invalidValue for a scale channel if the invalidDataMode is include for the channel. + * For scale channel with other invalidDataMode or non-scale channel, this value should be undefined. + */ + invalidValueRef: VgValueRef | undefined; + mainRefFn: (cDef: CD) => VgValueRef; +}): VgEncodeEntry { const condition = isConditionalDef(channelDef) && channelDef.condition; - const valueRef = refFn(channelDef); + + let valueRefs: VgValueRef[] = []; + if (condition) { const conditions = array(condition); - const vgConditions = conditions.map(c => { - const conditionValueRef = refFn(c); + valueRefs = conditions.map(c => { + const conditionValueRef = mainRefFn(c); if (isConditionalParameter(c)) { const {param, empty} = c; const test = parseSelectionPredicate(model, {param, empty}); @@ -31,10 +47,21 @@ export function wrapCondition 1) { + return {[vgChannel]: valueRefs}; + } else if (valueRefs.length === 1) { + return {[vgChannel]: valueRefs[0]}; + } + return {}; } diff --git a/src/compile/mark/encode/defined.ts b/src/compile/mark/encode/defined.ts index a73fcd4f99..71ae01bc16 100644 --- a/src/compile/mark/encode/defined.ts +++ b/src/compile/mark/encode/defined.ts @@ -1,49 +1,48 @@ -import {POSITION_SCALE_CHANNELS} from '../../../channel'; -import {ScaleChannel} from '../../../channel'; +import {isCountingAggregateOp} from '../../../aggregate'; +import {isScaleChannel} from '../../../channel'; import {Value} from '../../../channeldef'; -import {hasContinuousDomain} from '../../../scale'; -import {Dict, keys} from '../../../util'; import {VgEncodeEntry} from '../../../vega.schema'; -import {getMarkPropOrConfig, signalOrValueRef} from '../../common'; +import {signalOrValueRef} from '../../common'; +import {getScaleInvalidDataMode, shouldBreakPath} from '../../invalid/ScaleInvalidDataMode'; import {UnitModel} from '../../unit'; -import {fieldInvalidPredicate} from './valueref'; +import {fieldInvalidPredicate} from './invalid'; +/** + * Create Vega's "defined" encoding to break paths in a path mark for invalid values. + */ export function defined(model: UnitModel): VgEncodeEntry { const {config, markDef} = model; - const invalid = getMarkPropOrConfig('invalid', markDef, config); - if (invalid) { - const signal = allFieldsInvalidPredicate(model, {channels: POSITION_SCALE_CHANNELS}); + // For each channel (x/y), add fields to break path to a set first. + const fieldsToBreakPath = new Set(); - if (signal) { - return {defined: {signal}}; + model.forEachFieldDef((fieldDef, channel) => { + let scaleType; + if (!isScaleChannel(channel) || !(scaleType = model.getScaleType(channel))) { + // Skip if the channel is not a scale channel or does not have a scale + return; } - } - return {}; -} -function allFieldsInvalidPredicate( - model: UnitModel, - {invalid = false, channels}: {invalid?: boolean; channels: ScaleChannel[]} -) { - const filterIndex = channels.reduce((aggregator: Dict, channel) => { - const scaleComponent = model.getScaleComponent(channel); - if (scaleComponent) { - const scaleType = scaleComponent.get('type'); + const isCountAggregate = isCountingAggregateOp(fieldDef.aggregate); + const invalidDataMode = getScaleInvalidDataMode({ + scaleChannel: channel, + markDef, + config, + scaleType, + isCountAggregate + }); + if (shouldBreakPath(invalidDataMode)) { const field = model.vgField(channel, {expr: 'datum', binSuffix: model.stack?.impute ? 'mid' : undefined}); - - // While discrete domain scales can handle invalid values, continuous scales can't. - if (field && hasContinuousDomain(scaleType)) { - aggregator[field] = true; + if (field) { + fieldsToBreakPath.add(field); } } - return aggregator; - }, {}); + }); - const fields = keys(filterIndex); - if (fields.length > 0) { - const op = invalid ? '||' : '&&'; - return fields.map(field => fieldInvalidPredicate(field, invalid)).join(` ${op} `); + // If the set is not empty, return a defined signal. + if (fieldsToBreakPath.size > 0) { + const signal = [...fieldsToBreakPath].map(field => fieldInvalidPredicate(field, {invalid: false})).join(' && '); + return {defined: {signal}}; } return undefined; } diff --git a/src/compile/mark/encode/index.ts b/src/compile/mark/encode/index.ts index 966ffbbc3e..7f36b4dfbf 100644 --- a/src/compile/mark/encode/index.ts +++ b/src/compile/mark/encode/index.ts @@ -1,7 +1,6 @@ export {aria} from './aria'; export {baseEncodeEntry} from './base'; export {color} from './color'; -export {wrapCondition} from './conditional'; export {defined, valueIfDefined} from './defined'; export {nonPosition} from './nonposition'; export {pointPosition} from './position-point'; diff --git a/src/compile/mark/encode/invalid.ts b/src/compile/mark/encode/invalid.ts new file mode 100644 index 0000000000..18f7362a01 --- /dev/null +++ b/src/compile/mark/encode/invalid.ts @@ -0,0 +1,68 @@ +import {isString} from 'vega-util'; +import {FieldDef, FieldName, getFieldDef, vgField} from '../../../channeldef'; +import {fieldValidPredicate} from '../../../predicate'; +import {isCountingAggregateOp} from '../../../aggregate'; +import {ScaleChannel} from '../../../channel'; +import {ScaleInvalidDataIncludeAs, isScaleInvalidDataIncludeAsValue} from '../../../invalid'; +import {VgValueRef, isSignalRef} from '../../../vega.schema'; +import {getScaleInvalidDataMode, isScaleInvalidDataInclude} from '../../invalid/ScaleInvalidDataMode'; +import {ScaleComponent} from '../../scale/component'; +import {MidPointParams} from './valueref'; + +export function fieldInvalidPredicate(field: FieldName | FieldDef, {invalid = true}: {invalid: boolean}) { + return fieldValidPredicate(isString(field) ? field : vgField(field, {expr: 'datum'}), !invalid); +} + +export function getConditionalValueRefForIncludingInvalidValue({ + scaleChannel, + channelDef, + scale, + scaleName, + markDef, + config +}: { + scaleChannel: C; +} & Pick): VgValueRef | undefined { + const scaleType = scale?.get('type'); + + const fieldDef = getFieldDef(channelDef); + const isCountAggregate = isCountingAggregateOp(fieldDef?.aggregate); + + const invalidDataMode = getScaleInvalidDataMode({ + scaleChannel, + markDef, + config, + scaleType, + isCountAggregate + }); + + if (isScaleInvalidDataInclude(invalidDataMode)) { + return { + test: fieldInvalidPredicate(fieldDef, {invalid: true}), + ...refForInvalidValues(scaleChannel, invalidDataMode.includeAs, scale, scaleName) + }; + } + return undefined; +} + +function refForInvalidValues( + channel: C, + includeAs: ScaleInvalidDataIncludeAs, + scale: ScaleComponent, + scaleName: string +): VgValueRef { + if (isScaleInvalidDataIncludeAsValue(includeAs)) { + const {value} = includeAs; + return isSignalRef(value) ? {signal: value.signal} : {value}; + } + // If there is no specific value requested, return zero or min. + if (scaleName && scale.domainDefinitelyIncludesZero()) { + // If there + return {scale: scaleName, value: 0}; + } + + return channel === 'y' + ? {field: {group: 'height'}} + : // x / angle / radius can all use 0 + {value: 0}; +} diff --git a/src/compile/mark/encode/nonposition.ts b/src/compile/mark/encode/nonposition.ts index df56da2fb2..3f2380d4d1 100644 --- a/src/compile/mark/encode/nonposition.ts +++ b/src/compile/mark/encode/nonposition.ts @@ -6,6 +6,7 @@ import {getMarkPropOrConfig, signalOrValueRef} from '../../common'; import {UnitModel} from '../../unit'; import {wrapCondition} from './conditional'; import * as ref from './valueref'; +import {getConditionalValueRefForIncludingInvalidValue} from './invalid'; /** * Return encode for non-positional channels with scales. (Text doesn't have scale.) @@ -33,17 +34,34 @@ export function nonPosition( } const channelDef = encoding[channel]; + const commonProps = { + markDef, + config, + scaleName: model.scaleName(channel), + scale: model.getScaleComponent(channel) + }; - return wrapCondition(model, channelDef, vgChannel ?? channel, cDef => { + const invalidValueRef = getConditionalValueRefForIncludingInvalidValue({ + ...commonProps, + scaleChannel: channel, + channelDef + }); + + const mainRefFn = (cDef: typeof channelDef) => { return ref.midPoint({ + ...commonProps, channel, channelDef: cDef, - markDef, - config, - scaleName: model.scaleName(channel), - scale: model.getScaleComponent(channel), stack: null, // No need to provide stack for non-position as it does not affect mid point defaultRef }); + }; + + return wrapCondition({ + model, + channelDef, + vgChannel: vgChannel ?? channel, + invalidValueRef, + mainRefFn }); } diff --git a/src/compile/mark/encode/position-point.ts b/src/compile/mark/encode/position-point.ts index 1df089df32..e5e0fc136f 100644 --- a/src/compile/mark/encode/position-point.ts +++ b/src/compile/mark/encode/position-point.ts @@ -4,12 +4,11 @@ import { getSizeChannel, getVgPositionChannel, isXorY, + MainChannelOf, PolarPositionChannel, PositionChannel } from '../../../channel'; import {isFieldDef, isFieldOrDatumDef, TypedFieldDef} from '../../../channeldef'; -import {ScaleType} from '../../../scale'; -import {contains} from '../../../util'; import {VgValueRef} from '../../../vega.schema'; import {getMarkPropOrConfig} from '../../common'; import {ScaleComponent} from '../../scale/component'; @@ -70,7 +69,8 @@ export function pointPosition( stack, offset, defaultRef, - bandPosition: offsetType === 'encoding' ? 0 : undefined + bandPosition: offsetType === 'encoding' ? 0 : undefined, + model }); return valueRef ? {[vgChannel || channel]: valueRef} : undefined; @@ -85,6 +85,7 @@ export function pointPosition( export function positionRef( params: ref.MidPointParams & { channel: 'x' | 'y' | 'radius' | 'theta'; + model: UnitModel; } ): VgValueRef | VgValueRef[] { const {channel, channelDef, scaleName, stack, offset, markDef} = params; @@ -142,43 +143,14 @@ export function pointPositionDefaultRef({ switch (defaultPos) { case 'zeroOrMin': + return zeroOrMinMax({scaleName, scale, mode: 'min', mainChannel}); case 'zeroOrMax': - if (scaleName) { - const scaleType = scale.get('type'); - if (contains([ScaleType.LOG, ScaleType.TIME, ScaleType.UTC], scaleType)) { - // Log scales cannot have zero. - // Zero in time scale is arbitrary, and does not affect ratio. - // (Time is an interval level of measurement, not ratio). - // See https://en.wikipedia.org/wiki/Level_of_measurement for more info. - } else { - if (scale.domainDefinitelyIncludesZero()) { - return { - scale: scaleName, - value: 0 - }; - } - } - } - - if (defaultPos === 'zeroOrMin') { - return mainChannel === 'y' ? {field: {group: 'height'}} : {value: 0}; - } else { - // zeroOrMax - switch (mainChannel) { - case 'radius': - // max of radius is min(width, height) / 2 - return { - signal: `min(${model.width.signal},${model.height.signal})/2` - }; - case 'theta': - return {signal: '2*PI'}; - case 'x': - return {field: {group: 'width'}}; - case 'y': - return {value: 0}; - } - } - break; + return zeroOrMinMax({ + scaleName, + scale, + mode: {max: {widthSignal: model.width.signal, heightSignal: model.height.signal}}, + mainChannel + }); case 'mid': { const sizeRef = model[getSizeChannel(channel)]; return {...sizeRef, mult: 0.5}; @@ -188,3 +160,45 @@ export function pointPositionDefaultRef({ return undefined; }; } + +export function zeroOrMinMax({ + scaleName, + scale, + mode, + mainChannel +}: { + scaleName: string; + scale: ScaleComponent; + mode: 'min' | {max: {widthSignal: string; heightSignal: string}}; + mainChannel: MainChannelOf; +}): VgValueRef { + if (scaleName) { + if (scale.domainDefinitelyIncludesZero()) { + return { + scale: scaleName, + value: 0 + }; + } + } + + if (mode === 'min') { + return mainChannel === 'y' ? {field: {group: 'height'}} : {value: 0}; + } else { + // zeroOrMax + switch (mainChannel) { + case 'radius': { + const {widthSignal, heightSignal} = mode.max; + // max of radius is min(width, height) / 2 + return { + signal: `min(${widthSignal},${heightSignal})/2` + }; + } + case 'theta': + return {signal: '2*PI'}; + case 'x': + return {field: {group: 'width'}}; + case 'y': + return {value: 0}; + } + } +} diff --git a/src/compile/mark/encode/text.ts b/src/compile/mark/encode/text.ts index 845837290e..9b1bfc2241 100644 --- a/src/compile/mark/encode/text.ts +++ b/src/compile/mark/encode/text.ts @@ -9,7 +9,13 @@ import {wrapCondition} from './conditional'; export function text(model: UnitModel, channel: 'text' | 'href' | 'url' | 'description' = 'text') { const channelDef = model.encoding[channel]; - return wrapCondition(model, channelDef, channel, cDef => textRef(cDef, model.config)); + return wrapCondition({ + model, + channelDef, + vgChannel: channel, + mainRefFn: cDef => textRef(cDef, model.config), + invalidValueRef: undefined // text encoding doesn't have continuous scales and thus can't have invalid values + }); } export function textRef( diff --git a/src/compile/mark/encode/tooltip.ts b/src/compile/mark/encode/tooltip.ts index 53b5ae8f42..48f70ce3af 100644 --- a/src/compile/mark/encode/tooltip.ts +++ b/src/compile/mark/encode/tooltip.ts @@ -30,7 +30,7 @@ export function tooltip(model: UnitModel, opt: {reactiveGeom?: boolean} = {}) { return {tooltip: tooltipRefForEncoding({tooltip: channelDef}, stack, config, opt)}; } else { const datum = opt.reactiveGeom ? 'datum.datum' : 'datum'; - return wrapCondition(model, channelDef, 'tooltip', cDef => { + const mainRefFn = (cDef: Encoding['tooltip']) => { // use valueRef based on channelDef first const tooltipRefFromChannelDef = textRef(cDef, config, datum); if (tooltipRefFromChannelDef) { @@ -62,6 +62,14 @@ export function tooltip(model: UnitModel, opt: {reactiveGeom?: boolean} = {}) { } return undefined; + }; + + return wrapCondition({ + model, + channelDef, + vgChannel: 'tooltip', + mainRefFn, + invalidValueRef: undefined // tooltip encoding doesn't have continuous scales and thus can't have invalid values }); } } diff --git a/src/compile/mark/encode/valueref.ts b/src/compile/mark/encode/valueref.ts index 02f36523b3..52854dac48 100644 --- a/src/compile/mark/encode/valueref.ts +++ b/src/compile/mark/encode/valueref.ts @@ -2,28 +2,25 @@ * Utility files for producing Vega ValueRef for marks */ import type {SignalRef} from 'vega'; -import {isFunction, isString} from 'vega-util'; -import {isCountingAggregateOp} from '../../../aggregate'; +import {isFunction} from 'vega-util'; import {isBinned, isBinning} from '../../../bin'; -import {Channel, getMainRangeChannel, PolarPositionChannel, PositionChannel, X, X2, Y2} from '../../../channel'; +import {Channel, PolarPositionChannel, PositionChannel, X, X2, Y2, getMainRangeChannel} from '../../../channel'; import { - binRequiresRange, ChannelDef, DatumDef, - FieldDef, FieldDefBase, - FieldName, FieldRefOption, + SecondaryChannelDef, + SecondaryFieldDef, + TypedFieldDef, + Value, + binRequiresRange, getBandPosition, isDatumDef, isFieldDef, isFieldOrDatumDef, isTypedFieldDef, isValueDef, - SecondaryChannelDef, - SecondaryFieldDef, - TypedFieldDef, - Value, vgField } from '../../../channeldef'; import {Config} from '../../../config'; @@ -31,79 +28,34 @@ import {dateTimeToExpr, isDateTime} from '../../../datetime'; import {isExprRef} from '../../../expr'; import * as log from '../../../log'; import {Mark, MarkDef} from '../../../mark'; -import {fieldValidPredicate} from '../../../predicate'; -import {hasDiscreteDomain, isContinuousToContinuous} from '../../../scale'; +import {hasDiscreteDomain} from '../../../scale'; import {StackProperties} from '../../../stack'; import {TEMPORAL} from '../../../type'; import {contains, stringify} from '../../../util'; -import {isSignalRef, VgValueRef} from '../../../vega.schema'; -import {getMarkPropOrConfig, signalOrValueRef} from '../../common'; +import {VgValueRef, isSignalRef} from '../../../vega.schema'; +import {signalOrValueRef} from '../../common'; import {ScaleComponent} from '../../scale/component'; +import {getConditionalValueRefForIncludingInvalidValue} from './invalid'; export function midPointRefWithPositionInvalidTest( params: MidPointParams & { channel: PositionChannel | PolarPositionChannel; } -) { - const {channel, channelDef, markDef, scale, config} = params; - const ref = midPoint(params); +): VgValueRef | VgValueRef[] { + const {channel, channelDef, markDef, scale, scaleName, config} = params; + const scaleChannel = getMainRangeChannel(channel); + const mainRef = midPoint(params); - // Wrap to check if the positional value is invalid, if so, plot the point on the min value - if ( - // Only this for field def without counting aggregate (as count wouldn't be null) - isFieldDef(channelDef) && - !isCountingAggregateOp(channelDef.aggregate) && - // and only for continuous scale - scale && - isContinuousToContinuous(scale.get('type')) - ) { - return wrapPositionInvalidTest({ - fieldDef: channelDef, - channel, - markDef, - ref, - config - }); - } - return ref; -} - -export function wrapPositionInvalidTest({ - fieldDef, - channel, - markDef, - ref, - config -}: { - fieldDef: FieldDef; - channel: PositionChannel | PolarPositionChannel; - markDef: MarkDef; - ref: VgValueRef; - config: Config; -}): VgValueRef | VgValueRef[] { - const invalid = getMarkPropOrConfig('invalid', markDef, config); - if (invalid === null) { - // if there is no invalid filter, do the invalid test - return [fieldInvalidTestValueRef(fieldDef, channel), ref]; - } - return ref; -} - -export function fieldInvalidTestValueRef(fieldDef: FieldDef, channel: PositionChannel | PolarPositionChannel) { - const test = fieldInvalidPredicate(fieldDef, true); - - const mainChannel = getMainRangeChannel(channel) as PositionChannel | PolarPositionChannel; // we can cast here as the output can't be other things. - const zeroValueRef = - mainChannel === 'y' - ? {field: {group: 'height'}} - : // x / angle / radius can all use 0 - {value: 0}; - - return {test, ...zeroValueRef}; -} + const valueRefForIncludingInvalid = getConditionalValueRefForIncludingInvalidValue({ + scaleChannel, + channelDef, + scale, + scaleName, + markDef, + config + }); -export function fieldInvalidPredicate(field: FieldName | FieldDef, invalid = true) { - return fieldValidPredicate(isString(field) ? field : vgField(field, {expr: 'datum'}), !invalid); + return valueRefForIncludingInvalid !== undefined ? [valueRefForIncludingInvalid, mainRef] : mainRef; } export function datumDefToExpr(datumDef: DatumDef) { diff --git a/src/compile/mark/encode/zindex.ts b/src/compile/mark/encode/zindex.ts index d0df9e606e..d36f6b93b1 100644 --- a/src/compile/mark/encode/zindex.ts +++ b/src/compile/mark/encode/zindex.ts @@ -9,7 +9,13 @@ export function zindex(model: UnitModel) { const order = encoding.order; if (!isPathMark(mark) && isValueDef(order)) { - return wrapCondition(model, order, 'zindex', cd => signalOrValueRef(cd.value)); + return wrapCondition({ + model, + channelDef: order, + vgChannel: 'zindex', + mainRefFn: cd => signalOrValueRef(cd.value), + invalidValueRef: undefined // zindex encoding doesn't have continuous scales and thus can't have invalid values + }); } return {}; } diff --git a/src/compile/model.ts b/src/compile/model.ts index 4ecdf6f3fe..6dc99627d7 100644 --- a/src/compile/model.ts +++ b/src/compile/model.ts @@ -25,7 +25,7 @@ import {forEach, reduce} from '../encoding'; import {ExprRef, replaceExprRef} from '../expr'; import * as log from '../log'; import {Resolve} from '../resolve'; -import {hasDiscreteDomain} from '../scale'; +import {ScaleType, hasDiscreteDomain} from '../scale'; import {isFacetSpec} from '../spec'; import { extractCompositionLayout, @@ -635,6 +635,11 @@ export abstract class Model { return this.parent ? this.parent.getScaleComponent(channel) : undefined; } + public getScaleType(channel: ScaleChannel): ScaleType { + const scaleComponent = this.getScaleComponent(channel); + return scaleComponent ? scaleComponent.get('type') : undefined; + } + /** * Traverse a model's hierarchy to get a particular selection component. */ diff --git a/src/compile/scale/component.ts b/src/compile/scale/component.ts index ba0e305c0c..17b9e49c4b 100644 --- a/src/compile/scale/component.ts +++ b/src/compile/scale/component.ts @@ -3,7 +3,7 @@ import {isArray, isNumber} from 'vega-util'; import {ScaleChannel} from '../../channel'; import {Scale, ScaleType} from '../../scale'; import {ParameterExtent} from '../../selection'; -import {some} from '../../util'; +import {contains, some} from '../../util'; import {VgNonUnionDomain, VgScale} from '../../vega.schema'; import {Explicit, Split} from '../split'; @@ -34,6 +34,13 @@ export class ScaleComponent extends Split { * Whether the scale definitely includes zero in the domain */ public domainDefinitelyIncludesZero() { + if (contains([ScaleType.LOG, ScaleType.TIME, ScaleType.UTC], this.get('type'))) { + // Log scales cannot have zero. + // Zero in time scale is arbitrary, and does not affect ratio. + // (Time is an interval level of measurement, not ratio). + // See https://en.wikipedia.org/wiki/Level_of_measurement for more info. + return false; + } if (this.get('zero') !== false) { return true; } diff --git a/src/compile/scale/domain.ts b/src/compile/scale/domain.ts index b98669a66f..4a23487bf1 100644 --- a/src/compile/scale/domain.ts +++ b/src/compile/scale/domain.ts @@ -55,8 +55,10 @@ import {SignalRefWrapper} from '../signal'; import {Explicit, makeExplicit, makeImplicit, mergeValuesWithExplicit} from '../split'; import {UnitModel} from '../unit'; import {ScaleComponent, ScaleComponentIndex} from './component'; -import {isRectBasedMark} from '../../mark'; +import {isPathMark, isRectBasedMark} from '../../mark'; import {OFFSETTED_RECT_END_SUFFIX, OFFSETTED_RECT_START_SUFFIX} from '../data/timeunit'; +import {getScaleDataSourceForHandlingInvalidValues} from '../invalid/datasources'; +import {getMarkConfig} from '../common'; export function parseScaleDomain(model: Model) { if (isUnitModel(model)) { @@ -249,6 +251,11 @@ function parseSingleChannelDomain( const {type} = fieldOrDatumDef; const timeUnit = fieldOrDatumDef['timeUnit']; + const dataSourceTypeForScaleDomain = getScaleDataSourceForHandlingInvalidValues({ + invalid: getMarkConfig('invalid', markDef, config), + isPath: isPathMark(mark) + }); + if (isDomainUnionWith(domain)) { const defaultDomain = parseSingleChannelDomain(scaleType, undefined, model, channel); @@ -266,7 +273,7 @@ function parseSingleChannelDomain( return makeImplicit([[0, 1]]); } - const data = model.requestDataName(DataSourceType.Main); + const data = model.requestDataName(dataSourceTypeForScaleDomain); return makeImplicit([ { data, @@ -289,15 +296,14 @@ function parseSingleChannelDomain( const fieldDef = fieldOrDatumDef; // now we can be sure it's a fieldDef if (domain === 'unaggregated') { - const data = model.requestDataName(DataSourceType.Main); const {field} = fieldOrDatumDef; return makeImplicit([ { - data, + data: model.requestDataName(dataSourceTypeForScaleDomain), field: vgField({field, aggregate: 'min'}) }, { - data, + data: model.requestDataName(dataSourceTypeForScaleDomain), field: vgField({field, aggregate: 'max'}) } ]); @@ -315,7 +321,7 @@ function parseSingleChannelDomain( // If sort by aggregation of a specified sort field, we need to use RAW table, // so we can aggregate values for the scale independently from the main aggregation. data: util.isBoolean(sort) - ? model.requestDataName(DataSourceType.Main) + ? model.requestDataName(dataSourceTypeForScaleDomain) : model.requestDataName(DataSourceType.Raw), // Use range if we added it and the scale does not support computing a range as a signal. field: model.vgField(channel, binRequiresRange(fieldDef, channel) ? {binSuffix: 'range'} : {}), @@ -343,7 +349,7 @@ function parseSingleChannelDomain( } else { return makeImplicit([ { - data: model.requestDataName(DataSourceType.Main), + data: model.requestDataName(dataSourceTypeForScaleDomain), field: model.vgField(channel, {}) } ]); @@ -353,7 +359,7 @@ function parseSingleChannelDomain( const fieldDef2 = encoding[getSecondaryRangeChannel(channel)]; if (hasBandEnd(fieldDef, fieldDef2, markDef, config)) { - const data = model.requestDataName(DataSourceType.Main); + const data = model.requestDataName(dataSourceTypeForScaleDomain); const bandPosition = getBandPosition({fieldDef, fieldDef2, markDef, config}); const isRectWithOffset = isRectBasedMark(mark) && bandPosition !== 0.5 && isXorY(channel); @@ -375,7 +381,7 @@ function parseSingleChannelDomain( // If sort by aggregation of a specified sort field, we need to use RAW table, // so we can aggregate values for the scale independently from the main aggregation. data: util.isBoolean(sort) - ? model.requestDataName(DataSourceType.Main) + ? model.requestDataName(dataSourceTypeForScaleDomain) : model.requestDataName(DataSourceType.Raw), field: model.vgField(channel), sort @@ -384,7 +390,7 @@ function parseSingleChannelDomain( } else { return makeImplicit([ { - data: model.requestDataName(DataSourceType.Main), + data: model.requestDataName(dataSourceTypeForScaleDomain), field: model.vgField(channel) } ]); diff --git a/src/compositemark/boxplot.ts b/src/compositemark/boxplot.ts index d6e37c9efd..b11df271d8 100644 --- a/src/compositemark/boxplot.ts +++ b/src/compositemark/boxplot.ts @@ -4,7 +4,8 @@ import {getMarkPropOrConfig} from '../compile/common'; import {Config} from '../config'; import {Encoding, extractTransformsFromEncoding, normalizeEncoding} from '../encoding'; import * as log from '../log'; -import {isMarkDef, MarkDef, MarkInvalidMixins} from '../mark'; +import {isMarkDef, MarkDef} from '../mark'; +import {MarkInvalidMixins} from '../invalid'; import {NormalizerParams} from '../normalize'; import {GenericUnitSpec, NormalizedLayerSpec, NormalizedUnitSpec} from '../spec'; import {AggregatedFieldDef, CalculateTransform, JoinAggregateTransform, Transform} from '../transform'; diff --git a/src/data.ts b/src/data.ts index c8fff6fd56..e9065fa1f3 100644 --- a/src/data.ts +++ b/src/data.ts @@ -155,7 +155,9 @@ export enum DataSourceType { Main, Row, Column, - Lookup + Lookup, + PreFilterInvalid, + PostFilterInvalid } export type Generator = SequenceGenerator | SphereGenerator | GraticuleGenerator; diff --git a/src/invalid.ts b/src/invalid.ts new file mode 100644 index 0000000000..f336d7cbcd --- /dev/null +++ b/src/invalid.ts @@ -0,0 +1,70 @@ +import {SignalRef} from 'vega'; +import {ScaleChannel} from './channel'; +import {Mark, MarkDef} from './mark'; +import {isObject} from 'vega-util'; + +/** + * Mixins for Vega-Lite Spec's Mark Definiton (to add mark.invalid) + */ +export interface MarkInvalidMixins { + /** + * Defines how Vega-Lite should represent invalid values (`null` and `NaN` in continuous scales without defined output for invalid values) in the marks and their scale domains. + * + * - `"filter"` = *Exclude* all invalid values from the visualization's *marks* and *scales*. + * For path marks (for line, area, trail), this option will create paths that connect valid points only. + * + * - `"break-paths"` = Break path marks (for line, area, trail) at invalid values. For non-path marks, this is equivalent to `"filter"`. + * All *scale* domains will *exclude* these filtered data points. + * + * - `"break-paths-keep-domains"` = Break paths (for line, area, trail) at invalid values. Hide invalid values for non-path marks. + * All *scale* domains will *include* these filtered data points. + * + * - `"include"` | `null`. Include all data points in the marks and scale domains. + * By default, invalid values will output the same visual values as zeroes if zero is in the scale domain or otherwise the scale's min value. + * + * - `"break-and-keep-path-domains"` (default). This is equivalent to `"break-path-keep-domains` for path-based marks (line/area/trail) + * and `"filter"` for other marks. + */ + invalid?: MarkInvalidDataMode | null; +} + +export type MarkInvalidDataMode = + | 'filter' + | 'break-paths' + | 'break-paths-keep-domains' + | 'break-and-keep-path-domains' + | 'include'; + +/** + * Mixins for Vega-Lite Spec's config.scale + */ +export interface ScaleInvalidDataConfigMixins { + /** + * An object that defines scale outputs per channel for invalid values (null, NaN) on a continuous scale. + * - The keys in this object are the scale channels. + * - The values is either `"zero-or-min"`or a value definition `{value: ...}`. + * + * _Example:_ Setting this `config.scale.invalid` property to `{color: {value: '#aaa'}}` + * will make Vega-Lite color all invalid values with '#aaa'. + */ + invalid?: ScaleInvalidDataConfig; +} + +export type ScaleInvalidDataConfig = { + [c in ScaleChannel]?: ScaleInvalidDataIncludeAs; +}; + +export type ScaleInvalidDataIncludeAs = + | ScaleInvalidDataIncludeAsValue + | 'zero-or-min' + | 'min'; + +export type ScaleInvalidDataIncludeAsValue = { + value: MarkDef[C]; +}; + +export function isScaleInvalidDataIncludeAsValue( + invalidDataMode: ScaleInvalidDataIncludeAs +): invalidDataMode is ScaleInvalidDataIncludeAsValue { + return isObject(invalidDataMode) && 'value' in invalidDataMode; +} diff --git a/src/mark.ts b/src/mark.ts index a779ff0421..95d79f1c7d 100644 --- a/src/mark.ts +++ b/src/mark.ts @@ -3,6 +3,7 @@ import {CompositeMark, CompositeMarkDef} from './compositemark'; import {ExprRef} from './expr'; import {Flag, keys} from './util'; import {MapExcludeValueRefAndReplaceSignalWith} from './vega.schema'; +import {MarkInvalidMixins} from './invalid'; /** * All types of primitive marks. @@ -45,7 +46,11 @@ export function isMark(m: string): m is Mark { return m in Mark; } -export function isPathMark(m: Mark | CompositeMark): m is 'line' | 'area' | 'trail' { +export const PATH_MARKS = ['line', 'area', 'trail'] as const; + +export type PathMark = (typeof PATH_MARKS)[number]; + +export function isPathMark(m: Mark | CompositeMark): m is PathMark { return ['line', 'area', 'trail'].includes(m); } @@ -72,18 +77,6 @@ export interface TooltipContent { content: 'encoding' | 'data'; } -/** @hidden */ -export type Hide = 'hide'; - -export interface MarkInvalidMixins { - /** - * Defines how Vega-Lite should handle marks for invalid values (`null` and `NaN`). - * - If set to `"filter"` (default), all data items with null values will be skipped (for line, trail, and area marks) or filtered (for other marks). - * - If `null`, all data items are included. In this case, invalid values will be interpreted as zeroes. - */ - invalid?: 'filter' | Hide | null; -} - export interface VLOnlyMarkConfig extends ColorMixins, MarkInvalidMixins { /** * Whether the mark's color should be used as fill color instead of stroke color. diff --git a/src/scale.ts b/src/scale.ts index e0a0c06e35..a63af8da04 100644 --- a/src/scale.ts +++ b/src/scale.ts @@ -8,12 +8,13 @@ import { TimeInterval, TimeIntervalStep } from 'vega'; -import {isString} from 'vega-util'; import type {ColorScheme} from 'vega-typings'; +import {isString} from 'vega-util'; import * as CHANNEL from './channel'; import {Channel, isColorChannel} from './channel'; import {DateTime} from './datetime'; import {ExprRef} from './expr'; +import {ScaleInvalidDataConfigMixins} from './invalid'; import * as log from './log'; import {ParameterExtent} from './selection'; import {NOMINAL, ORDINAL, QUANTITATIVE, TEMPORAL, Type} from './type'; @@ -180,7 +181,7 @@ export function isContinuousToDiscrete(type: ScaleType): type is 'quantile' | 'q return CONTINUOUS_TO_DISCRETE_SCALES.has(type); } -export interface ScaleConfig { +export interface ScaleConfig extends ScaleInvalidDataConfigMixins { /** * If true, rounds numeric output values to integers. * This can be helpful for snapping to the pixel grid. @@ -333,7 +334,7 @@ export interface ScaleConfig { maxFontSize?: number; /** - * The default min value for mapping quantitative fields to tick's size/fontSize scale with zero=false + * The default min value for mapping quantitative fields to text's size/fontSize scale with zero=false * * __Default value:__ `8` * diff --git a/test/compile/data/filterinvalid.test.ts b/test/compile/data/filterinvalid.test.ts index 3eaddce808..f23412ef04 100644 --- a/test/compile/data/filterinvalid.test.ts +++ b/test/compile/data/filterinvalid.test.ts @@ -1,11 +1,16 @@ import {FilterInvalidNode} from '../../../src/compile/data/filterinvalid'; +import {getDataSourcesForHandlingInvalidValues} from '../../../src/compile/invalid/datasources'; import {UnitModel} from '../../../src/compile/unit'; import {NormalizedUnitSpec, TopLevel} from '../../../src/spec'; import {mergeDeep} from '../../../src/util'; import {parseUnitModelWithScale} from '../../util'; function parse(model: UnitModel) { - return FilterInvalidNode.make(null, model); + const dataSourcesForHandlingInvalidValues = getDataSourcesForHandlingInvalidValues({ + invalid: 'filter', + isPath: false + }); + return FilterInvalidNode.make(null, model, dataSourcesForHandlingInvalidValues); } describe('compile/data/filterinvalid', () => { diff --git a/test/compile/invalid/ChannelInvalidDataMode.test.ts b/test/compile/invalid/ChannelInvalidDataMode.test.ts new file mode 100644 index 0000000000..93ad0b5dc3 --- /dev/null +++ b/test/compile/invalid/ChannelInvalidDataMode.test.ts @@ -0,0 +1,164 @@ +import {NONPOSITION_SCALE_CHANNELS, POSITION_SCALE_CHANNELS, SCALE_CHANNELS} from '../../../src/channel'; +import {getScaleInvalidDataMode} from '../../../src/compile/invalid/ScaleInvalidDataMode'; +import {defaultConfig} from '../../../src/config'; +import {MarkInvalidDataMode} from '../../../src/invalid'; +import {PATH_MARKS, PRIMITIVE_MARKS} from '../../../src/mark'; + +describe('compile / invalid / ChannelInvalidDataMode / getChannelInvalidDataMode()', () => { + const ALL_MARK_INVALID_MODE: MarkInvalidDataMode[] = [ + 'filter', + 'break-paths', + 'break-paths-keep-domains', + 'include', + 'break-and-keep-path-domains' + ]; + + // TODO: add test for non-continuous scales, count + + describe.each([...PRIMITIVE_MARKS])('For all marks (%s)', mark => { + it.each(ALL_MARK_INVALID_MODE)( + 'should return the specified invalid output color for all invalid mode (%s)', + invalid => { + expect( + getScaleInvalidDataMode({ + markDef: {type: mark, invalid}, + scaleChannel: 'color', + scaleType: 'linear', + isCountAggregate: false, + config: { + ...defaultConfig, + scale: { + ...defaultConfig.scale, + invalid: {color: {value: 'red'}} + } + } + }) + ).toEqual({includeAs: {value: 'red'}}); + } + ); + + it.each(ALL_MARK_INVALID_MODE)('should return the specified invalid output size', invalid => { + expect( + getScaleInvalidDataMode({ + markDef: {type: mark, invalid}, + scaleChannel: 'size', + scaleType: 'linear', + isCountAggregate: false, + config: { + ...defaultConfig, + scale: { + ...defaultConfig.scale, + invalid: {size: {value: 4}} + } + } + }) + ).toEqual({includeAs: {value: 4}}); + }); + + describe.each(POSITION_SCALE_CHANNELS)('for all position scale channel (%s)', channel => { + it('should return {include: min} by default for include mode if scale invalid config is not specified', () => { + expect( + getScaleInvalidDataMode({ + markDef: {type: mark, invalid: 'include'}, + scaleChannel: channel, + scaleType: 'linear', + isCountAggregate: false, + config: { + ...defaultConfig, + scale: { + ...defaultConfig.scale, + invalid: {} + } + } + }) + ).toEqual({includeAs: 'min'}); + }); + }); + + describe.each(NONPOSITION_SCALE_CHANNELS)('for all non-position scale channel (%s)', channel => { + it('should return {include: zero-or-min} by default for include mode if scale invalid config is not specified', () => { + expect( + getScaleInvalidDataMode({ + markDef: {type: mark, invalid: 'include'}, + scaleChannel: channel, + scaleType: 'linear', + isCountAggregate: false, + config: { + ...defaultConfig, + scale: { + ...defaultConfig.scale, + invalid: {} + } + } + }) + ).toEqual({includeAs: 'zero-or-min'}); + }); + }); + + describe.each(SCALE_CHANNELS)('for all scale channel (%s)', channel => { + const OTHER_MODES: MarkInvalidDataMode[] = ['break-paths', 'break-paths-keep-domains', 'filter']; + + it.each(OTHER_MODES)('should return the mode (%s)', mode => { + expect( + getScaleInvalidDataMode({ + markDef: {type: mark, invalid: mode}, + scaleChannel: channel, + scaleType: 'linear', + isCountAggregate: false, + config: { + ...defaultConfig, + scale: { + ...defaultConfig.scale, + invalid: {} + } + } + }) + ).toEqual(mode); + }); + }); + }); + + describe.each(PATH_MARKS)('For all path marks (%s)', mark => { + describe.each(SCALE_CHANNELS)('for all scale channel (%s)', channel => { + it('should return the mode (%s)', () => { + expect( + getScaleInvalidDataMode({ + markDef: {type: mark, invalid: 'break-and-keep-path-domains'}, + scaleChannel: channel, + scaleType: 'linear', + isCountAggregate: false, + config: { + ...defaultConfig, + scale: { + ...defaultConfig.scale, + invalid: {} + } + } + }) + ).toBe('break-paths-keep-domains'); + }); + }); + }); + + describe.each(PATH_MARKS)('For all path marks (%s)', mark => { + describe.each(SCALE_CHANNELS)('for all scale channel (%s)', channel => { + it('should return the mode (%s)', () => { + expect( + getScaleInvalidDataMode({ + markDef: {type: mark, invalid: 'break-and-keep-path-domains'}, + scaleChannel: channel, + scaleType: 'linear', + isCountAggregate: false, + config: { + ...defaultConfig, + scale: { + ...defaultConfig.scale, + invalid: {} + } + } + }) + ).toBe('break-paths-keep-domains'); + }); + }); + }); +}); diff --git a/test/compile/mark/text.test.ts b/test/compile/mark/text.test.ts index 338b103183..286002bbdf 100644 --- a/test/compile/mark/text.test.ts +++ b/test/compile/mark/text.test.ts @@ -215,7 +215,7 @@ describe('Mark: Text', () => { }, data: {url: 'data/cars.json'}, config: { - mark: {invalid: 'hide'} + mark: {invalid: 'break-paths-keep-domains'} } }; const model = parseModelWithScale(spec); @@ -242,16 +242,10 @@ describe('Mark: Text', () => { }); it('should map color to fill', () => { - expect(props.fill).toEqual([ - { - test: '!isValid(datum["mean_Acceleration"]) || !isFinite(+datum["mean_Acceleration"])', - value: null - }, - { - scale: 'color', - field: 'mean_Acceleration' - } - ]); + expect(props.fill).toEqual({ + scale: 'color', + field: 'mean_Acceleration' + }); }); it('should map size to fontSize', () => { diff --git a/test/compile/selection/layers.test.ts b/test/compile/selection/layers.test.ts index 050b085935..244c92e8f6 100644 --- a/test/compile/selection/layers.test.ts +++ b/test/compile/selection/layers.test.ts @@ -43,7 +43,7 @@ describe('Layered Selections', () => { } } ], - config: {mark: {tooltip: null, invalid: 'hide'}} + config: {mark: {tooltip: null, invalid: 'break-paths-keep-domains'}} }); layers.parse(); @@ -85,16 +85,10 @@ describe('Layered Selections', () => { signal: '"Horsepower: " + (format(datum["Horsepower"], "")) + "; Miles_per_Gallon: " + (format(datum["Miles_per_Gallon"], "")) + "; Origin: " + (isValid(datum["Origin"]) ? datum["Origin"] : ""+datum["Origin"])' }, - fill: [ - { - test: '!isValid(datum["Horsepower"]) || !isFinite(+datum["Horsepower"]) || !isValid(datum["Miles_per_Gallon"]) || !isFinite(+datum["Miles_per_Gallon"])', - value: null - }, - { - scale: 'color', - field: 'Origin' - } - ], + fill: { + scale: 'color', + field: 'Origin' + }, shape: { value: 'circle' }, @@ -133,16 +127,10 @@ describe('Layered Selections', () => { signal: '"Horsepower: " + (format(datum["Horsepower"], "")) + "; Miles_per_Gallon: " + (format(datum["Miles_per_Gallon"], "")) + "; Origin: " + (isValid(datum["Origin"]) ? datum["Origin"] : ""+datum["Origin"])' }, - fill: [ - { - test: '!isValid(datum["Horsepower"]) || !isFinite(+datum["Horsepower"]) || !isValid(datum["Miles_per_Gallon"]) || !isFinite(+datum["Miles_per_Gallon"])', - value: null - }, - { - scale: 'color', - field: 'Origin' - } - ], + fill: { + scale: 'color', + field: 'Origin' + }, shape: { value: 'square' }, @@ -176,25 +164,13 @@ describe('Layered Selections', () => { signal: '"Horsepower: " + (format(datum["Horsepower"], "")) + "; Miles_per_Gallon: " + (format(datum["Miles_per_Gallon"], "")) + "; Origin: " + (isValid(datum["Origin"]) ? datum["Origin"] : ""+datum["Origin"])' }, - fill: [ - { - test: '!isValid(datum["Horsepower"]) || !isFinite(+datum["Horsepower"]) || !isValid(datum["Miles_per_Gallon"]) || !isFinite(+datum["Miles_per_Gallon"])', - value: null - }, - { - value: 'transparent' - } - ], - stroke: [ - { - test: '!isValid(datum["Horsepower"]) || !isFinite(+datum["Horsepower"]) || !isValid(datum["Miles_per_Gallon"]) || !isFinite(+datum["Miles_per_Gallon"])', - value: null - }, - { - scale: 'color', - field: 'Origin' - } - ], + fill: { + value: 'transparent' + }, + stroke: { + scale: 'color', + field: 'Origin' + }, x: { scale: 'x', field: 'Horsepower' diff --git a/test/compile/selection/timeunit.test.ts b/test/compile/selection/timeunit.test.ts index 60c96f51f2..24c94360fc 100644 --- a/test/compile/selection/timeunit.test.ts +++ b/test/compile/selection/timeunit.test.ts @@ -110,7 +110,7 @@ describe('Selection time unit', () => { y: {field: 'price', type: 'quantitative'} } }, - {mark: {invalid: 'hide'}} + {mark: {invalid: 'break-paths-keep-domains'}} ); const data0 = getData(model).filter(d => d.name === 'data_0')[0].transform; const data1 = getData(model).filter(d => d.name === 'data_1')[0].transform;