Is Vega performance good on large dataset?

Elastic Stack Kibana
1

I am working on a large timeseries dataset, around 120000 document approx. I want to know how the performance is of Vega on such large database. Does it take a lot of time to load to show charts and graphs. Or it works as smooth as other options like lens, TSVB etc.

I am using Vega for few visualization and it takes quite bit time to load if I select the time range as 1 yr. I want to know is this some limitation of Vega ?

2

The suspect slowness you are seeing is with the number of aggregation buckets from your vega query when using a large time range. Lens scales aggregation time interval to the time range. Are you using %autointerval% in your vega script to scale the the time interval?

3

Thanks for response @Nathan_Reese.
Yes, I am using %autointerval% in my Vega spec.

4

I am showing a time series chart in my visualization, which presents sum of power from each unique house number at a time.

{
  "$schema": "https://vega.github.io/schema/vega/v5.json",
  "description": "House Power",
  "data": [
    {
      "name": "saclass",
      "url": {
        "index": "class-metric*",
        "body": {
          "query": {
            "bool": {
              "must": [
                "%dashboard_context-must_clause%",
                {"exists": {"field": "house.power"}}
              ]
            }
          },
          "aggs": {
            "time_buckets": {
              "date_histogram": {
                "field": "@timestamp",
                "time_zone": "Asia/Dubai",
                "interval": {"%autointerval%": true},
                "extended_bounds": {
                  "min": {"%timefilter%": "min"},
                  "max": {"%timefilter%": "max"}
                }
              },
              "aggs": {
                "house_number": {
                  "terms": {"field": "house.number"},
                  "aggs": {
                    "power": {
                      "top_metrics": {
                        "metrics": {"field": "house.power"},
                        "sort": {"@timestamp": "desc"}
                      }
                    }
                  }
                }
              }
            }
          },
          "size": 0
        }
      },
      "format": {"property": "aggregations.time_buckets.buckets"}
    },
    {
      "name": "sumpower",
      "source": "saclass",
      "transform": [
        {"type": "flatten", "fields": ["house_number.buckets"], "as": ["nums"]},
        {
          "type": "formula",
          "expr": "datum[\"nums\"] && datum[\"nums\"][\"power\"] && datum[\"nums\"][\"power\"][\"top\"] && datum[\"nums\"][\"power\"][\"top\"][\"0\"] && datum[\"nums\"][\"power\"][\"top\"][\"0\"][\"metrics\"] && datum[\"nums\"][\"power\"][\"top\"][\"0\"][\"metrics\"][\"house.power\"]",
          "as": "nums.power.top.0.metrics.house.power"
        },
        {
          "type": "formula",
          "expr": "format(datum[\"nums.power.top.0.metrics.house.power\"]/1000, \".2f\")",
          "as": "pow"
        },
        {
          "type": "joinaggregate",
          "ops": ["sum"],
          "fields": ["pow"],
          "as": ["tpow"],
          "groupby": ["key"]
        },
        {"type": "extent", "field": "key", "signal": "xext"},
        {"type": "extent", "field": "tpow", "signal": "yext"}
      ]
    }
  ],
  "signals": [
    {"name": "xrange", "update": "[0, width]"},
    {"name": "yrange", "update": "[height, 0]"},
    {
      "name": "down",
      "value": null,
      "on": [
        {"events": "touchend", "update": "null"},
        {"events": "mousedown, touchstart", "update": "xy()"}
      ]
    },
    {
      "name": "xcur",
      "value": null,
      "on": [
        {"events": "mousedown, touchstart, touchend", "update": "slice(xdom)"}
      ]
    },
    {
      "name": "ycur",
      "value": null,
      "on": [
        {"events": "mousedown, touchstart, touchend", "update": "slice(ydom)"}
      ]
    },
    {
      "name": "delta",
      "value": [0, 0],
      "on": [
        {
          "events": [
            {
              "source": "window",
              "type": "mousemove",
              "consume": true,
              "between": [
                {"type": "mousedown"},
                {"source": "window", "type": "mouseup"}
              ]
            },
            {
              "type": "touchmove",
              "consume": true,
              "filter": "event.touches.length === 1"
            }
          ],
          "update": "down ? [down[0]-x(), y()-down[1]] : [0,0]"
        }
      ]
    },
    {
      "name": "anchor",
      "value": [0, 0],
      "on": [
        {
          "events": "wheel",
          "update": "[+invert('xscale', x()), invert('yscale', y())]"
        },
        {
          "events": {
            "type": "touchstart",
            "filter": "event.touches.length===2"
          },
          "update": "[(xdom[0] + xdom[1]) / 2, (ydom[0] + ydom[1]) / 2]"
        }
      ]
    },
    {
      "name": "zoom",
      "value": 1,
      "on": [
        {
          "events": "wheel!",
          "force": true,
          "update": "pow(1.001, event.deltaY * pow(16, event.deltaMode))"
        },
        {
          "events": {"signal": "dist2"},
          "force": true,
          "update": "dist1 / dist2"
        }
      ]
    },
    {
      "name": "dist1",
      "value": 0,
      "on": [
        {
          "events": {
            "type": "touchstart",
            "filter": "event.touches.length===2"
          },
          "update": "pinchDistance(event)"
        },
        {"events": {"signal": "dist2"}, "update": "dist2"}
      ]
    },
    {
      "name": "dist2",
      "value": 0,
      "on": [
        {
          "events": {
            "type": "touchmove",
            "consume": true,
            "filter": "event.touches.length===2"
          },
          "update": "pinchDistance(event)"
        }
      ]
    },
    {
      "name": "xdom",
      "update": "slice(xext)",
      "on": [
        {
          "events": {"signal": "delta"},
          "update": "[xcur[0] + span(xcur) * delta[0] / width, xcur[1] + span(xcur) * delta[0] / width]"
        },
        {
          "events": {"signal": "zoom"},
          "update": "[anchor[0] + (xdom[0] - anchor[0]) * zoom, anchor[0] + (xdom[1] - anchor[0]) * zoom]"
        }
      ]
    },
    {"name": "ydom", "update": "slice(yext)"},
    {"name": "hitAreaWidth", "value": 1},
    {
      "name": "hover",
      "value": {"x": 0, "y": 0},
      "on": [
        {
          "events": "@hit:mouseover",
          "update": "{x: scale('xscale', datum.x), y: scale('yscale', datum.y)}"
        }
      ]
    }
  ],
  "marks": [
    {
      "name": "hit",
      "type": "rect",
      "interactive": true,
      "clip": true,
      "from": {"data": "sumpower"},
      "encode": {
        "update": {
          "xc": {"scale": "xscale", "field": "key"},
          "width": {"signal": "hitAreaWidth"},
          "y": {"scale": "yscale", "value": 0}
        },
        "enter": {
          "y": {"value": 0},
          "height": {"signal": "height"},
          "fill": {"value": "transparent"}
        }
      }
    },
    {
      "type": "area",
      "from": {"data": "sumpower"},
      "clip": true,
      "interactive": false,
      "encode": {
        "enter": {"stroke": {"value": "#00B4AA"}, "fill": {"value": "#74c476"}},
        "update": {
          "x": {"scale": "xscale", "field": "key"},
          "y": {"scale": "yscale", "field": "tpow"},
          "y2": {"scale": "yscale", "value": 0}
        }
      }
    },
    {
      "type": "symbol",
      "shape": "circle",
      "size": 70,
      "from": {"data": "sumpower"},
      "encode": {
        "update": {
          "x": {"scale": "xscale", "field": "key"},
          "y": {"scale": "yscale", "field": "tpow"},
          "y2": {"value": 0},
          "fill": {"value": "#bbcddc"},
          "stroke": {"value": "#652c90"},
          "opacity": {"signal": 1}
        },
        "enter": {
          "tooltip": {
            "signal": "{'title': timeFormat(datum.key, '%d/%m/%Y %H:%M:%S'), 'Power': datum.tpow}",
            "nearest": true
          }
        }
      }
    },
    {
      "type": "rule",
      "interactive": false,
      "encode": {
        "enter": {
          "stroke": {"value": "orange"},
          "strokeWidth": {"value": 2},
          "y": {"signal": "height"},
          "y2": {"value": 0}
        },
        "update": {
          "x": {"signal": "hover.x"},
          "opacity": {"signal": "!hover.x && !hover.y ? 0 : 1"}
        }
      }
    }
  ],
  "scales": [
    {
      "name": "xscale",
      "type": "time",
      "zero": false,
      "domain": {"signal": "xdom"},
      "range": {"signal": "xrange"}
    },
    {
      "name": "yscale",
      "zero": true,
      "domain": {"signal": "ydom"},
      "range": {"signal": "yrange"}
    }
  ],
  "axes": [
    {
      "scale": "xscale",
      "orient": "bottom",
      "title": "Timestamp",
      "titlePadding": -2,
      "grid": true
    },
    {"scale": "yscale", "orient": "left", "title": "Power", "grid": true}
  ]
}

@Nathan_Reese Is there any optimization which I can do to my code ?

5

Anything on this I may know ?

Anyone ?

Thanks in advance.

Kibana vega visualisation

6

You could try profiling the elasticsearch query in dev tools search profiler. You could also use your browser's debugging tools to profile kibana to see whether the query is slow or the rendering is slow.

120000 documents is a pretty small data set. So maybe you need more nodes on the elasticsearch side.

7

Sure @Nathan_Reese I'll check.

8

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.