reeting
I have netflow data collection through Elastiflow/elasticsearch/kibana which display correctly sankey screenshot, on Visualize menu.
this is the request for selected timestamp:
{
"size": 0,
"aggs": {
"table": {
"composite": {
"size": 1000,
"sources": [
{
"stk1": {
"terms": {
"field": "flow.src.as.label"
}
}
},
{
"stk2": {
"terms": {
"field": "flow.dst.as.label"
}
}
}
]
},
"aggs": {
"bytes": {
"sum": {
"field": "network.bytes"
}
}
}
}
},
"query": {
"bool": {
"must": [
{
"range": {
"@timestamp": {
"gte": "2023-03-12T10:00:00.000Z",
"lte": "2023-03-13T10:15:58.158Z",
"format": "strict_date_optional_time"
}
}
}
],
"filter": [],
"should": [],
"must_not": []
}
}
}
and vega script:
{
"$schema": "https://vega.github.io/schema/vega/v3.0.json",
"data": [
{
"name": "rawData",
"url": {
"%context%": true,
"%timefield%": "@timestamp",
"index": "elastiflow-*",
"body": {
"size": 0,
"aggs": {
"table": {
"composite": {
"size": 1000,
"sources": [
{"stk1": {"terms": {"field": "flow.src.as.label"}}},
{"stk2": {"terms": {"field": "flow.dst.as.label"}}}
]
},
"aggs": {
"bytes": {
"sum": {
"field": "network.bytes"
}
}
}
}
}
}
},
"format": {"property": "aggregations.table.buckets"},
"transform": [
{"type": "formula", "expr": "datum.key.stk1", "as": "stk1"},
{"type": "formula", "expr": "datum.key.stk2", "as": "stk2"},
{"type": "formula", "expr": "datum.bytes.value", "as": "size"}
]
},
{
"name": "nodes",
"source": "rawData",
"transform": [
{
"type": "filter",
"expr": "!groupSelector || groupSelector.stk1 == datum.stk1 || groupSelector.stk2 == datum.stk2"
},
{"type": "formula", "expr": "datum.stk1+datum.stk2", "as": "key"},
{"type": "fold", "fields": ["stk1", "stk2"], "as": ["stack", "grpId"]},
{
"type": "formula",
"expr": "datum.stack == 'stk1' ? datum.stk1+datum.stk2 : datum.stk2+datum.stk1",
"as": "sortField"
},
{
"type": "stack",
"groupby": ["stack"],
"sort": {"field": "sortField", "order": "descending"},
"field": "size"
},
{"type": "formula", "expr": "(datum.y0+datum.y1)/2", "as": "yc"}
]
},
{
"name": "groups",
"source": "nodes",
"transform": [
{
"type": "aggregate",
"groupby": ["stack", "grpId"],
"fields": ["size"],
"ops": ["sum"],
"as": ["total"]
},
{
"type": "stack",
"groupby": ["stack"],
"sort": {"field": "grpId", "order": "descending"},
"field": "total"
},
{"type": "formula", "expr": "scale('y', datum.y0)", "as": "scaledY0"},
{"type": "formula", "expr": "scale('y', datum.y1)", "as": "scaledY1"},
{"type": "formula", "expr": "datum.stack == 'stk1'", "as": "rightLabel"},
{
"type": "formula",
"expr": "datum.total/domain('y')[1]",
"as": "percentage"
}
]
},
{
"name": "destinationNodes",
"source": "nodes",
"transform": [{"type": "filter", "expr": "datum.stack == 'stk2'"}]
},
{
"name": "edges",
"source": "nodes",
"transform": [
{"type": "filter", "expr": "datum.stack == 'stk1'"},
{
"type": "lookup",
"from": "destinationNodes",
"key": "key",
"fields": ["key"],
"as": ["target"]
},
{
"type": "linkpath",
"orient": "horizontal",
"shape": "diagonal",
"sourceY": {"expr": "scale('y', datum.yc)"},
"sourceX": {"expr": "scale('x', 'stk1') + bandwidth('x')"},
"targetY": {"expr": "scale('y', datum.target.yc)"},
"targetX": {"expr": "scale('x', 'stk2')"}
},
{
"type": "formula",
"expr": "range('y')[0]-scale('y', datum.size)",
"as": "strokeWidth"
},
{
"type": "formula",
"expr": "datum.size/domain('y')[1]",
"as": "percentage"
}
]
}
],
"scales": [
{
"name": "x",
"type": "band",
"range": "width",
"domain": ["stk1", "stk2"],
"paddingOuter": 0.01,
"paddingInner": 0.98
},
{
"name": "y",
"type": "linear",
"range": "height",
"domain": {"data": "nodes", "field": "y1"}
},
{
"name": "color",
"type": "ordinal",
"range": "category",
"domain": {"data": "rawData", "fields": ["stk1","stk2"]}
},
{
"name": "stackNames",
"type": "ordinal",
"range": ["Src AS", "Dst AS"],
"domain": ["stk1", "stk2"]
}
],
"axes": [
{
"orient": "bottom",
"scale": "x",
"labelColor": {
"value": "#444444"
},
"encode": {
"labels": {
"update": {
"text": {"scale": "stackNames", "field": "value"},
"fontSize": {"value": 14}
}
}
}
},
{
"orient": "left",
"scale": "y",
"labelColor": {
"value": "#444444"
},
"encode": {
"labels": {
"update": {
"text": {"signal": "format(datum.value, '.2s') + 'B'"},
"fontSize": {"value": 12}
}
}
}
}
],
"marks": [
{
"type": "path",
"name": "edgeMark",
"from": {"data": "edges"},
"clip": true,
"encode": {
"update": {
"stroke": [
{
"test": "groupSelector && groupSelector.stack=='stk1'",
"scale": "color",
"field": "stk2"
},
{"scale": "color", "field": "stk1"}
],
"strokeWidth": {"field": "strokeWidth"},
"path": {"field": "path"},
"strokeOpacity": {
"signal": "!groupSelector && (groupHover.stk1 == datum.stk1 || groupHover.stk2 == datum.stk2) ? 0.8 : 0.4"
},
"zindex": {
"signal": "!groupSelector && (groupHover.stk1 == datum.stk1 || groupHover.stk2 == datum.stk2) ? 1 : 0"
},
"tooltip": {
"signal": "datum.stk1 + ' → ' + datum.stk2 + ' ' + format(datum.size, '.2s') + 'B (' + format(datum.percentage, '.1%') + ')'"
}
},
"hover": {"strokeOpacity": {"value": 0.8}}
}
},
{
"type": "rect",
"name": "groupMark",
"from": {"data": "groups"},
"encode": {
"enter": {
"fill": {"scale": "color", "field": "grpId"},
"width": {"scale": "x", "band": 1}
},
"update": {
"x": {"scale": "x", "field": "stack"},
"y": {"field": "scaledY0"},
"y2": {"field": "scaledY1"},
"fillOpacity": {"value": 0.7},
"tooltip": {
"signal": "datum.grpId + ' ' + format(datum.total, '.2s') + 'B (' + format(datum.percentage, '.1%') + ')'"
}
},
"hover": {"fillOpacity": {"value": 1}}
}
},
{
"type": "text",
"from": {"data": "groups"},
"interactive": false,
"encode": {
"update": {
"x": {
"signal": "scale('x', datum.stack) + (datum.rightLabel ? bandwidth('x') + 8 : -8)"
},
"yc": {"signal": "(datum.scaledY0 + datum.scaledY1)/2"},
"align": {"signal": "datum.rightLabel ? 'left' : 'right'"},
"baseline": {"value": "middle"},
"fontWeight": {"value": "bold"},
"fontSize": {"value": 12},
"fill": {"value": "#222222"},
"text": {
"signal": "abs(datum.scaledY0-datum.scaledY1) > 10 ? datum.grpId : ''"
}
}
}
},
{
"type": "group",
"data": [
{
"name": "dataForShowAll",
"values": [{}],
"transform": [{"type": "filter", "expr": "groupSelector"}]
}
],
"encode": {
"enter": {
"xc": {"signal": "width/2"},
"y": {"value": 30},
"width": {"value": 100},
"height": {"value": 36}
}
},
"marks": [
{
"type": "group",
"name": "groupReset",
"from": {"data": "dataForShowAll"},
"encode": {
"enter": {
"cornerRadius": {"value": 3.5},
"fill": {"value": "#666666"},
"height": {"field": {"group": "height"}},
"width": {"field": {"group": "width"}}
},
"update": {"opacity": {"value": 1}},
"hover": {"fill": {"value": "#444444"}}
},
"marks": [
{
"type": "text",
"interactive": false,
"encode": {
"enter": {
"xc": {"field": {"group": "width"}, "mult": 0.5},
"yc": {"field": {"group": "height"}, "mult": 0.5, "offset": 1},
"align": {"value": "center"},
"baseline": {"value": "middle"},
"text": {"value": "Show All"},
"fontSize": {"value": 14},
"stroke": {"value": "#ecf0f1"}
}
}
}
]
}
]
}
],
"signals": [
{
"name": "groupHover",
"value": {},
"on": [
{
"events": "@groupMark:mouseover",
"update": "{stk1:datum.stack=='stk1' && datum.grpId, stk2:datum.stack=='stk2' && datum.grpId}"
},
{"events": "mouseout", "update": "{}"}
]
},
{
"name": "groupSelector",
"value": false,
"on": [
{
"events": "@groupMark:click!",
"update": "{stack:datum.stack, stk1:datum.stack=='stk1' && datum.grpId, stk2:datum.stack=='stk2' && datum.grpId}"
},
{
"events": [
{"type": "click", "markname": "groupReset"},
{"type": "dblclick"}
],
"update": "false"
}
]
}
]
}
As you see in vega script, field network.bytes (from netflow data) is in Bytes.
Do you think it is possible:
- to display in "bits/s" (Mbits/s or Kbits/s) instead of "KB" ou "MB" ?
- to change also unit on sankey display?
Thanks for any answer.
Jacques