My elasticsearch records store source and destination AS numbers. I wanted to visualize the top 10 source -> destination AS numbers. I wanted to know if this was possible to accomplish without having to store another field in my records mapping just this. Instead I wanted to do this through a script. Currently in kibana, when I try and visualize this, using this as my script value for aggregation:
{ "script" : "doc['srcAS.asn'].value + '-' + doc['dstAS.asn'].value" }
Kibana says 10 of 80 shards failed. When I take a look in the elastic search logs, there are a bunch of this error:
org.elasticsearch.transport.RemoteTransportException: [T_-QeCS][localhost:9300][indices:data/read/search[phase/query]]
Caused by: org.elasticsearch.search.query.QueryPhaseExecutionException: Query Failed [Failed to execute main query]
at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:414) ~[elasticsearch-5.5.1.jar:5.5.1]
at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:108) ~[elasticsearch-5.5.1.jar:5.5.1]
at org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:248) ~[elasticsearch-5.5.1.jar:5.5.1]
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:263) ~[elasticsearch-5.5.1.jar:5.5.1]
at org.elasticsearch.action.search.SearchTransportService$6.messageReceived(SearchTransportService.java:330) ~[elasticsearch-5.5.1.jar:5.5.1]
at org.elasticsearch.action.search.SearchTransportService$6.messageReceived(SearchTransportService.java:327) ~[elasticsearch-5.5.1.jar:5.5.1]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69) ~[elasticsearch-5.5.1.jar:5.5.1]
at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:644) [elasticsearch-5.5.1.jar:5.5.1]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638) [elasticsearch-5.5.1.jar:5.5.1]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.5.1.jar:5.5.1]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_131]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_131]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
Caused by: java.lang.ClassCastException
[2017-08-14T16:00:19,785][DEBUG][o.e.a.s.TransportSearchAction] [T_-QeCS] [sflow-2017.08.13][2], node[T_-QeCSeRcWiBArlDxC0LA], [P], s[STARTED], a[id=OtdkWpr-T9aQiUgDYvq2Qw]: Failed to execute [SearchRequest{searchType=QUERY_THEN_FETCH, indices=[sflow-*], indicesOptions=IndicesOptions[id=39, ignore_unavailable=true, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, allow_alisases_to_multiple_indices=true, forbid_closed_indices=true], types=[], routing='null', preference='1502738301561', requestCache=null, scroll=null, source={
"size" : 0,
"query" : {
"bool" : {
"must" : [
{
"query_string" : {
"query" : "*",
"fields" : [ ],
"use_dis_max" : true,
"tie_breaker" : 0.0,
"default_operator" : "or",
"auto_generate_phrase_queries" : false,
"max_determinized_states" : 10000,
"enable_position_increments" : true,
"fuzziness" : "AUTO",
"fuzzy_prefix_length" : 0,
"fuzzy_max_expansions" : 50,
"phrase_slop" : 0,
"analyze_wildcard" : true,
"escape" : false,
"split_on_whitespace" : true,
"boost" : 1.0
}
},
{
"range" : {
"@timestamp" : {
"from" : 1502424000000,
"to" : 1502683199999,
"include_lower" : true,
"include_upper" : true,
"format" : "epoch_millis",
"boost" : 1.0
}
}
}
],
"disable_coord" : false,
"adjust_pure_negative" : true,
"boost" : 1.0
}
},
"_source" : {
"includes" : [ ],
"excludes" : [ ]
},
"aggregations" : {
"2" : {
"terms" : {
"field" : "srcAS.asn",
"script" : {
"inline" : "doc['srcAS.asn'].value + '-' + doc['dstAS.asn'].value",
"lang" : "painless"
},
"size" : 5,
"min_doc_count" : 1,
"shard_min_doc_count" : 0,
"show_term_doc_count_error" : false,
"order" : [
{
"_count" : "desc"
},
{
"_term" : "asc"
}
]
}
}
}
}}] lastShard [true]
However, running this query as a CURL request gives me a response:
Curl Request:
curl -XGET "http://localhost:9200/index-*/_search?pretty&size=0" -d '
{
"aggs" : {
"asn_pairs" : {
"terms" : {
"script" : "doc['\''srcAS.asn'\''].value + '\''-'\'' + doc['\''dstAS.asn'\''].value"
}
}
}
}'
Curl Response:
{
"took" : 92548,
"timed_out" : false,
"_shards" : {
"total" : 80,
"successful" : 80,
"failed" : 0
},
"hits" : {
"total" : 586704849,
"max_score" : 0.0,
"hits" : [ ]
},
"aggregations" : {
"ip_pairs" : {
"doc_count_error_upper_bound" : 2806342,
"sum_other_doc_count" : 512704709,
"buckets" : [
{
"key" : "1-2",
"doc_count" : 10840742
},
{
"key" : "3-4",
"doc_count" : 8778455
},
{
"key" : "5-6",
"doc_count" : 8560268
},
{
"key" : "1-3",
"doc_count" : 7374190
},
{
"key" : "6-7",
"doc_count" : 6795917
},
{
"key" : "7-8",
"doc_count" : 6521930
},
{
"key" : "89-9",
"doc_count" : 6441702
},
{
"key" : "0-1",
"doc_count" : 6419702
},
{
"key" : "2-3",
"doc_count" : 6205563
},
{
"key" : "4-5",
"doc_count" : 6061671
}
]
}
}
}
Is it possible to make this work in Kibana graphing the doc_count response as the y value of the key x value?