Hi, I'm facing an issue from kibana. Trying to fetch the data of last 90 days I get a timeout. After that I tried to make a curl to the server using the full request that kibana runs in the inspect windows and curl worked perfectly fetching all the data.
Watching kibana logs I didn't see any error but maybe it tells you something.
{"type":"response","@timestamp":"2020-12-11T09:50:21Z","tags":[],"pid":1,"method":"post","statusCode":200,"req":{"url":"/api/ui_metric/report","method":"post","headers":{"connection":"upgrade","host":"url/kibana","x-real-ip":"1.1.1.1","x-forwarded-for":"1.1.1.1","x-forwarded-proto":"https","x-forwarded-host":"url/kibana","x-forwarded-port":"443","content-length":"122","user-agent":"Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0","accept":"*/*","accept-language":"es-ES,es;q=0.8,en-US;q=0.5,en;q=0.3","accept-encoding":"gzip, deflate, br","referer":"url/kibana","content-type":"application/json","kbn-version":"7.7.0","origin":"https://url/kibana"},"remoteAddress":"1.1.1.1","userAgent":"1.1.1.1","referer":"https://url/kibana"},"res":{"statusCode":200,"responseTime":987,"contentLength":9},"message":"POST /api/ui_metric/report 200 987ms - 9.0B"}
{"type":"response","@timestamp":"2020-12-11T09:50:29Z","tags":[],"pid":1,"method":"get","statusCode":200,"req":{"url":"/login","method":"get","headers":{"host":"1.1.1.1:5601","user-agent":"kube-probe/1.14+","accept-encoding":"gzip","connection":"close"},"remoteAddress":"1.1.1.1","userAgent":"1.1.1.1"},"res":{"statusCode":200,"responseTime":8,"contentLength":9},"message":"GET /login 200 8ms - 9.0B"}
{"type":"response","@timestamp":"2020-12-11T09:50:49Z","tags":[],"pid":1,"method":"get","statusCode":200,"req":{"url":"/login","method":"get","headers":{"host":"1.1.1.1:5601","user-agent":"kube-probe/1.14+","accept-encoding":"gzip","connection":"close"},"remoteAddress":"1.1.1.1","userAgent":"1.1.1.1"},"res":{"statusCode":200,"responseTime":9,"contentLength":9},"message":"GET /login 200 9ms - 9.0B"}
{"type":"response","@timestamp":"2020-12-11T09:51:09Z","tags":[],"pid":1,"method":"get","statusCode":200,"req":{"url":"/login","method":"get","headers":{"host":"1.1.1.1:5601","user-agent":"kube-probe/1.14+","accept-encoding":"gzip","connection":"close"},"remoteAddress":"1.1.1.1","userAgent":"1.1.1.1"},"res":{"statusCode":200,"responseTime":8,"contentLength":9},"message":"GET /login 200 8ms - 9.0B"}
{"type":"response","@timestamp":"2020-12-11T09:49:28Z","tags":[],"pid":1,"method":"post","statusCode":200,"req":{"url":"/internal/search/es","method":"post","headers":{"connection":"upgrade","host":"url","x-real-ip":"1.1.1.1","x-forwarded-for":"1.1.1.1","x-forwarded-proto":"https","x-forwarded-host":"url","x-forwarded-port":"443","content-length":"1883","user-agent":"Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0","accept":"*/*","accept-language":"es-ES,es;q=0.8,en-US;q=0.5,en;q=0.3","accept-encoding":"gzip, deflate, br","referer":"url/kibana","content-type":"application/json","kbn-version":"7.7.0","origin":"https://url/kibana"},"remoteAddress":"1.1.1.1","userAgent":"1.1.1.1","referer":"url/kibana"},"res":{"statusCode":200,"responseTime":120001,"contentLength":9},"message":"POST /internal/search/es 200 120001ms - 9.0B"}
{"type":"response","@timestamp":"2020-12-11T09:51:29Z","tags":[],"pid":1,"method":"get","statusCode":200,"req":{"url":"/login","method":"get","headers":{"host":"1.1.1.1:5601","user-agent":"kube-probe/1.14+","accept-encoding":"gzip","connection":"close"},"remoteAddress":"1.1.1.1","userAgent":"1.1.1.1"},"res":{"statusCode":200,"responseTime":9,"contentLength":9},"message":"GET /login 200 9ms - 9.0B"}
The query I'm kibana is trying to request is:
{
"version": true,
"size": "100",
"sort": [
{
"timestamp": {
"order": "desc",
"unmapped_type": "boolean"
}
}
],
"aggs": {
"2": {
"date_histogram": {
"field": "timestamp",
"calendar_interval": "1d",
"time_zone": "Europe/Madrid",
"min_doc_count": 1
}
}
},
"stored_fields": [
"*"
],
"script_fields": {},
"docvalue_fields": [
{
"field": "timestamp",
"format": "date_time"
},
{
"field": "data.aws.created-at",
"format": "date_time"
},
{
"field": "data.aws.createdAt",
"format": "date_time"
},
{
"field": "data.aws.end",
"format": "date_time"
},
{
"field": "data.aws.resource.instanceDetails.launchTime",
"format": "date_time"
},
{
"field": "data.aws.service.eventFirstSeen",
"format": "date_time"
},
{
"field": "data.aws.service.eventLastSeen",
"format": "date_time"
},
{
"field": "data.aws.start",
"format": "date_time"
},
{
"field": "data.aws.summary.Time Range.end",
"format": "date_time"
},
{
"field": "data.aws.summary.Time Range.start",
"format": "date_time"
},
{
"field": "data.aws.updatedAt",
"format": "date_time"
},
{
"field": "data.columns.datetime",
"format": "date_time"
},
{
"field": "data.columns.iso_8601",
"format": "date_time"
},
{
"field": "data.vulnerability.published",
"format": "date_time"
},
{
"field": "syscheck.mtime_after",
"format": "date_time"
},
{
"field": "syscheck.mtime_before",
"format": "date_time"
},
{
"field": "data.cis.timestamp",
"format": "date_time"
},
{
"field": "data.timestamp",
"format": "date_time"
}
],
"_source": {
"excludes": [
"@timestamp"
]
},
"query": {
"bool": {
"must": [],
"filter": [
{
"match_all": {}
},
{
"range": {
"timestamp": {
"gte": "2020-09-12T10:42:03.361Z",
"lte": "2020-12-11T11:42:03.361Z",
"format": "strict_date_optional_time"
}
}
}
],
"should": [],
"must_not": []
}
},
"highlight": {
"pre_tags": [
"@kibana-highlighted-field@"
],
"post_tags": [
"@/kibana-highlighted-field@"
],
"fields": {
"*": {}
},
"fragment_size": 2147483647
}
}
And curl run from Kibana's server is the following:
curl -k -u elastic -X POST "https://opendistro-server:9200/indice*/_search" -H 'Content-Type: application/json' -d '{"track_total_hits": true,"version":true,"size":"100","sort":[{"timestamp":{"order":"desc","unmapped_type":"boolean"}}],"aggs":{"2":{"date_histogram":{"field":"timestamp","calendar_interval":"1d","time_zone":"Europe/Madrid","min_doc_count":1}}},"stored_fields":["*"],"script_fields":{},"docvalue_fields":[{"field":"timestamp","format":"date_time"},{"field":"data.aws.created-at","format":"date_time"},{"field":"data.aws.createdAt","format":"date_time"},{"field":"data.aws.end","format":"date_time"},{"field":"data.aws.resource.instanceDetails.launchTime","format":"date_time"},{"field":"data.aws.service.eventFirstSeen","format":"date_time"},{"field":"data.aws.service.eventLastSeen","format":"date_time"},{"field":"data.aws.start","format":"date_time"},{"field":"data.aws.summary.TimeRange.end","format":"date_time"},{"field":"data.aws.summary.TimeRange.start","format":"date_time"},{"field":"data.aws.updatedAt","format":"date_time"},{"field":"data.columns.datetime","format":"date_time"},{"field":"data.columns.iso_8601","format":"date_time"},{"field":"data.vulnerability.published","format":"date_time"},{"field":"syscheck.mtime_after","format":"date_time"},{"field":"syscheck.mtime_before","format":"date_time"},{"field":"data.cis.timestamp","format":"date_time"},{"field":"data.timestamp","format":"date_time"}],"_source":{"excludes":["@timestamp"]},"query":{"bool":{"must":[],"filter":[{"match_all":{}},{"range":{"timestamp":{"gte":"2020-09-12T07:36:04.661Z","lte":"2020-12-11T08:36:04.661Z","format":"strict_date_optional_time"}}}],"should":[],"must_not":[]}},"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"fragment_size":2147483647}}'
The number of hits is about 500000000 so is a big amount of data but I'm not sure if it related since Kibana is asking for 100, and the number of total hits.
Any ideas about what can be happening??
Thanks in advance.
