Hello, I am having some trouble with Elasticsearch and kibana,
I am able to use the inspect element to get the raw request command of my desired query in Elasticsearch:
{
"version": true,
"size": 500,
"sort": [
{
"_score": {
"order": "desc"
}
}
],
"_source": {
"excludes": []
},
"aggs": {
"2": {
"date_histogram": {
"field": "timestamp",
"calendar_interval": "1h",
"time_zone": "Asia/Singapore",
"min_doc_count": 1
}
}
},
"stored_fields": [
"*"
],
"script_fields": {
"hour_of_day": {
"script": {
"source": "doc['timestamp'].value.getHour()",
"lang": "painless"
}
}
},
"docvalue_fields": [
{
"field": "@timestamp",
"format": "date_time"
},
{
"field": "timestamp",
"format": "date_time"
},
{
"field": "utc_time",
"format": "date_time"
}
],
"query": {
"bool": {
"must": [
{
"match_phrase": {
"clientip": {
"query": "218.148.135.12"
}
}
},
{
"range": {
"timestamp": {
"format": "strict_date_optional_time",
"gte": "2024-01-22T14:02:31.124Z",
"lte": "2024-01-24T14:02:31.124Z"
}
}
}
],
"filter": [
{
"match_all": {}
}
],
"should": [],
"must_not": []
}
},
"highlight": {
"pre_tags": [
"@kibana-highlighted-field@"
],
"post_tags": [
"@/kibana-highlighted-field@"
],
"fields": {
"*": {}
},
"fragment_size": 2147483647
}
}
My response is supposed to be:
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 2,
"hits": [
{
"_index": "kibana_sample_data_logs",
"_type": "_doc",
"_id": "ExYqG40BgWqgae_ZkH2N",
"_version": 1,
"_score": 2,
"_source": {
"agent": "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)",
"bytes": 6318,
"clientip": "218.148.135.12",
"extension": "",
"geo": {
"srcdest": "IN:IN",
"src": "IN",
"dest": "IN",
"coordinates": {
"lat": 59.50336056,
"lon": -139.6602261
}
},
"host": "www.elastic.co",
"index": "kibana_sample_data_logs",
"ip": "218.148.135.12",
"machine": {
"ram": 16106127360,
"os": "win xp"
},
"memory": null,
"message": "218.148.135.12 - - [2018-08-07T11:33:23.630Z] \"GET /enterprise HTTP/1.1\" 200 6318 \"-\" \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)\"",
"phpmemory": null,
"referer": "http://twitter.com/error/ronald-garan",
"request": "/enterprise",
"response": 200,
"tags": [
"error",
"security"
],
"timestamp": "2024-01-23T11:33:23.630Z",
"url": "https://www.elastic.co/downloads/enterprise",
"utc_time": "2024-01-23T11:33:23.630Z"
},
"fields": {
"@timestamp": [
"2024-01-23T11:33:23.630Z"
],
"utc_time": [
"2024-01-23T11:33:23.630Z"
],
"hour_of_day": [
11
],
"timestamp": [
"2024-01-23T11:33:23.630Z"
]
}
}
]
},
"aggregations": {
"2": {
"buckets": [
{
"key_as_string": "2024-01-23T19:00:00.000+08:00",
"key": 1706007600000,
"doc_count": 1
}
]
}
},
"status": 200
}
However, when i copied the json request into the kibana console, i am not receiving the correct output as expected? I am supposed to be only returning one response. (even the reponse dont match from the inspect page and the kibana dev console page)
My output response is:
{
"took" : 0,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 10000,
"relation" : "gte"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : "kibana_sample_data_logs",
"_type" : "_doc",
"_id" : "ahYqG40BgWqgae_ZjG4m",
"_score" : 1.0,
"_source" : {
"agent" : "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1",
"bytes" : 6219,
"clientip" : "223.87.60.27",
"extension" : "deb",
"geo" : {
"srcdest" : "IN:US",
"src" : "IN",
"dest" : "US",
"coordinates" : {
"lat" : 39.41042861,
"lon" : -88.8454325
}
},
"host" : "artifacts.elastic.co",
"index" : "kibana_sample_data_logs",
"ip" : "223.87.60.27",
"machine" : {
"ram" : 8589934592,
"os" : "win 8"
},
"memory" : null,
"message" : """223.87.60.27 - - [2018-07-22T00:39:02.912Z] "GET /elasticsearch/elasticsearch-6.3.2.deb_1 HTTP/1.1" 200 6219 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1"""",
"phpmemory" : null,
"referer" : "http://twitter.com/success/wendy-lawrence",
"request" : "/elasticsearch/elasticsearch-6.3.2.deb",
"response" : 200,
"tags" : [
"success",
"info"
],
"timestamp" : "2024-01-07T00:39:02.912Z",
"url" : "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.2.deb_1",
"utc_time" : "2024-01-07T00:39:02.912Z"
}
},
{
"_index" : "kibana_sample_data_logs",
"_type" : "_doc",
"_id" : "axYqG40BgWqgae_ZjG4m",
"_score" : 1.0,
"_source" : {
"agent" : "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1",
"bytes" : 6850,
"clientip" : "130.246.123.197",
"extension" : "",
"geo" : {
"srcdest" : "JP:IN",
"src" : "JP",
"dest" : "IN",
"coordinates" : {
"lat" : 38.58338806,
"lon" : -86.46248778
}
},
"host" : "www.elastic.co",
"index" : "kibana_sample_data_logs",
"ip" : "130.246.123.197",
"machine" : {
"ram" : 3221225472,
"os" : "win 8"
},
"memory" : null,
"message" : """130.246.123.197 - - [2018-07-22T03:26:21.326Z] "GET /beats/metricbeat_1 HTTP/1.1" 200 6850 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1"""",
"phpmemory" : null,
"referer" : "http://www.elastic-elastic-elastic.com/success/james-mcdivitt",
"request" : "/beats/metricbeat",
"response" : 200,
"tags" : [
"success",
"info"
],
"timestamp" : "2024-01-07T03:26:21.326Z",
"url" : "https://www.elastic.co/downloads/beats/metricbeat_1",
"utc_time" : "2024-01-07T03:26:21.326Z"
}
},
{
"_index" : "kibana_sample_data_logs",
"_type" : "_doc",
"_id" : "bBYqG40BgWqgae_ZjG4m",
"_score" : 1.0,
"_source" : {
"agent" : "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24",
"bytes" : 0,
"clientip" : "120.49.143.213",
"extension" : "css",
"geo" : {
"srcdest" : "CO:DE",
"src" : "CO",
"dest" : "DE",
"coordinates" : {
"lat" : 36.96015,
"lon" : -78.18499861
}
},
"host" : "cdn.elastic-elastic-elastic.org",
"index" : "kibana_sample_data_logs",
"ip" : "120.49.143.213",
"machine" : {
"ram" : 20401094656,
"os" : "ios"
},
"memory" : null,
"message" : """120.49.143.213 - - [2018-07-22T03:30:25.131Z] "GET /styles/main.css_1 HTTP/1.1" 503 0 "-" "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24"""",
"phpmemory" : null,
"referer" : "http://twitter.com/success/konstantin-feoktistov",
"request" : "/styles/main.css",
"response" : 503,
"tags" : [
"success",
"login"
],
"timestamp" : "2024-01-07T03:30:25.131Z",
"url" : "https://cdn.elastic-elastic-elastic.org/styles/main.css_1",
"utc_time" : "2024-01-07T03:30:25.131Z"
}
},
{
"_index" : "kibana_sample_data_logs",
"_type" : "_doc",
"_id" : "bRYqG40BgWqgae_ZjG4m",
"_score" : 1.0,
"_source" : {
"agent" : "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)",
"bytes" : 14113,
"clientip" : "99.74.118.237",
"extension" : "deb",
"geo" : {
"srcdest" : "LK:IN",
"src" : "LK",
"dest" : "IN",
"coordinates" : {
"lat" : 48.31140472,
"lon" : -114.2550694
}
},
"host" : "artifacts.elastic.co",
"index" : "kibana_sample_data_logs",
"ip" : "99.74.118.237",
"machine" : {
"ram" : 11811160064,
"os" : "ios"
},
"memory" : null,
"message" : """99.74.118.237 - - [2018-07-22T03:34:43.399Z] "GET /beats/metricbeat/metricbeat-6.3.2-amd64.deb_1 HTTP/1.1" 200 14113 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)"""",
"phpmemory" : null,
"referer" : "http://www.elastic-elastic-elastic.com/success/charles-camarda",
"request" : "/beats/metricbeat/metricbeat-6.3.2-amd64.deb",
"response" : 200,
"tags" : [
"success",
"info"
],
"timestamp" : "2024-01-07T03:34:43.399Z",
"url" : "https://artifacts.elastic.co/downloads/beats/metricbeat/metricbeat-6.3.2-amd64.deb_1",
"utc_time" : "2024-01-07T03:34:43.399Z"
}
},
{
"_index" : "kibana_sample_data_logs",
"_type" : "_doc",
"_id" : "bhYqG40BgWqgae_ZjG4m",
"_score" : 1.0,
"_source" : {
"agent" : "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)",
"bytes" : 2492,
"clientip" : "177.111.217.54",
"extension" : "",
"geo" : {
"srcdest" : "MZ:US",
"src" : "MZ",
"dest" : "US",
"coordinates" : {
"lat" : 46.77917333,
"lon" : -105.3047083
}
},
"host" : "www.elastic.co",
"index" : "kibana_sample_data_logs",
"ip" : "177.111.217.54",
"machine" : {
"ram" : 9663676416,
"os" : "win 7"
},
"memory" : null,
"message" : """177.111.217.54 - - [2018-07-22T03:37:04.863Z] "GET /enterprise_1 HTTP/1.1" 200 2492 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)"""",
"phpmemory" : null,
"referer" : "http://twitter.com/success/gregory-harbaugh",
"request" : "/enterprise",
"response" : 200,
"tags" : [
"success",
"info"
],
"timestamp" : "2024-01-07T03:37:04.863Z",
"url" : "https://www.elastic.co/downloads/enterprise_1",
"utc_time" : "2024-01-07T03:37:04.863Z"
}
},
{
"_index" : "kibana_sample_data_logs",
"_type" : "_doc",
"_id" : "bxYqG40BgWqgae_ZjG4m",
"_score" : 1.0,
"_source" : {
"agent" : "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1",
"bytes" : 0,
"clientip" : "106.225.58.146",
"extension" : "",
"geo" : {
"srcdest" : "EG:CN",
"src" : "EG",
"dest" : "CN",
"coordinates" : {
"lat" : 35.98531194,
"lon" : -85.80931806
}
},
"host" : "www.elastic.co",
"index" : "kibana_sample_data_logs",
"ip" : "106.225.58.146",
"machine" : {
"ram" : 17179869184,
"os" : "win 7"
},
"memory" : null,
"message" : """106.225.58.146 - - [2018-07-22T03:49:40.669Z] "GET /apm_1 HTTP/1.1" 503 0 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1"""",
"phpmemory" : null,
"referer" : "http://www.elastic-elastic-elastic.com/success/richard-o-covey",
"request" : "/apm",
"response" : 503,
"tags" : [
"success",
"security"
],
"timestamp" : "2024-01-07T03:49:40.669Z",
"url" : "https://www.elastic.co/downloads/apm_1",
"utc_time" : "2024-01-07T03:49:40.669Z"
}
},
{
"_index" : "kibana_sample_data_logs",
"_type" : "_doc",
"_id" : "cBYqG40BgWqgae_ZjG4m",
"_score" : 1.0,
"_source" : {
"agent" : "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1",
"bytes" : 1872,
"clientip" : "6.138.148.165",
"extension" : "deb",
"geo" : {
"srcdest" : "IN:NG",
"src" : "IN",
"dest" : "NG",
"coordinates" : {
"lat" : 34.89107083,
"lon" : -79.75905806
}
},
"host" : "artifacts.elastic.co",
"index" : "kibana_sample_data_logs",
"ip" : "6.138.148.165",
"machine" : {
"ram" : 9663676416,
"os" : "win 8"
},
"memory" : null,
"message" : """6.138.148.165 - - [2018-07-22T03:57:39.612Z] "GET /beats/metricbeat/metricbeat-6.3.2-amd64.deb_1 HTTP/1.1" 200 1872 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1"""",
"phpmemory" : null,
"referer" : "http://facebook.com/error/michael-p-anderson",
"request" : "/beats/metricbeat/metricbeat-6.3.2-amd64.deb",
"response" : 200,
"tags" : [
"success",
"info"
],
"timestamp" : "2024-01-07T03:57:39.612Z",
"url" : "https://artifacts.elastic.co/downloads/beats/metricbeat/metricbeat-6.3.2-amd64.deb_1",
"utc_time" : "2024-01-07T03:57:39.612Z"
}
},
{
"_index" : "kibana_sample_data_logs",
"_type" : "_doc",
"_id" : "cRYqG40BgWqgae_ZjG4m",
"_score" : 1.0,
"_source" : {
"agent" : "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1",
"bytes" : 4531,
"clientip" : "218.148.135.12",
"extension" : "gz",
"geo" : {
"srcdest" : "BR:ES",
"src" : "BR",
"dest" : "ES",
"coordinates" : {
"lat" : 32.73355611,
"lon" : -117.1896567
}
},
"host" : "artifacts.elastic.co",
"index" : "kibana_sample_data_logs",
"ip" : "218.148.135.12",
"machine" : {
"ram" : 11811160064,
"os" : "win 8"
},
"memory" : null,
"message" : """218.148.135.12 - - [2018-07-22T04:18:12.345Z] "GET /beats/filebeat/filebeat-6.3.2-linux-x86_64.tar.gz_1 HTTP/1.1" 200 4531 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1"""",
"phpmemory" : null,
"referer" : "http://www.elastic-elastic-elastic.com/success/gemini-6a",
"request" : "/beats/filebeat/filebeat-6.3.2-linux-x86_64.tar.gz",
"response" : 200,
"tags" : [
"warning",
"info"
],
"timestamp" : "2024-01-07T04:18:12.345Z",
"url" : "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.3.2-linux-x86_64.tar.gz_1",
"utc_time" : "2024-01-07T04:18:12.345Z"
}
},
{
"_index" : "kibana_sample_data_logs",
"_type" : "_doc",
"_id" : "chYqG40BgWqgae_ZjG4m",
"_score" : 1.0,
"_source" : {
"agent" : "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24",
"bytes" : 3629,
"clientip" : "216.9.22.134",
"extension" : "",
"geo" : {
"srcdest" : "CI:DZ",
"src" : "CI",
"dest" : "DZ",
"coordinates" : {
"lat" : 40.87480556,
"lon" : -81.88825
}
},
"host" : "www.elastic.co",
"index" : "kibana_sample_data_logs",
"ip" : "216.9.22.134",
"machine" : {
"ram" : 3221225472,
"os" : "win 8"
},
"memory" : null,
"message" : """216.9.22.134 - - [2018-07-22T05:27:11.939Z] "GET /beats/metricbeat_1 HTTP/1.1" 200 3629 "-" "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24"""",
"phpmemory" : null,
"referer" : "http://facebook.com/success/tracy-caldwell-dyson",
"request" : "/beats/metricbeat",
"response" : 200,
"tags" : [
"success",
"info"
],
"timestamp" : "2024-01-07T05:27:11.939Z",
"url" : "https://www.elastic.co/downloads/beats/metricbeat_1",
"utc_time" : "2024-01-07T05:27:11.939Z"
}
},
{
"_index" : "kibana_sample_data_logs",
"_type" : "_doc",
"_id" : "cxYqG40BgWqgae_ZjG4m",
"_score" : 1.0,
"_source" : {
"agent" : "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1",
"bytes" : 9797,
"clientip" : "12.132.31.17",
"extension" : "gz",
"geo" : {
"srcdest" : "BD:EC",
"src" : "BD",
"dest" : "EC",
"coordinates" : {
"lat" : 32.14308333,
"lon" : -111.1728611
}
},
"host" : "artifacts.elastic.co",
"index" : "kibana_sample_data_logs",
"ip" : "12.132.31.17",
"machine" : {
"ram" : 4294967296,
"os" : "osx"
},
"memory" : null,
"message" : """12.132.31.17 - - [2018-07-22T05:36:25.812Z] "GET /elasticsearch/elasticsearch-6.3.2.tar.gz_1 HTTP/1.1" 200 9797 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1"""",
"phpmemory" : null,
"referer" : "http://www.elastic-elastic-elastic.com/success/musa-manarov",
"request" : "/elasticsearch/elasticsearch-6.3.2.tar.gz",
"response" : 200,
"tags" : [
"success",
"security"
],
"timestamp" : "2024-01-07T05:36:25.812Z",
"url" : "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.2.tar.gz_1",
"utc_time" : "2024-01-07T05:36:25.812Z"
}
}
]
}
}
Eventually i will be using the api to query so i need to find a more efficient method to translate from the kibana dashboard to json