thanks, @stephenb !
Yea, I think I posted the results of the direct POST but now that you mentioned - not sure if I changed all IDs to avoid collision.... So I am inserting a fresh new doc today, into a brand new index for today (10-25) to make sure there are no collision possible ...
I'm not sure whether this part of the big event payload created by Filebeat and sent to ES should be a unique ID or not:
"event": {
"created": "2022-10-24T19:30:01.726Z",
"id": "59279bf715-5532412804884987"
}
so I changed it to be unique for at least today.
"event": {
"created": "2022-10-25T10:21:28.059Z",
"id": "10-25-id-1"
}
So here is my direct POST command:
POST ibc-parsed-logs-2022.10.25-000002/_doc/m_id_1025_1?pipeline=geoip-info
{
"@timestamp": "2022-10-25T10:21:26.956Z",
"input": {
"type": "gcp-pubsub"
},
"agent": {
"name": "mac-lt2-mpopova.fios-router.home",
"type": "filebeat",
"version": "8.4.3",
"ephemeral_id": "8f1c47c8-4a02-4e36-a4c9-fe8479ed7dae",
"id": "e0b4f8e6-d0c6-4c38-a62d-ac6ff81a555a"
},
"ecs": {
"version": "8.0.0"
},
"host": {
"os": {
"type": "macos",
"platform": "darwin",
"version": "12.3.1",
"family": "darwin",
"name": "macOS",
"kernel": "21.4.0",
"build": "21E258"
},
"name": "mac-lt2-mpopova.fios-router.home",
"id": "xxx443",
"ip": [
"xxx"
],
"mac": [
"xxx"
],
"hostname": "mac-lt2-mpopova.fios-router.home",
"architecture": "x86_64"
},
"event": {
"created": "2022-10-25T10:21:28.059Z",
"id": "10-25-id-1"
},
"message": {
"cid": "12345",
"remote_ip": "165.155.130.139",
"request_status": "500",
"event_timestamp_millis": "1666707272000",
"activity_date": "2022-10-25",
"request_method": "POST",
"response_size": "124",
"latency": "1.3",
"logstash_id": "m_id_1025_1",
"user_agent": "Mozilla/5.0 (X11; CrOS aarch64 13421.102.0) AppleWebKit/537.36 (KHTML, like Gecko)Chrome/86.0.4240.199 Safari/537.36",
"referer": "https://www.my.site1.com/",
"ref_param": "https://www.nyt.com",
"request_size": "52"
}
}
response:
{
"_index": "ibc-parsed-logs-2022.10.25-000002",
"_id": "m_id_1025_1",
"_version": 1,
"result": "created",
"_shards": {
"total": 3,
"successful": 1,
"failed": 0
},
"_seq_no": 0,
"_primary_term": 1
}
and now getting this doc by ID:
GET ibc-parsed-logs/_search
{
"query": {
"term": {
"message.logstash_id": {
"value": "m_id_1025_1"
}
}
}
}
result:
{
"took": 888,
"timed_out": false,
"_shards": {
"total": 6,
"successful": 6,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 0.2876821,
"hits": [
{
"_index": "ibc-parsed-logs-2022.10.25-000002",
"_id": "m_id_1025_1",
"_score": 0.2876821,
"_source": {
"input": {
"type": "gcp-pubsub"
},
"agent": {
"name": "mac-lt2-mpopova.fios-router.home",
"id": "e0b4f8e6-d0c6-4c38-a62d-ac6ff81a555a",
"type": "filebeat",
"ephemeral_id": "8f1c47c8-4a02-4e36-a4c9-fe8479ed7dae",
"version": "8.4.3"
},
"@timestamp": "2022-10-25T10:21:26.956Z",
"ecs": {
"version": "8.0.0"
},
"host": {
"hostname": "mac-lt2-mpopova.fios-router.home",
"os": {
"build": "21E258",
"kernel": "21.4.0",
"name": "macOS",
"type": "macos",
"family": "darwin",
"version": "12.3.1",
"platform": "darwin"
},
"ip": [
"xxx"
],
"name": "mac-lt2-mpopova.fios-router.home",
"id": "xxx443",
"mac": [
"xxx"
],
"architecture": "x86_64"
},
"event": {
"created": "2022-10-25T10:21:28.059Z",
"id": "10-25-id-1"
},
"message": {
"request_status": "500",
"referer": "https://www.my.site1.com/",
"ref_param": "https://www.nyt.com",
"remote_ip_geo": {
"continent_name": "North America",
"region_iso_code": "US-NY",
"city_name": "The Bronx",
"country_iso_code": "US",
"country_name": "United States",
"region_name": "New York",
"location": {
"lon": -73.8616,
"lat": 40.847
}
},
"latency": "1.3",
"activity_date": "2022-10-25",
"logstash_id": "m_id_1025_1",
"request_method": "POST",
"response_size": "124",
"remote_ip": "165.155.130.139",
"event_timestamp_millis": "1666707272000",
"request_size": "52",
"user_agent": "Mozilla/5.0 (X11; CrOS aarch64 13421.102.0) AppleWebKit/537.36 (KHTML, like Gecko)Chrome/86.0.4240.199 Safari/537.36",
"cid": "12345"
}
}
}
]
}
}
Now sending a new event through Filebeat (with pipeline in input):
{
"event_uuid":"m_id_1025_2",
"logstash_id":"m_id_1025_2",
"cid":"12345",
"event_timestamp_millis":"1666707272000",
"activity_date":"2022-10-25",
"remote_ip":"165.155.130.139",
"user_agent":"Mozilla/5.0 (X11; CrOS aarch64 13421.102.0) AppleWebKit/537.36 (KHTML, like Gecko)Chrome/86.0.4240.199 Safari/537.36",
"referer":"https://www.my.site1.com/",
"ref_param":"https://www.nyt.com",
"request_status":"500",
"request_method":"POST",
"request_size":"52",
"response_size":"124",
"latency":"1.3"
}
logs from Filebeat (not sure I got everything that is of interest ....):
{"log.level":"debug","@timestamp":"2022-10-25T10:25:44.936-0400","log.logger":"processors","log.origin":{"file.name":"processing/processors.go","file.line":210},"message":"Publish event: {\n \"@timestamp\": \"2022-10-25T14:25:44.022Z\",\n \"@metadata\": {\n \"beat\": \"filebeat\",\n \"type\": \"_doc\",\n \"version\": \"8.4.3\",\n \"_id\": \"m_id_1025_2\"\n },\n \"agent\": {\n \"version\": \"8.4.3\",\n \"ephemeral_id\": \"710d2939-cea1-4b6b-aa12-3d8c7767606f\",\n \"id\": \"e0b4f8e6-d0c6-4c38-a62d-ac6ff81a555a\",\n \"name\": \"mac-lt2-mpopova.fios-router.home\",\n \"type\": \"filebeat\"\n },\n \"event\": {\n \"id\": \"59279bf715-5532469019380529\",\n \"created\": \"2022-10-25T14:25:44.935Z\"\n },\n \"message\": {\n \"referer\": \"https://www.my.site1.com/\",\n \"request_method\": \"POST\",\n \"logstash_id\": \"m_id_1025_2\",\n \"ref_param\": \"https://www.nyt.com\",\n \"request_status\": \"500\",\n \"response_size\": \"124\",\n \"cid\": \"12345\",\n \"remote_ip\": \"165.155.130.139\",\n \"user_agent\": \"Mozilla/5.0 (X11; CrOS aarch64 13421.102.0) AppleWebKit/537.36 (KHTML, like Gecko)Chrome/86.0.4240.199 Safari/537.36\",\n \"event_timestamp_millis\": \"1666707272000\",\n \"activity_date\": \"2022-10-25\",\n \"request_size\": \"52\",\n \"latency\": \"1.3\"\n },\n \"input\": {\n \"type\": \"gcp-pubsub\"\n },\n \"ecs\": {\n \"version\": \"8.0.0\"\n },\n \"host\": {\n \"os\": {\n \"type\": \"macos\",\n \"platform\": \"darwin\",\n \"version\": \"12.3.1\",\n \"family\": \"darwin\",\n \"name\": \"macOS\",\n \"kernel\": \"21.4.0\",\n \"build\": \"21E258\"\n },\n \"id\": \"xxx443\",\n \"ip\": [\n \"fe80"\n ],\n \"name\": \"mac-lt2-mpopova.fios-router.home\",\n \"mac\": [\n \"82"\"\n ],\n \"hostname\": \"mac-lt2-mpopova.fios-router.home\",\n \"architecture\": \"x86_64\"\n }\n}","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-10-25T10:25:45.937-0400","log.logger":"publisher_pipeline_output","log.origin":{"file.name":"pipeline/client_worker.go","file.line":139},"message":"Connecting to backoff(elasticsearch(http://localhost:9200))","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2022-10-25T10:25:45.938-0400","log.logger":"esclientleg","log.origin":{"file.name":"eslegclient/connection.go","file.line":267},"message":"ES Ping(url=http://localhost:9200)","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2022-10-25T10:25:45.940-0400","log.logger":"esclientleg","log.origin":{"file.name":"transport/logging.go","file.line":42},"message":"Completed dialing successfully","service.name":"filebeat","network":"tcp","address":"localhost:9200","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2022-10-25T10:25:45.948-0400","log.logger":"esclientleg","log.origin":{"file.name":"eslegclient/connection.go","file.line":290},"message":"Ping status code: 200","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-10-25T10:25:45.948-0400","log.logger":"esclientleg","log.origin":{"file.name":"eslegclient/connection.go","file.line":291},"message":"Attempting to connect to Elasticsearch version 8.4.3","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2022-10-25T10:25:45.948-0400","log.logger":"esclientleg","log.origin":{"file.name":"eslegclient/connection.go","file.line":346},"message":"GET http://localhost:9200/_license?human=false <nil>","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2022-10-25T10:25:45.957-0400","log.logger":"esclientleg","log.origin":{"file.name":"eslegclient/connection.go","file.line":267},"message":"ES Ping(url=http://localhost:9200)","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2022-10-25T10:25:45.958-0400","log.logger":"esclientleg","log.origin":{"file.name":"eslegclient/connection.go","file.line":290},"message":"Ping status code: 200","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-10-25T10:25:45.958-0400","log.logger":"esclientleg","log.origin":{"file.name":"eslegclient/connection.go","file.line":291},"message":"Attempting to connect to Elasticsearch version 8.4.3","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2022-10-25T10:25:45.958-0400","log.logger":"esclientleg","log.origin":{"file.name":"eslegclient/connection.go","file.line":346},"message":"HEAD http://localhost:9200/_index_template/ibc-parsed-logs-template <nil>","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-10-25T10:25:45.973-0400","log.logger":"template_loader","log.origin":{"file.name":"template/load.go","file.line":115},"message":"Template \"ibc-parsed-logs-template\" already exists and will not be overwritten.","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-10-25T10:25:45.974-0400","log.logger":"index-management","log.origin":{"file.name":"idxmgmt/std.go","file.line":267},"message":"Loaded index template.","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2022-10-25T10:25:45.975-0400","log.logger":"esclientleg","log.origin":{"file.name":"eslegclient/connection.go","file.line":346},"message":"GET http://localhost:9200/ <nil>","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2022-10-25T10:25:45.976-0400","log.logger":"publisher_pipeline_output","log.origin":{"file.name":"pipeline/client_worker.go","file.line":147},"message":"Connection to backoff(elasticsearch(http://localhost:9200)) established","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2022-10-25T10:25:46.053-0400","log.logger":"elasticsearch","log.origin":{"file.name":"elasticsearch/client.go","file.line":247},"message":"PublishEvents: 1 events have been published to elasticsearch in 77.121136ms.","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2022-10-25T10:25:46.054-0400","log.logger":"publisher","log.origin":{"file.name":"memqueue/eventloop.go","file.line":498},"message":"broker ACK events: count=1, start-seq=1, end-seq=1\n","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2022-10-25T10:25:46.054-0400","log.logger":"acker","log.origin":{"file.name":"beater/acker.go","file.line":64},"message":"stateless ack","service.name":"filebeat","count":1,"ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2022-10-25T10:25:46.055-0400","log.logger":"publisher","log.origin":{"file.name":"memqueue/ackloop.go","file.line":95},"message":"ackloop: return ack to broker loop:1","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2022-10-25T10:25:46.055-0400","log.logger":"publisher","log.origin":{"file.name":"memqueue/ackloop.go","file.line":98},"message":"ackloop: done send ack","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2022-10-25T10:25:47.601-0400","log.origin":{"file.name":"numcpu/numcpu.go","file.line":41},"message":"Accurate CPU counts not available on platform, falling back to runtime.NumCPU for metrics","service.name":"filebeat","ecs.version":"1.6.0"}
{
getting this event from ES:
GET ibc-parsed-logs/_search
{
"query": {
"term": {
"message.logstash_id": {
"value": "m_id_1025_2"
}
}
}
}
result:
{
"took": 52,
"timed_out": false,
"_shards": {
"total": 6,
"successful": 6,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 0.6931471,
"hits": [
{
"_index": "ibc-parsed-logs-2022.10.25-000002",
"_id": "m_id_1025_2",
"_score": 0.6931471,
"_source": {
"@timestamp": "2022-10-25T14:25:44.022Z",
"host": {
"os": {
"family": "darwin",
"name": "macOS",
"kernel": "21.4.0",
"build": "21E258",
"type": "macos",
"platform": "darwin",
"version": "12.3.1"
},
"id": "xxx443",
"ip": [
"fe80"
],
"name": "mac-lt2-mpopova.fios-router.home",
"mac": [
"82"
],
"hostname": "mac-lt2-mpopova.fios-router.home",
"architecture": "x86_64"
},
"agent": {
"type": "filebeat",
"version": "8.4.3",
"ephemeral_id": "710d2939-cea1-4b6b-aa12-3d8c7767606f",
"id": "e0b4f8e6-d0c6-4c38-a62d-ac6ff81a555a",
"name": "mac-lt2-mpopova.fios-router.home"
},
"event": {
"id": "59279bf715-5532469019380529",
"created": "2022-10-25T14:25:44.935Z"
},
"message": {
"event_timestamp_millis": "1666707272000",
"activity_date": "2022-10-25",
"request_size": "52",
"latency": "1.3",
"referer": "https://www.my.site1.com/",
"request_method": "POST",
"logstash_id": "m_id_1025_2",
"ref_param": "https://www.nyt.com",
"cid": "12345",
"remote_ip": "165.155.130.139",
"user_agent": "Mozilla/5.0 (X11; CrOS aarch64 13421.102.0) AppleWebKit/537.36 (KHTML, like Gecko)Chrome/86.0.4240.199 Safari/537.36",
"request_status": "500",
"response_size": "124"
},
"input": {
"type": "gcp-pubsub"
},
"ecs": {
"version": "8.0.0"
}
}
}
]
}
}
no GEOIP info ...
thanks!!
- not sur eif related, but just to mention all custom setup....