Ok, so I've added the "ignore_failure" to my pipeline:
{
"geoip-no-missing": {
"description": "Add geoip info",
"processors": [
{
"geoip": {
"field": "message.remote_ip",
"target_field": "message.remote_ip_geo",
"ignore_failure": true
}
}
]
}
}
making sure this is the pipleine I use in filebeat:
# ============================== Filebeat inputs ===============================
filebeat.inputs:
- type: gcp-pubsub
enabled: true
project_id: ${PROJECT_ID}
topic: ${PUBSUB_INPUT_TOPIC}
subscription.name: ${SUBSCRIPTION_NAME}
fields_under_root: true
pipeline: "geoip-no-missing"
processed one event WITHOUT 'remote_ip' field - and see no GEO info added and no failures!
{
"took": 97,
"timed_out": false,
"_shards": {
"total": 12,
"successful": 12,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 0.2876821,
"hits": [
{
"_index": "ibc-parsed-logs-2022.10.25-000003",
"_id": "m_id_1025_9",
"_score": 0.2876821,
"_source": {
"input": {
"type": "gcp-pubsub"
},
"agent": {
"name": "dhcp-10-250-50-96.harvard.edu",
"id": "e0b4f8e6-d0c6-4c38-a62d-ac6ff81a555a",
"type": "filebeat",
"ephemeral_id": "de9db623-2d80-4455-b9a6-d59c2e43aeae",
"version": "8.4.3"
},
"@timestamp": "2022-10-25T21:35:51.419Z",
"ecs": {
"version": "8.0.0"
},
"host": {
"hostname": "dhcp-10-250-50-96.harvard.edu",
"os": {
"build": "21E258",
"kernel": "21.4.0",
"name": "macOS",
"family": "darwin",
"type": "macos",
"version": "12.3.1",
"platform": "darwin"
},
"ip": [
"fe80"
],
"name": "dhcp-10-250-50-96.harvard.edu",
"id": "xxx443",
"mac": [
"5e"
],
"architecture": "x86_64"
},
"event": {
"created": "2022-10-25T21:35:52.482Z",
"id": "59279bf715-6095478736584954"
},
"message": {
"request_status": "500",
"referer": "https://www.my.site1.com/",
"ref_param": "https://www.nyt.com",
"event_timestamp_millis": "1666707272000",
"latency": "1.3",
"activity_date": "2022-10-25",
"logstash_id": "m_id_1025_9",
"request_method": "POST",
"request_size": "52",
"user_agent": "Mozilla/5.0 (X11; CrOS aarch64 13421.102.0) AppleWebKit/537.36 (KHTML, like Gecko)Chrome/86.0.4240.199 Safari/537.36",
"response_size": "124",
"cid": "12345"
}
}
}
]
}
}
processed one event WITH remote_id field - GEO info is added correctly:
{
"took": 1022,
"timed_out": false,
"_shards": {
"total": 12,
"successful": 12,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 0.2876821,
"hits": [
{
"_index": "ibc-parsed-logs-2022.10.26-000004",
"_id": "m_id_1026_1",
"_score": 0.2876821,
"_source": {
"input": {
"type": "gcp-pubsub"
},
"agent": {
"name": "mac-lt2-mpopova.fios-router.home",
"id": "e0b4f8e6-d0c6-4c38-a62d-ac6ff81a555a",
"type": "filebeat",
"ephemeral_id": "3909273a-3f14-42ad-a5b7-a6c15691a912",
"version": "8.4.3"
},
"@timestamp": "2022-10-26T12:57:05.428Z",
"ecs": {
"version": "8.0.0"
},
"host": {
"hostname": "mac-lt2-mpopova.fios-router.home",
"os": {
"build": "21E258",
"kernel": "21.4.0",
"name": "macOS",
"type": "macos",
"family": "darwin",
"version": "12.3.1",
"platform": "darwin"
},
"ip": [
"fe80"
],
"name": "mac-lt2-mpopova.fios-router.home",
"id": "xxx443",
"mac": [
"7e"
],
"architecture": "x86_64"
},
"event": {
"created": "2022-10-26T12:57:06.629Z",
"id": "59279bf715-6098619314182042"
},
"message": {
"request_status": "500",
"ref_param": "https://www.nyt.com",
"referer": "https://www.my.site1.com/",
"remote_ip_geo": {
"continent_name": "North America",
"region_iso_code": "US-NY",
"city_name": "The Bronx",
"country_iso_code": "US",
"country_name": "United States",
"region_name": "New York",
"location": {
"lon": -73.8616,
"lat": 40.847
}
},
"latency": "1.3",
"activity_date": "2022-10-26",
"logstash_id": "m_id_1026_1",
"request_method": "POST",
"response_size": "124",
"remote_ip": "165.155.130.139",
"event_timestamp_millis": "1666788992000",
"request_size": "52",
"user_agent": "Mozilla/5.0 (X11; CrOS aarch64 13421.102.0) AppleWebKit/537.36 (KHTML, like Gecko)Chrome/86.0.4240.199 Safari/537.36",
"cid": "12345"
}
}
}
]
}
}
So the "ignore_failure" field works as expected!
thanks!
Marina