Filebeat monitoring metrics are "dropped" when a GEOIP pipeline is used

Elastic Stack Beats
,
24

Ok, so I've added the "ignore_failure" to my pipeline:

{
  "geoip-no-missing": {
    "description": "Add geoip info",
    "processors": [
      {
        "geoip": {
          "field": "message.remote_ip",
          "target_field": "message.remote_ip_geo",
          "ignore_failure": true
        }
      }
    ]
  }
}

making sure this is the pipleine I use in filebeat:

# ============================== Filebeat inputs ===============================
filebeat.inputs:
- type: gcp-pubsub
  enabled: true
  project_id: ${PROJECT_ID}
  topic: ${PUBSUB_INPUT_TOPIC}
  subscription.name: ${SUBSCRIPTION_NAME}
  fields_under_root: true
  pipeline: "geoip-no-missing"

processed one event WITHOUT 'remote_ip' field - and see no GEO info added and no failures!

{
  "took": 97,
  "timed_out": false,
  "_shards": {
    "total": 12,
    "successful": 12,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 1,
      "relation": "eq"
    },
    "max_score": 0.2876821,
    "hits": [
      {
        "_index": "ibc-parsed-logs-2022.10.25-000003",
        "_id": "m_id_1025_9",
        "_score": 0.2876821,
        "_source": {
          "input": {
            "type": "gcp-pubsub"
          },
          "agent": {
            "name": "dhcp-10-250-50-96.harvard.edu",
            "id": "e0b4f8e6-d0c6-4c38-a62d-ac6ff81a555a",
            "type": "filebeat",
            "ephemeral_id": "de9db623-2d80-4455-b9a6-d59c2e43aeae",
            "version": "8.4.3"
          },
          "@timestamp": "2022-10-25T21:35:51.419Z",
          "ecs": {
            "version": "8.0.0"
          },
          "host": {
            "hostname": "dhcp-10-250-50-96.harvard.edu",
            "os": {
              "build": "21E258",
              "kernel": "21.4.0",
              "name": "macOS",
              "family": "darwin",
              "type": "macos",
              "version": "12.3.1",
              "platform": "darwin"
            },
            "ip": [
              "fe80"
            ],
            "name": "dhcp-10-250-50-96.harvard.edu",
            "id": "xxx443",
            "mac": [
              "5e"
            ],
            "architecture": "x86_64"
          },
          "event": {
            "created": "2022-10-25T21:35:52.482Z",
            "id": "59279bf715-6095478736584954"
          },
          "message": {
            "request_status": "500",
            "referer": "https://www.my.site1.com/",
            "ref_param": "https://www.nyt.com",
            "event_timestamp_millis": "1666707272000",
            "latency": "1.3",
            "activity_date": "2022-10-25",
            "logstash_id": "m_id_1025_9",
            "request_method": "POST",
            "request_size": "52",
            "user_agent": "Mozilla/5.0 (X11; CrOS aarch64 13421.102.0) AppleWebKit/537.36 (KHTML, like Gecko)Chrome/86.0.4240.199 Safari/537.36",
            "response_size": "124",
            "cid": "12345"
          }
        }
      }
    ]
  }
}

processed one event WITH remote_id field - GEO info is added correctly:

{
  "took": 1022,
  "timed_out": false,
  "_shards": {
    "total": 12,
    "successful": 12,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 1,
      "relation": "eq"
    },
    "max_score": 0.2876821,
    "hits": [
      {
        "_index": "ibc-parsed-logs-2022.10.26-000004",
        "_id": "m_id_1026_1",
        "_score": 0.2876821,
        "_source": {
          "input": {
            "type": "gcp-pubsub"
          },
          "agent": {
            "name": "mac-lt2-mpopova.fios-router.home",
            "id": "e0b4f8e6-d0c6-4c38-a62d-ac6ff81a555a",
            "type": "filebeat",
            "ephemeral_id": "3909273a-3f14-42ad-a5b7-a6c15691a912",
            "version": "8.4.3"
          },
          "@timestamp": "2022-10-26T12:57:05.428Z",
          "ecs": {
            "version": "8.0.0"
          },
          "host": {
            "hostname": "mac-lt2-mpopova.fios-router.home",
            "os": {
              "build": "21E258",
              "kernel": "21.4.0",
              "name": "macOS",
              "type": "macos",
              "family": "darwin",
              "version": "12.3.1",
              "platform": "darwin"
            },
            "ip": [
              "fe80"
            ],
            "name": "mac-lt2-mpopova.fios-router.home",
            "id": "xxx443",
            "mac": [
              "7e"
            ],
            "architecture": "x86_64"
          },
          "event": {
            "created": "2022-10-26T12:57:06.629Z",
            "id": "59279bf715-6098619314182042"
          },
          "message": {
            "request_status": "500",
            "ref_param": "https://www.nyt.com",
            "referer": "https://www.my.site1.com/",
            "remote_ip_geo": {
              "continent_name": "North America",
              "region_iso_code": "US-NY",
              "city_name": "The Bronx",
              "country_iso_code": "US",
              "country_name": "United States",
              "region_name": "New York",
              "location": {
                "lon": -73.8616,
                "lat": 40.847
              }
            },
            "latency": "1.3",
            "activity_date": "2022-10-26",
            "logstash_id": "m_id_1026_1",
            "request_method": "POST",
            "response_size": "124",
            "remote_ip": "165.155.130.139",
            "event_timestamp_millis": "1666788992000",
            "request_size": "52",
            "user_agent": "Mozilla/5.0 (X11; CrOS aarch64 13421.102.0) AppleWebKit/537.36 (KHTML, like Gecko)Chrome/86.0.4240.199 Safari/537.36",
            "cid": "12345"
          }
        }
      }
    ]
  }
}

So the "ignore_failure" field works as expected!

thanks!
Marina