Configure ingest pipeline to add both GeoLite2-City AND GeoLite2-ASN fields

Elastic Stack Elasticsearch
1

I have and event stream that contains IP addresses. I wish to add meta-data from the GeoLite2 Max Mind databases. I can successfully add the city data. I can also successfully add the ASN data. Somehow, I cannot add both. If I include two geoip processors sequentially, only the last processor is used. More specifically...

My document:

[
  {
    "_id": "uiqk0IwBfixXPP18xyzX",
    "_index": ".event-stream-2024.01.01-000161",
    "_source": {
      "source": {
        "ip": "98.41.133.131"
      }
    }
  }
]

The first ingest pipeline configuration with the default city database listed last only gives the city data. Evidently it overwrites the ASN result.

{
  "version": 1,
  "description": "Pipeline to create consistent geo location data from source.ip.",
  "processors": [
    {
      "geoip": {
        "field": "source.ip",
        "target_field": "source.geo",
        "database_file": "GeoLite2-ASN.mmdb",
        "first_only": false
      }
    },
    {
      "geoip": {
        "field": "source.ip",
        "target_field": "source.geo",
        "first_only": false
      }
    },
    {
      "set": {
        "field": "dev",
        "value": "success"
      }
    }
  ],
  "on_failure": [
    {
      "set": {
        "field": "dev",
        "value": "failed"
      }
    }
  ]
}

The first result...

{
  "docs": [
    {
      "doc": {
        "_index": ".event-stream-2024.01.01-000161",
        "_id": "uiqk0IwBfixXPP18xyzX",
        "_version": "-3",
        "_source": {
          "dev": "success",
          "source": {
            "geo": {
              "continent_name": "North America",
              "region_iso_code": "US-CA",
              "city_name": "Sacramento",
              "country_iso_code": "US",
              "country_name": "United States",
              "region_name": "California",
              "location": {
                "lon": -121.5114,
                "lat": 38.6415
              }
            },
            "ip": "98.41.133.131"
          }
        },
        "_ingest": {
          "timestamp": "2024-01-03T19:30:51.015897141Z"
        }
      }
    }
  ]
}

Second ingest pipeline configuration with ASN coming last produces the opposite result.

{
  "version": 1,
  "description": "Pipeline to create consistent geo location data from source.ip.",
  "processors": [
    {
      "geoip": {
        "field": "source.ip",
        "target_field": "source.geo",
        "first_only": false
      }
    },
    {
      "geoip": {
        "field": "source.ip",
        "target_field": "source.geo",
        "database_file": "GeoLite2-ASN.mmdb",
        "first_only": false
      }
    },
    {
      "set": {
        "field": "dev",
        "value": "success"
      }
    }
  ],
  "on_failure": [
    {
      "set": {
        "field": "dev",
        "value": "failed"
      }
    }
  ]
}

The second result...

{
  "docs": [
    {
      "doc": {
        "_index": ".event-stream-2024.01.01-000161",
        "_id": "uiqk0IwBfixXPP18xyzX",
        "_version": "-3",
        "_source": {
          "dev": "success",
          "source": {
            "geo": {
              "ip": "98.41.133.131",
              "organization_name": "COMCAST-7922",
              "asn": 7922,
              "network": "98.40.0.0/14"
            },
            "ip": "98.41.133.131"
          }
        },
        "_ingest": {
          "timestamp": "2024-01-03T19:35:00.865475631Z"
        }
      }
    }
  ]
}

How can I get both added to the event?

2

Hi @jbrowe I suspect it is because the 2nd geoip is completely overwriting the target fields... Try 2 different target fields then you may need to self combine/copy to etc.. and clean up

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.