Geoip pipeline setting for Netflow (Map on Geo Location Dashboard)

Hi, we had added Netflow module to analyze traffic with Filebeat netflow module.
Everything works but cant activate Map settings.
As i found, we have to add new pipeline ( "geoip-info", for example) ad add "final_pipeline": "geoip-info" for filebeat index template.
Also we have desabled online mmdb files updates with "ingest.geoip.downloader.enabled" : false
And update "database_file" : "GeoLite2-City.mmdb".
But nothing changed, the Map on Geo Location Dashboard still emty.

May be you have step-by-step manual or could provide more detailes how to activate Map on dashboard?

Can you post how your geo data looks like?

I mean source.ip and destination.ip fields from Netflow index.

Example of fields (IPs are chamged)
{
"_index": "filebeat-7.17.3-2022.05.10-000009",
"_type": "_doc",
"_id": "fFsKsIABtoXZSYt0lLoJ",
"_version": 1,
"_score": 1,
"_source": {
"@timestamp": "2022-05-10T22:15:23.000Z",
"input": {
"type": "netflow"
},
"ecs": {
"version": "1.12.0"
},
"host": {
"containerized": false,
"ip": [
"2.2.2.2",
"fe80::250:56ff:fe8d:2fea",
"192.168.122.1"
],
"mac": [
"00:50:56:8d:2f:ea",
"52:54:00:65:57:71",
"52:54:00:65:57:71"
],
"hostname": "XX-YYY001",
"architecture": "x86_64",
"os": {
"codename": "Core",
"type": "linux",
"platform": "centos",
"version": "7 (Core)",
"family": "redhat",
"name": "CentOS Linux",
"kernel": "3.10.0-1160.42.2.el7.x86_64"
},
"name": "XX-YYY001",
"id": "bcde44628cbe47dba7570f0240e53be2"
},
"flow": {
"id": "KjAIVzzSrEQ",
"locality": "external"
},
"source": {
"ip": "1.1.0.1",
"locality": "external",
"port": 443,
"bytes": 28559,
"packets": 57
},
"network": {
"direction": "inbound",
"community_id": "1:2TP9g2WGvQftr/96Zu6Mu9BWVqs=",
"transport": "tcp",
"iana_number": 6,
"bytes": 28559,
"packets": 57
},
"event": {
"type": [
"connection"
],
"start": "2022-05-10T22:12:52.786Z",
"end": "2022-05-10T22:15:00.281Z",
"duration": 127495000000,
"created": "2022-05-10T22:15:23.352Z",
"kind": "event",
"category": [
"network_traffic",
"network"
],
"action": "netflow_flow"
},
"agent": {
"name": "XX-YYY001",
"type": "filebeat",
"version": "7.17.3",
"hostname": "XX-YYY001",
"ephemeral_id": "2b0392b6-80c6-421d-90db-b4a1df65e7ab",
"id": "5f031d52-e349-4e82-a32d-1fedcc6ad4d0"
},
"observer": {
"ip": "3.3.3.3"
},
"destination": {
"locality": "external",
"port": 51925,
"ip": "1.1.1.1"
},
"related": {
"ip": [
"1.1.0.1",
"1.1.1.1"
]
},
"netflow": {
"protocol_identifier": 6,
"forwarding_status": 64,
"bgp_source_as_number": 0,
"egress_interface": 105,
"destination_ipv4_prefix_length": 28,
"flow_direction": 0,
"bgp_next_hop_ipv4_address": "91.184.106.2",
"exporter": {
"source_id": 2097,
"version": 9,
"timestamp": "2022-05-10T22:15:23.000Z",
"uptime_millis": 3914061309,
"address": "3.3.3.3:19581"
},
"bgp_destination_as_number": 0,
"destination_transport_port": 51925,
"destination_ipv4_address": "1.1.1.1",
"flow_start_sys_up_time": 3913911095,
"packet_delta_count": 57,
"flow_end_sys_up_time": 3914038590,
"source_ipv4_prefix_length": 0,
"tcp_control_bits": 24,
"ip_class_of_service": 0,
"source_transport_port": 443,
"type": "netflow_flow",
"octet_delta_count": 28559,
"ingress_interface": 73,
"source_ipv4_address": "1.1.0.1"
}
},
"fields": {
"flow.id": [
"KjAIVzzSrEQ"
],
"event.category": [
"network_traffic",
"network"
],
"host.os.name.text": [
"CentOS Linux"
],
"host.hostname": [
"XX-YYY001"
],
"netflow.ip_class_of_service": [
0
],
"host.mac": [
"00:50:56:8d:2f:ea",
"52:54:00:65:57:71",
"52:54:00:65:57:71"
],
"netflow.source_transport_port": [
443
],
"netflow.tcp_control_bits": [
24
],
"netflow.exporter.version": [
9
],
"netflow.exporter.address": [
"3.3.3.3:19581"
],
"host.os.version": [
"7 (Core)"
],
"netflow.bgp_source_as_number": [
0
],
"host.os.name": [
"CentOS Linux"
],
"netflow.destination_ipv4_prefix_length": [
28
],
"source.ip": [
"1.1.0.1"
],
"agent.name": [
"XX-YYY001"
],
"host.name": [
"XX-YYY001"
],
"network.community_id": [
"1:2TP9g2WGvQftr/96Zu6Mu9BWVqs="
],
"event.kind": [
"event"
],
"source.packets": [
57
],
"host.os.type": [
"linux"
],
"network.packets": [
57
],
"netflow.flow_start_sys_up_time": [
3913911095
],
"netflow.destination_ipv4_address": [
"1.1.1.1"
],
"flow.locality": [
"external"
],
"netflow.source_ipv4_prefix_length": [
0
],
"input.type": [
"netflow"
],
"agent.hostname": [
"XX-YYY001"
],
"host.architecture": [
"x86_64"
],
"agent.id": [
"5f031d52-e349-4e82-a32d-1fedcc6ad4d0"
],
"source.port": [
443
],
"ecs.version": [
"1.12.0"
],
"host.containerized": [
false
],
"event.created": [
"2022-05-10T22:15:23.352Z"
],
"network.iana_number": [
"6"
],
"agent.version": [
"7.17.3"
],
"host.os.family": [
"redhat"
],
"event.start": [
"2022-05-10T22:12:52.786Z"
],
"netflow.bgp_next_hop_ipv4_address": [
"91.184.106.2"
],
"observer.ip": [
"3.3.3.3"
],
"netflow.type": [
"netflow_flow"
],
"netflow.source_ipv4_address": [
"1.1.0.1"
],
"destination.port": [
51925
],
"netflow.flow_end_sys_up_time": [
3914038590
],
"netflow.bgp_destination_as_number": [
0
],
"netflow.octet_delta_count": [
28559
],
"event.end": [
"2022-05-10T22:15:00.281Z"
],
"host.ip": [
"2.2.2.2",
"fe80::250:56ff:fe8d:2fea",
"192.168.122.1"
],
"agent.type": [
"filebeat"
],
"netflow.exporter.source_id": [
2097
],
"related.ip": [
"1.1.0.1",
"1.1.1.1"
],
"host.os.kernel": [
"3.10.0-1160.42.2.el7.x86_64"
],
"netflow.ingress_interface": [
73
],
"netflow.packet_delta_count": [
57
],
"network.bytes": [
28559
],
"network.direction": [
"inbound"
],
"host.id": [
"bcde44628cbe47dba7570f0240e53be2"
],
"netflow.exporter.uptime_millis": [
3914061309
],
"source.bytes": [
28559
],
"netflow.flow_direction": [
0
],
"destination.locality": [
"external"
],
"netflow.destination_transport_port": [
51925
],
"netflow.exporter.timestamp": [
"2022-05-10T22:15:23.000Z"
],
"host.os.codename": [
"Core"
],
"destination.ip": [
"1.1.1.1"
],
"source.locality": [
"external"
],
"network.transport": [
"tcp"
],
"event.duration": [
127495000000
],
"netflow.protocol_identifier": [
6
],
"event.action": [
"netflow_flow"
],
"@timestamp": [
"2022-05-10T22:15:23.000Z"
],
"host.os.platform": [
"centos"
],
"event.type": [
"connection"
],
"agent.ephemeral_id": [
"2b0392b6-80c6-421d-90db-b4a1df65e7ab"
],
"netflow.forwarding_status": [
64
],
"netflow.egress_interface": [
105
]
}
}

You geoip processor hasn't set properly. Check documentation.

You must have a structure like this:

    "source.ip": "89.160.20.128",
    "geoip": {
      "continent_name": "Europe",
      "country_name": "Sweden",
      "country_iso_code": "SE",
      "city_name" : "Linköping",
      "region_iso_code" : "SE-E",
      "region_name" : "Östergötland County",
      "location": { "lat": 58.4167, "lon": 15.6167 }
    }

Also the mapping for source.ip/destination.ip support GeoJSON, for example:

"geoip"  : {
  "dynamic": true,
  "properties" : {
    "ip": { "type": "ip" },
    "location" : { "type" : "geo_point" },
    "latitude" : { "type" : "half_float" },
    "longitude" : { "type" : "half_float" }
  }
}
1 Like

Thank you! But I'm not familiar with elastic, so I need more detailed info/manual how to set up geoip processor properly.