Geoip pipeline setting for Netflow (Map on Geo Location Dashboard)

Elastic Stack Kibana
1

Hi, we had added Netflow module to analyze traffic with Filebeat netflow module.
Everything works but cant activate Map settings.
As i found, we have to add new pipeline ( "geoip-info", for example) ad add "final_pipeline": "geoip-info" for filebeat index template.
Also we have desabled online mmdb files updates with "ingest.geoip.downloader.enabled" : false
And update "database_file" : "GeoLite2-City.mmdb".
But nothing changed, the Map on Geo Location Dashboard still emty.

May be you have step-by-step manual or could provide more detailes how to activate Map on dashboard?

2

Can you post how your geo data looks like?

3

I mean source.ip and destination.ip fields from Netflow index.

NF_1
NF_11316×446 43.2 KB

Example of fields (IPs are chamged)
{
"_index": "filebeat-7.17.3-2022.05.10-000009",
"_type": "_doc",
"_id": "fFsKsIABtoXZSYt0lLoJ",
"_version": 1,
"_score": 1,
"_source": {
"@timestamp": "2022-05-10T22:15:23.000Z",
"input": {
"type": "netflow"
},
"ecs": {
"version": "1.12.0"
},
"host": {
"containerized": false,
"ip": [
"2.2.2.2",
"fe80::250:56ff:fe8d:2fea",
"192.168.122.1"
],
"mac": [
"00:50:56:8d:2f:ea",
"52:54:00:65:57:71",
"52:54:00:65:57:71"
],
"hostname": "XX-YYY001",
"architecture": "x86_64",
"os": {
"codename": "Core",
"type": "linux",
"platform": "centos",
"version": "7 (Core)",
"family": "redhat",
"name": "CentOS Linux",
"kernel": "3.10.0-1160.42.2.el7.x86_64"
},
"name": "XX-YYY001",
"id": "bcde44628cbe47dba7570f0240e53be2"
},
"flow": {
"id": "KjAIVzzSrEQ",
"locality": "external"
},
"source": {
"ip": "1.1.0.1",
"locality": "external",
"port": 443,
"bytes": 28559,
"packets": 57
},
"network": {
"direction": "inbound",
"community_id": "1:2TP9g2WGvQftr/96Zu6Mu9BWVqs=",
"transport": "tcp",
"iana_number": 6,
"bytes": 28559,
"packets": 57
},
"event": {
"type": [
"connection"
],
"start": "2022-05-10T22:12:52.786Z",
"end": "2022-05-10T22:15:00.281Z",
"duration": 127495000000,
"created": "2022-05-10T22:15:23.352Z",
"kind": "event",
"category": [
"network_traffic",
"network"
],
"action": "netflow_flow"
},
"agent": {
"name": "XX-YYY001",
"type": "filebeat",
"version": "7.17.3",
"hostname": "XX-YYY001",
"ephemeral_id": "2b0392b6-80c6-421d-90db-b4a1df65e7ab",
"id": "5f031d52-e349-4e82-a32d-1fedcc6ad4d0"
},
"observer": {
"ip": "3.3.3.3"
},
"destination": {
"locality": "external",
"port": 51925,
"ip": "1.1.1.1"
},
"related": {
"ip": [
"1.1.0.1",
"1.1.1.1"
]
},
"netflow": {
"protocol_identifier": 6,
"forwarding_status": 64,
"bgp_source_as_number": 0,
"egress_interface": 105,
"destination_ipv4_prefix_length": 28,
"flow_direction": 0,
"bgp_next_hop_ipv4_address": "91.184.106.2",
"exporter": {
"source_id": 2097,
"version": 9,
"timestamp": "2022-05-10T22:15:23.000Z",
"uptime_millis": 3914061309,
"address": "3.3.3.3:19581"
},
"bgp_destination_as_number": 0,
"destination_transport_port": 51925,
"destination_ipv4_address": "1.1.1.1",
"flow_start_sys_up_time": 3913911095,
"packet_delta_count": 57,
"flow_end_sys_up_time": 3914038590,
"source_ipv4_prefix_length": 0,
"tcp_control_bits": 24,
"ip_class_of_service": 0,
"source_transport_port": 443,
"type": "netflow_flow",
"octet_delta_count": 28559,
"ingress_interface": 73,
"source_ipv4_address": "1.1.0.1"
}
},
"fields": {
"flow.id": [
"KjAIVzzSrEQ"
],
"event.category": [
"network_traffic",
"network"
],
"host.os.name.text": [
"CentOS Linux"
],
"host.hostname": [
"XX-YYY001"
],
"netflow.ip_class_of_service": [
0
],
"host.mac": [
"00:50:56:8d:2f:ea",
"52:54:00:65:57:71",
"52:54:00:65:57:71"
],
"netflow.source_transport_port": [
443
],
"netflow.tcp_control_bits": [
24
],
"netflow.exporter.version": [
9
],
"netflow.exporter.address": [
"3.3.3.3:19581"
],
"host.os.version": [
"7 (Core)"
],
"netflow.bgp_source_as_number": [
0
],
"host.os.name": [
"CentOS Linux"
],
"netflow.destination_ipv4_prefix_length": [
28
],
"source.ip": [
"1.1.0.1"
],
"agent.name": [
"XX-YYY001"
],
"host.name": [
"XX-YYY001"
],
"network.community_id": [
"1:2TP9g2WGvQftr/96Zu6Mu9BWVqs="
],
"event.kind": [
"event"
],
"source.packets": [
57
],
"host.os.type": [
"linux"
],
"network.packets": [
57
],
"netflow.flow_start_sys_up_time": [
3913911095
],
"netflow.destination_ipv4_address": [
"1.1.1.1"
],
"flow.locality": [
"external"
],
"netflow.source_ipv4_prefix_length": [
0
],
"input.type": [
"netflow"
],
"agent.hostname": [
"XX-YYY001"
],
"host.architecture": [
"x86_64"
],
"agent.id": [
"5f031d52-e349-4e82-a32d-1fedcc6ad4d0"
],
"source.port": [
443
],
"ecs.version": [
"1.12.0"
],
"host.containerized": [
false
],
"event.created": [
"2022-05-10T22:15:23.352Z"
],
"network.iana_number": [
"6"
],
"agent.version": [
"7.17.3"
],
"host.os.family": [
"redhat"
],
"event.start": [
"2022-05-10T22:12:52.786Z"
],
"netflow.bgp_next_hop_ipv4_address": [
"91.184.106.2"
],
"observer.ip": [
"3.3.3.3"
],
"netflow.type": [
"netflow_flow"
],
"netflow.source_ipv4_address": [
"1.1.0.1"
],
"destination.port": [
51925
],
"netflow.flow_end_sys_up_time": [
3914038590
],
"netflow.bgp_destination_as_number": [
0
],
"netflow.octet_delta_count": [
28559
],
"event.end": [
"2022-05-10T22:15:00.281Z"
],
"host.ip": [
"2.2.2.2",
"fe80::250:56ff:fe8d:2fea",
"192.168.122.1"
],
"agent.type": [
"filebeat"
],
"netflow.exporter.source_id": [
2097
],
"related.ip": [
"1.1.0.1",
"1.1.1.1"
],
"host.os.kernel": [
"3.10.0-1160.42.2.el7.x86_64"
],
"netflow.ingress_interface": [
73
],
"netflow.packet_delta_count": [
57
],
"network.bytes": [
28559
],
"network.direction": [
"inbound"
],
"host.id": [
"bcde44628cbe47dba7570f0240e53be2"
],
"netflow.exporter.uptime_millis": [
3914061309
],
"source.bytes": [
28559
],
"netflow.flow_direction": [
0
],
"destination.locality": [
"external"
],
"netflow.destination_transport_port": [
51925
],
"netflow.exporter.timestamp": [
"2022-05-10T22:15:23.000Z"
],
"host.os.codename": [
"Core"
],
"destination.ip": [
"1.1.1.1"
],
"source.locality": [
"external"
],
"network.transport": [
"tcp"
],
"event.duration": [
127495000000
],
"netflow.protocol_identifier": [
6
],
"event.action": [
"netflow_flow"
],
"@timestamp": [
"2022-05-10T22:15:23.000Z"
],
"host.os.platform": [
"centos"
],
"event.type": [
"connection"
],
"agent.ephemeral_id": [
"2b0392b6-80c6-421d-90db-b4a1df65e7ab"
],
"netflow.forwarding_status": [
64
],
"netflow.egress_interface": [
105
]
}
}

4

You geoip processor hasn't set properly. Check documentation.

You must have a structure like this:

    "source.ip": "89.160.20.128",
    "geoip": {
      "continent_name": "Europe",
      "country_name": "Sweden",
      "country_iso_code": "SE",
      "city_name" : "Linköping",
      "region_iso_code" : "SE-E",
      "region_name" : "Östergötland County",
      "location": { "lat": 58.4167, "lon": 15.6167 }
    }

Also the mapping for source.ip/destination.ip support GeoJSON, for example:

"geoip"  : {
  "dynamic": true,
  "properties" : {
    "ip": { "type": "ip" },
    "location" : { "type" : "geo_point" },
    "latitude" : { "type" : "half_float" },
    "longitude" : { "type" : "half_float" }
  }
}
1 Like
5

Thank you! But I'm not familiar with elastic, so I need more detailed info/manual how to set up geoip processor properly.

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.