Can't get geoip working

I know there are gazillion other posts but I have tried pretty much anything at this point.

Background: I am using AWS Opendistro and send events using Logstash both versions 7.4.0

Indexing template:

        {
  "panos-traffic" : {
    "order" : 0,
    "version" : 2,
    "index_patterns" : [
      "traffic*"
    ],
    "settings" : {
      "index" : {
        "refresh_interval" : "5s"
      }
    },
    "mappings" : {
      "properties" : {
        "RepeatCount" : {
          "type" : "long"
        },
        "type" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "DestinationPort" : {
          "type" : "integer"
        },
        "GeneratedTime" : {
          "format" : "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis",
          "type" : "date"
        },
        "Bytes" : {
          "type" : "long"
        },
        "path" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "SessionEndReason" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "SourceZone" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "TimeLogged" : {
          "format" : "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis",
          "type" : "date"
        },
        "SCTP_Received" : {
          "type" : "integer"
        },
        "SequenceNumber" : {
          "type" : "long"
        },
        "TunnelID_IMSI" : {
          "type" : "long"
        },
        "host" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "NATSourceIP" : {
          "type" : "ip"
        },
        "DestinationLocation" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "NATDestinationPort" : {
          "type" : "integer"
        },
        "RuleName" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "PacketsReceived" : {
          "type" : "long"
        },
        "PacketsSent" : {
          "type" : "long"
        },
        "syslog_host" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "MonitorTag_IMEI" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "StartTime" : {
          "format" : "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis",
          "type" : "date"
        },
        "DestinationIPGeo" : {
          "properties" : {
            "timezone" : {
              "ignore_above" : 256,
              "type" : "keyword"
            },
            "ip" : {
              "type" : "ip"
            },
            "latitude" : {
              "type" : "half_float"
            },
            "continent_code" : {
              "ignore_above" : 256,
              "type" : "keyword"
            },
            "city_name" : {
              "ignore_above" : 256,
              "type" : "keyword"
            },
            "country_code2" : {
              "ignore_above" : 256,
              "type" : "keyword"
            },
            "country_name" : {
              "ignore_above" : 256,
              "type" : "keyword"
            },
            "dma_code" : {
              "type" : "short"
            },
            "country_code3" : {
              "ignore_above" : 256,
              "type" : "keyword"
            },
            "location" : {
              "type" : "geo_point"
            },
            "region_name" : {
              "ignore_above" : 256,
              "type" : "keyword"
            },
            "postal_code" : {
              "type" : "short"
            },
            "longitude" : {
              "type" : "half_float"
            },
            "region_code" : {
              "ignore_above" : 256,
              "type" : "keyword"
            }
          }
        },
        "tags" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "DeviceGroupHierarchyLevel4" : {
          "type" : "short"
        },
        "DeviceGroupHierarchyLevel3" : {
          "type" : "short"
        },
        "LogForwardingProfile" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "SerialNumber" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "DeviceGroupHierarchyLevel2" : {
          "type" : "short"
        },
        "SourceUser" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "DeviceGroupHierarchyLevel1" : {
          "type" : "short"
        },
        "SourceVMUUID" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "ParentStartTime" : {
          "format" : "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis",
          "type" : "date"
        },
        "BytesSent" : {
          "type" : "long"
        },
        "Dest_geoLocation" : {
          "type" : "geo_point"
        },
        "ReceiveTime" : {
          "format" : "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis",
          "type" : "date"
        },
        "ActionFlags" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "DestinationZone" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "Application" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "SessionID" : {
          "type" : "long"
        },
        "NATSourcePort" : {
          "type" : "integer"
        },
        "SourceLocation" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "VirtualSystem" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "Action" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "VirtualSystemName" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "InboundInterface" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "SCTP_Sent" : {
          "type" : "integer"
        },
        "ParentSessionID" : {
          "type" : "integer"
        },
        "ElapsedTime" : {
          "type" : "long"
        },
        "SourcePort" : {
          "type" : "integer"
        },
        "Rule_UUID" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "URLCategory" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "OutboundInterface" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "SourceIP" : {
          "type" : "ip"
        },
        "Packets" : {
          "type" : "long"
        },
        "@version" : {
          "type" : "byte"
        },
        "SourceIPGeo" : {
          "properties" : {
            "timezone" : {
              "ignore_above" : 256,
              "type" : "keyword"
            },
            "ip" : {
              "type" : "ip"
            },
            "latitude" : {
              "type" : "half_float"
            },
            "continent_code" : {
              "ignore_above" : 256,
              "type" : "keyword"
            },
            "city_name" : {
              "ignore_above" : 256,
              "type" : "keyword"
            },
            "country_code2" : {
              "ignore_above" : 256,
              "type" : "keyword"
            },
            "country_name" : {
              "ignore_above" : 256,
              "type" : "keyword"
            },
            "dma_code" : {
              "type" : "long"
            },
            "country_code3" : {
              "ignore_above" : 256,
              "type" : "keyword"
            },
            "location" : {
              "type" : "geo_point"
            },
            "region_name" : {
              "ignore_above" : 256,
              "type" : "keyword"
            },
            "postal_code" : {
              "ignore_above" : 256,
              "type" : "keyword"
            },
            "longitude" : {
              "type" : "half_float"
            },
            "region_code" : {
              "ignore_above" : 256,
              "type" : "keyword"
            }
          }
        },
        "Protocol" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "Threat_ContentType" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "DestinationUser" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "geoip" : {
          "dynamic" : true,
          "properties" : {
            "ip" : {
              "type" : "ip"
            },
            "latitude" : {
              "type" : "half_float"
            },
            "location" : {
              "type" : "geo_point"
            },
            "longitude" : {
              "type" : "half_float"
            }
          }
        },
        "FUTURE_USE" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "TunnelType" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "DestinationVMUUID" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "SCTP_AssocID" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "NATDestinationIP" : {
          "type" : "ip"
        },
        "HTTP_Connection" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "Flags" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "BytesReceived" : {
          "type" : "long"
        },
        "DestinationIP" : {
          "type" : "ip"
        },
        "Type" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "@timestamp" : {
          "type" : "date"
        },
        "geoLocation" : {
          "type" : "geo_point"
        },
        "ActionSource" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "DeviceName" : {
          "ignore_above" : 256,
          "type" : "keyword"
        },
        "SCTP_Chunks" : {
          "type" : "integer"
        }
      }
    },
    "aliases" : { }
  }
}

filter:

geoip {
source => "DestinationIPGeo.location"
}
geoip {
source => "SourceIPGeo.location"
}

In Kibana:

If location is a field nested inside the DestinationIPGeo object, then in logstash that would be [DestinationIPGeo][location], not DestinationIPGeo.location. You would also need to set the target option, otherwise both filters will write to [geoip]

And it does not make sense that you are feeding a location to geoip. You should be feeding it an IP address in order to set a location.

1 Like

@Badger thank you for the fast response. So that is the weird part. As you can see in my screenshots on the original post from Kibana I have already did what you suggested and used target and indeed I have location created and can see the geo coordinates.. I also have these fields DestinationIPGeo & SourceIPGeo configured in my index template as type "geo-point".

Nevertheless the fields still being mapped in Kibana as normal strings...

Question - Does target also create the field or should I use add_field prior to geoip block??

geoip {
               source => "SourceIP"
               target => "SourceIPGeo"
          }

 geoip {
                source => "DestinationIP"
                target => "DestinationIPGeo"
            }

your mapping has location (type geo_point) nested inside DestinationIPGeo

so this

geoip {
                source => "DestinationIP"
                target => "DestinationIPGeo"
            }

should be

geoip {
                source => "[DestinationIP]"
                target => "[DestinationIPGeo][location]"
            }

I tried that too.

[org.logstash.execution.WorkerLoop][PAN-OS] Exception in pipelineworker, the pipeline stopped processing new events, please check your filter configuration and restart Logstash.
org.logstash.FieldReference$IllegalSyntaxException: Invalid FieldReference: `[SourceIPGeo] [location]`
	at org.logstash.FieldReference$StrictTokenizer.tokenize(org/logstash/FieldReference.java:283) ~[logstash-core.jar:?]
	at org.logstash.FieldReference.parse(org/logstash/FieldReference.java:184) ~[logstash-core.jar:?]
	at org.logstash.FieldReference.parseToCache(org/logstash/FieldReference.java:175) ~[logstash-core.jar:?]
	at org.logstash.FieldReference.from(org/logstash/FieldReference.java:107) ~[logstash-core.jar:?]
	at org.logstash.Event.getField(org/logstash/Event.java:152) ~[logstash-core.jar:?]
	at org.logstash.filters.GeoIPFilter.applyGeoData(org/logstash/filters/GeoIPFilter.java:175) ~[logstash-filter-geoip-6.0.0.jar:?]
	at org.logstash.filters.GeoIPFilter.handleEvent(org/logstash/filters/GeoIPFilter.java:167) ~[logstash-filter-geoip-6.0.0.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(jdk/internal/reflect/NativeMethodAccessorImpl.java:62) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(jdk/internal/reflect/DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:566) ~[?:?]
	at org.jruby.javasupport.JavaMethod.invokeDirectWithExceptionHandling(org/jruby/javasupport/JavaMethod.java:425) ~[jruby-complete-9.2.8.0.jar:?]
	at org.jruby.javasupport.JavaMethod.invokeDirect(org/jruby/javasupport/JavaMethod.java:292) ~[jruby-complete-9.2.8.0.jar:?]

Here is my entire filter as of now:

No errors in log but also can't get mapping to geo_point in ES

filter {
    if "PAN-OS_SysLog" in [tags] or "Prisma_SysLog" in [tags] {

	# Log types are "TRAFFIC", "THREAT", "CONFIG" and "SYSTEM". URL & Wildfire logs are inside Threat logs
    # Log fields: https://www.paloaltonetworks.com/documentation/80/pan-os/pan-os/monitoring/syslog-field-descriptions

        if ([message] =~ /TRAFFIC/) {
            csv {
                source => "message"
                columns => [
                    "FUTURE_USE", "ReceiveTime", "SerialNumber", "Type", "Threat_ContentType", "FUTURE_USE",
                    "GeneratedTime", "SourceIP", "DestinationIP", "NATSourceIP", "NATDestinationIP", "RuleName",
                    "SourceUser", "DestinationUser", "Application", "VirtualSystem", "SourceZone", "DestinationZone",
                    "InboundInterface", "OutboundInterface", "LogForwardingProfile", "TimeLogged", "SessionID",
                    "RepeatCount", "SourcePort", "DestinationPort", "NATSourcePort", "NATDestinationPort", "Flags",
                    "Protocol", "Action", "Bytes", "BytesSent", "BytesReceived", "Packets", "StartTime", "ElapsedTime",
                    "URLCategory", "FUTURE_USE", "SequenceNumber", "ActionFlags", "SourceLocation",
                    "DestinationLocation", "FUTURE_USE", "PacketsSent", "PacketsReceived", "SessionEndReason",
                    "DeviceGroupHierarchyLevel1", "DeviceGroupHierarchyLevel2", "DeviceGroupHierarchyLevel3",
                    "DeviceGroupHierarchyLevel4", "VirtualSystemName", "DeviceName", "ActionSource", "SourceVMUUID",
                    "DestinationVMUUID", "TunnelID_IMSI", "MonitorTag_IMEI", "ParentSessionID", "ParentStartTime",
                    "TunnelType", "SCTP_AssocID", "SCTP_Chunks", "SCTP_Sent", "SCTP_Received", "Rule_UUID", "HTTP_Connection"
                ]
            }

            mutate {
                convert => [ "Bytes", "integer" ]
                convert => [ "BytesReceived", "integer" ]
                convert => [ "BytesSent", "integer" ]
                convert => [ "ElapsedTime", "integer" ]
                convert => [ "GeoIP.dma_code", "integer" ]
                convert => [ "GeoIP.latitude", "float" ]
                convert => [ "GeoIP.longitude", "float" ]
                convert => [ "NATDestinationPort", "integer" ]
                convert => [ "NATSourcePort", "integer" ]
                convert => [ "Packets", "integer" ]
                convert => [ "PacketsReceived", "integer" ]
                convert => [ "PacketsSent", "integer" ]
                convert => [ "SequenceNumber", "integer" ]
                #convert => [ "SCTP_Chunks", "integer" ]
                #convert => [ "SCTP_Sent", "integer" ]
                #convert => [ "SCTP_Received", "integer" ]
                #add_field => [ "[Dest_geoLocation]", "geoLocation" ]
                remove_field => [ "SCTP_AssocID", "SCTP_AssocID", "SCTP_Sent", "SCTP_Received", "TunnelID_IMSI" ]
                add_tag => [ "PAN-OS_Traffic"]
		        tag_on_failure => []
            }
        }


        else if ([message] =~ /THREAT/) {
            csv {
                source => "message"
                columns => [
                    "FUTURE_USE", "ReceiveTime", "SerialNumber", "Type", "Threat_ContentType", "FUTURE_USE",
                    "GeneratedTime", "SourceIP", "DestinationIP", "NATSourceIP", "NATDestinationIP", "RuleName",
                    "SourceUser", "DestinationUser", "Application", "VirtualSystem", "SourceZone", "DestinationZone",
                    "InboundInterface", "OutboundInterface", "LogForwardingProfile", "FUTURE_USE", "SessionID",
                    "RepeatCount", "SourcePort", "DestinationPort", "NATSourcePort", "NATDestinationPort", "Flags",
                    "Protocol", "Action", "Miscellaneous", "ThreatID", "URLCategory", "Severity", "Direction",
                    "SequenceNumber", "ActionFlags", "SourceLocation", "DestinationLocation", "FUTURE_USE",
                    "ContentType", "PCAP_ID", "FileDigest", "Cloud", "URLIndex", "UserAgent", "FileType",
                    "X-Forwarded-For", "Referer", "Sender", "Subject", "Recipient", "ReportID",
                    "DeviceGroupHierarchyLevel1", "DeviceGroupHierarchyLevel2", "DeviceGroupHierarchyLevel3",
                    "DeviceGroupHierarchyLevel4", "VirtualSystemName", "DeviceName", "FUTURE_USE", "SourceVMUUID",
                    "DestinationVMUUID", "HTTPMethod", "TunnelID_IMSI", "MonitorTag_IMEI", "ParentSessionID",
                    "ParentStartTime", "TunnelType", "ThreatCategory", "ContentVersion", "FUTURE_USE", "SCTP_AssocID", "PayloadID", "HTTP_Headers", "URL_Categories", "Rule_UUID",
                    "HTTP_Connection"
                ]
            }

            mutate {
                convert => [ "GeoIP.dma_code", "integer" ]
                convert => [ "GeoIP.latitude", "float" ]
                convert => [ "GeoIP.longitude", "float" ]
                convert => [ "NATDestinationPort", "integer" ]
                convert => [ "NATSourcePort", "integer" ]
                convert => [ "SequenceNumber", "integer" ]

                add_tag => ["PAN-OS_Threat"]
            }
        }

	else if ([message] =~ /CONFIG/) {
	    csv {
		source => "message"
		columns => [
		    "FUTURE_USE", "Receive Time", "Serial Number", "Type", "Subtype", "FUTURE_USE", "Generated Time", "Host",
		    "Virtual System", "Command", "Admin", "Client", "Result", "Configuration Path", "Before Change Detail",
		    "After Change Detail", "Sequence Number", "Action Flags", "Device Group Hierarchy Level 1",
		    "Device Group Hierarchy Level 2", "Device Group Hierarchy Level 3", "Device Group Hierarchy Level 4",
		    "Virtual System Name", "Device Name"
		]
	    }

	    mutate {
                add_tag => [ "PAN-OS_Config"]
            }
        }

	else if ([message] =~ /SYSTEM/) {
	    csv {
		source => "message"
		columns => [
		    "FUTURE_USE", "Receive Time", "Serial Number", "Type", "Content/Threat Type", "FUTURE_USE", "Generated Time",
		    "Virtual System", "Event ID", "Object", "FUTURE_USE", "FUTURE_USE", "Module", "Severity", "Description",
		    "Sequence Number", "Action Flags", "Device Group Hierarchy Level 1", "Device Group Hierarchy Level 2",
		    "Device Group Hierarchy Level 3", "Device Group Hierarchy Level 4", "Virtual System Name", "Device Name"
		]
	    }

	    mutate {
                add_tag => [ "PAN-OS_System"]
            }
        }

        mutate {
            # Original message has been fully parsed, so remove it.
            remove_field => [ "message" ]
        }

        # Geolocate logs that have SourceIP if that SourceIP is a non-RFC1918 address
        if [SourceIP] and [SourceIP] !~ "(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)|(^169\.254\.)" {
            geoip {
               source => "[SourceIP]"
               target => "[GeoIP][location]"
          }

            # Delete 0,0 in SourceIPGeo.location if equal to 0,0
            if ([SourceIPGeo.location] and [SourceIPGeo.location] =~ "0,0") {
                mutate {
                    replace => [ "SourceIPGeo.location", "" ]
                }
            }
        }

        # Geolocate logs that have DestinationIP and if that DestinationIP is a non-RFC1918 address
        if [DestinationIP] and [DestinationIP] !~ "(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)|(^169\.254\.)" {
            geoip {
                source => "[DestinationIP]"
                target => "[GeoIP][location]"
            }

            # Delete 0,0 in DestinationIPGeo.location if equal to 0,0
            if ([DestinationIPGeo.location] and [DestinationIPGeo.location] =~ "0,0") {
                mutate {
                    replace => [ "DestinationIPGeo.location", "" ]
                }
            }
        }

        # Takes the 5-tuple of source address, source port, destination address, destination port, and protocol and does a SHA1 hash to fingerprint the flow.  This is a useful
        # way to be able to do top N terms queries on flows, not just on one field.
        if [SourceIP] and [DestinationIP] {
            fingerprint {
                concatenate_sources => true
                method => "SHA1"
                key => "logstash"
                source => [ "SourceIP", "SourcePort", "DestinationIP", "DestinationPort", "Protocol" ]
            }
        }

}


ruby {
                code => "
                        hash = event.to_hash
                        hash.each do |k,v|
                                if v == nil
                                        event.remove(k)
                                end
                        end
                "
        }

}

Yes, it does.

The error message you quote

Invalid FieldReference: `[SourceIPGeo] [location]`

has a space between the field names. You need to remove that.

The configuration you quote has many references to fields that are never created, such as SourceIPGeo.location. It is unclear whether you intended that to reference the field created by the geoip filter

target => "[GeoIP][location]"

@Badger sorry for the confusion. I changed the target afterwards to

"[GeoIP][location]"

I will change again as you suggested and see if there is any difference.

still no-go..

I attach also one doc as indexed (scraped values):

"_version": 1,
"_score": null,
"_source": {
"NATDestinationIP":
"DestinationPort":
"type": "syslog",
"SessionEndReason": "aged-out",
"ParentSessionID": "0",
"NATSourceIP": "0.0.0.0",
"DestinationIP":
"DeviceGroupHierarchyLevel3": "0",
"DestinationZone":
"Packets": 2,
"TimeLogged": "2020/06/25 16:55:53",
"SourcePort":
"ReceiveTime": "2020/06/25 16:55:53",
"NATSourcePort": 0,
"Application": "dns",
"BytesSent": ,
"FUTURE_USE": "0",
"Type": "TRAFFIC",
"NATDestinationPort": 0,
"PacketsSent": 1,
"host": "
"URLCategory": "any",
"DeviceGroupHierarchyLevel1": "0",
"SCTP_Chunks": "0",
"Bytes": 271,
"StartTime": "2020/06/25 16:55:22",
"DestinationLocation": "
"ActionSource": "from-policy",
"TunnelType": "N/A",
"fingerprint": "
"DeviceGroupHierarchyLevel2": "0",
"tags": [
"PAN-OS_SysLog",
"PAN-OS_Traffic"
],
"@version": "1",
"Threat_ContentType": "end",
"SequenceNumber": ,
"Action": "allow",
"SourceUser":
"ActionFlags": "0x0",
"LogForwardingProfile": "ELK log forwarding",
"RuleName": "allow-Internal-Zones-DNS",
"OutboundInterface":
"InboundInterface": "tunnel.1",
"@timestamp": "
"VirtualSystem":
"GeneratedTime": "
"Protocol": "udp",
"SourceIP":
"BytesReceived": 185,
"SourceLocation":
"Flags": "0x19",
"ElapsedTime": 0,
"RepeatCount": "1",
"SerialNumber":
"SessionID": "2916676",
"SourceZone":
"DeviceGroupHierarchyLevel4": "0",
"PacketsReceived": 1,
"DeviceName": "
"Rule_UUID":
"HTTP_Connection": "0"
},

Ok this is resolved. One thing that was overlooked here is the fact I was already mapping per specific index, so even when making these changes right, you still need to do one of the following:

-either reindex the data for getting geo_point mapped properly

-OR, create a new index entirely for start mapping with geo_point

Thank you all for pointing me in the right direction.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.