Logstash Private IP Address - geoip

Elastic Stack Logstash
1

I can successfully get geo-location data from my IIS weblogs into Elastic if it's a public IP, but there are some private IP addresses that I want to explicitly set to a certain geo-locations and it's not working quite right. Any thoughts?

input {
 file {
    #type => "iis"
    path => "C:/logs/*.log"
    start_position => "beginning" 
  }
}

filter {

  #ignore log comments
  if [message] =~ "^#" {
    drop {}
  }

  grok {
    # check that fields match your IIS log settings
    match => ["message", "%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:s-sitename} %{IPORHOST:s-ip} %{WORD:cs-method} %{URIPATH:cs-uri-stem} %{NOTSPACE:cs-uri-query} %{NUMBER:s-port} %{NOTSPACE:cs-username} %{IPORHOST:c-ip} %{WORD:cs-version}"]
  }

  #Set the Event Timesteamp from the log
    date {
    match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss" ]
      timezone => "Etc/UTC"
  }	

  geoip {
    source => "c-ip"
    target => "geoip"
    add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
    add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]

    if [c-ip] =~ /^10\./ {
	    mutate { replace => { "[geoip][timezone]" => "Pacific/Auckland" } }
    mutate { replace => { "[geoip][country_name]" => "University of Otago" } }
    mutate { replace => { "[geoip][country_code2]" => "UO" } }
    mutate { replace => { "[geoip][country_code3]" => "UoO" } }
    mutate { remove_field => [ "[geoip][location]" ] }
    mutate { add_field => { "[geoip][location]" => "170.525" } }
    mutate { add_field => { "[geoip][location]" => "-45.865" } }
    mutate { convert => [ "[geoip][location]", "float" ] }
    mutate { replace => [ "[geoip][latitude]", -45.856 ] }
    mutate { convert => [ "[geoip][latitude]", "float" ] }
    mutate { replace => [ "[geoip][longitude]", 170.525 ] }
    mutate { convert => [ "[geoip][longitude]", "float" ] }
    }
  }

  mutate {
    convert => [ "[geoip][coordinates]", "float" ]
  }	

  mutate {
    remove_field => [ "log_timestamp"]
  }
}

# See documentation for different protocols:
# http://logstash.net/docs/1.4.2/outputs/elasticsearch
output {
  # stdout { codec => rubydebug }
    elasticsearch { hosts => ["localhost:9200"] }
}
2

In what way is it not working quite right?

3

Logstash errors with the below when I execute the config.

[2017-02-22T05:19:08,476][ERROR][logstash.agent ] fetched an invalid config {:config=>"input {\n file {\n #type => "iis"\n path => "C:/logs/*.log"\n start_position => "beginning" \n }\n}\n\nfilter {\n\n #ignore log comments\n if [message] =~ "^#" {\n drop {}\n }\n\n grok {\n # check that fields match your IIS log settings\n match => ["message", "%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:s-sitename} %{IPORHOST:s-ip} %{WORD:cs-method} %{URIPATH:cs-uri-stem} %{NOTSPACE:cs-uri-query} %{NUMBER:s-port} %{NOTSPACE:cs-username} %{IPORHOST:c-ip} %{WORD:cs-version}"]\n }\n\n #Set the Event Timesteamp from the log\n date {\n match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss" ]\n timezone => "Etc/UTC"\n }\t\n\n geoip {\n source => "c-ip"\n target => "geoip"\n add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]\n add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]\n\n if [c-ip] =~ /^10\./ {\n\t mutate { replace => { "[geoip][timezone]" => "Pacific/Auckland" } }\n mutate { replace => { "[geoip][country_name]" => "University of Otago" } }\n mutate { replace => { "[geoip][country_code2]" => "UO" } }\n mutate { replace => { "[geoip][country_code3]" => "UoO" } }\n mutate { remove_field => [ "[geoip][location]" ] }\n mutate { add_field => { "[geoip][location]" => "170.525" } }\n mutate { add_field => { "[geoip][location]" => "-45.865" } }\n mutate { convert => [ "[geoip][location]", "float" ] }\n mutate { replace => [ "[geoip][latitude]", -45.856 ] }\n mutate { convert => [ "[geoip][latitude]", "float" ] }\n mutate { replace => [ "[geoip][longitude]", 170.525 ] }\n mutate { convert => [ "[geoip][longitude]", "float" ] }\n }\n }\n\n mutate {\n convert => [ "[geoip][coordinates]", "float" ]\n }\t\n\n mutate {\n remove_field => [ "log_timestamp"]\n }\n}\n\n# See documentation for different protocols:\n# http://logstash.net/docs/1.4.2/outputs/elasticsearch\noutput {\n # stdout { codec => rubydebug }\n elasticsearch { hosts => ["localhost:9200"] }\n}\n", :reason=>"Expected one of #, => at line 33, column 8 (byte 830) after filter {\n\n #ignore log comments\n if [message] =~ "^#" {\n drop {}\n }\n\n grok {\n # check that fields match your IIS log settings\n match => ["message", "%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:s-sitename} %{IPORHOST:s-ip} %{WORD:cs-method} %{URIPATH:cs-uri-stem} %{NOTSPACE:cs-uri-query} %{NUMBER:s-port} %{NOTSPACE:cs-username} %{IPORHOST:c-ip} %{WORD:cs-version}"]\n }\n\n #Set the Event Timesteamp from the log\n date {\n match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss" ]\n timezone => "Etc/UTC"\n }\t\n\n geoip {\n source => "c-ip"\n target => "geoip"\n add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]\n add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]\n\n if "}

4

You're not closing your geoip filter.

5

Where am I missing it? It looks like I have all the open/close in there.

6

Don't do that, it's pointless. LS creates a geoip.location field for you.

Are you really on 1.4?

7

I'm using logstash 5.2.0.

8

Where am I missing it? It looks like I have all the open/close in there.

No. Your geoip filter looks like this:

  geoip {
    source => "c-ip"
    target => "geoip"
    add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
    add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]

The next line is the conditional and there's no closing brace in between.

9

Ah thank you, looks to be working now!