template mapping snippit
"geoip": {
"properties": {
"city_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"continent_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_code2": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_code3": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dma_code": {
"type": "long"
},
"ip": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"latitude": {
"type": "float"
},
"location": {
"properties": {
"lat": {
"type": "float"
},
"lon": {
"type": "float"
}
}
},
"longitude": {
"type": "float"
},
"postal_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"region_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"region_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"timezone": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
Logstash configuration file
input {
beats {
port => 5044
type => "log"
}
}
filter {
if [fileset][module] == "iis" {
if [message] =~ "^#" { drop { } }
if [fileset][name] == "access" {
if "/ApplicationServer/AuthenticationService.asmx/CheckConnectivity" in [message] { drop { } }
grok {
match => ["message", "%{TIMESTAMP_ISO8601:log_timestamp} %{IP:serverIP} %{WORD:method} %{URIPATH:uriStem} %{NOTSPACE:uriQuery} %{NUMBER:port} %{NOTSPACE:username} %{IPORHOST:clientIP} %{NOTSPACE:userAgent} %{NOTSPACE:referer} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:win32response} %{NUMBER:timetaken}"]
}
# Set the Event Timestamp from the log
date {
match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss" ]
timezone => "Etc/UTC"
target => "@timestamp"
}
mutate {
convert => ["timetaken", "integer"]
}
useragent {
source=> "useragent"
prefix=> "browser"
}
geoip {
source => "clientIP"
}
}
if [fileset][name] == "error" {
grok{
match => ["message", "%{TIMESTAMP_ISO8601:log_timestamp} %{IPORHOST:clientIP} %{NUMBER:clientPort} %{NOTSPACE:serverIP} %{NOTSPACE:serverPort} %{NOTSPACE:clientHTTPVersion} %{NOTSPACE:method} %{NOTSPACE:uriStem} %{NOTSPACE:status} %{NOTSPACE:siteID} %{NOTSPACE:reason} %{NOTSPACE:applicationPoolName}"]
}
date {
match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss" ]
timezone => "Etc/UTC"
target => "@timestamp"
}
}
}
}
output {
elasticsearch {
hosts => "localhost:9200"
manage_template => true
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
}
}
After getting all of this set. I activated the plugin, refreshed the indexes, restarted all components of the elk stack, deleted old index data (because this is just a test run right now), restarted the entire elk stack, refreshed the indexes again, deleted the index, recreated the index, refreshed the new index, and restarted the entire elk stack again. Wow that was a run on! At this point I am at a loss of what to do next. I am doing something wrong so any help is appreciated!
Thank you!