Hi,
this is my first time working with elastic so i think im missing something.
Im trying to setup the Elastic Stack to analyze Cowrie Logs. Problem is there are different sources for the Input field.
In the Cowrie Logs Input is the entered command. (Does only occour in an important subset of logs).
Filebeat adds a field input.type = "log".
When i try to filter the input.type field i drop every input field. But i need the input field with the command data.
I get the following Error:
[WARN ] 2022-04-20 10:05:27.891 [[main]>worker1] elasticsearch - Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"cowrie-logstash", :routing=>nil}, {"timestamp"=>"2022-04-20T10:05:26.417373Z", "geoip"=>{"ip"=>"<redacted-my-ip>", "geo"=>{"continent_code"=>"EU", "timezone"=>"Europe/Berlin", "location"=>{"lat"=><redacted-my-lat>, "lon"=><redacted-my-lat>}, "city_name"=>"<redacted-my-city>", "country_iso_code"=>"DE", "region_name"=>"redacted", "country_name"=>"Germany", "region_iso_code"=>"redacted", "postal_code"=>"redacted"}}, "host"=>{"ip"=>["10.0.10.43", "fe80::42:12ff:feb9:46e"], "name"=>"ip-10-0-10-43", "containerized"=>false, "hostname"=>"ip-10-0-10-43", "architecture"=>"x86_64", "os"=>{"kernel"=>"5.13.0-1021-aws", "name"=>"Ubuntu", "codename"=>"focal", "version"=>"20.04.3 LTS (Focal Fossa)", "type"=>"linux", "platform"=>"ubuntu", "family"=>"debian"}, "id"=>"511573f801fe4397bf51ff7b53295618", "mac"=>["02:42:12:b9:04:6e"]}, "input"=>"ls", "log"=>{"file"=>{}}, "cloud"=>{"account"=>{"id"=>"redacted"}, "machine"=>{"type"=>"t2.small"}, "region"=>"redacted", "image"=>{"id"=>"ami-0d527b8c289b4af7f"}, "provider"=>"aws", "instance"=>{"id"=>"i-0743a76b841fbbe68"}, "availability_zone"=>"redacted", "service"=>{"name"=>"EC2"}}, "eventid"=>"cowrie.command.input", "fields"=>{"event"=>{"type"=>"cowried"}}, "session"=>"246117b121a4", "tags"=>[], "message"=>"CMD: ls", "agent"=>{"ephemeral_id"=>"d500bf92-7620-4601-8548-8093a2d4907e", "name"=>"ip-10-0-10-43", "id"=>"053ae52a-d7df-4b8e-b58f-5bf941e855c4", "type"=>"filebeat", "version"=>"8.1.2"}, "src_host"=>"<redacted-my-ip>", "@timestamp"=>2022-04-20T10:05:26.417Z, "event"=>{"original"=>"{\"eventid\":\"cowrie.command.input\",\"input\":\"ls\",\"message\":\"CMD: ls\",\"sensor\":\"ip-10-0-10-43\",\"timestamp\":\"2022-04-20T10:05:26.417373Z\",\"src_ip\":\"<redacted-my-ip>\",\"session\":\"246117b121a4\"}"}, "src_ip"= >"<redacted-my-ip>", "ecs"=>{"version"=>"8.0.0"}, "@version"=>"1", "type"=>"cowrie", "sensor"=>"ip-10-0-10-43"}], :response=>{"index"=>{"_index"=>"cowrie-logstash", "_id"=>"5fpvRoABqN69WikeGePG", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"object mapping for [input] tried to parse field [input] as object, but found a concrete value"}}}}
This is a log with input field:
{
"timestamp" => "2022-04-20T10:05:24.939836Z",
"geoip" => {
"ip" => "<redacted-my-ip>",
"geo" => {
"continent_code" => "EU",
"timezone" => "Europe/Berlin",
"location" => {
"lat" => <redacted-my-lat>,
"lon" => <redacted-my-lat>
},
"city_name" => "<redacted-my-city>",
"country_iso_code" => "DE",
"region_name" => "<redacted-my-region>",
"country_name" => "Germany",
"region_iso_code" => "redacted",
"postal_code" => "redacted"
}
},
"host" => {
"ip" => [
[0] "10.0.10.43",
[1] "fe80::42:12ff:feb9:46e"
],
"name" => "ip-10-0-10-43",
"containerized" => false,
"architecture" => "x86_64",
"os" => {
"family" => "debian",
"name" => "Ubuntu",
"kernel" => "5.13.0-1021-aws",
"version" => "20.04.3 LTS (Focal Fossa)",
"codename" => "focal",
"type" => "linux",
"platform" => "ubuntu"
},
"id" => "511573f801fe4397bf51ff7b53295618",
"hostname" => "ip-10-0-10-43",
"mac" => [
[0] "02:42:12:b9:04:6e"
]
},
"input" => "ls -la",
"log" => {
"file" => {}
},
"cloud" => {
"machine" => {
"type" => "t2.small"
},
"account" => {
"id" => "redacted"
},
"region" => "redacted",
"image" => {
"id" => "ami-0d527b8c289b4af7f"
},
"instance" => {
"id" => "i-0743a76b841fbbe68"
},
"provider" => "aws",
"availability_zone" => "redacteda",
"service" => {
"name" => "EC2"
}
},
"eventid" => "cowrie.command.input",
"fields" => {
"event" => {
"type" => "cowried"
}
},
"session" => "246117b121a4",
"tags" => [],
"message" => "CMD: ls -la",
"agent" => {
"ephemeral_id" => "d500bf92-7620-4601-8548-8093a2d4907e",
"name" => "ip-10-0-10-43",
"id" => "053ae52a-d7df-4b8e-b58f-5bf941e855c4",
"type" => "filebeat",
"version" => "8.1.2"
},
"src_host" => "<redacted-my-ip>",
"@timestamp" => 2022-04-20T10:05:24.939Z,
"event" => {
"original" => "{\"eventid\":\"cowrie.command.input\",\"input\":\"ls -la\",\"message\":\"CMD: ls -la\",\"sensor\":\"ip-10-0-10-43\",\"timestamp\":\"2022-04-20T10:05:24.939836Z\",\"src_ip\":\"<redacted-my-ip>\",\"session\":\"246117b121a4\"}"
},
"src_ip" => "<redacted-my-ip>",
"ecs" => {
"version" => "8.0.0"
},
"@version" => "1",
"type" => "cowrie",
"sensor" => "ip-10-0-10-43"
}
This is with input.type field
{
"sensor" => "ip-10-0-10-43",
"timestamp" => "2022-04-20T10:08:22.712960Z",
"geoip" => {
"ip" => "<redacted-my-ip>",
"geo" => {
"continent_code" => "EU",
"timezone" => "Europe/Berlin",
"location" => {
"lat" => <redacted-my-lat>,
"lon" => <redacted-my-lat>
},
"city_name" => "<redacted-my-city>",
"country_iso_code" => "DE",
"region_name" => "redacted",
"country_name" => "Germany",
"region_iso_code" => "redacted",
"postal_code" => "redacted"
}
},
"host" => {
"ip" => [
[0] "10.0.10.43",
[1] "fe80::42:12ff:feb9:46e"
],
"name" => "ip-10-0-10-43",
"containerized" => false,
"architecture" => "x86_64",
"os" => {
"family" => "debian",
"name" => "Ubuntu",
"kernel" => "5.13.0-1021-aws",
"version" => "20.04.3 LTS (Focal Fossa)",
"codename" => "focal",
"type" => "linux",
"platform" => "ubuntu"
},
"id" => "511573f801fe4397bf51ff7b53295618",
"hostname" => "ip-10-0-10-43",
"mac" => [
[0] "02:42:12:b9:04:6e"
]
},
"input" => {
"type" => "log"
},
"log" => {
"file" => {}
},
"cloud" => {
"machine" => {
"type" => "t2.small"
},
"account" => {
"id" => "redacted"
},
"region" => "redacted",
"image" => {
"id" => "ami-0d527b8c289b4af7f"
},
"provider" => "aws",
"instance" => {
"id" => "i-0743a76b841fbbe68"
},
"availability_zone" => "redacteda",
"service" => {
"name" => "EC2"
}
},
"eventid" => "cowrie.session.closed",
"fields" => {
"event" => {
"type" => "cowried"
}
},
"session" => "246117b121a4",
"tags" => [],
"message" => "Connection lost after 182 seconds",
"agent" => {
"ephemeral_id" => "d500bf92-7620-4601-8548-8093a2d4907e",
"name" => "ip-10-0-10-43",
"id" => "053ae52a-d7df-4b8e-b58f-5bf941e855c4",
"type" => "filebeat",
"version" => "8.1.2"
},
"src_host" => "<redacted-my-ip>",
"@timestamp" => 2022-04-20T10:08:22.712Z,
"event" => {
"original" => "{\"eventid\":\"cowrie.session.closed\",\"duration\":182.86497902870178,\"message\":\"Connection lost after 182 seconds\",\"sensor\":\"ip-10-0-10-43\",\"timestamp\":\"2022-04-20T10:08:22.712960Z\",\"src_ip\":\"<redacted-my-ip>\",\"session\":\"246117b121a4\"}"
},
"src_ip" => "<redacted-my-ip>",
"ecs" => {
"version" => "8.0.0"
},
"@version" => "1",
"type" => "cowrie",
"duration" => 182.86497902870178
}
And here is my logstash conf.
ubuntu@ip-10-0-8-196:~$ sudo cat /etc/logstash/conf.d/logstash-cowrie.conf
input {
# filebeats
beats {
port => 5044
type => "cowrie"
}
# if you don't want to use filebeat: this is the actual live log file to monitor
#file {
# path => ["/home/cowrie/cowrie-git/log/cowrie.json"]
# codec => json
# type => "cowrie"
#}
}
filter {
if [type] == "cowrie" {
json {
source => message
}
date {
match => [ "timestamp", "ISO8601" ]
}
if [src_ip] {
mutate {
add_field => { "src_host" => "%{src_ip}" }
}
dns {
reverse => [ "src_host" ]
nameserver => [ "8.8.8.8", "8.8.4.4" ]
action => "replace"
hit_cache_size => 4096
hit_cache_ttl => 900
failed_cache_size => 512
failed_cache_ttl => 900
}
geoip {
source => "src_ip"
target => "geoip"
database => "/opt/logstash/vendor/geoip/GeoLite2-City.mmdb"
}
}
# if [input][type] == "log" { mutate { remove_field => [ "input" ] }}
# ändert goar nix:
mutate {
# remove_field => ["[input][type]" ]
# cut out useless tags/fields
# rename => ["input", "%{[type]}" ]
remove_tag => [ "beats_input_codec_plain_applied"]
remove_field => [ "[log][file][path]", "[log][offset]" ]
}
}
}
output {
if [type] == "cowrie" {
elasticsearch {
hosts => ["localhost:9200"]
ilm_enabled => auto
ilm_rollover_alias => "cowrie-logstash"
}
#file {
# path => "/tmp/cowrie-logstash.log"
# codec => json
#}
stdout {
codec => rubydebug
}
}
}
I also tried having the log with bot the input field and input.type field come in first. Then only the input field shows up in the saved doc. And when trying to log logs with the input.type field i get the following error.
[WARN ] 2022-04-20 10:16:58.940 [[main]>worker1] elasticsearch - Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"cowrie-logstash", :routing=>nil}, {"shasum"=>"466e20ca6d44133f771978fe6582745f1445e5333beb47371a51bc2d2ded978a", "eventid"=>"cowrie.log.closed", "cloud"=>{"machine"=>{"type"=>"t2.small"}, "account"=>{"id"=>"redacted"}, "region"=>"redacted", "image"=>{"id"=>"ami-0d527b8c289b4af7f"}, "instance"=>{"id"=>"i-0743a76b841fbbe68"}, "provider"=>"aws", "availability_zone"=>"redacteda", "service"=>{"name"=>"EC2"}}, "session"=>"49d6e3589b7f", "agent"=>{"ephemeral_id"=>"d500bf92-7620-4601-8548-8093a2d4907e", "name"=>"ip-10-0-10-43", "id"=>"053ae52a-d7df-4b8e-b58f-5bf941e855c4", "type"=>"filebeat", "version"=>"8.1.2"}, "ttylog"=>"var/lib/cowrie/tty/466e20ca6d44133f771978fe6582745f1445e5333beb47371a51bc2d2ded978a", "event"=>{"original"=>"{\"eventid\":\"cowrie.log.closed\",\"ttylog\":\"var/lib/cowrie/tty/466e20ca6d44133f771978fe6582745f1445e5333beb47371a51bc2d2ded978a\",\"size\":689,\"shasum\":\"466e20ca6d44133f771978fe6582745f1445e5333beb47371a51bc2d2ded978a\",\"duplicate\":false,\"duration\":78.79388785362244,\"message\":\"Closing TTY Log: var/lib/cowrie/tty/466e20ca6d44133f771978fe6582745f1445e5333beb47371a51bc2d2ded978a after 78 seconds\",\"sensor\":\"ip-10-0-10-43\",\"timestamp\":\"2022-04-20T10:16:56.326244Z\",\"src_ip\":\"redacted\",\"session\":\"49d6e3589b7f\"}"}, "src_ip"=>"redacted", "timestamp"=>"2022-04-20T10:16:56.326244Z", "@version"=>"1", "type"=>"cowrie", "sensor"=>"ip-10-0-10-43", "duration"=>0.7879388785362244e2, "size"=>689, "host"=>{"ip"=>["10.0.10.43", "fe80::42:12ff:feb9:46e"], "name"=>"ip-10-0-10-43", "containerized"=>false, "os"=>{"kernel"=>"5.13.0-1021-aws", "name"=>"Ubuntu", "platform"=>"ubuntu", "version"=>"20.04.3 LTS (Focal Fossa)", "family"=>"debian", "codename"=>"focal", "type"=>"linux"}, "hostname"=>"ip-10-0-10-43", "id"=>"511573f801fe4397bf51ff7b53295618", "architecture"=>"x86_64", "mac"=>["02:42:12:b9:04:6e"]}, "input"=>{"type"=>"log"}, "log"=>{"file"=>{}}, "fields"=>{"event"=>{"type"=>"cowried"}}, "tags"=>[], "message"=>"Closing TTY Log: var/lib/cowrie/tty/466e20ca6d44133f771978fe6582745f1445e5333beb47371a51bc2d2ded978a after 78 seconds", "duplicate"=>false, "src_host"=>"redacted", "@timestamp"=>2022-04-20T10:16:56.326Z, "ecs"=>{"version"=>"8.0.0"}, "geoip"=>{"ip"=>"redacted", "geo"=>{"continent_code"=>"EU", "timezone"=>"Europe/Berlin", "location"=>{"lat"=>redacted, "lon"=>redacted}, "city_name"=>"redacted", "country_iso_code"=>"DE", "region_name"=>"redacted", "country_name"=>"Germany", "region_iso_code"=>"redacted", "postal_code"=>"redacted"}}}], :response=>{"index"=>{"_index"=>"cowrie-logstash", "_id"=>"8Pp5RoABqN69WikepeMy", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse field [input] of type [text] in document with id '8Pp5RoABqN69WikepeMy'. Preview of field's value: '{type=log}'", "caused_by"=>{"type"=>"illegal_state_exception", "reason"=>"Can't get text on a START_OBJECT at 1:1762"}}}}}