So I am trying to use beats and logstash to ship to splunk as well as a number of other places. Splunk requires an "event" field to be the outer most of the json. I am having a very hard time doing this. Any help would be appreciated as I am only days into using logstash. My current issue my sub json items are getting wrapped in quotes and thus not parsed by splunk correctly. I have tried not using the mapping but then it is missing the top event field it needs. Here is a way I am getting data into splunk but getting all nested json wrapped in quotes and thus being treated like a string.
Logstash.conf
input {
beats {
port => 5044
}
}
filter {
}
output {
if [@metadata][beat] == "filebeat" {
stdout { codec => rubydebug }
http {
http_method => "post"
url => "https://http-inputs-inforfr.splunkcloud.com:443/services/collector"
headers => {
"Authorization" => "Splunk id"
"X-Splunk-Request-Channel" => ""
}
mapping => {
"event" => {
"message" => "%{message}"
"log" => "%{log}"
"prospector" => "%{prospector}"
"beat" => "%{beat}"
"fields" => "%{fields}"
"tags" => "%{tags}"
"input" => "%{input}"
"host" => "%{host}"
"meta" => "%{meta}"
}
}
}
}
}
raw output (all sub json items are wrapped in quotes)
{"message":"Feb 1 17:18:10 ip-172-18-0-56 telegraf: 2019-02-01T17:18:10Z E! [agent] Error writing to output [influxdb]: could not write any address","log":"{"file":{"path":"/var/log/messages"}}","prospector":"{"type":"log"}","beat":"{"name":"ip-172-18-0-56.mgmt.inforbc.com","version":"6.6.0","hostname":"ip-172-18-0-56.mgmt.inforbc.com"}","fields":"{"env":"test"}","tags":"test-tag,beats_input_codec_plain_applied","input":"{"type":"log"}","host":"{"name":"ip-172-18-0-56.mgmt.inforbc.com","id":"089940e35302405aac620f176b27e384","os":{"name":"CentOS Linux","family":"redhat","version":"7 (Core)","platform":"centos","codename":"Core"},"architecture":"x86_64","containerized":true}","meta":"{"cloud":{"machine_type":"m5.xlarge","availability_zone":"us-gov-west-1a","instance_id":"i-0dc1828ced35769e6","region":"us-gov-west-1","provider":"ec2"}}"}
desired output
{
"message": "Jan 31 19:06:50 ip-172-18-0-56 telegraf: 2019-01-31T19:06:50Z E! [agent] Error writing to output [influxdb]: could not write any address",
"log": {
"file": {
"path": "/var/log/messages"
}
},
"prospector": {
"type": "log"
},
"beat": {
"name": "ip-172-18-0-56.mgmt.inforbc.com",
"version": "6.6.0",
"hostname": "ip-172-18-0-56.mgmt.inforbc.com"
},
"fields": {
"env": "test"
},
"tags": "test-tag,beats_input_codec_plain_applied",
"input": {
"type": "log"
},
"host": {
"name": "ip-172-18-0-56.mgmt.inforbc.com",
"id": "089940e35302405aac620f176b27e384",
"os": {
"name": "CentOS Linux",
"family": "redhat",
"version": "7 (Core)",
"platform": "centos",
"codename": "Core"
},
"architecture": "x86_64",
"containerized": true
},
"meta": {
"cloud": {
"machine_type": "m5.xlarge",
"availability_zone": "us-gov-west-1a",
"instance_id": "i-0dc1828ced35769e6",
"region": "us-gov-west-1",
"provider": "ec2"
}
}
}