Hello.
I am new to elasticsearch and I am trying to send json data with filebeat to logstash -> elasticsearch.
#filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/json-data2.log
json.message_key: log
output.logstash:
hosts: ["localhost:5044"]
#logstash.conf
input {
beats {
port => 5044
codec => json
}
}
filter {
json {
source => "message"
}
mutate {
remove_field => ["message"]
}
}
output {
elasticsearch {
hosts => "http://localhost:9200"
index => "demo-index5"
}
stdout {}
}
#json-data2.log
{"id":2,"timestamp":"2019-08-11T17:55:56Z","paymentType":"Visa","name":"Darby Dacks","gender":"Female","ip_address":"77.72.239.47","purpose":"Shoes","country":"Poland","age":55}
{"id":3,"timestamp":"2019-07-14T04:48:25Z","paymentType":"Visa","name":"Harri Cayette","gender":"Female","ip_address":"227.6.210.146","purpose":"Sports","country":"Canada","age":27}
{"id":4,"timestamp":"2020-02-29T12:41:59Z","paymentType":"Mastercard","name":"Regan Stockman","gender":"Male","ip_address":"139.224.15.154","purpose":"Home","country":"Indonesia","age":34}
I get an endless loop of:
" "@timestamp" => 2021-02-17T13:34:42.186Z,
"fileset" => {
"module" => "system",
"name" => "syslog"
},
"beat" => {
"name" => "localhost.localdomain",
"hostname" => "localhost.localdomain",
"version" => "6.8.14"
},
"tags" => [
[0] "_jsonparsefailure",
[1] "beats_input_codec_json_applied"
],
"source" => "/var/log/messages",
"event" => {
"dataset" => "system.syslog"
},
"prospector" => {
"type" => "log"
}
}
{
"offset" => 24662899,
"@version" => "1",
"host" => {
"architecture" => "x86_64",
"name" => "localhost.localdomain",
"id" => "bc275b96cbd50a4d9c4c359e91d9a4a1",
"containerized" => false,
"os" => {
"platform" => "centos",
"name" => "CentOS Linux",
"codename" => "Core",
"version" => "7 (Core)",
"family" => "redhat"
}
},
"input" => {
"type" => "log"
},
"log" => {
"file" => {
"path" => "/var/log/messages"
}
},
"@timestamp" => 2021-02-17T13:34:42.186Z,
"fileset" => {
"name" => "syslog",
"module" => "system"
},
"beat" => {
"name" => "localhost.localdomain",
"hostname" => "localhost.localdomain",
"version" => "6.8.14"
},
"tags" => [
[0] "_jsonparsefailure",
[1] "beats_input_codec_json_applied"
],
"source" => "/var/log/messages",
"event" => {
"dataset" => "system.syslog"
},
"prospector" => {
"type" => "log"
}
[main][a9321cb3f911597a581039589786cdf9fdcdc1442e0bd6752257a5c28c656582] Error p\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\"[truncated 12226 bytes]; line: 1, column: 7]>}\\\\\\\\\\\\\\\", :exception=>#<LogStash::Json::ParserError: Unexpected character ('-' (code 45)): was expecting comma to separate Array entries\\\\\\\\n at [Source: (byte[])\\\\\\\\\\\\\\\"[2021-02-17T13:22:09,208][WARN ][logstash.filters.json ][main][a9321cb3f911597a581039589786cdf9fdcdc1442e0bd6752257a5c28c656582] Error parsing json {:source=>\\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\", :raw=>\\\\\\\\\\\\\\\"[2021-02-17T13:19:27,199][WARN ][logstash.filters.json ][main][a9321cb3f911597a581039589786cdf9fdcdc1442e0bd6752257a5c28c656582] Error parsing json {:source=>\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\", :raw=>\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\"[2021-02-17T13:18:14,653][WARN ][logstash.filters.json ][main][a9321cb3f911597a581039589786cdf9fdcdc1442e0bd6752257a5c28c656582] Error p\\\\\\\\\\\\\\\"[truncated 18529 bytes]; line: 1, column: 7]>}\\\\\\\", :exception=>#<LogStash::Json::ParserError: Unexpected character ('-' (code 45)): was expecting comma to separate Array entries\\\\n at [Source: (byte[])\\\\\\\"[2021-02-17T13:23:15,084][WARN ][logstash.filters.json ][main][a9321cb3f911597a581039589786cdf9fdcdc1442e0bd6752257a5c28c656582] Error parsing json {:source=>\\\\\\\"message\\\\\\\", :raw=>\\\\\\\"[2021-02-17T13:22:09,208][WARN ][logstash.filters.json ][main][a9321cb3f911597a581039589786cdf9fdcdc1442e0bd6752257a5c28c656582] Error parsing json {:source=>\\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\", :raw=>\\\\\\\\\\\\\\\"[2021-02-17T13:19:27,199][WARN ][logstash.filters.json ][main][a9321cb3f911597a581039589786cdf9fdcdc1442e0bd6752257a5c28c656582] Error p\\\\\\\"[truncated 30272 bytes]; line: 1, column: 7]>}\\\", :exception=>#<LogStash::Json::ParserError: Unexpected character ('-' (code 45)): was expecting comma to separate Array entries\\n at [Source: (byte[])\\\"[2021-02-17T13:26:26,015][WARN ][logstash.filters.json ][main][a9321cb3f911597a581039589786cdf9fdcdc1442e0bd6752257a5c28c656582] Error parsing json {:source=>\\\"message\\\", :raw=>\\\"[2021-02-17T13:23:15,084][WARN ][logstash.filters.json ][main][a9321cb3f911597a581039589786cdf9fdcdc1442e0bd6752257a5c28c656582] Error parsing json {:source=>\\\\\\\"message\\\\\\\", :raw=>\\\\\\\"[2021-02-17T13:22:09,208][WARN ][logstash.filters.json ][main][a9321cb3f911597a581039589786cdf9fdcdc1442e0bd6752257a5c28c656582] Error p\\\"[truncated 52895 bytes]; line: 1, column: 7]>}\", :exception=>#<LogStash::Json::ParserError: Unexpected character ('-' (code 45)): was expecting comma to separate Array entries\n at [Source: (byte[])\"[2021-02-17T14:12:18,411][WARN ][logstash.filters.json ][main][444b3f75f975aa0ad1c9043b2d077d7cfe9213d7f2597a4c67ea530310289589] Error parsing json {:source=>\"message\", :raw=>\"[2021-02-17T13:26:26,015][WARN ][logstash.filters.json ][main][a9321cb3f911597a581039589786cdf9fdcdc1442e0bd6752257a5c28c656582] Error parsing json {:source=>\\\"message\\\", :raw=>\\\"[2021-02-17T13:23:15,084][WARN ][logstash.filters.json ][main][a9321cb3f911597a581039589786cdf9fdcdc1442e0bd6752257a5c28c656582] Error p\"[truncated 97278 bytes]; line: 1, column: 7]>}"}
[FATAL] 2021-02-17 16:14:55.423 [LogStash::Runner] Logstash -
org.jruby.exceptions.ThreadKill: null
[WARN ] 2021-02-17 16:14:55.353 [[main]>worker1] json - Error parsing json {:source=>"message", :raw=>"[2021-02-17T14:13:40,649][ERROR][logstash.codecs.json ][main][5721c497d08e156a27467a3ca9766b8b9724daeb6cf0799842e699c5f2304fc7] JSON parse error, original data now in message field {:error=>#<LogStash::Json::ParserError: Unrecognized token 'Feb': was expecting ('true', 'false' or 'null')\n at [Source: (String)\"Feb 17 14:13:12 localhost logstash: [2021-02-17T14:13:12,750][ERROR][logstash.codecs.json ][main][5721c497d08e156a27467a3ca9766b8b9724daeb6cf0799842e699c5f2304fc7] JSON parse error, original data now in message field {:error=>#<LogStash::Json::ParserError: Unexpected character ('-' (code 45)): was expecting comma to separate Array entries\"; line: 1, column: 4]>, :data=>\"Feb 17 14:13:12 localhost logstash: [2021-02-17T14:13:12,750][ERROR][logstash.codecs.json ][main][5721c497d08e156a27467a3ca9766b8b9724daeb6cf0799842e699c5f2304fc7] JSON parse error, original data now in message field {:error=>#<LogStash::Json::ParserError: Unexpected character ('-' (code 45)): was expecting comma to separate Array entries\"}", :exception=>#<LogStash::Json::ParserError: Unexpected character ('-' (code 45)): was expecting comma to separate Array entries
at [Source: (byte[])"[2021-02-17T14:13:40,649][ERROR][logstash.codecs.json ][main][5721c497d08e156a27467a3ca9766b8b9724daeb6cf0799842e699c5f2304fc7] JSON parse error, original data now in message field {:error=>#<LogStash::Json::ParserError: Unrecognized token 'Feb': was expecting ('true', 'false' or 'null')
at [Source: (String)"Feb 17 14:13:12 localhost logstash: [2021-02-17T14:13:12,750][ERROR][logstash.codecs.json ][main][5721c497d08e156a27467a3ca9766b8b9724daeb6cf0799842e699c5f2304fc7] JSON parse error,"[truncated 538 bytes]; line: 1, column: 7]>}
When i send json file without filebeat through logstash, it works like a charm, but with filebeat I just cant get it to work.