Parsing Json File with Logstash and Filebeat

Hi I am new to ES. I was trying to parse json file with logstash but I am getting error like below. Please help me to solve this issue

Error

<LogStash::Json::ParserError: Unexpected end-of-input: expected close marker for OBJECT (from [Source: [B@4df0e025; line: 1, column: 1])
at [Source: [B@4df0e025; line: 37, column: 868]>}
[2017-09-10T16:57:18,382][WARN ][logstash.filters.json ] Error parsing json {:source=>"message", :raw=>"{\n "sensor-time" : {\n "timezone" : "UTC",\n "time" : "2017-09-07T08:00:02Z"\n },\n "status" : {\n "code" : "OK"\n },\n "content" : {\n "element" : [ {\n "element-id" : 0,\n "element-name" : "Line 0",\n "sensor-type" : "SINGLE_SENSOR",\n "data-type" : "LINE",\n "from" : "2017-09-07T07:00:00Z",\n "to" : "2017-09-07T08:00:00Z",\n "resolution" : "ONE_HOUR",\n "measurement" : [ {\n "from" : "2017-09-07T07:00:00Z",\n "to" : "2017-09-07T08:00:00Z",\n "value" : [ {\n "value" : 31,\n "label" : "fw"\n }, {\n "value" : 27,\n "label" : "bw"\n } ]\n } ]\n } ]\n },\n "sensor-info" : {\n "serial-number" : "00:1E:C0:9D:41:ED",\n "ip-address" : "10.0.100.12",\n "name" : "Wireless Sensor 01",\n "group" : "SDS Wireless Lab",\n "device-type" : "PC2R"\n }", :exception=>#<LogStash::Json::ParserError: Unexpected end-of-input: expected close marker for OBJECT (from [Source: [B@5e21f900; line: 1, column: 1])
at [Source: [B@5e21f900; line: 37, column: 868]>}
[2017-09-10T16:57:18,383][WARN ][logstash.filters.json ] Error parsing json {:source=>"message", :raw=>"{\n "sensor-time" : {\n "timezone" : "UTC",\n "time" : "2017-09-06T13:00:02Z"\n },\n "status" : {\n "code" : "OK"\n },\n "content" : {\n "element" : [ {\n "element-id" : 0,\n "element-name" : "Line 0",\n "sensor-type" : "SINGLE_SENSOR",\n "data-type" : "LINE",\n "from" : "2017-09-06T12:00:00Z",\n "to" : "2017-09-06T13:00:00Z",\n "resolution" : "ONE_HOUR",\n "measurement" : [ {\n "from" : "2017-09-06T12:00:00Z",\n "to" : "2017-09-06T13:00:00Z",\n "value" : [ {\n "value" : 32,\n "label" : "fw"\n }, {\n "value" : 30,\n "label" : "bw"\n } ]\n } ]\n } ]\n },\n "sensor-info" : {\n "serial-number" : "00:1E:C0:9D:41:ED",\n "ip-address" : "10.0.100.12",\n "name" : "Wireless Sensor 01",\n "group" : "SDS Wireless Lab",\n "device-type" : "PC2R"\n }", :exception=>#<LogStash::Json::ParserError: Unexpected end-of-input: expected close marker for OBJECT (from [Source: [B@280b8ded; line: 1, column: 1])
at [Source: [B@280b8ded; line: 37, column: 868]>}
[2017-09-10T16:57:18,383][WARN ][logstash.filters.json ] Error parsing json {:source=>"message", :raw=>"{\n "sensor-time" : {\n "timezone" : "UTC",\n "time" : "2017-09-07T01:00:02Z"\n },\n "status" : {\n "code" : "OK"\n },\n "content" : {\n "element" : [ {\n "element-id" : 0,\n "element-name" : "Line 0",\n "sensor-type" : "SINGLE_SENSOR",\n "data-type" : "LINE",\n "from" : "2017-09-07T00:00:00Z",\n "to" : "2017-09-07T01:00:00Z",\n "resolution" : "ONE_HOUR",\n "measurement" : [ {\n "from" : "2017-09-07T00:00:00Z",\n "to" : "2017-09-07T01:00:00Z",\n "value" : [ {\n "value" : 0,\n "label" : "fw"\n }, {\n "value" : 0,\n "label" : "bw"\n } ]\n } ]\n } ]\n },\n "sensor-info" : {\n "serial-number" : "00:1E:C0:9D:41:ED",\n "ip-address" : "10.0.100.12",\n "name" : "Wireless Sensor 01",\n "group" : "SDS Wireless Lab",\n "device-type" : "PC2R"\n }", :exception=>#<LogStash::Json::ParserError: Unexpected end-of-input: expected close marker for OBJECT (from [Source: [B@4f638d4; line: 1, column: 1])
at [Source: [B@4f638d4; line: 37, column: 866]>}

sensor1.json

{
"sensor-time" : {
"timezone" : "UTC",
"time" : "2017-09-06T08:00:02Z"
},
"status" : {
"code" : "OK"
},
"content" : {
"element" : [ {
"element-id" : 0,
"element-name" : "Line 0",
"sensor-type" : "SINGLE_SENSOR",
"data-type" : "LINE",
"from" : "2017-09-06T07:00:00Z",
"to" : "2017-09-06T08:00:00Z",
"resolution" : "ONE_HOUR",
"measurement" : [ {
"from" : "2017-09-06T07:00:00Z",
"to" : "2017-09-06T08:00:00Z",
"value" : [ {
"value" : 10,
"label" : "fw"
}, {
"value" : 10,
"label" : "bw"
} ]
} ]
} ]
},
"sensor-info" : {
"serial-number" : "00:1E:C0:9D:41:ED",
"ip-address" : "10.0.100.12",
"name" : "Wireless Sensor 01",
"group" : "SDS Wireless Lab",
"device-type" : "PC2R"
}
}

Pipeline.conf

input {
beats {
type=> beats
port => "5043"
codec=> json
}
}

filter{
json{
source => "message"
}
mutate {
remove_field => [ "type", "tags"]
add_field => {
"time" => "%{[sensor-time][time]}"
"serial-number" => "%{[sensor-info][serial-number]}"
"ip-address" => "%{[sensor-info][ip-address]}"
"group" => "%{[sensor-info][group]}"
}
}
}

output {
elasticsearch {
hosts => ["localhost:9200"]
index => "sensor"
}
stdout { codec => rubydebug { metadata => true }}
}

filebeat.yml

filebeat.prospectors:

  • input_type: log
    paths:
    • C:\Users\mufashid.manantavid\Desktop\Xovis Json*.json
      multiline.pattern: '^{'
      multiline.negate: true
      multiline.match: after
      #fields_under_root: true
      #json.keys_under_root: true
      #json.add_error_key: true
      #fields:
      #tags: ['json']

output.logstash:
hosts: ["localhost:5043"]
#template.name: filebeat
template.path: filebeat.template.json

This approach does not always work.

In the filebeat, the negated pattern is ^{. It accumulates lines of data and because there is no { following the last } (and perhaps also no newline after the last }) the last } is never sent to logstash. This means that the JSON is malformed as the error message says.

Try adding a newline after the last }.
I am not very familiar with the exact algorithm that filebeat uses (it was once coded very similar to LS multiline code, but I think the beats team have made some improvements).

How do I do that.

1 Like

if I add new line to 1 json file it is working fine but I am using more than 1 json file(Data is pusing from the external device). Is there any solution for add new line from filebeat or logstash.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.