Parsing Json File with Logstash and Filebeat

Elastic Stack Logstash
1

Hi I am new to ES. I was trying to parse json file with logstash but I am getting error like below. Please help me to solve this issue

Error

<LogStash::Json::ParserError: Unexpected end-of-input: expected close marker for OBJECT (from [Source: [B@4df0e025; line: 1, column: 1])
at [Source: [B@4df0e025; line: 37, column: 868]>}
[2017-09-10T16:57:18,382][WARN ][logstash.filters.json ] Error parsing json {:source=>"message", :raw=>"{\n "sensor-time" : {\n "timezone" : "UTC",\n "time" : "2017-09-07T08:00:02Z"\n },\n "status" : {\n "code" : "OK"\n },\n "content" : {\n "element" : [ {\n "element-id" : 0,\n "element-name" : "Line 0",\n "sensor-type" : "SINGLE_SENSOR",\n "data-type" : "LINE",\n "from" : "2017-09-07T07:00:00Z",\n "to" : "2017-09-07T08:00:00Z",\n "resolution" : "ONE_HOUR",\n "measurement" : [ {\n "from" : "2017-09-07T07:00:00Z",\n "to" : "2017-09-07T08:00:00Z",\n "value" : [ {\n "value" : 31,\n "label" : "fw"\n }, {\n "value" : 27,\n "label" : "bw"\n } ]\n } ]\n } ]\n },\n "sensor-info" : {\n "serial-number" : "00:1E:C0:9D:41:ED",\n "ip-address" : "10.0.100.12",\n "name" : "Wireless Sensor 01",\n "group" : "SDS Wireless Lab",\n "device-type" : "PC2R"\n }", :exception=>#<LogStash::Json::ParserError: Unexpected end-of-input: expected close marker for OBJECT (from [Source: [B@5e21f900; line: 1, column: 1])
at [Source: [B@5e21f900; line: 37, column: 868]>}
[2017-09-10T16:57:18,383][WARN ][logstash.filters.json ] Error parsing json {:source=>"message", :raw=>"{\n "sensor-time" : {\n "timezone" : "UTC",\n "time" : "2017-09-06T13:00:02Z"\n },\n "status" : {\n "code" : "OK"\n },\n "content" : {\n "element" : [ {\n "element-id" : 0,\n "element-name" : "Line 0",\n "sensor-type" : "SINGLE_SENSOR",\n "data-type" : "LINE",\n "from" : "2017-09-06T12:00:00Z",\n "to" : "2017-09-06T13:00:00Z",\n "resolution" : "ONE_HOUR",\n "measurement" : [ {\n "from" : "2017-09-06T12:00:00Z",\n "to" : "2017-09-06T13:00:00Z",\n "value" : [ {\n "value" : 32,\n "label" : "fw"\n }, {\n "value" : 30,\n "label" : "bw"\n } ]\n } ]\n } ]\n },\n "sensor-info" : {\n "serial-number" : "00:1E:C0:9D:41:ED",\n "ip-address" : "10.0.100.12",\n "name" : "Wireless Sensor 01",\n "group" : "SDS Wireless Lab",\n "device-type" : "PC2R"\n }", :exception=>#<LogStash::Json::ParserError: Unexpected end-of-input: expected close marker for OBJECT (from [Source: [B@280b8ded; line: 1, column: 1])
at [Source: [B@280b8ded; line: 37, column: 868]>}
[2017-09-10T16:57:18,383][WARN ][logstash.filters.json ] Error parsing json {:source=>"message", :raw=>"{\n "sensor-time" : {\n "timezone" : "UTC",\n "time" : "2017-09-07T01:00:02Z"\n },\n "status" : {\n "code" : "OK"\n },\n "content" : {\n "element" : [ {\n "element-id" : 0,\n "element-name" : "Line 0",\n "sensor-type" : "SINGLE_SENSOR",\n "data-type" : "LINE",\n "from" : "2017-09-07T00:00:00Z",\n "to" : "2017-09-07T01:00:00Z",\n "resolution" : "ONE_HOUR",\n "measurement" : [ {\n "from" : "2017-09-07T00:00:00Z",\n "to" : "2017-09-07T01:00:00Z",\n "value" : [ {\n "value" : 0,\n "label" : "fw"\n }, {\n "value" : 0,\n "label" : "bw"\n } ]\n } ]\n } ]\n },\n "sensor-info" : {\n "serial-number" : "00:1E:C0:9D:41:ED",\n "ip-address" : "10.0.100.12",\n "name" : "Wireless Sensor 01",\n "group" : "SDS Wireless Lab",\n "device-type" : "PC2R"\n }", :exception=>#<LogStash::Json::ParserError: Unexpected end-of-input: expected close marker for OBJECT (from [Source: [B@4f638d4; line: 1, column: 1])
at [Source: [B@4f638d4; line: 37, column: 866]>}

sensor1.json

{
"sensor-time" : {
"timezone" : "UTC",
"time" : "2017-09-06T08:00:02Z"
},
"status" : {
"code" : "OK"
},
"content" : {
"element" : [ {
"element-id" : 0,
"element-name" : "Line 0",
"sensor-type" : "SINGLE_SENSOR",
"data-type" : "LINE",
"from" : "2017-09-06T07:00:00Z",
"to" : "2017-09-06T08:00:00Z",
"resolution" : "ONE_HOUR",
"measurement" : [ {
"from" : "2017-09-06T07:00:00Z",
"to" : "2017-09-06T08:00:00Z",
"value" : [ {
"value" : 10,
"label" : "fw"
}, {
"value" : 10,
"label" : "bw"
} ]
} ]
} ]
},
"sensor-info" : {
"serial-number" : "00:1E:C0:9D:41:ED",
"ip-address" : "10.0.100.12",
"name" : "Wireless Sensor 01",
"group" : "SDS Wireless Lab",
"device-type" : "PC2R"
}
}

Pipeline.conf

input {
beats {
type=> beats
port => "5043"
codec=> json
}
}

filter{
json{
source => "message"
}
mutate {
remove_field => [ "type", "tags"]
add_field => {
"time" => "%{[sensor-time][time]}"
"serial-number" => "%{[sensor-info][serial-number]}"
"ip-address" => "%{[sensor-info][ip-address]}"
"group" => "%{[sensor-info][group]}"
}
}
}

output {
elasticsearch {
hosts => ["localhost:9200"]
index => "sensor"
}
stdout { codec => rubydebug { metadata => true }}
}

filebeat.yml

filebeat.prospectors:

  • input_type: log
    paths:
    • C:\Users\mufashid.manantavid\Desktop\Xovis Json*.json
      multiline.pattern: '^{'
      multiline.negate: true
      multiline.match: after
      #fields_under_root: true
      #json.keys_under_root: true
      #json.add_error_key: true
      #fields:
      #tags: ['json']

output.logstash:
hosts: ["localhost:5043"]
#template.name: filebeat
template.path: filebeat.template.json

2

This approach does not always work.

In the filebeat, the negated pattern is ^{. It accumulates lines of data and because there is no { following the last } (and perhaps also no newline after the last }) the last } is never sent to logstash. This means that the JSON is malformed as the error message says.

Try adding a newline after the last }.
I am not very familiar with the exact algorithm that filebeat uses (it was once coded very similar to LS multiline code, but I think the beats team have made some improvements).

3

How do I do that.

1 Like
4

if I add new line to 1 json file it is working fine but I am using more than 1 json file(Data is pusing from the external device). Is there any solution for add new line from filebeat or logstash.

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.