Splitting an array of objects using Logstash

Elastic Stack Logstash
1

Hello everyone,

I'm trying to split the following array of objects into multiple log events:

[
  {
    "time": "*",
    "twkMessageId": "*",
    "environmentName": "*",
    "virtualhostName": "default",
    "apiproxyName": "*",
    "proxyBasepath": "*",
    "uri": "*",
    "clientIp": "*",
    "clientCountry": null,
    "clientHost": "*",
    "currentFlowName": "PreFlow",
    "logLevel": "INFO",
    "logMessage": {
      "httpStatusCode": "",
      "verb": "POST",
      "target": "",
      "targetUri": "",
      "clientFirstName": "",
      "clientLastName": "",
      "clientUsername": "",
      "clientEmail": "",
      "productName": "",
      "appId": "",
      "appName": "",
      "appStatus": "",
      "appExpiresAt": "",
      "headers": {
        "Accept": "****",
        "Accept-Encoding": "****",
        "Access-Control-Allow-Credentials": "****",
        "Access-Control-Allow-Headers": "****",
        "Access-Control-Allow-Methods": "****",
        "Access-Control-Allow-Origin": "****",
        "Access-Control-Max-Age": "****",
        "Authorization": "****",
        "Content-Length": "****",
        "Content-Type": "****",
        "Host": "****",
        "Postman-Token": "****",
        "User-Agent": "****",
        "X-Content-Type-Options": "****",
        "X-Forwarded-For": "****",
        "X-Forwarded-Port": "****",
        "X-Forwarded-Proto": "****",
        "X-XSS-Protection": "****"
      },
      "hashedHeaders": "*",
      "queryParams": "",
      "payload": {
        "post.statusId": "****",
        "post.body": "****",
        "post.mediaId": "****",
        "post.actions.0.id": "****",
        "post.actions.0.order": "****",
        "post.actions.0.label": "****",
        "post.actions.0.params.0.key": "****",
        "post.actions.0.params.0.value": "****",
        "post.actions.0.value": "****"
      },
      "hashedPayload": "*",
      "organizationName": "*",
      "clientReceivedStartTime": "*",
      "clientReceivedEndTime": "*",
      "duration": 5,
      "status": ""
    }
  },
  {
    "time": "*",
    "twkMessageId": "*",
    "environmentName": "*",
    "virtualhostName": "*",
    "apiproxyName": "*",
    "proxyBasepath": "*",
    "uri": "*",
    "clientIp": "*",
    "clientCountry": null,
    "clientHost": "*",
    "currentFlowName": "PostFlow",
    "logLevel": "INFO",
    "logMessage": {
      "httpStatusCode": "",
      "verb": "POST",
      "target": "*",
      "targetUri": "",
      "clientFirstName": "*",
      "clientLastName": "*",
      "clientUsername": "",
      "clientEmail": "*",
      "productName": "",
      "appId": "*",
      "appName": "",
      "appStatus": "approved",
      "appExpiresAt": "",
      "headers": {
        "Accept": "****",
        "Accept-Encoding": "****",
        "Access-Control-Allow-Credentials": "****",
        "Access-Control-Allow-Headers": "****",
        "Access-Control-Allow-Methods": "****",
        "Access-Control-Allow-Origin": "****",
        "Access-Control-Max-Age": "****",
        "Authorization": "****",
        "Content-Length": "****",
        "Content-Type": "****",
        "Host": "****",
        "Postman-Token": "****",
        "User-Agent": "****",
        "X-Content-Type-Options": "****",
        "X-Forwarded-For": "****",
        "X-Forwarded-Port": "****",
        "X-Forwarded-Proto": "****",
        "X-XSS-Protection": "****"
      },
      "hashedHeaders": "*",
      "queryParams": "",
      "payload": {
        "post.statusId": "****",
        "post.body": "****",
        "post.mediaId": "****",
        "post.actions.0.id": "****",
        "post.actions.0.order": "****",
        "post.actions.0.label": "****",
        "post.actions.0.params.0.key": "****",
        "post.actions.0.params.0.value": "****",
        "post.actions.0.value": "****"
      },
      "hashedPayload": "*",
      "organizationName": "*",
      "clientReceivedStartTime": "*",
      "clientReceivedEndTime": "*",
      "duration": 5,
      "status": ""
    }
  }
]

Here is my config file:

input {
  beats {
    port => 5044
  }
}
filter {
    json {
       source => "message"
       target => "logs"
         }
split {
        field => "logs"
         }
}
output {
  elasticsearch {
    hosts => [localhost:9200"]
    user => '*'
    password => '*'
    index => "logsatsh_test-%{+YYYY.MM.dd}"
  }
}

In Kibana i got the logs mapped almost correctly but it's still being logged as single event with the error that reads:

Error decoding JSON: json: cannot unmarshal array into Go value of type map[string]interface {}

Note: I'm using Logstash with Filebeat.

2

That is a Go error, which means it comes from filebeat, not logstash. You must be trying to parse the JSON in filebeat, so this is a filebeat question, not a logstash question.

3

yes, but i also commented out output.elasticsearch in the filebeat yaml. Any idea on why is it showing that error message?

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.