Want to parse a nested json form filebeat to elasticsearch

Elastic Stack Beats
1

Hello!
I am having a hard time finding the right configuration for Filebeat to be able to correctly parse nested JSON log lines.
filbeat.yml file
</
filebeat.inputs:

  • type: log
    enabled: true
    paths:
    • D:/Development_Avecto/test-log/tLog-file.log
      input_type: log
      json.keys_under_root: true
      json.add_error_key: true
      processors:
  • decode_json_fields:
    fields: ["message.projectId", "message.projectData.Title", "message.ProjectTeam.projectId", "message.ProjectTeam.userProfileId"]
    process_array: false
    output.elasticsearch:
    hosts: ["localhost:9200"]
    index: "project-access-%{+yyyy.MM.dd}"
    username: "elastic"
    password: "elastic575841"
    setup.template:
    name: 'project-access'
    pattern: 'project-access-*'
    enabled: true

Below, here is my sample log file

<
{"message":[{"projectId":"3333","projectData":{"Title":"TNA3"},"ProjectTeam":{"projectId":"3333","userProfileId":"735185"}},{"projectId":"4444","projectData":{"Title":"Sprint1"},"ProjectTeam":{"projectId":"4444","userProfileId":"735185"}},{"projectId":"5555","projectData":{"Title":"Wave2"},"ProjectTeam":{"projectId":"5555","userProfileId":"735185"}}],"level":"info"}
{"message":[{"projectId":"3333","projectData":{"Title":"TNA3"},"ProjectTeam":{"projectId":"3333","userProfileId":"735185"}},{"projectId":"4444","projectData":{"Title":"Sprint1"},"ProjectTeam":{"projectId":"4444","userProfileId":"735185"}},{"projectId":"5555","projectData":{"Title":"Wave2"},"ProjectTeam":{"projectId":"5555","userProfileId":"735185"}}],"level":"info"}

Here is my elasticsearch log error
<
2019-06-27T11:28:01,442][DEBUG][o.e.a.b.TransportShardBulkAction] [hsE0JyO] [project-access-2019.06.27][3] failed to execute bulk item (index) index {[project-access-2019.06.27][doc][kNiDl2sBZj3h-CAiH6bS], source[{"@timestamp":"2019-06-27T05:57:58.912Z","offset":740,"log":{"file":{"path":"D:\Development_Avecto\test-log\tLog-file.log"}},"prospector":{"type":"log"},"host":{"name":"PC387296"},"level":"info","message":[{"projectData":{"Title":"TNA3"},"ProjectTeam":{"projectId":"3333","userProfileId":"735185"},"projectId":"3333"},{"ProjectTeam":{"projectId":"4444","userProfileId":"735185"},"projectId":"4444","projectData":{"Title":"Sprint1"}},{"projectData":{"Title":"Wave2"},"ProjectTeam":{"projectId":"5555","userProfileId":"735185"},"projectId":"5555"}],"source":"D:\Development_Avecto\test-log\tLog-file.log","input":{"type":"log"},"beat":{"name":"PC387296","hostname":"PC387296","version":"6.6.2"}}]}
org.elasticsearch.index.mapper.MapperParsingException: failed to parse field [message] of type [text]
at org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:303) ~[elasticsearch-6.6.2.jar:6.6.2]

Anyone pls help me.

Thanks.

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.