Reading containers logs using FileBeat and logstash

Elastic Stack Logstash
1

I am trying to read docker containers logs using Filebeat and logstash. right now, i got multiple outputs like following:

{
      "@version" => "1",
         "event" => {
        "original" => "233.61.46.84 - - [22/Jun/2023:16:27:07 +0000] \"HEAD /solutions/monetize HTTP/1.0\" 404 18502"
    },
       "message" => "233.61.46.84 - - [22/Jun/2023:16:27:07 +0000] \"HEAD /solutions/monetize HTTP/1.0\" 404 18502",
         "agent" => {
             "version" => "8.8.1",
        "ephemeral_id" => "c3d1e355-9d97-44c9-9ca3-21e36021a8f7",
                  "id" => "2fc7e211-da52-4f9a-bd44-7dd36953ee88",
                "type" => "filebeat",
                "name" => "filebeat"
    },
           "ecs" => {
        "version" => "8.0.0"
    },
     "container" => {
        "id" => "lib"
    },
          "host" => {
        "name" => "filebeat"
    },
         "input" => {
        "type" => "container"
    },
    "@timestamp" => 2023-06-22T16:27:07.139Z,
          "json" => {},
          "tags" => [
        [0] "beats_input_codec_plain_applied"
    ],
           "log" => {
          "file" => {
            "path" => "/usr/share/filebeat/host_docker/lib/docker/containers/61e3b35465485c9e85308b024ea500afef3f1ebafca6742279deb31405d41d15/61e3b35465485c9e85308b024ea500afef3f1ebafca6742279deb31405d41d15-json.log"
        },
        "offset" => 175718
    },
        "stream" => "stdout"
}

my logstash.conf is

input {
  beats {
    port => 5044
    host => "0.0.0.0"
  }
}

filter {
    json {
      source => "message"
    }
}

output {
  stdout { codec => rubydebug }
}

however, i got

2023-06-22T17:08:31,772][WARN ][logstash.filters.json    ][main][119cb09ed8928c3e63724e778fa18c71f01f518e8fa07fc8d7e819d6d799c9c2] Error parsing json {:source=>"message", :raw=>"19.137.28.198 - - [22/Jun/2023:16:27:07 +0000] \"HEAD /holistic/aggregate/synergize/grow HTTP/2.0\" 203 20383", :exception=>#<LogStash::Json::ParserError: Unexpected character ('.' (code 46)): Expected space separating root-level values
 at [Source: (byte[])"19.137.28.198 - - [22/Jun/2023:16:27:07 +0000] "HEAD /holistic/aggregate/synergize/grow HTTP/2.0" 203 20383"; line: 1, column: 8]>}
[2023-06-22T17:08:31,773][WARN ][logstash.filters.json    ][main][119cb09ed8928c3e63724e778fa18c71f01f518e8fa07fc8d7e819d6d799c9c2] Error parsing json {:source=>"message", :raw=>"151.79.131.120 - lowe4281 [22/Jun/2023:16:27:07 +0000] \"PUT /turn-key/cross-platform HTTP/1.0\" 304 27783", :exception=>#<LogStash::Json::ParserError: Unexpected character ('.' (code 46)): Expected space separating root-level values
 at [Source: (byte[])"151.79.131.120 - lowe4281 [22/Jun/2023:16:27:07 +0000] "PUT /turn-key/cross-platform HTTP/1.0" 304 27783"; line: 1, column: 8]>}
[2023-06-22T17:08:31,773][WARN ][logstash.filters.json    ][main][119cb09ed8928c3e63724e778fa18c71f01f518e8fa07fc8d7e819d6d799c9c2] Error parsing json {:source=>"message", :raw=>"175.192.20.43 - sauer6437 [22/Jun/2023:16:27:07 +0000] \"HEAD /deliverables/integrated/interfaces HTTP/1.1\" 404 10678", :exception=>#<LogStash::Json::ParserError: Unexpected character ('.' (code 46)): Expected space separating root-level values

is there a correct way to file beats output to save message, timesstamp, container ID and use grok to push data to ADX?

Thank you all.

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.