Create new Filebeat

Elastic Stack Beats
18

Hey @saffatechy,

I have taken a look to your module and it needs some changes:

  • thread and category need to be prefixed with bizswitch.
  • timestamp should be parsed and stored in @timestamp.
  • Log message should be stored in message, or at least in a known field.
  • loglevel should be stored in log.level (or in some known field).

Find here a patch with some changes that work for me:

diff --git a/filebeat/module/bizswitch/log/ingest/pipeline.json b/filebeat/module/bizswitch/log/ingest/pipeline.json
index 43bf104344..d9b674433b 100644
--- a/filebeat/module/bizswitch/log/ingest/pipeline.json
+++ b/filebeat/module/bizswitch/log/ingest/pipeline.json
@@ -8,13 +8,21 @@
         "grok": {
           "field": "message",
           "patterns":
-["%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:loglevel} (%{SPACE})?(?<thread>(%{IP}[:]%{INT})?(-)?(Thread-%{INT})?(\\(%{INT}\\)-)?%{WORD}(:%{WORD})?([\\-])?(%{SPACE})?(([a-zA-Z]+)?(%{SPACE})?(%{WORD})?([:])?)(%{INT})?([:])?(%{INT})?([\\-])?(%{INT})?) \\[%{WORD:category}\\] %{GREEDYDATA:bizlog}"],
+["%{TIMESTAMP_ISO8601:_tmp.timestamp} %{LOGLEVEL:log.level} (%{SPACE})?(?<bizswitch.thread>(%{IP}[:]%{INT})?(-)?(Thread-%{INT})?(\\(%{INT}\\)-)?%{WORD}(:%{WORD})?([\\-])?(%{SPACE})?(([a-zA-Z]+)?(%{SPACE})?(%{WORD})?([:])?)(%{INT})?([:])?(%{INT})?([\\-])?(%{INT})?) \\[%{WORD:bizswitch.category}\\] %{GREEDYDATA:message}"],
           "ignore_missing":  true
           }
         },
+       {
+         "date": {
+            "field": "_tmp.timestamp",
+            "target_field": "@timestamp",
+            "formats": ["yyyy-MM-dd HH:mm:ss,SSS"],
+            "timezone": "{{ event.timezone }}"
+          }
+       },
         {
           "remove": {
-            "field": "message"
+            "field": "_tmp"
           }
         }
     ],