Parse json in Log field to get individual fields for visualization

Elastic Stack Logstash
7

You can use Ingest Processors in Elastic. Here is a simulation so you can see how it's done. Run in Dev Tools in Kibana.

POST _ingest/pipeline/_simulate
{
  "pipeline": {
    "description": "...",
    "processors": [
      {
        "grok": {
          "field": "message",
          "patterns": [
            """
              %{DATE}T%{TIME} %{NOTSPACE:log_level} \[%{DATA:worker}\] %{DATA:message_type} - %{GREEDYDATA:json_message}
            """
          ]
        }
      },
      {
        "json": {
          "field": "json_message",
          "add_to_root": true
        }
      },
      {
        "remove": {
          "field": [
            "message",
            "json_message"
          ]
        }
      }
    ]
  },
  "docs": [
    {
      "_source": {
        "message": """
          2021-12-08T14:02:49.899 INFO [bwEngThread:In-Memory Process Worker-1] c.t.b.p.g.L.E.LogMessage - {"env":"<mark>DEV</mark>","appName":"GenLogs","transactionID":"449d8241-392f-4877-a5ea-dddeebed3c29","timestamp":"1638972169775","srcApplication":"EPIC","operation":"testLog","type":"INFO","message":"New message logged.","payload":"<timer:TimerOutputSchema xmlns:timer=\"http://tns.tibco.com/bw/activity/timer/xsd/output\"><Now>1638972169328</Now><Hour>2</Hour><Minute>2</Minute><Second>49</Second><Week>50</Week><Month>12</Month><Year>2021</Year><Date>2021-12-08</Date><Time>2:02:49 PM</Time><DayOfMonth>8</DayOfMonth></timer:TimerOutputSchema>"}
        """
      }
    }
  ]
}