Logstash rename json fields

Elastic Stack Logstash
1

Hi community,

We have a json log as below

{
  "Timestamp": "2023-02-09T17:41:54.5320239+03:00",
  "Level": "",
  "MessageTemplate": "",
  "Properties": {
    "responsetime": 4758,
    "SourceContext": "",
    "Username": null,
    "Url": "",
    "HttpMethod": "POST",
    "TraceId": "",
    "UserIP": "",
    "UserAgent": "",
    "RequestBody": "",
    "RequestQueryString": "",
    "ResponseBody": "",
    "RequestId": "",
    "RequestPath": "",
    "ConnectionId": "",
    "ApplicationName": ""
  }
}

We need to ship this json to any source as below

{
  "Timestamp": "2023-02-09T17:41:54.5320239+03:00",
  "Level": "",
  "MessageTemplate": "",
  "M1": 4758,
  "M3": "",
  "M4": null,
  "M5": "",
  "M6": "POST",
  "M7": "",
  "M9": "",
  "N8": "",
  "N6": "",
  "Y2": "",
  "...":"...."
}

so we need to change all fields name.

Thanks any help.

2

Sounds like a great case for the logstash and the mutate filter.

Something like this:

 mutate {
      rename => { "responsetime" => "M1" }
      rename => { "SourceContext" => "M3" }
    }

You'll also want to flatten out the json as well.

Good luck!

3

Hi Andrew,

My config is

input {
    file {
        path => ["/usr/share/logstash/pipeline/logs/*.log"]
        codec => json
        sincedb_path => "/dev/null"
        start_position => "beginning"
    }
}

filter {
    mutate {
      rename => { "responsetime" => "M1" }
      rename => { "SourceContext" => "M3" }
    }
}

output {
    file {
        path => ["/usr/share/logstash/pipeline/logs_migration/"]
    }
}

output is

{
  "event": {
    "original": "{\"Timestamp\":\"2023-02-09T17:41:54.5320239+03:00\",\"Level\":\"Information\",\"MessageTemplate\":\"Request finished in {responsetime} ms.\",\"Properties\":{\"responsetime\":4758,\"SourceContext\":\"VPOS.Application.RequestResponseLoggingMiddleware\",\"Username\":null,\"Url\":\"http://localhost:5187/api/token\",\"HttpMethod\":\"POST\",\"TraceId\":\"0HMOALF12PNG6:00000002\",\"UserIP\":\"::1\",\"UserAgent\":\"PostmanRuntime/7.30.0\",\"RequestBody\":\"{\\r\\n  \\\"username\\\": \\\"testadmin\\\",\\r\\n  \\\"password\\\": \\\"******\\\"\\r\\n}\",\"RequestQueryString\":\"\",\"ResponseBody\":\"{\\\"token\\\":\\\"4q23PcpOKO5YKVxzwvoTN0hQnp3GIZ328qWiIHUN2ueO2MwHa8N5RFu6P2ou3g\\\",\\\"created\\\":\\\"2023-02-09T17:41:54.510778+03:00\\\"}\",\"RequestId\":\"0HMOALF12PNG6:00000002\",\"RequestPath\":\"/api/token\",\"ConnectionId\":\"0HMOALF12PNG6\",\"ApplicationName\":\"VPOS.Auth.Api, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null\"}}\r"
  },
  "host": { "name": "de5133b1867a" },
  "log": { "file": { "path": "/usr/share/logstash/pipeline/logs/logs.log" } },
  "Level": "Information",
  "Properties": {
    "SourceContext": "",
    "RequestBody": "",
    "TraceId": "0HMOALF12PNG6:00000002",
    "RequestQueryString": "",
    "HttpMethod": "POST",
    "RequestPath": "",
    "Username": null,
    "ResponseBody": "",
    "ConnectionId": "0HMOALF12PNG6",
    "ApplicationName": "VPOS.Auth.Api, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null",
    "responsetime": 4758,
    "UserAgent": "PostmanRuntime/7.30.0",
    "Url": "",
    "UserIP": "::1",
    "RequestId": "0HMOALF12PNG6:00000002"
  },
  "@version": "1",
  "Timestamp": "2023-02-09T17:41:54.5320239+03:00",
  "MessageTemplate": "Request finished in {responsetime} ms.",
  "@timestamp": "2023-02-13T16:56:44.582486700Z"
}
4

The field name is wrong, you do not have a responsetime or SourceContext field, both are nested inder the Properties field, so you need to use the correct field name.

Try the following:

filter {
    mutate {
      rename => { "[Properties][responsetime]" => "M1" }
      rename => { "[Properties][SourceContext]" => "M3" }
    }
}
5

Oh yeah, probably wasn’t very clear by my "flatten out the json" comment but you’ll want to access the nested values like @leandrojmp says.

6

Hi all,

Thank you so much, it works fine.

7

You're welcome. Glad to hear.