Logstash rename json fields

Hi community,

We have a json log as below

{
  "Timestamp": "2023-02-09T17:41:54.5320239+03:00",
  "Level": "",
  "MessageTemplate": "",
  "Properties": {
    "responsetime": 4758,
    "SourceContext": "",
    "Username": null,
    "Url": "",
    "HttpMethod": "POST",
    "TraceId": "",
    "UserIP": "",
    "UserAgent": "",
    "RequestBody": "",
    "RequestQueryString": "",
    "ResponseBody": "",
    "RequestId": "",
    "RequestPath": "",
    "ConnectionId": "",
    "ApplicationName": ""
  }
}

We need to ship this json to any source as below

{
  "Timestamp": "2023-02-09T17:41:54.5320239+03:00",
  "Level": "",
  "MessageTemplate": "",
  "M1": 4758,
  "M3": "",
  "M4": null,
  "M5": "",
  "M6": "POST",
  "M7": "",
  "M9": "",
  "N8": "",
  "N6": "",
  "Y2": "",
  "...":"...."
}

so we need to change all fields name.

Thanks any help.

Sounds like a great case for the logstash and the mutate filter.

Something like this:

 mutate {
      rename => { "responsetime" => "M1" }
      rename => { "SourceContext" => "M3" }
    }

You'll also want to flatten out the json as well.

Good luck!

Hi Andrew,

My config is

input {
    file {
        path => ["/usr/share/logstash/pipeline/logs/*.log"]
        codec => json
        sincedb_path => "/dev/null"
        start_position => "beginning"
    }
}

filter {
    mutate {
      rename => { "responsetime" => "M1" }
      rename => { "SourceContext" => "M3" }
    }
}

output {
    file {
        path => ["/usr/share/logstash/pipeline/logs_migration/"]
    }
}

output is

{
  "event": {
    "original": "{\"Timestamp\":\"2023-02-09T17:41:54.5320239+03:00\",\"Level\":\"Information\",\"MessageTemplate\":\"Request finished in {responsetime} ms.\",\"Properties\":{\"responsetime\":4758,\"SourceContext\":\"VPOS.Application.RequestResponseLoggingMiddleware\",\"Username\":null,\"Url\":\"http://localhost:5187/api/token\",\"HttpMethod\":\"POST\",\"TraceId\":\"0HMOALF12PNG6:00000002\",\"UserIP\":\"::1\",\"UserAgent\":\"PostmanRuntime/7.30.0\",\"RequestBody\":\"{\\r\\n  \\\"username\\\": \\\"testadmin\\\",\\r\\n  \\\"password\\\": \\\"******\\\"\\r\\n}\",\"RequestQueryString\":\"\",\"ResponseBody\":\"{\\\"token\\\":\\\"4q23PcpOKO5YKVxzwvoTN0hQnp3GIZ328qWiIHUN2ueO2MwHa8N5RFu6P2ou3g\\\",\\\"created\\\":\\\"2023-02-09T17:41:54.510778+03:00\\\"}\",\"RequestId\":\"0HMOALF12PNG6:00000002\",\"RequestPath\":\"/api/token\",\"ConnectionId\":\"0HMOALF12PNG6\",\"ApplicationName\":\"VPOS.Auth.Api, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null\"}}\r"
  },
  "host": { "name": "de5133b1867a" },
  "log": { "file": { "path": "/usr/share/logstash/pipeline/logs/logs.log" } },
  "Level": "Information",
  "Properties": {
    "SourceContext": "",
    "RequestBody": "",
    "TraceId": "0HMOALF12PNG6:00000002",
    "RequestQueryString": "",
    "HttpMethod": "POST",
    "RequestPath": "",
    "Username": null,
    "ResponseBody": "",
    "ConnectionId": "0HMOALF12PNG6",
    "ApplicationName": "VPOS.Auth.Api, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null",
    "responsetime": 4758,
    "UserAgent": "PostmanRuntime/7.30.0",
    "Url": "",
    "UserIP": "::1",
    "RequestId": "0HMOALF12PNG6:00000002"
  },
  "@version": "1",
  "Timestamp": "2023-02-09T17:41:54.5320239+03:00",
  "MessageTemplate": "Request finished in {responsetime} ms.",
  "@timestamp": "2023-02-13T16:56:44.582486700Z"
}

The field name is wrong, you do not have a responsetime or SourceContext field, both are nested inder the Properties field, so you need to use the correct field name.

Try the following:

filter {
    mutate {
      rename => { "[Properties][responsetime]" => "M1" }
      rename => { "[Properties][SourceContext]" => "M3" }
    }
}
1 Like

Oh yeah, probably wasn’t very clear by my "flatten out the json" comment but you’ll want to access the nested values like @leandrojmp says.

Hi all,

Thank you so much, it works fine.

You're welcome. Glad to hear.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.