Rename fields in logstash

Elastic Stack Logstash
1

Hello team,
from below log i am trying to remove below fields from logstash but it is not working as expected.

Can you please help me on this:

This is how field is coming in kibana:

image
image1520×422 41.7 KB

value.com.softwareag.um.server:brokerName=umserver,type=Broker.PublicKey

I had tried with below code but not working:

mutate {
      remove_field => ["[value][com][softwareag][um][server\:brokerName\=umserver\,type\=Broker][PublicKey]"]
	                     
    }

Also:

 mutate {
      remove_field => ["[value][com][softwareag][um][server:brokerName=umserver,type=Broker][PublicKey]"]
	                     
    }

mutate {
      remove_field => ["value.com.softwareag.um.server:brokerName=umserver,type=Broker.PublicKey"]
	                     
    }
2

Please share how the json of the document looks like.

In Kibana expand the document and go to the json tab to see how it looks like.

Also, what does your parse looks like? It seems that something is wrong to create fields like that.

3

Hi @leandrojmp
PFB sample json
Here i need to rename ""com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/dispatch/CallActivityReturn,destinationType=Topic,subtype=Configuration,type=Broker":"

{
    "request": {
        "mbean": "com.softwareag.um.server:brokerName=umserver,destinationName=*,destinationType=Topic,subtype=*,type=Broker",
        "type": "read"
    },
    "value": {
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/dispatch/CallActivityReturn,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "/wm/is/wm/prt/dispatch/CallActivityReturn",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/dispatch/CallActivityStart,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "/wm/is/wm/prt/dispatch/CallActivityStart",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/dispatch/ProcessTransitionVol,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "/wm/is/wm/prt/dispatch/ProcessTransitionVol",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/dispatch/nullDoc,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "/wm/is/wm/prt/dispatch/nullDoc",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/dispatch/ProcessTransition,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "/wm/is/wm/prt/dispatch/ProcessTransition",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=InvoiceTopic,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "InvoiceTopic",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/dispatch/SubprocessReturn,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "/wm/is/wm/prt/dispatch/SubprocessReturn",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/BR92bc8a40afd54a9b9b0655a215103a81_J_jRQtEolDEurgAVIwwwADXaZQs__RequestReplyClient,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "/wm/is/BR92bc8a40afd54a9b9b0655a215103a81_J_jRQtEolDEurgAVIwwwADXaZQs__RequestReplyClient",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/webservices/WebserviceReply,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "/wm/is/wm/prt/webservices/WebserviceReply",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/dispatch/SubprocessStart,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "/wm/is/wm/prt/dispatch/SubprocessStart",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/status/ChangeCommand,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "/wm/is/wm/prt/status/ChangeCommand",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=TestTopic,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "TestTopic",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/dispatch/SubprocessDone,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "/wm/is/wm/prt/dispatch/SubprocessDone",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/dispatch/Signal,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "/wm/is/wm/prt/dispatch/Signal",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/webservices/WebserviceRequest,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "/wm/is/wm/prt/webservices/WebserviceRequest",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/debug/ProcessComplete,destinationType=Topic,subtype=Configuration,type=Broker": {
            "CacheOnReload": true,
            "TypeName": "UM TOPIC",
            "SynOnEachWrite": false,
            "SyncMaxBatchSize": 0,
            "StampDictionary": "default",
            "PerformAutoMaintenance": true,
            "MultiFileEventsPerSpindle": 50000,
            "AbsolutePath": "/wm/is/wm/prt/debug/ProcessComplete",
            "UsesMergeEngine": false,
            "UsesJMSEngine": true,
            "TTL": 0,
            "ClusterWide": false,
            "HonorCapacityWhenFull": true,
            "StoreType": "Mixed",
            "Type": 2,
            "SyncBatchTime": 0,
            "Capacity": 0,
            "EnableCaching": true,
            "External": false,
            "DeadEventStore": "None"
        }
    },
    "status": 200,
    "timestamp": 1730263910
}
4

In logstash that would be called

[value][com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/dispatch/ProcessTransitionVol,destinationType=Topic,subtype=Configuration,type=Broker]

Your fields names have . embedded in them. kibana uses the name a.b for both { "a": { "b": 1 } } and { "a.b": 2 }, logstash does not. (It uses [a][b] and [a.b].)

What do you want to do with the keys within the [value] hash? If you delete them you won't have any data left. Do you want to rename them to something shorter?

5

HI @Badger ,
Thanks for the info .
My requirment is from below json line i need parse NoOfEvents, CurrentConnections ... fields seperatly without any json array and need to parse all fields seperatly from

Purpose: They need filter in dashboard for brokerName , destinationName and accordingly below details should be change

com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/debug/ProcessComplete,destinationType=Topic,type=Broker

like broker name , destination name

NOTE:
currently i have 2 values but realtime it may more than 100 .
Sample Log line:

{
    "request": {
        "mbean": "com.softwareag.um.server:brokerName=umserver,destinationName=*,destinationType=Topic,type=Broker",
        "type": "read"
    },
    "value": {
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/debug/ProcessComplete,destinationType=Topic,type=Broker": {
            "NoOfEvents": 0,
            "CurrentConnections": 0,
            "LastEID": 0,
            "FanoutLatency": 0,
            "UsedSpace": 800024,
            "MemoryUsage": 0,
            "ClusterWide": false,
            "Name": "/wm/is/wm/prt/debug/ProcessComplete",
            "TotalNoOfConnections": 0,
            "TotalConsumed": 0,
            "TotalPublished": 0,
            "PercentFree": 0,
            "CacheHitRatio": 0
        },
        "com.softwareag.um.server:brokerName=umserver,destinationName=/wm/is/wm/prt/status/ChangeCommand,destinationType=Topic,type=Broker": {
            "NoOfEvents": 0,
            "CurrentConnections": 0,
            "LastEID": 0,
            "FanoutLatency": 0,
            "UsedSpace": 800024,
            "MemoryUsage": 0,
            "ClusterWide": false,
            "Name": "/wm/is/wm/prt/status/ChangeCommand",
            "TotalNoOfConnections": 0,
            "TotalConsumed": 0,
            "TotalPublished": 0,
            "PercentFree": 0,
            "CacheHitRatio": 0
        }
		 },
    "status": 200,
    "timestamp": 1730263696
}
6

When you run a JMS query you get the [value] hash, which has multiple entries where the key is the object name and the value is a hash of data.

JMS object names are complicated. It may seem like the only thing you care about is the destinationName, but I would retain everything in case that changes. To do this I would split [value] so that each response is a separate document. Then I parse the object name into several fields in that document. If [value][subtype] is always configuration in your use case you lose very little by retaining it, and if it varies then you lose a lot by deleting it.

If you use

    # Put the full object name inside the object details
    ruby {
        code => '
            data = event.get("value")
            newData = []
            data.each { |k, v|
                v["fullName"] = k
                newData << v
            }
            event.set("value", newData)
        '
    }
    # Have one object name per event
    split { field => "value" }

    # Get the various parts of the object name into the object details
    mutate { rename => { "[value][fullName]" => "[@metadata][fullName]" } }

    mutate {
        split => { "[@metadata][fullName]" => ":" }
        add_field => { "[value][serverName]" => "%{[@metadata][fullName][0]}" }
    }

    kv { field_split => "," source => "[@metadata][fullName][1]" target => "[@metadata][value]" }
    mutate { merge => { "[value]" => "[@metadata][value]" } }

you will end up with events like

   "request" => {
    "mbean" => "com.softwareag.um.server:brokerName=umserver,destinationName=*,destinationType=Topic,subtype=*,type=Broker",
     "type" => "read"
},
 "timestamp" => 1730263910,
     "value" => {
    "MultiFileEventsPerSpindle" => 50000,
              "UsesMergeEngine" => false,
                 "AbsolutePath" => "/wm/is/wm/prt/dispatch/Signal",
                    "StoreType" => "Mixed",
                   "brokerName" => "umserver",
              "destinationType" => "Topic",
                   "serverName" => "com.softwareag.um.server",

etc. etc.

7

HI @Badger
Thanks for the quick help. Above code is worked for me.