How to drop any fields containing the word "PublicKey" from logstash

Elastic Stack Logstash
1

Hello team,
I wanted to drop all fields which containaing PublicKey word in the field name.

date=2019-05-10 time=11:37:47 PublicKey="abc" type.PublicKey="xyz" PublicKey.subtype="pqr" level="notice" vd="vdom1" eventtime=1557513467369913239

I want to drop all fields which contains PublicKey.

I have tried below Ruby code but it is not working:

 ruby {
    code => '
      event.to_hash.keys.each { |k| event.remove(k) if k.include?("PublicKey") }
    '
  }

Also:

ruby {
    code => '
      event.to_hash.keys.each { |k| 
        if k.include?("PublicKey") 
          event.remove(k) 
        end 
      }
    '
  }
2 
input { generator { count => 1 lines => [ 'date=2019-05-10 time=11:37:47 PublicKey="abc" type.PublicKey="xyz" PublicKey.subtype="pqr" level="notice" vd="vdom1" eventtime=1557513467369913239' ] } }
output { stdout { codec => rubydebug { metadata => false } } }
filter {
    kv {}
    ruby {
        code => '
            event.to_hash.keys.each { |k| event.remove(k) if k.include?("PublicKey") }
        '
    }
}

works as expected for me. Can you provide a reproducible example?

1 Like
3

Hi @Badger
Please find below log line where i need to drop all fields where PublicKey is preset. I tried your code but it is not working

input {
beats {
    port => 5044
  }
}


filter {

json {
source => "message"
}

ruby {
        code => '
            event.to_hash.keys.each { |k| event.remove(k) if k.include?("PublicKey") }
        '
    }
  
}

output {


elasticsearch {
                     hosts => ["http://localhost:9200"]  
                        index => "abc"
        
                }
				stdout {}


}



{"request":{"mbean":"com.softwareag.um.server:brokerName=*,type=Broker","type":"read"},"value":{"com.softwareag.um.server:brokerName=umserver,type=Broker":{"CurrentConnections":15,"RealmAdapters":"nhp:\/\/LI-LT654-HPG8:900","HasPublishingBeenPaused":false,"PublicKey":[48,-126,4,61,48,-126,3,37,-96,3,2,1,2,2,4,103,29,18,79,48,13,6,9,42,-122,72,-122,-9,13,1,1,11,5,0,48,-127,-50,49,17,48,15,6,3,85,4,3,12,8,117,109,115,101,114,118,101,114,49,109,48,107,6,3,85,4,11,12,100,65,117,116,111,109,97,116,105,99,97,108,108,121,32,103,101,110,101,114,97,116,101,100,32,98,121,32,116,104,101,32,114,101,97,108,109,32,115,101,114,118,101,114,44,32,102,111,114,32,109,111,114,101,32,105,110,102,111,114,109,97,116,105,111,110,32,99,111,110,116,97,99,116,32,109,121,45,99,104,97,110,110,101,108,115,44,32,43,52,52,50,48,55,51,55,53,55,52,48,48,49,20,48,18,6,3,85,4,10,12,11,109,121,45,67,104,97,110,110,101,108,115,49,15,48,13,6,3,85,4,7,12,6,76,111,110,100,111,110,49,22,48,20,6,3,85,4,8,12,13,71,114,101,97,116,32,66,114,105,116,97,105,110,49,11,48,9,6,3,85,4,6,19,2,71,66,48,30,23,13,50,52,49,48,50,54,49,54,48,49,49,57,90,23,13,51,52,49,48,50,52,49,54,48,49,49,57,90,48,-127,-50,49,17,48,15,6,3,85,4,3,12,8,117,109,115,101,114,118,101,114,49,109,48,107,6,3,85,4,11,12,100,65,117,116,111,109,97,116,105,99,97,108,108,121,32,103,101,110,101,114,97,116,101,100,32,98,121,32,116,104,101,32,114,101,97,108,109,32,115,101,114,118,101,114,44,32,102,111,114,32,109,111,114,101,32,105,110,102,111,114,109,97,116,105,111,110,32,99,111,110,116,97,99,116,32,109,121,45,99,104,97,110,110,101,108,115,44,32,43,52,52,50,48,55,51,55,53,55,52,48,48,49,20,48,18,6,3,85,4,10,12,11,109,121,45,67,104,97,110,110,101,108,115,49,15,48,13,6,3,85,4,7,12,6,76,111,110,100,111,110,49,22,48,20,6,3,85,4,8,12,13,71,114,101,97,116,32,66,114,105,116,97,105,110,49,11,48,9,6,3,85,4,6,19,2,71,66,48,-126,1,34,48,13,6,9,42,-122,72,-122,-9,13,1,1,1,5,0,3,-126,1,15,0,48,-126,1,10,2,-126,1,1,0,-83,-17,-30,-69,-21,-94,97,77,18,-49,30,59,58,-58,40,-123,10,-107,105,-88,30,-11,-47,92,17,121,-93,-56,126,56,110,-122,88,67,36,4,114,68,33,-42,-95,97,122,-80,-83,8,-2,15,-55,-20,-10,-113,95,-72,-36,-5,-64,-125,42,105,-52,127,92,-32,57,-7,-21,-25,60,52,-85,88,-8,-83,109,78,-48,-49,66,-69,4,116,-69,31,87,-6,75,15,-73,-57,-56,-85,-1,-57,-14,90,71,127,31,-128,-59,37,-7,-37,-27,56,-42,-128,35,-110,115,-69,63,-115,28,122,-101,-24,-88,-126,60,-105,79,-74,93,7,48,-41,-58,-98,34,-50,-77,0,115,-8,-77,30,7,-85,-13,124,-82,0,-116,-4,104,-64,-30,114,-33,-26,-63,-72,12,12,-15,20,24,86,3,-29,49,-116,-117,74,-77,6,82,48,-18,-51,88,63,-46,67,73,-7,-121,-94,37,-61,25,-56,80,-99,85,3,-50,27,31,-23,-105,-5,66,52,22,17,119,-93,99,-18,-19,67,-46,81,-34,67,-71,-46,-11,-31,-65,-15,85,33,-3,-36,27,26,92,-115,82,-37,-106,33,-126,121,-11,7,-126,25,-112,-116,41,105,-80,-46,-110,-89,7,50,-17,-110,51,-58,-60,50,-115,126,-25,50,-119,-126,-44,-29,2,3,1,0,1,-93,33,48,31,48,29,6,3,85,29,14,4,22,4,20,106,28,-6,-17,-29,114,45,79,121,-8,17,67,37,19,-27,71,-45,-112,-123,46,48,13,6,9,42,-122,72,-122,-9,13,1,1,11,5,0,3,-126,1,1,0,22,-42,11,100,24,76,74,-1,-34,68,-74,-75,90,-25,-4,-122,85,107,47,74,5,-89,-92,-97,122,-10,80,-112,85,21,-107,-15,-18,81,-83,11,-73,2,-94,119,-67,67,103,-65,-45,-128,98,106,-55,84,-3,-53,-1,36,61,-63,18,-66,81,-110,45,102,-12,-92,-58,-59,52,-108,112,-92,-45,1,54,-119,-102,-22,12,22,-43,77,90,-21,70,20,55,-84,-112,-78,80,6,69,121,-122,37,20,79,-69,5,-120,38,-69,-116,-35,-97,73,103,-25,15,-40,-114,0,76,21,-104,105,76,-78,-90,22,-116,127,-21,-1,107,-126,-89,63,-30,125,88,-27,-104,-117,35,54,-6,-102,-25,-34,26,-3,-17,85,55,-102,58,4,-56,97,98,-39,93,-23,-72,-61,-60,98,-116,126,-14,-125,-122,-60,9,122,-6,-30,1,125,105,14,-67,106,-76,-58,39,-12,-35,91,-85,-8,17,-84,43,-81,30,92,-115,-104,12,5,42,78,58,-59,50,60,74,6,-97,-125,-32,-21,112,-112,-25,-76,-13,49,-7,100,-96,-114,2,47,-21,87,118,16,-9,17,39,14,49,53,-18,-39,73,50,-58,-122,-57,13,-103,-7,127,18,50,126,-22,-41,-57,-21,-5,-116,10,-56,-45,-4,93,-95,-44,86,81,-19,82],"TotalConsumedCount":0,"CPU":0.5853521227836609,"DirectMemory":867,"DiskUsage":228268,"TotalPublishedEvents":0,"NumberOfKnownRealms":1,"ZoneType":"","MemoryFree":963028840,"ZoneName":"","ClientAdapters":"nhp:\/\/LI-LT654-HPG8:900","BytesOut":10316131,"HeapMemory":918,"BytesIn":233742,"MemoryMax":1073741824,"TotalConnections":15,"NumberOfStores":18}},"status":200,"timestamp":1730263431}
4

My original answer just tests the top-level fields. If you want to remove fields nested inside of fields then try

    ruby {
        init => '
            def doSomething(object, name, event)
                if object   # Remove this if your use-case needs to process nil objects
                    # If we need to handle non-leaf nodes then test this first
                    if name.include?("PublicKey")
                        event.remove(name)
                    elsif object.kind_of?(Hash) and object != {}
                        object.each { |k, v| doSomething(v, "#{name}[#{k}]", event) }
                    elsif object.kind_of?(Array) and object != []
                        object.each_index { |i|
                            doSomething(object[i], "#{name}[#{i}]", event)
                        }
                    end
                end
            end

        '
        code => '
            event.to_hash.each { |k, v|
                doSomething(v, "[#{k}]", event)
            }
        '
    }

It seems like half the times I use this type of loop I find something that breaks my original version and variants. This time it was the need to run the test against non-leaf nodes :rofl:

5

Hi @Badger
Thanks for the quick help . It is working as expected now