Removing empty fields from an event (2026 edition!)

Elastic Stack Logstash
1

Continuing the discussion from Removing empty fields from an event (2024 edition!):

It now removes any empty objects left over after removing empty fields:

filter {
  ruby {
    id => "remove empty fields"
    path => "/usr/share/logstash/ruby/remove_empty_fields.rb"
  }
}

def _find_empty_fields(event, path = [], &blk)
  remove_self = true
  event.each do |k, v|
    curpath = path + [k]
    case v
    when nil
      yield curpath
    when ''
      yield curpath
    when '-'
      yield curpath
    when []
      yield curpath
    when {}
      yield curpath
    when Hash
      # return value tells us if we found at least one non-empty value below this level
      remove_self = _find_empty_fields(v, curpath, &blk) && remove_self
    else
      # we found at least one non-empty value at this level
      remove_self = false
    end
  end
  yield path if remove_self
  remove_self
end

def filter(event)
  _find_empty_fields(event.to_hash) { |path| event.remove('[' + path.join('][') + ']') }
  [event]
end

test 'with 3-nesting empty fields' do
  in_event {
    {
      'hello' => 'world',
      'foo' => {
        'bar' => {
          'baz' => ''
        }
      }
    }
  }

  expect('returns one event')               { |events| events.size == 1 }
  expect('removes the deepest field')       { |events| events[0].get('[foo][bar][baz]') == nil }
  expect('removes the mid field')           { |events| events[0].get('[foo][bar]') == nil }
  expect('removes the now-empty top field') { |events| events[0].get('[foo]') == nil }
  expect('leaves another one alone')        { |events| events[0].get('[hello]') == 'world' }
end

test 'with only deeper empty fields' do
  in_event {
    {
      'hello' => 'world',
      'foo' => {
        'name' => 'moose',
        'bar' => {
          'baz' => ''
        }
      }
    }
  }

  expect('returns one event')              { |events| events.size == 1 }
  expect('removes the deepest field')      { |events| events[0].get('[foo][bar][baz]') == nil }
  expect('removes the mid field')          { |events| events[0].get('[foo][bar]') == nil }
  expect('leaves the non-empty top field') { |events| events[0].get('[foo]') == { 'name' => 'moose' } }
  expect('leaves another one alone')       { |events| events[0].get('[hello]') == 'world' }
end

test 'with various empty-ish fields' do
  in_event {
    {
      'hello' => 'world',
      'blank' => '',
      'dash' => '-',
      'nil' => nil,
      'array' => [],
      'hash' => {},
      'one' => 1,
      'tardis' => ['space']
    }
  }

  expect('returns one event')              { |events| events.size == 1 }
  expect('removes the empty string')       { |events| events[0].get('[blank]') == nil }
  expect('removes the single-dash string') { |events| events[0].get('[dash]') == nil }
  expect('removes the null field')         { |events| events[0].to_hash.keys.include?('nil') == false }
  expect('removes the empty array')        { |events| events[0].get('[array]') == nil }
  expect('removes the empty hash')         { |events| events[0].get('[hash]') == nil }
  expect('leaves digits alone')            { |events| events[0].get('[one]') == 1 }
  expect('leaves strings alone')           { |events| events[0].get('[hello]') == 'world' }
  expect('leaves arrays alone')            { |events| events[0].get('[tardis]') == ['space'] }
end

and has tests!