Translate filter trouble

@yaauie

Thank you for your reply.

The reason I am looking to go down this road is because we ran into performance issues when attempting to use a JDBC_static filter that essentially had two SQL queries that were polled every 30 minutes and stored in memory.

Here is a snippet of our previous config that was causing extreme queue depths:

filter {

  jdbc_static {

    loaders => [ 

      {

        id => "remote-clients"

        query => "QUERY"

        local_table => "clients"

      },

      {

        id => "remote-client-details"

        query => "QUERY"

        local_table => "details"

      }      

    ]

    local_db_objects => [ 

      {

        name => "clients"

        index_columns => ["machinename"]

        columns => [

          ["machinename", "varchar(30)"],

          ["collectionname", "varchar(100)"]

        ]

      },

      {

        name => "details"

        index_columns => ["name"]

        columns => [

          ["name", "varchar(255)"],

          ["model", "varchar(255)"],

          ["systemtype", "varchar(255)"],

          ["resourceid", "varchar(255)"],

          ["oscaption", "varchar(255)"],

          ["osbuildnumber", "varchar(255)"],

          ["osversion", "varchar(255)"]

        ]

      }      

    ]

    local_lookups => [ 

      {

        id => "local-clients"

        query => "select machinename,collectionname from clients WHERE machinename = :beat_name"

        parameters => { beat_name => "[host][name]" }

        target => "client"

  default_hash => {

    "earlyadopter" => "NO"

  }

      },

      {

        id => "local-details"

        query => "select name, model, systemtype, resourceid, oscaption, osbuildnumber, osversion from details WHERE name = :beat_name"

        parameters => { beat_name => "[host][name]" }

        target => "details" 

      }      

    ]   

  add_field => { "[sentry_earlyadopter]" => "%{[client][0][collectionname]}" }

    add_field => { "[model]" => "%{[details][0][model]}" }

    add_field => { "[systemtype]" => "%{[details][0][systemtype]}" }

    add_field => { "[os_caption]" => "%{[details][0][oscaption]}" }

    add_field => { "[os_buildnumber]" => "%{[details][0][osbuildnumber]}" }

    add_field => { "[os_version]" => "%{[details][0][osversion]}" }   

    remove_field => "[details]"  

    remove_field => "[client]" 

    jdbc_user => "USER"

    jdbc_password => "PASSWORD"

    jdbc_driver_class => "Java::net.sourceforge.jtds.jdbc.Driver"

    jdbc_connection_string => "jdbc:jtds:sqlserver://SQL/CM_SEP;domain=domain.com;username=USER;password=PASSWORD"

    jdbc_driver_library => "/usr/share/logstash/jtds-1.3.1-dist/jtds-1.3.1.jar"

    loader_schedule => "35 * * * *"

  }
  }

What got me thinking about using translate were these two threads: 1 2.