Thank you for your reply.
The reason I am looking to go down this road is because we ran into performance issues when attempting to use a JDBC_static filter that essentially had two SQL queries that were polled every 30 minutes and stored in memory.
Here is a snippet of our previous config that was causing extreme queue depths:
filter {
jdbc_static {
loaders => [
{
id => "remote-clients"
query => "QUERY"
local_table => "clients"
},
{
id => "remote-client-details"
query => "QUERY"
local_table => "details"
}
]
local_db_objects => [
{
name => "clients"
index_columns => ["machinename"]
columns => [
["machinename", "varchar(30)"],
["collectionname", "varchar(100)"]
]
},
{
name => "details"
index_columns => ["name"]
columns => [
["name", "varchar(255)"],
["model", "varchar(255)"],
["systemtype", "varchar(255)"],
["resourceid", "varchar(255)"],
["oscaption", "varchar(255)"],
["osbuildnumber", "varchar(255)"],
["osversion", "varchar(255)"]
]
}
]
local_lookups => [
{
id => "local-clients"
query => "select machinename,collectionname from clients WHERE machinename = :beat_name"
parameters => { beat_name => "[host][name]" }
target => "client"
default_hash => {
"earlyadopter" => "NO"
}
},
{
id => "local-details"
query => "select name, model, systemtype, resourceid, oscaption, osbuildnumber, osversion from details WHERE name = :beat_name"
parameters => { beat_name => "[host][name]" }
target => "details"
}
]
add_field => { "[sentry_earlyadopter]" => "%{[client][0][collectionname]}" }
add_field => { "[model]" => "%{[details][0][model]}" }
add_field => { "[systemtype]" => "%{[details][0][systemtype]}" }
add_field => { "[os_caption]" => "%{[details][0][oscaption]}" }
add_field => { "[os_buildnumber]" => "%{[details][0][osbuildnumber]}" }
add_field => { "[os_version]" => "%{[details][0][osversion]}" }
remove_field => "[details]"
remove_field => "[client]"
jdbc_user => "USER"
jdbc_password => "PASSWORD"
jdbc_driver_class => "Java::net.sourceforge.jtds.jdbc.Driver"
jdbc_connection_string => "jdbc:jtds:sqlserver://SQL/CM_SEP;domain=domain.com;username=USER;password=PASSWORD"
jdbc_driver_library => "/usr/share/logstash/jtds-1.3.1-dist/jtds-1.3.1.jar"
loader_schedule => "35 * * * *"
}
}
What got me thinking about using translate were these two threads: 1 2.