Aggregate filter for nested data types

I have an INDEX test with the below mappings. Employees are nested arrays within departments which are themselves nested arrays within the organization document. When dealing with the below, how can an aggregate filter be created in logstash be created to load both departments and employees within departments.

PUT test
{
"mappings": {
"organization": {
"properties": {
"org_id": {"type":"integer"},
"org_name": {"type":"text"},
"departments": {
"type": "nested",
"properties": {
"dept_id": {"type":"integer"},
"dept_name": {"type":"text"},
"employees": {
"type":"nested",
"properties": {
"emp_id": {"type":"integer"},
"emp_name": {"type":"text"}
}
}
}
}
}
}
}
}

input {
jdbc {
jdbc_driver_library => "/../lib/postgresql-42.1.4.jar"
jdbc_driver_class => "org.postgresql.Driver"
jdbc_connection_string => "jdbc:postgresql://localhost:5432/postgres"
jdbc_user => "postgres"
jdbc_password => "postgres"
#jdbc_paging_enabled => "true"
#jdbc_page_size => "1"
statement => "SELECT o.org_id org_id, o.org_name org_name, d.dept_id dept_id, d.dept_name dept_name FROM organization o INNER JOIN department d ON d.org_id=o.org_id "
}
}
filter
{
aggregate {
task_id => "%{org_id}"
code => "
map['org_id']=event.get('org_id')
map['org_name']=event.get('org_name')
map['departments']||= []
map['departments'] <<
{
'dept_id'=> event.get('dept_id') ,
'dept_name'=> event.get('dept_name')
}
event.cancel()"
push_previous_map_as_event => true
timeout => 3
}

}

output {
elasticsearch {
hosts => ["localhost:9200"]
index => "test"
document_type => "organization"
document_id => "%{org_id}"
action =>"update"
}
#stdout { codec => rubydebug }
}

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.