I'm trying to aggregated some results from my statement and i was following this example over here: https://www.javacodegeeks.com/2017/10/aggregate-index-data-elasticsearch-using-logstash-jdbc.html
But I'm getting some error's
Where's the logstash config
# file: simple-out.conf
input {
jdbc {
jdbc_connection_string => "jdbc:mysql://localhost:3306/mestradods"
jdbc_user => "root"
jdbc_password => ""
jdbc_validate_connection => true
jdbc_driver_library => "C:\Program Files (x86)\MySQL\Connector.J 5.1\mysql-connector-java-5.1.41-bin.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
statement => "SELECT p.*, f.* FROM mestradods.projetos p join mestradods.fileuploaded f on p.projeto_id = f.projeto_id where p.projeto_id > :sql_last_value order by p.projeto_id;"
use_column_value => true
tracking_column => projeto_id
schedule => "*/5 * * * * *"
last_run_metadata_path => "G:\logstash\es_jdbc_last_run_metadata.log"
}
}
filter {
aggregate {
task_id => "%{projeto_id}"
code => "
map['projeto_id'] = event.get('projeto_id')
map['projeto_datacriacao'] = event.get('projeto_datacriacao')
map['projeto_sessao'] = event.get('projeto_sessao')
map['projeto_nome'] = event.get('projeto_nome')
map['projeto_path'] = event.get('projeto_path')
map['fileuploaded_list'] ||= []
map['fileuploaded'] ||= []
if(event.get('file_id') != nil)
if!(map['fileuploaded_list'].include? event.get('file_id'))
map['fileuploaded_list'] << event.get('file_id')
map['fileuploaded'] << {
'file_id' => event.get('file_id'),
'file_name' => event.get('file_name'),
'file_name_encode' => event.get('file_name_encode'),
'file_datacriacao' => event.get('file_datacriacao'),
'file_size' => event.get('file_size'),
'file_path' => event.get('file_path'),
'file_type' => event.get('file_type')
}
end
end
event.cancel()
"
push_previous_map_as_event => true
timeout => 150000
}
}
output {
#stdout { codec => json_lines }
elasticsearch {
index => "mestrado"
document_type => "infos"
document_id => "%{projeto_id}"
hosts => "localhost"
}
}
ERROR 1:
[2018-06-29T02:34:02,635][ERROR][org.logstash.Logstash ] java.lang.IllegalStateException: org.jruby.exceptions.RaiseException: (SyntaxError) (aggregate filter code):24: syntax error, unexpected kEND
end
After i removed one end I got this ERROR:
[2018-06-29T02:20:19,971][ERROR][logstash.filters.aggregate] Aggregate exception occurred {:error=>#<NoMethodError: undefined methodif!' for #LogStash::Filters::Aggregate:0x1caefe75>, :code=>" \n map['projeto_id'] = event.get('projeto_id')\n map['projeto_datacriacao'] = event.get('projeto_datacriacao')\n map['projeto_sessao'] = event.get('projeto_sessao')\n\t map['projeto_nome'] = event.get('projeto_nome')\n\t map['projeto_path'] = event.get('projeto_path')\n\t \n\t map['fileuploaded_list'] ||= []\n\t map['fileuploaded'] ||= []\t \n\t if(event.get('file_id') != nil)\n\t\t if!(map['fileuploaded_list'].include? event.get('file_id'))\n\t\t\t map['fileuploaded_list'] << event.get('file_id')\n\t\t\t \n\t\t\t map['fileuploaded'] << {\n\t\t\t\t 'file_id' => event.get('file_id'),\n\t\t\t\t 'file_name' => event.get('file_name'),\n\t\t\t\t 'file_name_encode' => event.get('file_name_encode'),\n\t\t\t\t 'file_datacriacao' => event.get('file_datacriacao'),\n\t\t\t\t 'file_size' => event.get('file_size'),\n\t\t\t\t 'file_path' => event.get('file_path'),\n\t\t\t\t 'file_type' => event.get('file_type')\n\t\t\t }\n\t\t end\n\t\tevent.cancel()\n ", :map=>{"projeto_id"=>8, "projeto_datacriacao"=>2017-08-13T04:33:54.000Z, "projeto_sessao"=>"3QoTLWI-pXsXNDcvhIzuB3_WmxIugKPRykiuPAca", "projeto_nome"=>"Teste", "projeto_path"=>"C:\Users\Bruno\Documents\Arquivos\oVWeV2qCi5dMvytKXpY1709APsHAMnqCpeUN2yqd\8", "fileuploaded_list"=>[], "fileuploaded"=>[]}, :event_data=>{"file_path"=>"C:\Users\Bruno\Documents\Arquivos\oVWeV2qCi5dMvytKXpY1709APsHAMnqCpeUN2yqd\8\APkvoLW5x", "projeto_path"=>"C:\Users\Bruno\Documents\Arquivos\oVWeV2qCi5dMvytKXpY1709APsHAMnqCpeUN2yqd\8", "file_name_encode"=>"APkvoLW5x", "file_name"=>"16_S16_L001_R1_001", "projeto_nome"=>"Teste", "file_datacriacao"=>2017-08-13T04:35:03.000Z, "file_size"=>"158,2 MB", "@timestamp"=>2018-06-29T05:20:16.860Z, "projeto_id"=>8, "projeto_sessao"=>"3QoTLWI-pXsXNDcvhIzuB3_WmxIugKPRykiuPAca", "file_type"=>"fastq", "file_id"=>191, "@version"=>"1", "projeto_datacriacao"=>2017-08-13T04:33:54.000Z}}
Can you guys help me with that?
