Logstash aggregated error


(Bruno Andrade) #1

I'm trying to aggregated some results from my statement and i was following this example over here: https://www.javacodegeeks.com/2017/10/aggregate-index-data-elasticsearch-using-logstash-jdbc.html

But I'm getting some error's

Where's the logstash config

# file: simple-out.conf
input {
    jdbc {
     
        jdbc_connection_string => "jdbc:mysql://localhost:3306/mestradods"
        jdbc_user => "root"
		jdbc_password => ""
		jdbc_validate_connection => true
        jdbc_driver_library => "C:\Program Files (x86)\MySQL\Connector.J 5.1\mysql-connector-java-5.1.41-bin.jar"       
        jdbc_driver_class => "com.mysql.jdbc.Driver"       
        statement => "SELECT p.*, f.* FROM mestradods.projetos p join mestradods.fileuploaded f on p.projeto_id = f.projeto_id where p.projeto_id > :sql_last_value order by p.projeto_id;"
		use_column_value => true
		tracking_column => projeto_id
		schedule => "*/5 * * * * *"
		last_run_metadata_path => "G:\logstash\es_jdbc_last_run_metadata.log"
    }
}
filter {    

  aggregate {
    task_id => "%{projeto_id}"

    code => "     
      map['projeto_id'] = event.get('projeto_id')
      map['projeto_datacriacao'] = event.get('projeto_datacriacao')
      map['projeto_sessao'] = event.get('projeto_sessao')
	  map['projeto_nome'] = event.get('projeto_nome')
	  map['projeto_path'] = event.get('projeto_path')
	  
	  map['fileuploaded_list'] ||= []
	  map['fileuploaded'] ||= []	  
	  if(event.get('file_id') != nil)
		  if!(map['fileuploaded_list'].include? event.get('file_id'))
			  map['fileuploaded_list'] << event.get('file_id')
			  
			  map['fileuploaded'] << {
				  'file_id' => event.get('file_id'),
				  'file_name' => event.get('file_name'),
				  'file_name_encode' => event.get('file_name_encode'),
				  'file_datacriacao' => event.get('file_datacriacao'),
				  'file_size' => event.get('file_size'),
				  'file_path' => event.get('file_path'),
				  'file_type' => event.get('file_type')
			  }
		  end
		  end
		event.cancel()
    "
    push_previous_map_as_event => true
    timeout => 150000
  }
	
}
output {
    #stdout { codec => json_lines }
	elasticsearch {
		index => "mestrado"
		document_type => "infos"
		document_id => "%{projeto_id}"
		hosts => "localhost"
	}
	
}

ERROR 1:

[2018-06-29T02:34:02,635][ERROR][org.logstash.Logstash    ] java.lang.IllegalStateException: org.jruby.exceptions.RaiseException: (SyntaxError) (aggregate filter code):24: syntax error, unexpected kEND
                  end

After i removed one end I got this ERROR:

[2018-06-29T02:20:19,971][ERROR][logstash.filters.aggregate] Aggregate exception occurred {:error=>#<NoMethodError: undefined methodif!' for #LogStash::Filters::Aggregate:0x1caefe75>, :code=>" \n map['projeto_id'] = event.get('projeto_id')\n map['projeto_datacriacao'] = event.get('projeto_datacriacao')\n map['projeto_sessao'] = event.get('projeto_sessao')\n\t map['projeto_nome'] = event.get('projeto_nome')\n\t map['projeto_path'] = event.get('projeto_path')\n\t \n\t map['fileuploaded_list'] ||= []\n\t map['fileuploaded'] ||= []\t \n\t if(event.get('file_id') != nil)\n\t\t if!(map['fileuploaded_list'].include? event.get('file_id'))\n\t\t\t map['fileuploaded_list'] << event.get('file_id')\n\t\t\t \n\t\t\t map['fileuploaded'] << {\n\t\t\t\t 'file_id' => event.get('file_id'),\n\t\t\t\t 'file_name' => event.get('file_name'),\n\t\t\t\t 'file_name_encode' => event.get('file_name_encode'),\n\t\t\t\t 'file_datacriacao' => event.get('file_datacriacao'),\n\t\t\t\t 'file_size' => event.get('file_size'),\n\t\t\t\t 'file_path' => event.get('file_path'),\n\t\t\t\t 'file_type' => event.get('file_type')\n\t\t\t }\n\t\t end\n\t\tevent.cancel()\n ", :map=>{"projeto_id"=>8, "projeto_datacriacao"=>2017-08-13T04:33:54.000Z, "projeto_sessao"=>"3QoTLWI-pXsXNDcvhIzuB3_WmxIugKPRykiuPAca", "projeto_nome"=>"Teste", "projeto_path"=>"C:\Users\Bruno\Documents\Arquivos\oVWeV2qCi5dMvytKXpY1709APsHAMnqCpeUN2yqd\8", "fileuploaded_list"=>[], "fileuploaded"=>[]}, :event_data=>{"file_path"=>"C:\Users\Bruno\Documents\Arquivos\oVWeV2qCi5dMvytKXpY1709APsHAMnqCpeUN2yqd\8\APkvoLW5x", "projeto_path"=>"C:\Users\Bruno\Documents\Arquivos\oVWeV2qCi5dMvytKXpY1709APsHAMnqCpeUN2yqd\8", "file_name_encode"=>"APkvoLW5x", "file_name"=>"16_S16_L001_R1_001", "projeto_nome"=>"Teste", "file_datacriacao"=>2017-08-13T04:35:03.000Z, "file_size"=>"158,2 MB", "@timestamp"=>2018-06-29T05:20:16.860Z, "projeto_id"=>8, "projeto_sessao"=>"3QoTLWI-pXsXNDcvhIzuB3_WmxIugKPRykiuPAca", "file_type"=>"fastq", "file_id"=>191, "@version"=>"1", "projeto_datacriacao"=>2017-08-13T04:33:54.000Z}}

Can you guys help me with that?


#2

You need a space after each if. Yes, seriously.


(Bruno Andrade) #3

Lol, Thank's :smiley:

Now I'm facing another problem, I'm starting the logstash with de command logstash -w 1 -f db.conf, but the logstash isn't pushing all the records into elasticsearch, sometimes there's 4 files to put into fileuploaded map, but logstash only pushes one. Do you know what can be done to fix this?


(system) #4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.