Input JDBC with mongo

Elastic Stack Logstash
1

Hello,
I have a config to get data from mongo , logstash get doc , but only one doc is in ES
logstash conf

input {
		jdbc {
			 jdbc_driver_library => "/usr/share/logstash/logstash-core/lib/jars/mongojdbc1.8.jar"
			 jdbc_driver_class => "com.dbschema.MongoJdbcDriver"
			 jdbc_connection_string => "jdbc:mongodb://0000000:27018/toto"
			 jdbc_validate_connection => true
			 jdbc_user => ""
			 jdbc_password => ""
			 schedule => "/15 * * * * *"
			 statement => "db.getCollection('people').aggregate([
			 		{ $project: {
							'_id': {$toString: '$_id'},
							'name' : 1,
    					'gender' : 1,
    					'birthDate' : 1,
    					'type' : 1,
    					'origin' : 1,
    					'created_at' : 1,
    					'updated_at' : 1
						}
					}
					]);"
			}
}

filter {
    fingerprint {
        key => "1234ABCD"
        method => "SHA256"
        source => ["document._id"]
        target => "[@metadata][generated_id]"
        concatenate_sources => true
    }
}


output {
	elasticsearch {
		hosts => "elasticsearch:9200"
		user => "elastic"
		password => "changeme"
		document_id => "%{[@metadata][generated_id]}"
		index => "pdn_test_people_dd"

	}
	stdout { codec => rubydebug }
}

logstash output

{
"document" => {
"name" => "xcsssgds",
"birthDate" => "1909",
"origin" => "Connaissance commune",
"type" => "Adulte",
"created_at" => 2020-03-27T11:24:52.231Z,
"_id" => "5e7de284df83870033dfd705",
"gender" => "femme"
},
"@version" => "1",
"@timestamp" => 2020-04-01T20:24:45.781Z
} ...... to 108 docs

in ES only last is show .

General

Health yellow

Primaries 1

Docs Count 1

why logstash set document{.......} i would like a structure like this
{
"name" => "xcsssgds",
"birthDate" => "1909",
"origin" => "Connaissance commune",
"type" => "Adulte",
"created_at" => 2020-03-27T11:24:52.231Z,
"_id" => "5e7de284df83870033dfd705",
"gender" => "femme",
"@version" => "1",
"@timestamp" => 2020-04-01T20:24:45.781Z

}
thanks for your help , if you have a better solution to connect mongo to ES , you welcome :slight_smile:

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.