Error ingestion two csv in elastic search

Elastic Stack Logstash
1

Hello it is possible to ingest of two csv files in elastic using file beat and logstsh with the same index

2

Yes.

3

I use this config logstsh

input {
beats {
host => "localhost"
port => "5044"
 }
}                  

filter {
csv {
    separator => ";"
	columns => [ "chaine", "job", "date_plan", "statut", "date_debut", "date_fin", "serveur", "numero_passage", "application", "sous_application", "date fraicheur" ]
}

    if [sous_application] == "DWH PRODUCTION" {
     mutate {
     add_field => { "UNIVERS" => "Entrepôt de données Production" }
     }
}
     if [statut] == "OK" {
     mutate {
     add_field => { "statut_globale" => "1" }
     }
     }

   if [statut] == "EN-ERREUR" {
     mutate {
     add_field => { "statut_globale" => "0" }
     }	  	  	  
    }

     if [statut] == "EN-ATTENTE" {
     mutate {
     add_field => { "statut_globale" => "3" }
     }	  	  	  
  }

   date {
   match => [ "date_plan" , "YYYY-MM-dd" ]
   timezone => "Europe/Paris"
   }

 date {
 match => [ "date_debut" , "YYYY-MM-dd HH:mm:ss" ]
 timezone => "Europe/Paris"
}

date {
match => [ "date_fin" , "YYYY-MM-dd HH:mm:ss" ]
timezone => "Europe/Paris"
 }

date {
match => [ "date fraicheur" , "YYYY-MM-dd" ]
timezone => "Europe/Paris"
}

}

output {
stdout {
codec => rubydebug
}
   elasticsearch { 
   hosts => "http://localhost:9200" 
   index => "v00"
   }
 }
4

I have a erroe: the the column date fraicheur not creat in elastic search

5

Bonjour :wink:

May I recommend you change date fraicheur to date_fraicheur.

6

I tested this but the date_fraicheur column is not created in elastic

7

Can you share the full logstash configuration file and the first 10 lines of your CSV file?

8

this is the first CSV file

csv1
csv1764×48 1 KB

9

this is the second CSV file

Capturecsv3
Capturecsv31434×122 10.9 KB

10

and the full logstash configuration file is :

input {
beats {
host => "localhost"
port => "5044"
 }
}                  

filter {
csv {
separator => ";"
columns => [ "chaine", "job", "date_plan", "statut", "date_debut", "date_fin", "serveur", "numero_passage", "application", "sous_application", "date fraicheur" ]
}

if [sous_application] == "DWH PRODUCTION" {
 mutate {
 add_field => { "UNIVERS" => "Entrepôt de données Production" }
 }
} 
 if [statut] == "OK" {
  mutate {
 add_field => { "statut_globale" => "1" }
 }
 }

if [statut] == "EN-ERREUR" {
 mutate {
 add_field => { "statut_globale" => "0" }
 }	  	  	  
}

 if [statut] == "EN-ATTENTE" {
 mutate {
 add_field => { "statut_globale" => "3" }
 }	  	  	  
 }

 date {
 match => [ "date_plan" , "YYYY-MM-dd" ]
 timezone => "Europe/Paris"
 }

date {
match => [ "date_debut" , "YYYY-MM-dd HH:mm:ss" ]
timezone => "Europe/Paris"
}

 date {
 match => [ "date_fin" , "YYYY-MM-dd HH:mm:ss" ]
 timezone => "Europe/Paris"
}

date {
match => [ "date fraicheur" , "YYYY-MM-dd" ]
timezone => "Europe/Paris"
}

}

 output {
 stdout {
 codec => rubydebug
}
   elasticsearch { 
   hosts => "http://localhost:9200" 
   index => "v00"
 }
}
11

Any idea???

12

Any Idea???

13

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.