How to ingest multiple files


(Ahmed HADDAD) #1

Hello, I need to ingest multiple files with the same categorty in the {index, document_type}
so i tried to use "*" to do that but it dosen't work (the pipline is runinng but it dosen't ingest the data).
this is my configuration file:

input{
 file{
path=>"C:\Users\Ahmed\Desktop\ElasticSearch\cabs_trajectories\green\green_tripdata_2013-*.csv"
            start_position=>"beginning"
             sincedb_path=>NUL
         }
}
filter{
    csv{
          separator=>","
          columns=>["VendorID","lpep_pickup_datetime","Lpep_dropoff_datetime","Store_and_fwd_flag","RateCodeID","Pickup_longitude","Pickup_latitude","Dropoff_longitude","Dropoff_latitude","Passenger_count","Trip_distance","Fare_amount","Extra","MTA_tax","Tip_amount","Tolls_amount","Ehail_fee","Total_amount","Payment_type","Trip_type"]  

       }
     mutate{
     convert=>["Pickup_latitude","float"]
 convert=>["Pickup_longitude","float"]  
 convert=>["Dropoff_latitude","float"]
 convert=>["Dropoff_longitude","float"]
 
 }      
 mutate{
 rename=>["Pickup_latitude","Pickup_location[lat]"]
 rename=>["Pickup_longitude","Pickup_location[lon]"]
 rename=>["Dropoff_latitude","Dropoff_location[lat]"]
 rename=>["Dropoff_longitude","Dropoff_location[lon]"]
 }
 date{ 
  match=>["lpep_pickup_datetime","MM/dd/yyyy HH:mm"]
  target=>"PickupDate"  
  }
}

output{ 
elasticsearch {
   hosts=> "localhost"
   index=> "cabs_pick"
   document_type=>"traj2013"
 }        
 stdout{codec=>rubydebug}
}

(Magnus B├Ąck) #2

I believe you need to use forward slashes in your filename pattern, not backslashes.


(Ahmed HADDAD) #3

i didn't know why this happened , it works with one file with \ !.


(system) #4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.