Hello, I need to ingest multiple files with the same categorty in the {index, document_type}
so i tried to use "*" to do that but it dosen't work (the pipline is runinng but it dosen't ingest the data).
this is my configuration file:
input{
file{
path=>"C:\Users\Ahmed\Desktop\ElasticSearch\cabs_trajectories\green\green_tripdata_2013-*.csv"
start_position=>"beginning"
sincedb_path=>NUL
}
}
filter{
csv{
separator=>","
columns=>["VendorID","lpep_pickup_datetime","Lpep_dropoff_datetime","Store_and_fwd_flag","RateCodeID","Pickup_longitude","Pickup_latitude","Dropoff_longitude","Dropoff_latitude","Passenger_count","Trip_distance","Fare_amount","Extra","MTA_tax","Tip_amount","Tolls_amount","Ehail_fee","Total_amount","Payment_type","Trip_type"]
}
mutate{
convert=>["Pickup_latitude","float"]
convert=>["Pickup_longitude","float"]
convert=>["Dropoff_latitude","float"]
convert=>["Dropoff_longitude","float"]
}
mutate{
rename=>["Pickup_latitude","Pickup_location[lat]"]
rename=>["Pickup_longitude","Pickup_location[lon]"]
rename=>["Dropoff_latitude","Dropoff_location[lat]"]
rename=>["Dropoff_longitude","Dropoff_location[lon]"]
}
date{
match=>["lpep_pickup_datetime","MM/dd/yyyy HH:mm"]
target=>"PickupDate"
}
}
output{
elasticsearch {
hosts=> "localhost"
index=> "cabs_pick"
document_type=>"traj2013"
}
stdout{codec=>rubydebug}
}