Hi everyone,
Background info:
I am trying to read CSV files' data and transfer it to an Elasticsearch index.
Issue:
Elasticsearch index shows only one CSV path in that specific index. I called it "rosindex". Refreshing the page at https://***.found.io:9243/rosindex/_search?pretty will show different JSON content for different CSV paths, but I don't want it it this way. I want all CSV files to show up at once in the index.
Is there anyway i can show all CSV datas at once?
Logstash.conf file:
input {
file {
type => ".csv"
start_position => "beginning"
path => "/home/kourosh/Documents/metrics/files/*.csv"
}
}
filter {
....
}
output{
elasticsearch{
hosts => ["https://***.found.io:9243"]
user => "..."
password => "..."
index => "rosindex"
workers => 1
}
stdout {
codec => rubydebug
}
}
Elasticsearch index:
{
"took" : 0,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 10000,
"relation" : "gte"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : "rosindex",
"_type" : "_doc",
"_id" : "wF2dSGsB9yYBmDbPHzh1",
"_score" : 1.0,
"_source" : {
"data" : 7.40000009537,
"path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
"message" : "1557444210630278341,7.40000009537",
"@timestamp" : "2019-06-11T22:16:25.004Z",
"host" : "itecoation-Lenovo-ideapad-330S-15IKB",
"type" : ".csv",
"rosbagTimestamp" : 1557444210630278341,
"@version" : "1"
}
},
{
"_index" : "rosindex",
"_type" : "_doc",
"_id" : "wV2dSGsB9yYBmDbPHzh1",
"_score" : 1.0,
"_source" : {
"data" : 6.5,
"path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
"message" : "1557444211714413452,6.5",
"@timestamp" : "2019-06-11T22:16:25.004Z",
"host" : "itecoation-Lenovo-ideapad-330S-15IKB",
"type" : ".csv",
"rosbagTimestamp" : 1557444211714413452,
"@version" : "1"
}
},
{
"_index" : "rosindex",
"_type" : "_doc",
"_id" : "wl2dSGsB9yYBmDbPHzh1",
"_score" : 1.0,
"_source" : {
"data" : 8.30000019073,
"path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
"message" : "1557444212803535300,8.30000019073",
"@timestamp" : "2019-06-11T22:16:25.004Z",
"host" : "itecoation-Lenovo-ideapad-330S-15IKB",
"type" : ".csv",
"rosbagTimestamp" : 1557444212803535300,
"@version" : "1"
}
},
{
"_index" : "rosindex",
"_type" : "_doc",
"_id" : "w12dSGsB9yYBmDbPHzh1",
"_score" : 1.0,
"_source" : {
"data" : 6.40000009537,
"path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
"message" : "1557444213892452573,6.40000009537",
"@timestamp" : "2019-06-11T22:16:25.004Z",
"host" : "itecoation-Lenovo-ideapad-330S-15IKB",
"type" : ".csv",
"rosbagTimestamp" : 1557444213892452573,
"@version" : "1"
}
},
{
"_index" : "rosindex",
"_type" : "_doc",
"_id" : "xF2dSGsB9yYBmDbPHzh1",
"_score" : 1.0,
"_source" : {
"data" : 9.19999980927,
"path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
"message" : "1557444214981876244,9.19999980927",
"@timestamp" : "2019-06-11T22:16:25.005Z",
"host" : "itecoation-Lenovo-ideapad-330S-15IKB",
"type" : ".csv",
"rosbagTimestamp" : 1557444214981876244,
"@version" : "1"
}
},
{
"_index" : "rosindex",
"_type" : "_doc",
"_id" : "xV2dSGsB9yYBmDbPHzh1",
"_score" : 1.0,
"_source" : {
"data" : 6.40000009537,
"path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
"message" : "1557444216066947192,6.40000009537",
"@timestamp" : "2019-06-11T22:16:25.005Z",
"host" : "itecoation-Lenovo-ideapad-330S-15IKB",
"type" : ".csv",
"rosbagTimestamp" : 1557444216066947192,
"@version" : "1"
}
},
{
"_index" : "rosindex",
"_type" : "_doc",
"_id" : "xl2dSGsB9yYBmDbPHzh1",
"_score" : 1.0,
"_source" : {
"data" : 9.10000038147,
"path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
"message" : "1557444217166283953,9.10000038147",
"@timestamp" : "2019-06-11T22:16:25.005Z",
"host" : "itecoation-Lenovo-ideapad-330S-15IKB",
"type" : ".csv",
"rosbagTimestamp" : 1557444217166283953,
"@version" : "1"
}
},
{
"_index" : "rosindex",
"_type" : "_doc",
"_id" : "x12dSGsB9yYBmDbPHzh1",
"_score" : 1.0,
"_source" : {
"data" : 6.5,
"path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
"message" : "1557444218247984032,6.5",
"@timestamp" : "2019-06-11T22:16:25.005Z",
"host" : "itecoation-Lenovo-ideapad-330S-15IKB",
"type" : ".csv",
"rosbagTimestamp" : 1557444218247984032,
"@version" : "1"
}
}
}
}
]
}
}