Logstash does not transfer all CSV files to an Elasticsearch index

Elastic Stack Logstash
1

Hi everyone,

Background info:
I am trying to read CSV files' data and transfer it to an Elasticsearch index.

Issue:
Elasticsearch index shows only one CSV path in that specific index. I called it "rosindex". Refreshing the page at https://***.found.io:9243/rosindex/_search?pretty will show different JSON content for different CSV paths, but I don't want it it this way. I want all CSV files to show up at once in the index.

Is there anyway i can show all CSV datas at once?

Logstash.conf file:

input {
    file {
        type => ".csv"
        start_position => "beginning"
        path => "/home/kourosh/Documents/metrics/files/*.csv"
    }
}

filter {
 ....
}

output{
    elasticsearch{
        hosts => ["https://***.found.io:9243"]
        user => "..."
        password => "..."
        index => "rosindex"
        workers => 1
    }
    stdout { 
        codec => rubydebug 
    }
}

Elasticsearch index:

{
  "took" : 0,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "rosindex",
        "_type" : "_doc",
        "_id" : "wF2dSGsB9yYBmDbPHzh1",
        "_score" : 1.0,
        "_source" : {
          "data" : 7.40000009537,
          "path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
          "message" : "1557444210630278341,7.40000009537",
          "@timestamp" : "2019-06-11T22:16:25.004Z",
          "host" : "itecoation-Lenovo-ideapad-330S-15IKB",
          "type" : ".csv",
          "rosbagTimestamp" : 1557444210630278341,
          "@version" : "1"
        }
      },
      {
        "_index" : "rosindex",
        "_type" : "_doc",
        "_id" : "wV2dSGsB9yYBmDbPHzh1",
        "_score" : 1.0,
        "_source" : {
          "data" : 6.5,
          "path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
          "message" : "1557444211714413452,6.5",
          "@timestamp" : "2019-06-11T22:16:25.004Z",
          "host" : "itecoation-Lenovo-ideapad-330S-15IKB",
          "type" : ".csv",
          "rosbagTimestamp" : 1557444211714413452,
          "@version" : "1"
        }
      },
      {
        "_index" : "rosindex",
        "_type" : "_doc",
        "_id" : "wl2dSGsB9yYBmDbPHzh1",
        "_score" : 1.0,
        "_source" : {
          "data" : 8.30000019073,
          "path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
          "message" : "1557444212803535300,8.30000019073",
          "@timestamp" : "2019-06-11T22:16:25.004Z",
          "host" : "itecoation-Lenovo-ideapad-330S-15IKB",
          "type" : ".csv",
          "rosbagTimestamp" : 1557444212803535300,
          "@version" : "1"
        }
      },
      {
        "_index" : "rosindex",
        "_type" : "_doc",
        "_id" : "w12dSGsB9yYBmDbPHzh1",
        "_score" : 1.0,
        "_source" : {
          "data" : 6.40000009537,
          "path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
          "message" : "1557444213892452573,6.40000009537",
          "@timestamp" : "2019-06-11T22:16:25.004Z",
          "host" : "itecoation-Lenovo-ideapad-330S-15IKB",
          "type" : ".csv",
          "rosbagTimestamp" : 1557444213892452573,
          "@version" : "1"
        }
      },
      {
        "_index" : "rosindex",
        "_type" : "_doc",
        "_id" : "xF2dSGsB9yYBmDbPHzh1",
        "_score" : 1.0,
        "_source" : {
          "data" : 9.19999980927,
          "path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
          "message" : "1557444214981876244,9.19999980927",
          "@timestamp" : "2019-06-11T22:16:25.005Z",
          "host" : "itecoation-Lenovo-ideapad-330S-15IKB",
          "type" : ".csv",
          "rosbagTimestamp" : 1557444214981876244,
          "@version" : "1"
        }
      },
      {
        "_index" : "rosindex",
        "_type" : "_doc",
        "_id" : "xV2dSGsB9yYBmDbPHzh1",
        "_score" : 1.0,
        "_source" : {
          "data" : 6.40000009537,
          "path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
          "message" : "1557444216066947192,6.40000009537",
          "@timestamp" : "2019-06-11T22:16:25.005Z",
          "host" : "itecoation-Lenovo-ideapad-330S-15IKB",
          "type" : ".csv",
          "rosbagTimestamp" : 1557444216066947192,
          "@version" : "1"
        }
      },
      {
        "_index" : "rosindex",
        "_type" : "_doc",
        "_id" : "xl2dSGsB9yYBmDbPHzh1",
        "_score" : 1.0,
        "_source" : {
          "data" : 9.10000038147,
          "path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
          "message" : "1557444217166283953,9.10000038147",
          "@timestamp" : "2019-06-11T22:16:25.005Z",
          "host" : "itecoation-Lenovo-ideapad-330S-15IKB",
          "type" : ".csv",
          "rosbagTimestamp" : 1557444217166283953,
          "@version" : "1"
        }
      },
      {
        "_index" : "rosindex",
        "_type" : "_doc",
        "_id" : "x12dSGsB9yYBmDbPHzh1",
        "_score" : 1.0,
        "_source" : {
          "data" : 6.5,
          "path" : "/home/kourosh/Documents/metrics/files/_slash_diagnostics_slash_cpu_monitor_slash_cpu.csv",
          "message" : "1557444218247984032,6.5",
          "@timestamp" : "2019-06-11T22:16:25.005Z",
          "host" : "itecoation-Lenovo-ideapad-330S-15IKB",
          "type" : ".csv",
          "rosbagTimestamp" : 1557444218247984032,
          "@version" : "1"
        }
      }
      }
      }
    ]
  }
}
2

Bump, I'm still having problems with this. Any suggestions?

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.