NFS issues

My database team are exporting some documents couple of times per day and I read them as soon as possible and put them in Elasticsearch. Files (CSV) are mounted by NFS and I can see them on my local drive. Logstash is running all the time. All files are fine (double checked), but for some reason some of them don't get properly into ES.

input{
    file{
        id => "live_import_transaction_files"
        path => "/logstash/transactions/transactions_*"
        start_position => "beginning"
        sincedb_path => "/etc/logstash/sincedb"
        file_completed_action => "log"
        file_completed_log_path => "/etc/logstash/file_completed"
    }
}

filter {
   csv {
        columns => ["account_id", "order_id", "date_executed", "date_initialized", "amount", "description", "vpnb", "dpnb" ]
        separator => ","
        skip_empty_columns => true
        convert => {
                "dpnb" => "float"
        }
    }

    grok {
      match => [
         "date_executed", "(?<index_year>%{YEAR})"
      ]
    }

    mutate {
        add_field => {
            "[@metadata][index_year]" => "%{index_year}"
        }
    }

    mutate {
        remove_field => [ "index_year", "@version", "message", "@timestamp", "host", "path"]
    }
}
output{
    elasticsearch {
        hosts => ["https://localhost:9200"]
        user => "elastic"
        password => "XXXX"
        ssl => true
        #ssl_certificate_verification => false
        cacert => '/etc/logstash/ServerCA4.cer.pem'
        index => "transactions_%{[@metadata][index_year]}"
        routing => "%{account_id}"
        document_id => "%{order_id}"
    }

}

I have same 2 logstash scripts running on 2 machines, and I have errors on both of them, but on different files (lines). I can see that in logs and in some random created indices on ES (transactions_%{index_year}, transactions_99, transactions_01 etc.).

[2019-11-03T23:21:12,009][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>"%{order_id}", :index=>"transactions%{index_year}", :_type=>"_doc", :routing=>"%{account_id}"}, #LogStash::Event:0x4a54bd65], :response=>{"index"=>{"index"=>"transactions%{index_year}", "_type"=>"_doc", "_id"=>"%{order_id}", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse field [dpnb] of type [double] in document with id '%{order_id}'. Preview of field's value: 'XXXXXXXXXXXXXXXXXX'", "caused_by"=>{"type"=>"number_format_exception", "reason"=>"For input string: "XXXXXXXXXXXXXXXXXXX""}}}}}

If I run second time logstash on that file, it will be ok (for the most of the time). I guess problem is with NFS, but don't know what is the solution? Can I do some tweeks on NFS or in my logstash script. I noticed that if my file has 1000 lines and error happens on line 800, it will also fail to process next 200 lines.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.