Logstash s3 input reads file multiple times

Elastic Stack Logstash
1

hello,
I have configured the following s3 input in logstash, and it keeps reading the same file from the given S3 bucket. I noticed that the given sincedb file ${DATA_DIR}/.sincedb_activities.txt is being used and it contains the date and time corresponding to the last modified value of the file on S3.

What am I missing? Do I need to set the parameter delete to true ? As far as I understand, that should not be necessary, as the sincedb mechanism should guarantee files are processed only once. Thanks

#############
#  INPUT  #
#############
input {

  s3 {
    access_key_id => "${AWS_ACCESS_KEY_ID}"
    secret_access_key => "${AWS_SECRET_ACCESS_KEY}"
    endpoint => "${S3_ENDPOINT}"
    bucket => "${INPUT_BUCKET}"
    temporary_directory => "/tmp/logstash"
    prefix => "activities"
    additional_settings => {
      force_path_style => true
      follow_redirects => false
    }
    codec => multiline {
      charset => "UTF-8"
      auto_flush_interval => 5
      pattern => '^([0-9]+|id_activ),'
      negate => true
      what => "previous"
    }
    sincedb_path => "${DATA_DIR}/.sincedb_activities.txt"
    add_field => {
      "type_fichier" => "db_activity"
      "[@metadata][index]" => "${ACT_IDX}"
      "[@metadata][pipeline]" => "ACT"
    }
  }
}
2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.