S3 input plugin point to today's date or skip old data

Elastic Stack Logstash
1

Hi,
I want to ignore the old data in the bucket and point to start from today's date for example: 2022-08-31T01:00:*

Please let me know how i can point to that timestamp or latest timestamp.

bash-4.2$ cat sincedb_3109c2cd6a5baeddde56f2f1c8547efe
2022-08-30 13:32:18 UTC

input {
  s3 {
    type => "xxx_errors"
    codec => "json"
    bucket => "abc-errors-bucket"
    backup_to_bucket => "abc-errors-bucket"
    prefix => "errors-json-backup"
    # this is an arbitrary prefix that is used to track the files already processed by logstash
    backup_add_prefix =>  "errors-json-backup/"
    delete => true
  }
}

output {

  amazon_es {
    hosts => ["${ELASTICSEARCH_HOST}"]
    region => "us-east-1"
    index => "${ELASTICSEARCH_INDEX}-%{+YYYY.MM.dd}"
    document_id => "%{id}"
  }
}

I want to point s3 input to latest date. I

Json blob from kibana

{
  "_index": "logstash_xxx_errors-2022.08.31",
  "_type": "xxx_errors",
  "_id": "c1852e92-3894-4f28-8fbd-9ecb6582f116",
  "_score": null,
  "_source": {
    "date": "2022-08-30T13:39:55+0000",
    "referer": "https://www.xxx.com/",
    "request": "OPTIONS",
    "server": "cache-iad-kcgs7200042-IAD",
    "is_https": true,
    "transfer-encoding": "",
    "lon": -84.41,
    "tls_version": "TLSv1.2",
    "type": "xxx_errors",
    "host": "us.api.xxx.com",
    "@version": "1",
    "client_ip": "xx.xx.xxx.x",
    "id": "c1852e92-3894-4f28-8fbd-9ecb6582f116",
    "lat": 39.27,
    "X-User-Id": "",
    "query": "",
    "bytes_read": 0,
    "cache_status": "23",
    "url": "/api/v2/xxx/xxx/7905",
    "X-IHR-Profile-ID": "",
    "country_code": "US",
    "@timestamp": "2022-08-31T01:54:00.983Z",
    "header_bytes": 523,
    "response": "OK",
    "proto": "HTTP/1.1",
    "time_elapsed_ms": 0,
    "tls": false,
    "x-hostname": "",
    "status": 200,
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36"
  },
  "fields": {
    "date": [
      1661866795000
    ],
    "@timestamp": [
      1661910840983
    ]
  },
  "sort": [
    1661866795000
  ]
}

Thanks!

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.