Hello All,
I'm trying to get data from oracle db and parsing it through logstash pipeline and finally to elastic.
while checking the data in discover in kibana ,observed that the data gets automaticaly deleted after few minutes.what could be causing this?...cross checked pipeline and associated policy.
In output filter id is unique key in db that might not cause this issue .
Below are the details,Can someone help out with this?
input{
jdbc {
jdbc_connection_string => "jdbc:oracle:thin:@td08appl002.group.kpti:1991/cis"
jdbc_user => "MIAADMIN"
jdbc_password => "MIAADMIN"
jdbc_driver_library => "../lib/ojdbc8-12.2.0.1.jar"
jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
jdbc_paging_enabled => true
last_run_metadata_path => "../config/lastrun-audit.yml"
schedule => "*/25 * * * * *"
connection_retry_attempts => 5
connection_retry_attempts_wait_time => 10
statement=> "select ID, STATUS_EVENT_DATA, from_tz(CAST (CREATION_DATE AS TIMESTAMP), 'UTC') as CREATION_DATE from SYSAUDIT where CREATION_DATE >= SYS_EXTRACT_UTC(:sql_last_value )"
type => "audit"
}
}
output{
Elasticsearch {
hosts => "http://td08appl002.group.kpti:9200"
ilm_pattern => "{now/d}-000001"
"doc_as_upsert" => true
ilm_rollover_alias => "job-audit"
ilm_policy => "audit-policy"
"document_id" => "%{id}" ---------------unique key in table
}
}
PUT _ilm/policy/audit-policy
{
"policy": {
"phases": {
"hot": {
"min_age": "0ms",
"actions": {
"rollover": {
"max_age": "1d"
},
"set_priority": {
"priority": null
}
}
},
"delete": {
"min_age": "30d",
"actions": {
"delete": {
"delete_searchable_snapshot": true
}
}
}
}
}
}
Thanx