Hello, i'm trying to understand ILM. I have created a policy
"dev-focus-rollover" : {
"version" : 3,
"modified_date" : "2020-02-06T10:21:55.920Z",
"policy" : {
"phases" : {
"warm" : {
"min_age" : "0ms",
"actions" : {
"allocate" : {
"include" : { },
"exclude" : { },
"require" : {
"node_id" : "osx04437"
}
},
"shrink" : {
"number_of_shards" : 1
},
"set_priority" : {
"priority" : 50
}
}
},
"cold" : {
"min_age" : "7d",
"actions" : {
"allocate" : {
"number_of_replicas" : 1,
"include" : { },
"exclude" : { },
"require" : {
"node_id" : "hostname"
}
},
"set_priority" : {
"priority" : 0
}
}
},
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "10gb",
"max_age" : "14d",
"max_docs" : 100000
},
"set_priority" : {
"priority" : 100
}
}
}
}
}
}
And also i have a 14 index templates, that are assigned to this policy. One of them looks like this:
{
"dev-focus-audit-log--card_assignment_change" : {
"order" : 0,
"index_patterns" : [
"dev-*--card_assignment_change*"
],
"settings" : {
"index" : {
"lifecycle" : {
"name" : "dev-focus-rollover",
"rollover_alias" : "dev-focus-audit-log--card_assignment_change"
}
}
},
"mappings" : { },
"aliases" : {
"dev-focus-audit-log--card_assignment_change" : { }
}
}
}
I have running logstash, that collects data from postgresql and write output to elasticsearch. All events are writen to index alias.
input {
jdbc {
jdbc_driver_class => "Java::org.postgresql.Driver"
jdbc_connection_string => "jdbc:postgresql://dbhost:dbport/dbname"
jdbc_user => "dbuser"
jdbc_password => "${dbpassword}"
#jdbc_default_timezone => "CET"
#plugin_timezone => "utc"
statement => "select * from DB_TABLE_NAME where id > :sql_last_value"
use_column_value => true
tracking_column => id
tracking_column_type => "numeric"
#clean_run => true
schedule => "* * * * *"
last_run_metadata_path => "/path_to_file/.logstash_jdbc_last_run"
}
}
filter {
json {
source => "value"
target => "event"
}
mutate {
add_field => {
"[@metadata][event_type]" => "%{[event][event_type]}"
}
}
mutate {
lowercase => ["[@metadata][event_type]"]
}
date {
timezone => "Europe/Warsaw"
match => ["[event][event_date]", "YYYY-MM-dd HH:mm:ss.SSS", "ISO8601"]
target => "@timestamp"
}
}
output {
elasticsearch {
hosts => "elastic_host:9200"
user => elasticuser
password => "${elasticpassword}"
index => "dev-focus-audit-log--%{[@metadata][event_type]}"
document_id => "%{id}"
}
# stdout {
# codec => "rubydebug"
# }
}
I get this error
illegal_argument_exception: Rollover alias [dev-focus-audit-log--death_date_assignment] can point to multiple indices, found duplicated alias [[dev-focus-audit-log--death_date_assignment]] in index template [dev-focus-audit-log--death_date_assignment]
Anyone know how to set this up correctly?
And another one question. It is possible to write a template, that will create new index (that will be autoincremented) for new event_type (as shown in logstash configuration) and will assignee this index to existing ILM policy?