Hey all, I'm currently in the process of porting from another logging system over to elasticsearch/kibana using filebeat as the shipper and ingest pipelines on the cloud to parse the data. What I want to accomplish is to have a seperate index for each fileset module. I'm able to do that by specifying 'index: "test-000001"' in the config file for each module fileset and I can get ILM to pick that up and rotate the log file. The trouble is that filebeat keeps pushing to the same named index with "000001" and it doesn't rotate. So what I'm looking for is help to get ILM working properly so that I can write to one index from filebeat and have ILM work to rotate the writable index. This is what I have so far for config and other settings but the ILM part is not working.
test/config/test.yml
type: log
paths:
{{ range $i, $path := .paths }}
- {{$path}}
{{ end }}
fields:
tags: ["test"]
multiline:
pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}
negate: true
match: after
index: "test"
GET test*/_ilm/explain
{
"indices" : {
"test" : {
"index" : "test",
"managed" : true,
"policy" : "test_ilm",
"lifecycle_date_millis" : 1597947506212,
"age" : "10.67m",
"phase" : "hot",
"phase_time_millis" : 1597947506545,
"action" : "rollover",
"action_time_millis" : 1597947605451,
"step" : "check-rollover-ready",
"step_time_millis" : 1597947605451,
"phase_execution" : {
"policy" : "test_ilm",
"phase_definition" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "20gb",
"max_age" : "1m"
},
"set_priority" : {
"priority" : 100
}
}
},
"version" : 10,
"modified_date_in_millis" : 1597947487738
}
}
}
}
POST _aliases
{
"actions" : [
{ "add" : { "index" : "test", "alias" : "test-000001", "is_write_index" : "true" } }
]
}
GET _alias/test*
{
"test" : {
"aliases" : {
"test-000001" : {
"is_write_index" : true
}
}
}
}