Hello,
I'd like to use datastream +ilm policy to roll index on daily basis and delete them after 1 month.
It's important for me to roll index on daily basis without fail ( 10-20 mins delay doesn't doesn't affect me).
For testing purposes I created a datastream with the following configuration.
And I'm pushing test data continuously at every 5th minute to the datastream.
PUT _ilm/policy/test-ilm-policy
{
"policy": {
"phases": {
"hot": {
"min_age": "0s",
"actions": {
"set_priority": {
"priority": 100
},
"rollover": {
"max_age": "1h"
},
}
},
"warm": {
"min_age": "0s",
"actions": {
"set_priority": {
"priority": 50
}
}
},
"delete": {
"min_age": "12h",
"actions": {
"delete": {}
}
}
}
}
}
PUT /_index_template/it_test_ilm
{
"index_patterns" : ["test-ilm-*"],
"priority" : 1,
"data_stream": { },
"template": {
"settings" : {
"index" : {
"lifecycle" : {
"name" : "test-ilm-policy"
},
"default_pipeline": "ip_timestamp",
"number_of_replicas": 0
}
}
}
}
In this config I encountered the following observation/concerns:
-
The new index after rollover is skipping sequence number:
.ds-test-ilm-index-2023.01.30-000011 .ds-test-ilm-index-2023.01.30-000013 .ds-test-ilm-index-2023.01.30-000016 .ds-test-ilm-index-2023.01.30-000017 .ds-test-ilm-index-2023.01.30-000019 .ds-test-ilm-index-2023.01.30-000020 .ds-test-ilm-index-2023.01.30-000021 .ds-test-ilm-index-2023.01.30-000023Why is this happening ? Will this affect the working of datastream ?
-
The scheduled rollover event is not accurate:
.ds-test-ilm-index-2023.01.30-000011 //created at : Mon Jan 30 2023 01:27:40 , .ds-test-ilm-index-2023.01.30-000013 //created at, index 11 rolled over at: Mon Jan 30 2023 02:47:40 .ds-test-ilm-index-2023.01.30-000016 //created at, index 13 rolled over at: Mon Jan 30 2023 05:47:40 .ds-test-ilm-index-2023.01.30-000017 //created at, index 16 rolled over at: Mon Jan 30 2023 06:57:40 .ds-test-ilm-index-2023.01.30-000019 //created at, index 17 rolled over at: Mon Jan 30 2023 08:18:54 .ds-test-ilm-index-2023.01.30-000020 //created at, index 19 rolled over at: Mon Jan 30 2023 09:28:54 .ds-test-ilm-index-2023.01.30-000021 //created at, index 20 rolled over at: Mon Jan 30 2023 11:48:54 .ds-test-ilm-index-2023.01.30-000023 //created at, index 21 rolled over at: Mon Jan 30 2023 13:38:54
The index 13 rolled over after 3 hours & index 20 rolled over after 2 hourse, this can be a blocker from my usecase.
I am aware of the 10mins polling time, but the delay is exceeding hours in my case.
I have gone through the logs, didn't find anything justifying the delay.
I have tried cluster level ilm configuration : indices.lifecycle.rollover.only_if_has_documents= false, with this the rollover worked fine (rolling over at 1hr 10mins every time).
Is there any miss configuration or fault in understanding here ?
Can anyone help me out with this ?
Attaching Ilm policy explain output with this:
{
"indices": {
".ds-test-ilm-index-2023.01.30-000011": {
"index": ".ds-test-ilm-index-2023.01.30-000011",
"managed": true,
"policy": "test-ilm-policy",
"index_creation_date_millis": 1675042060302,
"time_since_index_creation": "12.35h",
"lifecycle_date_millis": 1675046860168,
"age": "11.01h",
"phase": "warm",
"phase_time_millis": 1675046861368,
"action": "complete",
"action_time_millis": 1675046862369,
"step": "complete",
"step_time_millis": 1675046862369,
"phase_execution": {
"policy": "test-ilm-policy",
"phase_definition": {
"min_age": "0s",
"actions": {
"readonly": {},
"set_priority": {
"priority": 50
}
}
},
"version": 1,
"modified_date_in_millis": 1674983234456
}
},
".ds-test-ilm-index-2023.01.30-000013": {
"index": ".ds-test-ilm-index-2023.01.30-000013",
"managed": true,
"policy": "test-ilm-policy",
"index_creation_date_millis": 1675046860243,
"time_since_index_creation": "11.01h",
"lifecycle_date_millis": 1675057660244,
"age": "8.01h",
"phase": "warm",
"phase_time_millis": 1675057661645,
"action": "complete",
"action_time_millis": 1675057662645,
"step": "complete",
"step_time_millis": 1675057662645,
"phase_execution": {
"policy": "test-ilm-policy",
"phase_definition": {
"min_age": "0s",
"actions": {
"readonly": {},
"set_priority": {
"priority": 50
}
}
},
"version": 1,
"modified_date_in_millis": 1674983234456
}
},
".ds-test-ilm-index-2023.01.30-000016": {
"index": ".ds-test-ilm-index-2023.01.30-000016",
"managed": true,
"policy": "test-ilm-policy",
"index_creation_date_millis": 1675057660297,
"time_since_index_creation": "8.01h",
"lifecycle_date_millis": 1675061860235,
"age": "6.85h",
"phase": "warm",
"phase_time_millis": 1675061861435,
"action": "complete",
"action_time_millis": 1675061862436,
"step": "complete",
"step_time_millis": 1675061862436,
"phase_execution": {
"policy": "test-ilm-policy",
"phase_definition": {
"min_age": "0s",
"actions": {
"readonly": {},
"set_priority": {
"priority": 50
}
}
},
"version": 1,
"modified_date_in_millis": 1674983234456
}
},
".ds-test-ilm-index-2023.01.30-000017": {
"index": ".ds-test-ilm-index-2023.01.30-000017",
"managed": true,
"policy": "test-ilm-policy",
"index_creation_date_millis": 1675061860295,
"time_since_index_creation": "6.85h",
"lifecycle_date_millis": 1675066734697,
"age": "5.49h",
"phase": "warm",
"phase_time_millis": 1675066735898,
"action": "complete",
"action_time_millis": 1675066737712,
"step": "complete",
"step_time_millis": 1675066737712,
"phase_execution": {
"policy": "test-ilm-policy",
"phase_definition": {
"min_age": "0s",
"actions": {
"readonly": {},
"set_priority": {
"priority": 50
}
}
},
"version": 1,
"modified_date_in_millis": 1674983234456
}
},
".ds-test-ilm-index-2023.01.30-000019": {
"index": ".ds-test-ilm-index-2023.01.30-000019",
"managed": true,
"policy": "test-ilm-policy",
"index_creation_date_millis": 1675066734763,
"time_since_index_creation": "5.49h",
"lifecycle_date_millis": 1675070934516,
"age": "4.33h",
"phase": "warm",
"phase_time_millis": 1675070935918,
"action": "complete",
"action_time_millis": 1675070937119,
"step": "complete",
"step_time_millis": 1675070937119,
"phase_execution": {
"policy": "test-ilm-policy",
"phase_definition": {
"min_age": "0s",
"actions": {
"readonly": {},
"set_priority": {
"priority": 50
}
}
},
"version": 1,
"modified_date_in_millis": 1674983234456
}
},
".ds-test-ilm-index-2023.01.30-000020": {
"index": ".ds-test-ilm-index-2023.01.30-000020",
"managed": true,
"policy": "test-ilm-policy",
"index_creation_date_millis": 1675070934641,
"time_since_index_creation": "4.33h",
"lifecycle_date_millis": 1675079334811,
"age": "1.99h",
"phase": "warm",
"phase_time_millis": 1675079336412,
"action": "complete",
"action_time_millis": 1675079337413,
"step": "complete",
"step_time_millis": 1675079337413,
"phase_execution": {
"policy": "test-ilm-policy",
"phase_definition": {
"min_age": "0s",
"actions": {
"readonly": {},
"set_priority": {
"priority": 50
}
}
},
"version": 1,
"modified_date_in_millis": 1674983234456
}
},
".ds-test-ilm-index-2023.01.30-000021": {
"index": ".ds-test-ilm-index-2023.01.30-000021",
"managed": true,
"policy": "test-ilm-policy",
"index_creation_date_millis": 1675079334867,
"time_since_index_creation": "1.99h",
"lifecycle_date_millis": 1675085934567,
"age": "9.93m",
"phase": "warm",
"phase_time_millis": 1675085935968,
"action": "complete",
"action_time_millis": 1675085936976,
"step": "complete",
"step_time_millis": 1675085936976,
"phase_execution": {
"policy": "test-ilm-policy",
"phase_definition": {
"min_age": "0s",
"actions": {
"readonly": {},
"set_priority": {
"priority": 50
}
}
},
"version": 1,
"modified_date_in_millis": 1674983234456
}
},
".ds-test-ilm-index-2023.01.30-000023": {
"index": ".ds-test-ilm-index-2023.01.30-000023",
"managed": true,
"policy": "test-ilm-policy",
"index_creation_date_millis": 1675085934613,
"time_since_index_creation": "9.93m",
"lifecycle_date_millis": 1675085934613,
"age": "9.93m",
"phase": "hot",
"phase_time_millis": 1675085935167,
"action": "rollover",
"action_time_millis": 1675085935768,
"step": "check-rollover-ready",
"step_time_millis": 1675085935768,
"phase_execution": {
"policy": "test-ilm-policy",
"phase_definition": {
"min_age": "0s",
"actions": {
"set_priority": {
"priority": 100
},
"rollover": {
"max_age": "1h"
}
}
},
"version": 1,
"modified_date_in_millis": 1674983234456
}
}
}
}