Dear All, I'm currently using elasticsearch and kibana in 8.5.2 version; I used the stack as a centralized logging platform. Everything works fine, logstash is able to send me tousant of logs through elastic data stream mechanism.
Everything is setup according to the documentation, BUT I don't understand why my rollover (currently set to a value of 30Gb for max_primary_shard_size) is not applied until the index reach 120 Gb ; If I force the rollover
manually, the index rotation is instant.
Is anybody struggle with ilm and data stream and encounters the same issue with the rollover as well ?
Please find below my dev console output with the following commands:
GET _cat/indices/.ds-logs*eis*/?v&h=index,store.size,docs.count
GET _data_stream/logs-lu-ec-app_eis-default
GET _data_stream/logs-lu-ec-app_eis-default/_stats
GET /_ilm/status
GET .ds-logs-lu-ec-app_eis-default-2023.02.23-000001/_ilm/explain
GET .ds-logs-lu-ec-app_eis-default-2023.02.25-000002/_ilm/explain
GET .ds-logs-lu-ec-app_eis-default-2023.02.27-000003/_ilm/explain
GET _ilm/policy/logs
These commands output, show us that ilm policy is well linked to the datastream, and that the size of each index (1 shard, 1 replica shard) reached 120Gb for the 1st index. the 2nd reach 80+ and I used _rollover api to force the index rotation.
The behaviour I expect is the index to rotation arround 30gb, I don't care about a couple of gb more but x4 is not what I wanted. Any ideas ?
Thanks in advance for your time and help.
kr,
nb: full dev console output:
# GET _cat/indices/.ds-logs*eis*/?v&h=index,store.size,docs.count 200 OK
index store.size docs.count
.ds-logs-lu-ec-app_eis-default-2023.02.23-000001 120.6gb 473374006
.ds-logs-lu-ec-app_eis-default-2023.02.25-000002 88.3gb 345972631
.ds-logs-lu-ec-app_eis-default-2023.02.27-000003 3.1gb 9883481
# GET _data_stream/logs-lu-ec-app_eis-default 200 OK
{
"data_streams": [
{
"name": "logs-lu-ec-app_eis-default",
"timestamp_field": {
"name": "@timestamp"
},
"indices": [
{
"index_name": ".ds-logs-lu-ec-app_eis-default-2023.02.23-000001",
"index_uuid": "uf08u4rkTyK2LSievjdMuQ"
},
{
"index_name": ".ds-logs-lu-ec-app_eis-default-2023.02.25-000002",
"index_uuid": "u1BIUrGwRparNPhD6alSFA"
},
{
"index_name": ".ds-logs-lu-ec-app_eis-default-2023.02.27-000003",
"index_uuid": "FLY2npvtQUOGAQcJ5tKwcQ"
}
],
"generation": 3,
"_meta": {
"owner": "xxxxxxxxxxxxxxxxxxxxxxxxx",
"description": "Template used with .ds-logs-lu-ec-* data_streams"
},
"status": "GREEN",
"template": "index_template_elkg_migration",
"ilm_policy": "logs",
"hidden": false,
"system": false,
"allow_custom_routing": false,
"replicated": false
}
]
}
# GET _data_stream/logs-lu-ec-app_eis-default/_stats 200 OK
{
"_shards": {
"total": 12,
"successful": 12,
"failed": 0
},
"data_stream_count": 1,
"backing_indices": 3,
"total_store_size_bytes": 227775423955,
"data_streams": [
{
"data_stream": "logs-lu-ec-app_eis-default",
"backing_indices": 3,
"store_size_bytes": 227775423955,
"maximum_timestamp": 1677511641612
}
]
}
# GET /_ilm/status 200 OK
{
"operation_mode": "RUNNING"
}
# GET .ds-logs-lu-ec-app_eis-default-2023.02.23-000001/_ilm/explain 200 OK
{
"indices": {
".ds-logs-lu-ec-app_eis-default-2023.02.23-000001": {
"index": ".ds-logs-lu-ec-app_eis-default-2023.02.23-000001",
"managed": true,
"policy": "logs",
"index_creation_date_millis": 1677155907081,
"time_since_index_creation": "4.11d",
"lifecycle_date_millis": 1677358492906,
"age": "1.77d",
"phase": "hot",
"phase_time_millis": 1677155909555,
"action": "complete",
"action_time_millis": 1677358493707,
"step": "complete",
"step_time_millis": 1677358493707,
"phase_execution": {
"policy": "logs",
"phase_definition": {
"min_age": "0ms",
"actions": {
"rollover": {
"max_primary_shard_size": "30gb",
"max_age": "30d"
}
}
},
"version": 5,
"modified_date_in_millis": 1677072035871
}
}
}
}
# GET .ds-logs-lu-ec-app_eis-default-2023.02.25-000002/_ilm/explain 200 OK
{
"indices": {
".ds-logs-lu-ec-app_eis-default-2023.02.25-000002": {
"index": ".ds-logs-lu-ec-app_eis-default-2023.02.25-000002",
"managed": true,
"policy": "logs",
"index_creation_date_millis": 1677358493003,
"time_since_index_creation": "1.77d",
"lifecycle_date_millis": 1677506165140,
"age": "1.53h",
"phase": "hot",
"phase_time_millis": 1677358493307,
"action": "complete",
"action_time_millis": 1677506177350,
"step": "complete",
"step_time_millis": 1677506177350,
"phase_execution": {
"policy": "logs",
"phase_definition": {
"min_age": "0ms",
"actions": {
"rollover": {
"max_primary_shard_size": "30gb",
"max_age": "30d"
}
}
},
"version": 5,
"modified_date_in_millis": 1677072035871
}
}
}
}
# GET .ds-logs-lu-ec-app_eis-default-2023.02.27-000003/_ilm/explain 200 OK
{
"indices": {
".ds-logs-lu-ec-app_eis-default-2023.02.27-000003": {
"index": ".ds-logs-lu-ec-app_eis-default-2023.02.27-000003",
"managed": true,
"policy": "logs",
"index_creation_date_millis": 1677506165303,
"time_since_index_creation": "1.53h",
"lifecycle_date_millis": 1677506165303,
"age": "1.53h",
"phase": "hot",
"phase_time_millis": 1677506165740,
"action": "rollover",
"action_time_millis": 1677506165940,
"step": "check-rollover-ready",
"step_time_millis": 1677506165940,
"phase_execution": {
"policy": "logs",
"phase_definition": {
"min_age": "0ms",
"actions": {
"rollover": {
"max_primary_shard_size": "30gb",
"max_age": "30d"
}
}
},
"version": 5,
"modified_date_in_millis": 1677072035871
}
}
}
}
# GET _ilm/policy/logs 200 OK
{
"logs": {
"version": 5,
"modified_date": "2023-02-22T13:20:35.871Z",
"policy": {
"phases": {
"hot": {
"min_age": "0ms",
"actions": {
"rollover": {
"max_primary_shard_size": "30gb",
"max_age": "30d"
}
}
},
"delete": {
"min_age": "60d",
"actions": {
"delete": {
"delete_searchable_snapshot": true
},
"wait_for_snapshot": {
"policy": "daily_all.json"
}
}
}
},
"_meta": {
"description": "default policy for the logs index template installed by x-pack",
"managed": true
}
},
"in_use_by": {
"indices": [
".ds-logs-lu-ec-tel_expressway-default-2023.02.23-000001",
".ds-logs-lu-ec-sec_network_sda-default-2023.02.23-000001",
".ds-logs-lu-ec-sec_network_aci-default-2023.02.23-000001",
".ds-logs-lu-ec-sec_pulse-default-2023.02.23-000001",
".ds-logs-lu-ec-sec_network_legacy-default-2023.02.23-000001",
".ds-logs-lu-ec-sys_netapp-default-2023.02.23-000001",
".ds-logs-lu-ec-sec_cisco_ise-default-2023.02.23-000001",
".ds-logs-lu-ec-sys_vcsa6-default-2023.02.23-000001",
".ds-logs-lu-ec-sys_printing-default-2023.02.23-000001",
".ds-logs-lu-ec-sec_netscaler-default-2023.02.23-000001",
".ds-logs-lu-ec-sec_netskope-default-2023.02.23-000001",
".ds-logs-lu-ec-sec_testnetskope-default-2023.02.23-000001",
".ds-logs-lu-ec-app_lovion-default-2023.02.23-000001",
".ds-logs-lu-ec-sys_adaxes-default-2023.02.23-000001",
".ds-logs-lu-ec-app_eis-default-2023.02.27-000003",
. . .