Hi, I'm having an issue where certain index is being deleted after 28 days, I already removed the lifecycle policy which has 60 days delete phase and it is still being deleted after 28 days. Does anyone has experience on this? thanks
Which version of Elasticsearch are you using? Which index is being deleted? Do you by any chance have Curator running in addition to ILM?
Thank you for replying, we are using v7.16.2 on a Linux VM, this setup was done by another team so im still figuring out things, I checked elasticsearch VMs for any curator service and its not installed there. All indexes for different applications are 28 days, some don't have lifecycle policy by default but still only 28 days of logs remains.
Please share the index settings of the indices that should not be deleted after 28 days but still are. Please also share the configuration of any ILM policies that may be applying.
1 Like
Here is the index settings :
{
"settings": {
"index": {
"routing": {
"allocation": {
"include": {
"_tier_preference": "data_cold,data_warm,data_hot"
},
"require": {
"_id": "yyHhORzXQ5SVHT578O0NKQ"
}
}
},
"number_of_shards": "5",
"provided_name": "my-index-2023.08.09",
"creation_date": "1691539203997",
"priority": "0",
"number_of_replicas": "1",
"uuid": "D_mJUU-zSBm07Dr22-idKg",
"version": {
"created": "7160299"
}
}
},
"defaults": {
"index": {
"flush_after_merge": "512mb",
"final_pipeline": "_none",
"max_inner_result_window": "100",
"unassigned": {
"node_left": {
"delayed_timeout": "1m"
}
},
"max_terms_count": "65536",
"rollup": {
"source": {
"name": "",
"uuid": ""
}
},
"lifecycle": {
"name": "",
"parse_origination_date": "false",
"step": {
"wait_time_threshold": "12h"
},
"indexing_complete": "false",
"rollover_alias": "",
"origination_date": "-1"
},
"routing_partition_size": "1",
"force_memory_term_dictionary": "false",
"max_docvalue_fields_search": "100",
"merge": {
"scheduler": {
"max_thread_count": "2",
"auto_throttle": "true",
"max_merge_count": "7"
},
"policy": {
"floor_segment": "2mb",
"max_merge_at_once_explicit": "30",
"max_merge_at_once": "10",
"max_merged_segment": "5gb",
"expunge_deletes_allowed": "10.0",
"segments_per_tier": "10.0",
"deletes_pct_allowed": "33.0"
}
},
"max_refresh_listeners": "1000",
"max_regex_length": "1000",
"load_fixed_bitset_filters_eagerly": "true",
"number_of_routing_shards": "5",
"write": {
"wait_for_active_shards": "1"
},
"verified_before_close": "false",
"mapping": {
"coerce": "false",
"nested_fields": {
"limit": "50"
},
"depth": {
"limit": "20"
},
"field_name_length": {
"limit": "9223372036854775807"
},
"total_fields": {
"limit": "1000"
},
"nested_objects": {
"limit": "10000"
},
"ignore_malformed": "false",
"dimension_fields": {
"limit": "16"
}
},
"source_only": "false",
"soft_deletes": {
"enabled": "true",
"retention": {
"operations": "0"
},
"retention_lease": {
"period": "12h"
}
},
"max_script_fields": "32",
"query": {
"default_field": [
"*"
],
"parse": {
"allow_unmapped_fields": "true"
}
},
"format": "0",
"frozen": "false",
"sort": {
"missing": [],
"mode": [],
"field": [],
"order": []
},
"codec": "default",
"max_rescore_window": "10000",
"max_adjacency_matrix_filters": "100",
"analyze": {
"max_token_count": "10000"
},
"gc_deletes": "60s",
"top_metrics_max_size": "10",
"optimize_auto_generated_id": "true",
"max_ngram_diff": "1",
"hidden": "false",
"translog": {
"generation_threshold_size": "64mb",
"flush_threshold_size": "512mb",
"sync_interval": "5s",
"retention": {
"size": "-1",
"age": "-1"
},
"durability": "REQUEST"
},
"auto_expand_replicas": "false",
"mapper": {
"dynamic": "true"
},
"recovery": {
"type": ""
},
"requests": {
"cache": {
"enable": "true"
}
},
"data_path": "",
"highlight": {
"max_analyzed_offset": "1000000"
},
"routing": {
"rebalance": {
"enable": "all"
},
"allocation": {
"include": {
"_tier": ""
},
"disk": {
"watermark": {
"ignore": "false"
}
},
"exclude": {
"_tier": ""
},
"require": {
"_tier": ""
},
"enable": "all",
"total_shards_per_node": "-1"
}
},
"search": {
"slowlog": {
"level": "TRACE",
"threshold": {
"fetch": {
"warn": "-1",
"trace": "-1",
"debug": "-1",
"info": "-1"
},
"query": {
"warn": "-1",
"trace": "-1",
"debug": "-1",
"info": "-1"
}
}
},
"idle": {
"after": "30s"
},
"throttled": "false"
},
"fielddata": {
"cache": "node"
},
"default_pipeline": "_none",
"max_slices_per_scroll": "1024",
"shard": {
"check_on_startup": "false"
},
"xpack": {
"watcher": {
"template": {
"version": ""
}
},
"version": "",
"ccr": {
"following_index": "false"
}
},
"percolator": {
"map_unmapped_fields_as_text": "false"
},
"allocation": {
"max_retries": "5",
"existing_shards_allocator": "gateway_allocator"
},
"refresh_interval": "1s",
"indexing": {
"slowlog": {
"reformat": "true",
"threshold": {
"index": {
"warn": "-1",
"trace": "-1",
"debug": "-1",
"info": "-1"
}
},
"source": "1000",
"level": "TRACE"
}
},
"compound_format": "0.1",
"blocks": {
"metadata": "false",
"read": "false",
"read_only_allow_delete": "false",
"read_only": "false",
"write": "false"
},
"max_result_window": "10000",
"store": {
"stats_refresh_interval": "10s",
"type": "",
"fs": {
"fs_lock": "native"
},
"preload": [],
"snapshot": {
"snapshot_name": "",
"index_uuid": "",
"cache": {
"prewarm": {
"enabled": "true"
},
"enabled": "true",
"excluded_file_types": []
},
"repository_uuid": "",
"uncached_chunk_size": "-1b",
"index_name": "",
"partial": "false",
"blob_cache": {
"metadata_files": {
"max_length": "64kb"
}
},
"repository_name": "",
"snapshot_uuid": ""
}
},
"queries": {
"cache": {
"enabled": "true"
}
},
"shard_limit": {
"group": "normal"
},
"warmer": {
"enabled": "true"
},
"max_shingle_diff": "3",
"query_string": {
"lenient": "false"
}
}
}
}
grep for for "delete" on elasticsearch logs
k8s-cluster.log:[2023-09-06T00:00:15,474][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.03] from [{"phase":"warm","action":"complete","name":"complete"}] to [{"phase":"delete","action":"delete","name":"wait-for-shard-history-leases"}] in policy [7-days-default]
k8s-cluster.log:[2023-09-06T00:09:19,331][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.03] from [{"phase":"delete","action":"delete","name":"wait-for-shard-history-leases"}] to [{"phase":"delete","action":"delete","name":"cleanup-snapshot"}] in policy [7-days-default]
k8s-cluster.log:[2023-09-06T00:09:19,785][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.03] from [{"phase":"delete","action":"delete","name":"cleanup-snapshot"}] to [{"phase":"delete","action":"delete","name":"delete"}] in policy [7-days-default]
k8s-cluster.log:[2023-09-06T00:09:19,900][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [events-metricbeat-2023.09.03/1MEQ9YqcSRm3fh8bUua6uQ] deleting index
k8s-cluster.log:[2023-09-06T01:00:00,006][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [.monitoring-es-7-2023.08.30/CL-MnfZBQcOrYXtxxP76JA] deleting index
k8s-cluster.log:[2023-09-06T01:00:55,815][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index1-2023.08.09/eqVAEC8IQ9qMz7dfeTxujw] deleting index
k8s-cluster.log:[2023-09-06T01:00:55,815][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index2-2023.08.09/yT-AJgYORiagQO7_fu6RGQ] deleting index
k8s-cluster.log:[2023-09-06T01:00:55,815][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index3-2023.08.09/D_mJUU-zSBm07Dr22-idKg] deleting index
k8s-cluster.log:[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index4-2023.08.09/0Dorl8RaQiyY_lIsOGnxjg] deleting index
k8s-cluster.log:[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index5-2023.08.09/0DnFHE8ZRBumSXNxLZUzew] deleting index
k8s-cluster.log:[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index6-2023.08.09/Z4N7AEPkSZ6962tnMb79ug] deleting index
k8s-cluster.log:[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index7-2023.08.09/ym_66nYRQKGpgxAWfPqqVg] deleting index
k8s-cluster.log:[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index8-2023.08.09/PaxmbXpxRRGYYqpYSO8sUA] deleting index
k8s-cluster.log:[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index9-2023.08.09/UubobaEdTr6q92-R_RYkoA] deleting index
k8s-cluster.log:[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index10-2023.08.09/8oYrK9MYSu67uRwM1UT1YA] deleting index
k8s-cluster.log:[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index11-2023.08.09/pbFGzPJvR1CTXcK1Xjopfw] deleting index
k8s-cluster.log:[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index12-2023.08.09/7F66pVfBQbWfD7K5fiXZnA] deleting index
k8s-cluster.log:[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index13-2023.08.09/PdmK61_tSEWnGnPNuSdDfA] deleting index
k8s-cluster.log:[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index14-2023.08.09/439UYR1GRAKkOIIwUBQWAw] deleting index
k8s-cluster.log:[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index15-2023.08.09/PrK1e_XhRtiUbV7pNhHXAg] deleting index
k8s-cluster.log:[2023-09-06T01:25:00,009][INFO ][o.e.x.m.a.TransportDeleteExpiredDataAction] [vm-ip-address] Deleting expired data
k8s-cluster.log:[2023-09-06T01:25:00,024][INFO ][o.e.x.m.j.r.UnusedStatsRemover] [vm-ip-address] Successfully deleted [0] unused stats documents
k8s-cluster.log:[2023-09-06T01:25:00,025][INFO ][o.e.x.m.a.TransportDeleteExpiredDataAction] [vm-ip-address] Completed deletion of expired ML data
k8s-cluster.log:[2023-09-06T01:25:00,025][INFO ][o.e.x.m.MlDailyMaintenanceService] [vm-ip-address] Successfully completed [ML] maintenance task: triggerDeleteExpiredDataTask
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T00:00:15,474Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "moving index [events-metricbeat-2023.09.03] from [{\"phase\":\"warm\",\"action\":\"complete\",\"name\":\"complete\"}] to [{\"phase\":\"delete\",\"action\":\"delete\",\"name\":\"wait-for-shard-history-leases\"}] in policy [7-days-default]", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T00:09:19,331Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "moving index [events-metricbeat-2023.09.03] from [{\"phase\":\"delete\",\"action\":\"delete\",\"name\":\"wait-for-shard-history-leases\"}] to [{\"phase\":\"delete\",\"action\":\"delete\",\"name\":\"cleanup-snapshot\"}] in policy [7-days-default]", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T00:09:19,785Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "moving index [events-metricbeat-2023.09.03] from [{\"phase\":\"delete\",\"action\":\"delete\",\"name\":\"cleanup-snapshot\"}] to [{\"phase\":\"delete\",\"action\":\"delete\",\"name\":\"delete\"}] in policy [7-days-default]", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T00:09:19,900Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[events-metricbeat-2023.09.03/1MEQ9YqcSRm3fh8bUua6uQ] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:00,006Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[.monitoring-es-7-2023.08.30/CL-MnfZBQcOrYXtxxP76JA] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:55,815Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[index1-2023.08.09/eqVAEC8IQ9qMz7dfeTxujw] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:55,815Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[index2-2023.08.09/yT-AJgYORiagQO7_fu6RGQ] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:55,815Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[index3-2023.08.09/D_mJUU-zSBm07Dr22-idKg] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:55,816Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[index4-2023.08.09/0Dorl8RaQiyY_lIsOGnxjg] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:55,816Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[index5-2023.08.09/0DnFHE8ZRBumSXNxLZUzew] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:55,816Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[index6-2023.08.09/Z4N7AEPkSZ6962tnMb79ug] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:55,816Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[index7-2023.08.09/ym_66nYRQKGpgxAWfPqqVg] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:55,816Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[index8-2023.08.09/PaxmbXpxRRGYYqpYSO8sUA] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:55,816Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[index9-2023.08.09/UubobaEdTr6q92-R_RYkoA] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:55,816Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[index10-2023.08.09/8oYrK9MYSu67uRwM1UT1YA] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:55,816Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[index11-2023.08.09/pbFGzPJvR1CTXcK1Xjopfw] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:55,816Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[index12-2023.08.09/7F66pVfBQbWfD7K5fiXZnA] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:55,816Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[index13-2023.08.09/PdmK61_tSEWnGnPNuSdDfA] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:55,816Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[index14-2023.08.09/439UYR1GRAKkOIIwUBQWAw] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:00:55,816Z", "level": "INFO", "component": "o.e.c.m.MetadataDeleteIndexService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "[index15-2023.08.09/PrK1e_XhRtiUbV7pNhHXAg] deleting index", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:25:00,009Z", "level": "INFO", "component": "o.e.x.m.a.TransportDeleteExpiredDataAction", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "Deleting expired data", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:25:00,024Z", "level": "INFO", "component": "o.e.x.m.j.r.UnusedStatsRemover", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "Successfully deleted [0] unused stats documents", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:25:00,025Z", "level": "INFO", "component": "o.e.x.m.a.TransportDeleteExpiredDataAction", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "Completed deletion of expired ML data", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
k8s-cluster_server.json:{"type": "server", "timestamp": "2023-09-06T01:25:00,025Z", "level": "INFO", "component": "o.e.x.m.MlDailyMaintenanceService", "cluster.name": "k8s-cluster", "node.name": "vm-ip-address", "message": "Successfully completed [ML] maintenance task: triggerDeleteExpiredDataTask", "cluster.uuid": "Rd65RBeuTMuTzO9JLntixA", "node.id": "NNn-IhAJR1OUGe7DwhjtRQ" }
[2023-09-06T00:00:00,266][INFO ][o.e.c.m.MetadataCreateIndexService] [vm-ip-address] [events-metricbeat-2023.09.06] creating index, cause [auto(bulk api)], templates [container-events-me
[2023-09-06T00:00:01,691][INFO ][o.e.c.m.MetadataCreateIndexService] [vm-ip-address] [index12-2023.09.06] creating index, cause [auto(bulk api)], templates [], shards [1]/[1]
[2023-09-06T00:00:02,047][INFO ][o.e.c.m.MetadataCreateIndexService] [vm-ip-address] [index14-2023.09.06] creating index, cause [auto(bulk api)], templates [index14
[2023-09-06T00:00:02,374][INFO ][o.e.c.m.MetadataCreateIndexService] [vm-ip-address] [index7-2023.09.06] creating index, cause [auto(bulk api)], templates [], shards [1]/[1]
[2023-09-06T00:00:02,648][INFO ][o.e.c.m.MetadataCreateIndexService] [vm-ip-address] [index3-2023.09.06] creating index, cause [auto(bulk api)], templates [aia_vitality_mpv_ve_vpf_template],
[2023-09-06T00:00:03,270][INFO ][o.e.c.m.MetadataCreateIndexService] [vm-ip-address] [.monitoring-es-7-2023.09.06] creating index, cause [auto(bulk api)], templates [.monitoring-es], shards [1]/[0]
[2023-09-06T00:00:03,276][INFO ][o.e.c.r.a.AllocationService] [vm-ip-address] updating number_of_replicas to [1] for indices [.monitoring-es-7-2023.09.06]
[2023-09-06T00:00:03,833][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:03,977][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:04,024][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:04,063][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:04,142][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:04,194][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:04,268][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:04,311][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:04,354][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:04,465][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:04,503][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:04,562][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:05,049][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index7-2023.09.06/W4oGq-VoRnm4aEa913nGmA] create_mapping [_doc]
[2023-09-06T00:00:05,168][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:05,175][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index12-2023.09.06/Mp29r8kvSAidDmqF5HyzOA] create_mapping [_doc]
[2023-09-06T00:00:05,918][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:06,066][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:06,129][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:06,209][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:06,219][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index3-2023.09.06/VcD-YuWsR3OCSntBJ9nIHQ] update_mapping [_doc]
[2023-09-06T00:00:06,254][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:06,290][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:06,920][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:07,034][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:07,166][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:07,197][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:07,384][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:07,412][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:07,472][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:07,787][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:08,023][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:08,071][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:08,684][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:08,744][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:08,773][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:08,847][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:08,871][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:08,926][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:08,934][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index7-2023.09.06/W4oGq-VoRnm4aEa913nGmA] update_mapping [_doc]
[2023-09-06T00:00:08,956][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:09,430][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:09,529][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:09,563][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:09,648][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:09,723][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:10,056][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:10,102][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:10,166][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:10,270][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:10,616][INFO ][o.e.c.m.MetadataCreateIndexService] [vm-ip-address] [index15-2023.09.06] creating index, cause [auto(bulk api)], templates [], shards [1]/[1]
[2023-09-06T00:00:11,255][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:11,304][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index9-2023.09.06/mJLPcC0ZQJKAV7ZbftI0Wg] update_mapping [_doc]
[2023-09-06T00:00:11,428][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index1-2023.09.06/r4fWU_4DRXmeDAGYNEFsxQ] update_mapping [_doc]
[2023-09-06T00:00:12,123][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:12,195][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:12,240][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index15-2023.09.06/78xZrurxRSusxq16AeNAig] create_mapping [_doc]
[2023-09-06T00:00:12,269][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [events-metricbeat-2023.09.06/0bgRxD-dRairRug9TsEyPA] update_mapping [_doc]
[2023-09-06T00:00:12,974][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:13,399][INFO ][o.e.c.m.MetadataCreateIndexService] [vm-ip-address] [index11-2023.09.06] creating index, cause [auto(bulk api)], templates [], shards [1]/[1]
[2023-09-06T00:00:13,793][INFO ][o.e.c.m.MetadataCreateIndexService] [vm-ip-address] [index6-2023.09.06] creating index, cause [auto(bulk api)], templates [], shards [1]/[1]
[2023-09-06T00:00:14,147][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index11-2023.09.06/gXTi-IWBRSmUX86A3Bh-xg] create_mapping [_doc]
[2023-09-06T00:00:14,965][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:15,014][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index6-2023.09.06/zfQm8rC-RJCiBmQJPv7VKQ] create_mapping [_doc]
[2023-09-06T00:00:15,474][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.06] from [null] to [{"phase":"new","action":"complete","name":"comp
[2023-09-06T00:00:15,474][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.03] from [{"phase":"warm","action":"complete","name":"complete"}] ty-leases"}] in policy [7-days-default]
[2023-09-06T00:00:15,475][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.05] from [{"phase":"hot","action":"complete","name":"complete"}] to policy [7-days-default]
[2023-09-06T00:00:16,719][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:17,795][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.06] from [{"phase":"new","action":"complete","name":"complete"}] to7-days-default]
[2023-09-06T00:00:17,796][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.05] from [{"phase":"warm","action":"set_priority","name":"set_priorte-index"}] in policy [7-days-default]
[2023-09-06T00:00:17,986][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.05] from [{"phase":"warm","action":"readonly","name":"check-not-wri"}] in policy [7-days-default]
[2023-09-06T00:00:18,175][INFO ][o.e.c.m.MetadataIndexStateService] [vm-ip-address] adding block write to indices [[events-metricbeat-2023.09.05/ydTSg9vxQ7OcQ31lA2NYdg]]
[2023-09-06T00:00:18,396][INFO ][o.e.c.m.MetadataIndexStateService] [vm-ip-address] completed adding block write to indices [events-metricbeat-2023.09.05]
[2023-09-06T00:00:18,472][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.05] from [{"phase":"warm","action":"readonly","name":"readonly"}] ton"}] in policy [7-days-default]
[2023-09-06T00:00:18,569][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.05] from [{"phase":"warm","action":"migrate","name":"branch-check-s"}] in policy [7-days-default]
[2023-09-06T00:00:19,027][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.05] from [{"phase":"warm","action":"migrate","name":"migrate"}] to licy [7-days-default]
[2023-09-06T00:00:19,187][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.05] from [{"phase":"warm","action":"migrate","name":"check-migratioerge-check-prerequisites"}] in policy [7-days-default]
[2023-09-06T00:00:19,188][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.05] from [{"phase":"warm","action":"forcemerge","name":"branch-forcge","name":"check-not-write-index"}] in policy [7-days-default]
[2023-09-06T00:00:19,189][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.05] from [{"phase":"warm","action":"forcemerge","name":"check-not-wonly"}] in policy [7-days-default]
[2023-09-06T00:00:19,286][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.05] from [{"phase":"warm","action":"forcemerge","name":"readonly"}]policy [7-days-default]
[2023-09-06T00:00:20,032][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:21,378][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:44,616][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:00:46,470][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index12-2023.09.06/Mp29r8kvSAidDmqF5HyzOA] update_mapping [_doc]
[2023-09-06T00:02:59,491][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T00:09:19,331][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.03] from [{"phase":"delete","action":"delete","name":"wait-for-sharcleanup-snapshot"}] in policy [7-days-default]
[2023-09-06T00:09:19,785][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.03] from [{"phase":"delete","action":"delete","name":"cleanup-snapspolicy [7-days-default]
[2023-09-06T00:09:19,900][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [events-metricbeat-2023.09.03/1MEQ9YqcSRm3fh8bUua6uQ] deleting index
[2023-09-06T00:15:52,954][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.05] from [{"phase":"warm","action":"forcemerge","name":"forcemerge"] in policy [7-days-default]
[2023-09-06T00:19:19,330][INFO ][o.e.x.i.IndexLifecycleTransition] [vm-ip-address] moving index [events-metricbeat-2023.09.05] from [{"phase":"warm","action":"forcemerge","name":"segment-cou policy [7-days-default]
[2023-09-06T01:00:00,000][INFO ][o.e.x.m.e.l.LocalExporter] [vm-ip-address] cleaning up [1] old indices
[2023-09-06T01:00:00,006][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [.monitoring-es-7-2023.08.30/CL-MnfZBQcOrYXtxxP76JA] deleting index
[2023-09-06T01:00:08,415][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T01:00:16,189][INFO ][o.e.c.m.MetadataMappingService] [vm-ip-address] [index14-2023.09.06/qYaD5MTTQMKhpW7-WinZdA] update_mapping [_doc]
[2023-09-06T01:00:55,815][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index1-2023.08.09/eqVAEC8IQ9qMz7dfeTxujw] deleting index
[2023-09-06T01:00:55,815][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index2-2023.08.09/yT-AJgYORiagQO7_fu6RGQ] deleting index
[2023-09-06T01:00:55,815][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index3-2023.08.09/D_mJUU-zSBm07Dr22-idKg] deleting index
[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index4-2023.08.09/0Dorl8RaQiyY_lIsOGnxjg] deleting index
[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index5-2023.08.09/0DnFHE8ZRBumSXNxLZUzew] deleting index
[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index6-2023.08.09/Z4N7AEPkSZ6962tnMb79ug] deleting index
[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index7-2023.08.09/ym_66nYRQKGpgxAWfPqqVg] deleting index
[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index8-2023.08.09/PaxmbXpxRRGYYqpYSO8sUA] deleting index
[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index9-2023.08.09/UubobaEdTr6q92-R_RYkoA] deleting index
[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index10-2023.08.09/8oYrK9MYSu67uRwM1UT1YA] deleting index
[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index11-2023.08.09/pbFGzPJvR1CTXcK1Xjopfw] deleting index
[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index12-2023.08.09/7F66pVfBQbWfD7K5fiXZnA] deleting index
[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index13-2023.08.09/PdmK61_tSEWnGnPNuSdDfA] deleting index
[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index14-2023.08.09/439UYR1GRAKkOIIwUBQWAw] deleting index
[2023-09-06T01:00:55,816][INFO ][o.e.c.m.MetadataDeleteIndexService] [vm-ip-address] [index15-2023.08.09/PrK1e_XhRtiUbV7pNhHXAg] deleting index
Go to the Kibana dev console and run GET _ilm/policy to see which policies exist in the cluster. Has the one you removed been deleted? Are there any other policies that could be applying to the indices you see being deleted?
1 Like
thank you for your response! this is the output of the command :
".alerts-ilm-policy" : {
"version" : 103,
"modified_date" : "2023-08-23T10:17:34.008Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
}
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
".deprecation-indexing-ilm-policy" : {
"version" : 1,
"modified_date" : "2021-12-17T03:55:17.357Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "10gb",
"max_age" : "14d"
}
}
}
}
},
"in_use_by" : {
"indices" : [
".ds-.logs-deprecation.elasticsearch-default-2022.09.09-000020",
".ds-.logs-deprecation.elasticsearch-default-2022.04.08-000009",
".ds-.logs-deprecation.elasticsearch-default-2022.05.20-000012",
".ds-.logs-deprecation.elasticsearch-default-2023.05.05-000037",
".ds-.logs-deprecation.elasticsearch-default-2022.10.21-000023",
".ds-.logs-deprecation.elasticsearch-default-2022.02.25-000006",
".ds-.logs-deprecation.elasticsearch-default-2023.03.24-000034",
".ds-.logs-deprecation.elasticsearch-default-2023.01.27-000030",
".ds-.logs-deprecation.elasticsearch-default-2022.07.01-000015",
".ds-.logs-deprecation.elasticsearch-default-2022.12.30-000028",
".ds-.logs-deprecation.elasticsearch-default-2023.08.11-000044",
".ds-.logs-deprecation.elasticsearch-default-2023.07.28-000043",
".ds-.logs-deprecation.elasticsearch-default-2022.10.07-000022",
".ds-.logs-deprecation.elasticsearch-default-2022.07.15-000016",
".ds-.logs-deprecation.elasticsearch-default-2023.03.10-000033",
".ds-.logs-deprecation.elasticsearch-default-2023.07.14-000042",
".ds-.logs-deprecation.elasticsearch-default-2022.11.18-000025",
".ds-.logs-deprecation.elasticsearch-default-2022.03.11-000007",
".ds-.logs-deprecation.elasticsearch-default-2022.06.17-000014",
".ds-.logs-deprecation.elasticsearch-default-2022.04.22-000010",
".ds-.logs-deprecation.elasticsearch-default-2022.06.03-000013",
".ds-.logs-deprecation.elasticsearch-default-2023.06.02-000039",
".ds-.logs-deprecation.elasticsearch-default-2023.04.21-000036",
".ds-.logs-deprecation.elasticsearch-default-2022.03.25-000008",
".ds-.logs-deprecation.elasticsearch-default-2023.02.24-000032",
".ds-.logs-deprecation.elasticsearch-default-2022.01.28-000004",
".ds-.logs-deprecation.elasticsearch-default-000001",
".ds-.logs-deprecation.elasticsearch-default-2022.07.29-000017",
".ds-.logs-deprecation.elasticsearch-default-2022.08.12-000018",
".ds-.logs-deprecation.elasticsearch-default-2022.11.04-000024",
".ds-.logs-deprecation.elasticsearch-default-2022.01.14-000003",
".ds-.logs-deprecation.elasticsearch-default-2023.08.25-000045",
".ds-.logs-deprecation.elasticsearch-default-2023.04.07-000035",
".ds-.logs-deprecation.elasticsearch-default-2022.02.11-000005",
".ds-.logs-deprecation.elasticsearch-default-2021.12.31-000002",
".ds-.logs-deprecation.elasticsearch-default-2023.01.13-000029",
".ds-.logs-deprecation.elasticsearch-default-2022.12.16-000027",
".ds-.logs-deprecation.elasticsearch-default-2023.06.30-000041",
".ds-.logs-deprecation.elasticsearch-default-2023.02.10-000031",
".ds-.logs-deprecation.elasticsearch-default-2022.09.23-000021",
".ds-.logs-deprecation.elasticsearch-default-2022.05.06-000011",
".ds-.logs-deprecation.elasticsearch-default-2022.12.02-000026",
".ds-.logs-deprecation.elasticsearch-default-2023.06.16-000040",
".ds-.logs-deprecation.elasticsearch-default-2023.05.19-000038",
".ds-.logs-deprecation.elasticsearch-default-2022.08.26-000019"
],
"data_streams" : [
".logs-deprecation.elasticsearch-default"
],
"composable_templates" : [
".deprecation-indexing-template"
]
}
},
".fleet-actions-results-ilm-policy" : {
"version" : 1,
"modified_date" : "2021-12-17T03:46:01.697Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "300gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
".items-default" : {
"version" : 1,
"modified_date" : "2021-06-24T03:16:02.726Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb"
}
}
}
}
},
"in_use_by" : {
"indices" : [
".items-default-000001"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
".lists-default" : {
"version" : 1,
"modified_date" : "2021-06-24T03:16:02.503Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb"
}
}
}
}
},
"in_use_by" : {
"indices" : [
".lists-default-000001"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"180-days-default" : {
"version" : 1,
"modified_date" : "2021-12-17T03:55:16.616Z",
"policy" : {
"phases" : {
"warm" : {
"min_age" : "2d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"shrink" : {
"number_of_shards" : 1
}
}
},
"cold" : {
"min_age" : "30d",
"actions" : { }
},
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "180d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot, warm, and cold phases with a retention of 180 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"30-days-default" : {
"version" : 14,
"modified_date" : "2023-07-24T07:37:06.073Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"set_priority" : {
"priority" : 100
}
}
},
"delete" : {
"min_age" : "60d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
},
"warm" : {
"min_age" : "2d",
"actions" : {
"allocate" : {
"number_of_replicas" : 1,
"include" : { },
"exclude" : { },
"require" : { }
},
"set_priority" : {
"priority" : 50
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot and warm phases with a retention of 30 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [
"filebeat-d-prod-2023.08.13",
"filebeat-d-prod-2023.08.14",
"filebeat-d-prod-2023.08.12",
"filebeat-s-prod-2023.09.08",
"filebeat-s-prod-2023.09.06",
"filebeat-s-prod-2023.09.07",
"filebeat-s-prod-2023.08.13",
"filebeat-s-prod-2023.08.12",
"filebeat-s-prod-2023.08.15",
"filebeat-s-prod-2023.08.14",
"filebeat-d-prod-2023.09.07",
"filebeat-d-prod-2023.09.08",
"filebeat-d-prod-2023.09.05",
"filebeat-d-prod-2023.09.06",
"filebeat-s-prod-2023.08.20",
"filebeat-d-prod-2023.08.31",
"filebeat-s-prod-2023.08.22",
"filebeat-s-prod-2023.08.21",
"filebeat-d-prod-2023.08.30",
"filebeat-s-prod-2023.08.24",
"filebeat-d-prod-2023.09.03",
"filebeat-d-prod-2023.09.04",
"filebeat-s-prod-2023.08.23",
"filebeat-s-prod-2023.08.26",
"filebeat-d-prod-2023.09.01",
"filebeat-d-prod-2023.09.02",
"filebeat-s-prod-2023.08.25",
"filebeat-s-prod-2023.08.17",
"filebeat-d-prod-2023.08.28",
"filebeat-s-prod-2023.08.16",
"filebeat-d-prod-2023.08.29",
"filebeat-s-prod-2023.08.19",
"filebeat-d-prod-2023.08.26",
"filebeat-s-prod-2023.08.18",
"filebeat-d-prod-2023.08.27",
"filebeat-d-prod-2023.08.20",
"filebeat-s-prod-2023.08.31",
"filebeat-d-prod-2023.08.21",
"filebeat-s-prod-2023.09.01",
"filebeat-s-prod-2023.08.30",
"filebeat-s-prod-2023.09.04",
"filebeat-d-prod-2023.08.24",
"filebeat-s-prod-2023.09.05",
"filebeat-d-prod-2023.08.25",
"filebeat-s-prod-2023.09.02",
"filebeat-d-prod-2023.08.22",
"filebeat-d-prod-2023.08.23",
"filebeat-s-prod-2023.09.03",
"filebeat-s-prod-2023.08.28",
"filebeat-d-prod-2023.08.17",
"filebeat-d-prod-2023.08.18",
"filebeat-s-prod-2023.08.27",
"filebeat-d-prod-2023.08.15",
"filebeat-s-prod-2023.08.29",
"filebeat-d-prod-2023.08.16",
"filebeat-d-prod-2023.08.19"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"365-days-default" : {
"version" : 1,
"modified_date" : "2021-12-17T03:55:17.230Z",
"policy" : {
"phases" : {
"warm" : {
"min_age" : "2d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"shrink" : {
"number_of_shards" : 1
}
}
},
"cold" : {
"min_age" : "30d",
"actions" : { }
},
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "365d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot, warm, and cold phases with a retention of 365 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"60-days-default" : {
"version" : 10,
"modified_date" : "2023-08-30T07:47:42.759Z",
"policy" : {
"phases" : {
"warm" : {
"min_age" : "7d",
"actions" : {
"set_priority" : {
"priority" : 50
},
"shrink" : {
"number_of_shards" : 2
}
}
},
"cold" : {
"min_age" : "14d",
"actions" : {
"set_priority" : {
"priority" : 0
}
}
},
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "100gb",
"max_age" : "40d"
},
"set_priority" : {
"priority" : 100
}
}
},
"delete" : {
"min_age" : "60d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot, warm, and cold phases with a retention of 180 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"7-days-default" : {
"version" : 6,
"modified_date" : "2023-04-03T03:07:52.087Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : { }
},
"delete" : {
"min_age" : "3d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
},
"warm" : {
"min_age" : "1d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"readonly" : { },
"set_priority" : {
"priority" : 50
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot and warm phases with a retention of 7 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [
"events-metricbeat-2023.09.08",
"events-metricbeat-2023.09.06",
"events-metricbeat-2023.09.07"
],
"data_streams" : [ ],
"composable_templates" : [
"events-metricbeat"
]
}
},
"90-days-default" : {
"version" : 1,
"modified_date" : "2021-12-17T03:55:16.920Z",
"policy" : {
"phases" : {
"warm" : {
"min_age" : "2d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"shrink" : {
"number_of_shards" : 1
}
}
},
"cold" : {
"min_age" : "30d",
"actions" : { }
},
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot, warm, and cold phases with a retention of 90 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"application_custom__policy" : {
"version" : 20,
"modified_date" : "2023-08-18T08:20:29.350Z",
"policy" : {
"phases" : {
"warm" : {
"min_age" : "2d",
"actions" : {
"allocate" : {
"number_of_replicas" : 1,
"include" : { },
"exclude" : { },
"require" : { }
},
"set_priority" : {
"priority" : 50
}
}
},
"cold" : {
"min_age" : "30d",
"actions" : {
"set_priority" : {
"priority" : 0
}
}
},
"hot" : {
"min_age" : "0ms",
"actions" : {
"set_priority" : {
"priority" : 100
}
}
},
"delete" : {
"min_age" : "60d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"ilm-history-ilm-policy" : {
"version" : 1,
"modified_date" : "2020-10-29T06:45:12.538Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
".ds-ilm-history-5-2023.08.09-000038",
".ds-ilm-history-5-2023.09.08-000039",
"ilm-history-2-000035",
"ilm-history-2-000034",
".ds-ilm-history-5-2023.07.10-000035",
"ilm-history-2-000033",
"ilm-history-2-000032",
".ds-ilm-history-5-2023.06.10-000034"
],
"data_streams" : [
"ilm-history-5"
],
"composable_templates" : [
"ilm-history"
]
}
},
"kibana-event-log-policy" : {
"version" : 1,
"modified_date" : "2020-10-29T10:47:15.943Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
".kibana-event-log-7.9.3-000035",
".kibana-event-log-7.16.1-000019",
".kibana-event-log-7.9.3-000034",
".kibana-event-log-7.9.3-000033",
".kibana-event-log-7.9.3-000032",
".kibana-event-log-7.16.2-000018",
".kibana-event-log-7.16.2-000017",
".kibana-event-log-7.16.2-000019",
".kibana-event-log-7.16.1-000022",
".kibana-event-log-7.16.1-000021",
".kibana-event-log-7.16.1-000020",
".kibana-event-log-7.16.2-000016"
],
"data_streams" : [ ],
"composable_templates" : [
".kibana-event-log-7.16.1-template",
".kibana-event-log-7.16.2-template"
]
}
},
"kibana-reporting" : {
"version" : 1,
"modified_date" : "2021-12-17T04:35:08.935Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : { }
}
}
},
"in_use_by" : {
"indices" : [
".reporting-2023-09-03",
".reporting-2022-10-23",
".reporting-2022-12-04",
".reporting-2022-05-29",
".reporting-2023-03-12",
".reporting-2022-04-17",
".reporting-2023-06-04",
".reporting-2023-04-23",
".reporting-2022-01-23",
".reporting-2023-08-20",
".reporting-2023-08-27",
".reporting-2023-07-16",
".reporting-2023-02-19",
".reporting-2022-10-30",
".reporting-2023-01-08",
".reporting-2022-12-11",
".reporting-2022-02-20",
".reporting-2022-06-26",
".reporting-2023-01-01",
".reporting-2022-09-18",
".reporting-2023-03-26",
".reporting-2022-05-15",
".reporting-2023-02-12",
".reporting-2022-07-31",
".reporting-2022-02-27",
".reporting-2022-09-11",
".reporting-2023-04-30",
".reporting-2023-08-13",
".reporting-2022-11-13",
".reporting-2023-04-09",
".reporting-2022-10-09",
".reporting-2023-02-26",
".reporting-2023-01-15",
".reporting-2023-04-02",
".reporting-2023-05-14",
".reporting-2023-06-25",
".reporting-2022-07-24",
".reporting-2022-05-01",
".reporting-2023-08-06",
".reporting-2022-11-20",
".reporting-2021-12-19",
".reporting-2023-01-29",
".reporting-2023-03-05",
".reporting-2022-04-24",
".reporting-2022-03-13",
".reporting-2023-04-16",
".reporting-2022-06-05",
".reporting-2022-10-16",
".reporting-2022-11-27",
".reporting-2023-01-22",
".reporting-2022-02-06",
".reporting-2022-08-21",
".reporting-2022-07-10",
".reporting-2023-05-21"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"logs" : {
"version" : 1,
"modified_date" : "2020-10-29T06:45:12.440Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
}
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
"logs"
]
}
},
"metricbeat" : {
"version" : 2,
"modified_date" : "2020-10-30T03:30:18.471Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
}
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"metrics" : {
"version" : 1,
"modified_date" : "2020-10-29T06:45:12.390Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
}
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
"metrics"
]
}
},
"ml-size-based-ilm-policy" : {
"version" : 1,
"modified_date" : "2020-10-29T06:45:12.326Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb"
}
}
}
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
".ml-state",
".ml-stats"
]
}
},
"slm-history-ilm-policy" : {
"version" : 1,
"modified_date" : "2020-10-29T06:45:12.588Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
".slm-history"
]
}
},
"synthetics" : {
"version" : 1,
"modified_date" : "2021-12-17T03:55:17.065Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
}
},
"_meta" : {
"description" : "default policy for the synthetics index template installed by x-pack",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
"synthetics"
]
}
},
"watch-history-ilm-policy" : {
"version" : 1,
"modified_date" : "2020-10-29T06:45:12.483Z",
"policy" : {
"phases" : {
"delete" : {
"min_age" : "7d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
".watch-history-13"
]
}
}
}
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.