Hi,
We are running a small deployment in the Elastic Cloud ingesting very low events per second, however every couple of days we receive an email advising that a node has been restarted as it is out of memory.
Checking the node in kibana for the given time period shows very low memory usage and certainly nothing which would indicate a restart was required.
Has anyone else had this issue? Or have any ideas why it's happening?
Thanks
What is the output from the _cluster/stats?pretty&human API?
Thanks. Here's the output, I redacted the cluster name and uuid (but can message if you need them).
{
"_nodes" : {
"total" : 3,
"successful" : 3,
"failed" : 0
},
"cluster_name" : "XXXXXXXXX",
"cluster_uuid" : "XXXXXXXXX",
"timestamp" : 1644536696188,
"status" : "green",
"indices" : {
"count" : 211,
"shards" : {
"total" : 422,
"primaries" : 211,
"replication" : 1.0,
"index" : {
"shards" : {
"min" : 2,
"max" : 2,
"avg" : 2.0
},
"primaries" : {
"min" : 1,
"max" : 1,
"avg" : 1.0
},
"replication" : {
"min" : 1.0,
"max" : 1.0,
"avg" : 1.0
}
}
},
"docs" : {
"count" : 80568730,
"deleted" : 18028
},
"store" : {
"size" : "42gb",
"size_in_bytes" : 45115827793,
"total_data_set_size" : "42gb",
"total_data_set_size_in_bytes" : 45115827793,
"reserved" : "0b",
"reserved_in_bytes" : 0
},
"fielddata" : {
"memory_size" : "1mb",
"memory_size_in_bytes" : 1101616,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "16.5mb",
"memory_size_in_bytes" : 17311302,
"total_count" : 441989417,
"hit_count" : 4488396,
"miss_count" : 437501021,
"cache_size" : 25342,
"cache_count" : 66198,
"evictions" : 40856
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 1710,
"memory" : "26.5mb",
"memory_in_bytes" : 27848978,
"terms_memory" : "18.2mb",
"terms_memory_in_bytes" : 19098936,
"stored_fields_memory" : "936.4kb",
"stored_fields_memory_in_bytes" : 958912,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "189.1kb",
"norms_memory_in_bytes" : 193728,
"points_memory" : "0b",
"points_memory_in_bytes" : 0,
"doc_values_memory" : "7.2mb",
"doc_values_memory_in_bytes" : 7597402,
"index_writer_memory" : "26.8mb",
"index_writer_memory_in_bytes" : 28103292,
"version_map_memory" : "2.2mb",
"version_map_memory_in_bytes" : 2336682,
"fixed_bit_set" : "18mb",
"fixed_bit_set_memory_in_bytes" : 18914032,
"max_unsafe_auto_id_timestamp" : 1644491316062,
"file_sizes" : { }
},
"mappings" : {
"field_types" : [
{
"name" : "alias",
"count" : 392,
"index_count" : 53,
"script_count" : 0
},
{
"name" : "binary",
"count" : 7,
"index_count" : 7,
"script_count" : 0
},
{
"name" : "boolean",
"count" : 2415,
"index_count" : 111,
"script_count" : 0
},
{
"name" : "byte",
"count" : 36,
"index_count" : 36,
"script_count" : 0
},
{
"name" : "constant_keyword",
"count" : 166,
"index_count" : 56,
"script_count" : 0
},
{
"name" : "date",
"count" : 4116,
"index_count" : 171,
"script_count" : 0
},
{
"name" : "date_nanos",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "date_range",
"count" : 7,
"index_count" : 7,
"script_count" : 0
},
{
"name" : "double",
"count" : 547,
"index_count" : 14,
"script_count" : 0
},
{
"name" : "double_range",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "flattened",
"count" : 764,
"index_count" : 52,
"script_count" : 0
},
{
"name" : "float",
"count" : 860,
"index_count" : 82,
"script_count" : 0
},
{
"name" : "float_range",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "geo_point",
"count" : 590,
"index_count" : 87,
"script_count" : 0
},
{
"name" : "geo_shape",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "half_float",
"count" : 70,
"index_count" : 28,
"script_count" : 0
},
{
"name" : "histogram",
"count" : 29,
"index_count" : 29,
"script_count" : 0
},
{
"name" : "integer",
"count" : 108,
"index_count" : 33,
"script_count" : 0
},
{
"name" : "integer_range",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "ip",
"count" : 1661,
"index_count" : 93,
"script_count" : 0
},
{
"name" : "ip_range",
"count" : 6,
"index_count" : 6,
"script_count" : 0
},
{
"name" : "keyword",
"count" : 82123,
"index_count" : 188,
"script_count" : 0
},
{
"name" : "long",
"count" : 17103,
"index_count" : 129,
"script_count" : 0
},
{
"name" : "long_range",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "match_only_text",
"count" : 1223,
"index_count" : 20,
"script_count" : 0
},
{
"name" : "nested",
"count" : 723,
"index_count" : 67,
"script_count" : 0
},
{
"name" : "object",
"count" : 23983,
"index_count" : 147,
"script_count" : 0
},
{
"name" : "scaled_float",
"count" : 403,
"index_count" : 54,
"script_count" : 0
},
{
"name" : "shape",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "short",
"count" : 530,
"index_count" : 14,
"script_count" : 0
},
{
"name" : "text",
"count" : 3670,
"index_count" : 121,
"script_count" : 0
},
{
"name" : "version",
"count" : 7,
"index_count" : 7,
"script_count" : 0
},
{
"name" : "wildcard",
"count" : 323,
"index_count" : 23,
"script_count" : 0
}
],
"runtime_field_types" : [ ]
},
"analysis" : {
"char_filter_types" : [ ],
"tokenizer_types" : [ ],
"filter_types" : [ ],
"analyzer_types" : [ ],
"built_in_char_filters" : [ ],
"built_in_tokenizers" : [ ],
"built_in_filters" : [ ],
"built_in_analyzers" : [
{
"name" : "simple",
"count" : 12,
"index_count" : 6
},
{
"name" : "whitespace",
"count" : 1,
"index_count" : 1
}
]
},
"versions" : [
{
"version" : "7.15.2",
"index_count" : 113,
"primary_shard_count" : 113,
"total_primary_size" : "8gb",
"total_primary_bytes" : 8648461201
},
{
"version" : "7.16.1",
"index_count" : 98,
"primary_shard_count" : 98,
"total_primary_size" : "12.7gb",
"total_primary_bytes" : 13664178328
}
]
},
"nodes" : {
"count" : {
"total" : 3,
"coordinating_only" : 0,
"data" : 0,
"data_cold" : 0,
"data_content" : 2,
"data_frozen" : 0,
"data_hot" : 2,
"data_warm" : 0,
"ingest" : 2,
"master" : 3,
"ml" : 0,
"remote_cluster_client" : 2,
"transform" : 2,
"voting_only" : 1
},
"versions" : [
"7.16.1"
],
"os" : {
"available_processors" : 6,
"allocated_processors" : 6,
"names" : [
{
"name" : "Linux",
"count" : 3
}
],
"pretty_names" : [
{
"pretty_name" : "Ubuntu 20.04.3 LTS",
"count" : 3
}
],
"architectures" : [
{
"arch" : "amd64",
"count" : 3
}
],
"mem" : {
"total" : "9gb",
"total_in_bytes" : 9663676416,
"free" : "1gb",
"free_in_bytes" : 1097826304,
"used" : "7.9gb",
"used_in_bytes" : 8565850112,
"free_percent" : 11,
"used_percent" : 89
}
},
"process" : {
"cpu" : {
"percent" : 24
},
"open_file_descriptors" : {
"min" : 722,
"max" : 1780,
"avg" : 1403
}
},
"jvm" : {
"max_uptime" : "58.7d",
"max_uptime_in_millis" : 5071684357,
"versions" : [
{
"version" : "17.0.1",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "17.0.1+12",
"vm_vendor" : "Eclipse Adoptium",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 3
}
],
"mem" : {
"heap_used" : "2.8gb",
"heap_used_in_bytes" : 3109087344,
"heap_max" : "4gb",
"heap_max_in_bytes" : 4336910336
},
"threads" : 454
},
"fs" : {
"total" : "252gb",
"total_in_bytes" : 270582939648,
"free" : "206.9gb",
"free_in_bytes" : 222249746432,
"available" : "206.9gb",
"available_in_bytes" : 222249746432
},
"plugins" : [
{
"name" : "repository-s3",
"version" : "7.16.1",
"elasticsearch_version" : "7.16.1",
"java_version" : "1.8",
"description" : "The S3 repository plugin adds S3 repositories",
"classname" : "org.elasticsearch.repositories.s3.S3RepositoryPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
}
],
"network_types" : {
"transport_types" : {
"security4" : 3
},
"http_types" : {
"security4" : 3
}
},
"discovery_types" : {
"zen" : 3
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "docker",
"count" : 3
}
],
"ingest" : {
"number_of_pipelines" : 86,
"processor_stats" : {
"append" : {
"count" : 173206,
"failed" : 0,
"current" : 0,
"time" : "1.6s",
"time_in_millis" : 1695
},
"conditional" : {
"count" : 3139541,
"failed" : 0,
"current" : 0,
"time" : "24.6m",
"time_in_millis" : 1481701
},
"convert" : {
"count" : 511492,
"failed" : 0,
"current" : 0,
"time" : "7.7s",
"time_in_millis" : 7767
},
"date" : {
"count" : 2442419,
"failed" : 0,
"current" : 0,
"time" : "5.2m",
"time_in_millis" : 316930
},
"dot_expander" : {
"count" : 697248,
"failed" : 0,
"current" : 0,
"time" : "8.3s",
"time_in_millis" : 8327
},
"foreach" : {
"count" : 11102,
"failed" : 72,
"current" : 0,
"time" : "337ms",
"time_in_millis" : 337
},
"geoip" : {
"count" : 232888,
"failed" : 0,
"current" : 0,
"time" : "54.1s",
"time_in_millis" : 54190
},
"grok" : {
"count" : 3031806,
"failed" : 0,
"current" : 0,
"time" : "11.5m",
"time_in_millis" : 695839
},
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"json" : {
"count" : 87156,
"failed" : 0,
"current" : 0,
"time" : "5s",
"time_in_millis" : 5069
},
"pipeline" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"remove" : {
"count" : 4207786,
"failed" : 0,
"current" : 0,
"time" : "44s",
"time_in_millis" : 44060
},
"rename" : {
"count" : 7401416,
"failed" : 0,
"current" : 0,
"time" : "2m",
"time_in_millis" : 124483
},
"script" : {
"count" : 224308,
"failed" : 0,
"current" : 0,
"time" : "52.3s",
"time_in_millis" : 52327
},
"set" : {
"count" : 12733673,
"failed" : 0,
"current" : 0,
"time" : "9m",
"time_in_millis" : 543457
},
"set_security_user" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"split" : {
"count" : 2243662,
"failed" : 0,
"current" : 0,
"time" : "35.4s",
"time_in_millis" : 35471
},
"uri_parts" : {
"count" : 357708,
"failed" : 407,
"current" : 0,
"time" : "15.5s",
"time_in_millis" : 15553
},
"urldecode" : {
"count" : 115394,
"failed" : 0,
"current" : 0,
"time" : "3.7s",
"time_in_millis" : 3741
},
"user_agent" : {
"count" : 115394,
"failed" : 0,
"current" : 0,
"time" : "6.8s",
"time_in_millis" : 6871
}
}
}
}
}
Nope, all good.
The fist thing I can see is that your average shard size is about 100MB, which is super small and likely going to be inefficient.
What sort of data do you have in the cluster?
Thanks Mark,
Data mainly consists of Windows event logs, IIS logs and some database events pulled via logstash/JDBC. Others are metricbeat for reporting purposes and APM which Elastic Cloud seems to send by default.
Is increasing the shard size fairly straightforward?
Cool, you're on a recent version so you should really be using ILM for things, can you confirm that? Cause it's by far the easiest way to manage this.
Yes, we use ILM. It's pretty basic at the moment and is mostly using the hot phase with some older heartbeats being sent to the delete phase.
Can you share the policy?
Ideally you want to be aiming for 30-50GB per shard.
Here's the current policy. Note that a few different people have experimented with this so it's all over the place. It would be good to know the required settings to keep the shards at a decent size. Many of these are default policies, but we have our own policies for heartbeat, winlogbeat, jdbc (and maybe a few others).
{
".alerts-ilm-policy" : {
"version" : 5,
"modified_date" : "2022-01-26T15:59:22.375Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
}
}
},
"in_use_by" : {
"indices" : [
".internal.alerts-observability.metrics.alerts-default-000002",
".internal.alerts-observability.metrics.alerts-default-000001"
],
"data_streams" : [ ],
"composable_templates" : [
".alerts-observability.metrics.alerts-default-index-template"
]
}
},
".deprecation-indexing-ilm-policy" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.791Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "10gb",
"max_age" : "14d"
}
}
}
}
},
"in_use_by" : {
"indices" : [
".ds-.logs-deprecation.elasticsearch-default-2022.01.11-000003",
".ds-.logs-deprecation.elasticsearch-default-2021.12.14-000001",
".ds-.logs-deprecation.elasticsearch-default-2021.12.28-000002",
".ds-.logs-deprecation.elasticsearch-default-2022.02.08-000005",
".ds-.logs-deprecation.elasticsearch-default-2022.01.25-000004"
],
"data_streams" : [
".logs-deprecation.elasticsearch-default"
],
"composable_templates" : [
".deprecation-indexing-template"
]
}
},
".fleet-actions-results-ilm-policy" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.815Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "300gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "default policy for fleet action results indices",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
".items-default" : {
"version" : 1,
"modified_date" : "2021-11-23T03:50:00.609Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb"
}
}
}
}
},
"in_use_by" : {
"indices" : [
".items-default-000001"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
".lists-default" : {
"version" : 1,
"modified_date" : "2021-11-23T03:50:00.381Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb"
}
}
}
}
},
"in_use_by" : {
"indices" : [
".lists-default-000001"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
".siem-signals-default" : {
"version" : 1,
"modified_date" : "2021-11-23T03:46:49.893Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
}
}
},
"in_use_by" : {
"indices" : [
".siem-signals-default-000001",
".siem-signals-default-000002",
".siem-signals-default-000003"
],
"data_streams" : [ ],
"composable_templates" : [
".siem-signals-default"
]
}
},
".siem-signals-default-migration-cleanup" : {
"version" : 1,
"modified_date" : "2021-11-23T03:46:49.742Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "30d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"180-days-default" : {
"version" : 1,
"modified_date" : "2021-12-14T06:43:19.674Z",
"policy" : {
"phases" : {
"warm" : {
"min_age" : "2d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"shrink" : {
"number_of_shards" : 1
}
}
},
"cold" : {
"min_age" : "30d",
"actions" : { }
},
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "180d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot, warm, and cold phases with a retention of 180 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"30-days-default" : {
"version" : 1,
"modified_date" : "2021-12-14T06:43:16.995Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "30d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
},
"warm" : {
"min_age" : "2d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"shrink" : {
"number_of_shards" : 1
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot and warm phases with a retention of 30 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"365-days-default" : {
"version" : 1,
"modified_date" : "2021-12-14T06:43:15.964Z",
"policy" : {
"phases" : {
"warm" : {
"min_age" : "2d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"shrink" : {
"number_of_shards" : 1
}
}
},
"cold" : {
"min_age" : "30d",
"actions" : { }
},
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "365d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot, warm, and cold phases with a retention of 365 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"7-days-default" : {
"version" : 1,
"modified_date" : "2021-12-14T06:43:18.265Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "7d"
}
}
},
"delete" : {
"min_age" : "7d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
},
"warm" : {
"min_age" : "2d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"shrink" : {
"number_of_shards" : 1
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot and warm phases with a retention of 7 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"90-days-default" : {
"version" : 1,
"modified_date" : "2021-12-14T06:43:14.466Z",
"policy" : {
"phases" : {
"warm" : {
"min_age" : "2d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"shrink" : {
"number_of_shards" : 1
}
}
},
"cold" : {
"min_age" : "30d",
"actions" : { }
},
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot, warm, and cold phases with a retention of 90 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"apm-rollover-30-days" : {
"version" : 1,
"modified_date" : "2021-11-23T02:02:39.061Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
},
"set_priority" : {
"priority" : 100
}
}
},
"warm" : {
"min_age" : "30d",
"actions" : {
"readonly" : { },
"set_priority" : {
"priority" : 50
}
}
}
}
},
"in_use_by" : {
"indices" : [
"apm-7.15.2-transaction-000001",
"apm-7.15.2-transaction-000003",
"apm-7.15.2-transaction-000002",
"apm-7.16.1-metric-000001",
"apm-7.16.1-error-000002",
"apm-7.16.1-error-000001",
"apm-7.15.2-span-000001",
"apm-7.15.2-error-000002",
"apm-7.15.2-error-000003",
"apm-7.15.2-metric-000003",
"apm-7.15.2-span-000003",
"apm-7.15.2-metric-000002",
"apm-7.15.2-error-000001",
"apm-7.15.2-span-000002",
"apm-7.15.2-metric-000001",
"apm-7.16.1-transaction-000001",
"apm-7.16.1-metric-000002",
"apm-7.15.2-profile-000003",
"apm-7.15.2-profile-000001",
"apm-7.16.1-profile-000001",
"apm-7.16.1-profile-000002",
"apm-7.15.2-profile-000002",
"apm-7.16.1-span-000001",
"apm-7.16.1-span-000002",
"apm-7.16.1-transaction-000002"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"elastic-cloud-logs" : {
"version" : 6,
"modified_date" : "2021-12-02T04:11:07.502Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_primary_shard_size" : "10gb",
"max_age" : "2d"
}
}
},
"delete" : {
"min_age" : "2d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
"elastic-cloud-logs-7-2022.02.10-000039",
"elastic-cloud-logs-7-2022.02.08-000038"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"ent-search-api-ecs-ilm-logs" : {
"version" : 1,
"modified_date" : "2021-11-23T02:02:52.679Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"readonly" : { },
"rollover" : {
"max_size" : "10gb",
"max_age" : "7d"
},
"set_priority" : {
"priority" : 50
}
}
},
"delete" : {
"min_age" : "7d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
".ent-search-api-ecs-ilm-logs-production-2022.02.08-000012",
".ent-search-api-ecs-ilm-logs-production-2022.02.01-000011"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"ent-search-app-search-analytics-ecs-ilm-logs" : {
"version" : 1,
"modified_date" : "2021-11-23T02:02:57.665Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"readonly" : { },
"rollover" : {
"max_size" : "3gb",
"max_age" : "30d"
},
"set_priority" : {
"priority" : 100
}
}
},
"delete" : {
"min_age" : "180d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
".ent-search-app-search-analytics-ecs-ilm-logs-production-2021.12.23-000002",
".ent-search-app-search-analytics-ecs-ilm-logs-production-2021.11.23-000001",
".ent-search-app-search-analytics-ecs-ilm-logs-production-2022.01.22-000003"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"ent-search-crawler-ecs-ilm-logs" : {
"version" : 1,
"modified_date" : "2021-11-23T02:02:58.514Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"readonly" : { },
"rollover" : {
"max_size" : "10gb",
"max_age" : "14d"
},
"set_priority" : {
"priority" : 100
}
}
},
"delete" : {
"min_age" : "14d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
".ent-search-crawler-ecs-ilm-logs-production-2022.01.18-000005",
".ent-search-crawler-ecs-ilm-logs-production-2022.02.01-000006"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"ent-search-workplace-search-analytics-ecs-ilm-logs" : {
"version" : 1,
"modified_date" : "2021-11-23T02:03:01.616Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"readonly" : { },
"rollover" : {
"max_size" : "3gb",
"max_age" : "30d"
},
"set_priority" : {
"priority" : 109
}
}
},
"delete" : {
"min_age" : "180d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
".ent-search-workplace-search-analytics-ecs-ilm-logs-production-2022.01.22-000003",
".ent-search-workplace-search-analytics-ecs-ilm-logs-production-2021.11.23-000001",
".ent-search-workplace-search-analytics-ecs-ilm-logs-production-2021.12.23-000002"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"ent-search-workplace-search-content-events-ecs-ilm-logs" : {
"version" : 1,
"modified_date" : "2021-11-23T02:03:04.012Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"readonly" : { },
"rollover" : {
"max_size" : "3gb",
"max_age" : "30d"
},
"set_priority" : {
"priority" : 109
}
}
},
"delete" : {
"min_age" : "180d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
".ent-search-workplace-search-content-events-ecs-ilm-logs-production-2021.12.23-000002",
".ent-search-workplace-search-content-events-ecs-ilm-logs-production-2022.01.22-000003",
".ent-search-workplace-search-content-events-ecs-ilm-logs-production-2021.11.23-000001"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"filebeat" : {
"version" : 1413,
"modified_date" : "2022-02-10T22:10:05.416Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
}
}
},
"in_use_by" : {
"indices" : [
"filebeat-7.15.2-2021.12.23-000002",
"filebeat-7.16.1-2022.01.13-000002",
"filebeat-7.15.2-2021.11.23-000001",
"filebeat-7.15.2-2022.01.22-000003",
"filebeat-7.16.1-2021.12.14-000001"
],
"data_streams" : [ ],
"composable_templates" : [
"filebeat-7.16.1"
]
}
},
"github" : {
"version" : 1,
"modified_date" : "2021-11-23T03:18:36.090Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
},
"set_priority" : {
"priority" : 100
}
}
}
}
},
"in_use_by" : {
"indices" : [
"github-000003",
"github-000002",
"github-000001"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"heartbeat" : {
"version" : 4,
"modified_date" : "2021-12-02T04:59:37.763Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_primary_shard_size" : "5gb",
"max_age" : "2d"
}
}
},
"delete" : {
"min_age" : "2d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
"heartbeat-7.16.2-2022.02.10-000020",
"heartbeat-7.15.2-2022.02.10-000037",
"heartbeat-7.16.1-2022.02.10-000030",
"heartbeat-7.16.2-2022.02.08-000019",
"heartbeat-7.16.1-2022.02.08-000029",
"heartbeat-7.15.2-2022.02.08-000036"
],
"data_streams" : [ ],
"composable_templates" : [
"heartbeat-7.16.1",
"heartbeat-7.16.2"
]
}
},
"ilm-history-ilm-policy" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.741Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "default policy for the ILM history indices",
"managed" : true
}
},
"in_use_by" : {
"indices" : [
".ds-ilm-history-5-2021.12.23-000002",
".ds-ilm-history-5-2021.11.23-000001",
".ds-ilm-history-5-2022.01.22-000003"
],
"data_streams" : [
"ilm-history-5"
],
"composable_templates" : [
"ilm-history"
]
}
},
"jdbc" : {
"version" : 1,
"modified_date" : "2021-11-23T03:05:04.035Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
},
"set_priority" : {
"priority" : 100
}
}
}
}
},
"in_use_by" : {
"indices" : [
"jdbc-prod-db-audit-000002",
"jdbc-test-event-000003",
"jdbc-test-event-000002",
"jdbc-prod-db-audit-000001",
"jdbc-test-event-000001",
"jdbc-test-error-000001",
"jdbc-test-error-000003",
"jdbc-test-error-000002",
"jdbc-test-auth-000002",
"jdbc-test-auth-000003",
"jdbc-test-auth-000001",
"jdbc-prod-db-audit-000003"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"kibana-event-log-policy" : {
"version" : 1,
"modified_date" : "2021-11-23T02:02:21.979Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
".kibana-event-log-7.15.2-000003",
".kibana-event-log-7.15.2-000002",
".kibana-event-log-7.16.1-000002",
".kibana-event-log-7.15.2-000001",
".kibana-event-log-7.16.1-000001"
],
"data_streams" : [ ],
"composable_templates" : [
".kibana-event-log-7.16.1-template"
]
}
},
"kibana-reporting" : {
"version" : 1,
"modified_date" : "2021-11-23T02:02:22.016Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : { }
}
}
},
"in_use_by" : {
"indices" : [
".reporting-2021-12-05"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"logs" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.684Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
}
},
"_meta" : {
"description" : "default policy for the logs index template installed by x-pack",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
".logs-endpoint.actions",
"logs-elastic_agent.metricbeat",
"logs-elastic_agent.fleet_server",
".logs-endpoint.action.responses",
"logs-system.system",
"logs-system.application",
"logs-elastic_agent.osquerybeat",
"logs-endpoint.events.security",
"logs-endpoint.events.registry",
"logs-elastic_agent.auditbeat",
"logs",
"logs-endpoint.events.network",
"logs-endpoint.events.process",
"logs-elastic_agent.apm_server",
"logs-system.security",
"logs-system.auth",
"logs-elastic_agent.filebeat",
"logs-elastic_agent.packetbeat",
"logs-elastic_agent.endpoint_security",
"logs-endpoint.events.library",
"logs-elastic_agent.heartbeat",
"logs-system.syslog",
"logs-endpoint.alerts",
"logs-elastic_agent",
"logs-endpoint.events.file"
]
}
},
"logs-endpoint.collection-diagnostic" : {
"version" : 1,
"modified_date" : "2021-12-14T07:09:23.165Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "1gb",
"max_age" : "7d",
"max_docs" : 10000
}
}
},
"delete" : {
"min_age" : "10m",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
".logs-endpoint.diagnostic.collection"
]
}
},
"logstash-syslog" : {
"version" : 1,
"modified_date" : "2021-11-23T03:06:03.987Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
},
"set_priority" : {
"priority" : 100
}
}
}
}
},
"in_use_by" : {
"indices" : [
"logstash-syslog-000001",
"logstash-syslog-000002",
"logstash-syslog-000003"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"metricbeat" : {
"version" : 5,
"modified_date" : "2021-12-02T05:00:58.663Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_primary_shard_size" : "5gb",
"max_age" : "7d"
}
}
},
"delete" : {
"min_age" : "2d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
"metricbeat-7.16.1-2022.02.08-000009"
],
"data_streams" : [ ],
"composable_templates" : [
"metricbeat-7.16.1"
]
}
},
"metrics" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.620Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
}
},
"_meta" : {
"description" : "default policy for the metrics index template installed by x-pack",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
"metrics-system.process",
"metrics-elastic_agent.packetbeat",
"metrics-system.fsstat",
"metrics-elastic_agent.osquerybeat",
"metrics-elastic_agent.endpoint_security",
"metrics-elastic_agent.apm_server",
"metrics-system.memory",
"metrics-endpoint.metadata",
"metrics-system.socket_summary",
"metrics-endpoint.metrics",
"metrics-elastic_agent.elastic_agent",
"metrics-elastic_agent.fleet_server",
"metrics-system.load",
"metrics-system.core",
"metrics-elastic_agent.filebeat",
"metrics-system.uptime",
"metrics-system.process.summary",
"metrics-system.cpu",
"metrics-elastic_agent.heartbeat",
"metrics-system.diskio",
"metrics-endpoint.policy",
"metrics-elastic_agent.metricbeat",
"metrics-elastic_agent.auditbeat",
"metrics-system.network",
"metrics-system.filesystem",
"metrics"
]
}
},
"ml-size-based-ilm-policy" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.566Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb"
}
}
}
},
"_meta" : {
"description" : "default policy for machine learning state and stats indices",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
".ml-state",
".ml-stats"
]
}
},
"slm-history-ilm-policy" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.766Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "default policy for the SLM history indices",
"managed" : true
}
},
"in_use_by" : {
"indices" : [
".ds-.slm-history-5-2021.12.23-000002",
".ds-.slm-history-5-2021.11.23-000001",
".ds-.slm-history-5-2022.01.22-000003"
],
"data_streams" : [
".slm-history-5"
],
"composable_templates" : [
".slm-history"
]
}
},
"synthetics" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.651Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
}
},
"_meta" : {
"description" : "default policy for the synthetics index template installed by x-pack",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
"synthetics"
]
}
},
"watch-history-ilm-policy" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.711Z",
"policy" : {
"phases" : {
"delete" : {
"min_age" : "7d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "default policy for the watcher history indices",
"managed" : true
}
},
"in_use_by" : {
"indices" : [
".watcher-history-13-2022.02.09",
".watcher-history-13-2022.02.11",
".watcher-history-13-2022.02.10"
],
"data_streams" : [ ],
"composable_templates" : [
".watch-history-13"
]
}
},
"winlogbeat" : {
"version" : 1,
"modified_date" : "2021-11-23T03:59:27.609Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
}
}
},
"in_use_by" : {
"indices" : [
"winlogbeat-7.15.2-2021.11.25-000001",
"winlogbeat-7.15.2-2021.12.25-000002",
"winlogbeat-7.15.1-2021.11.23-000001",
"winlogbeat-7.16.1-2021.12.15-000001",
"winlogbeat-7.16.1-2022.01.14-000002",
"winlogbeat-7.15.1-2021.12.23-000002",
"winlogbeat-7.15.2-2022.01.24-000003",
"winlogbeat-7.15.1-2022.01.22-000003"
],
"data_streams" : [ ],
"composable_templates" : [
"winlogbeat-7.16.1"
]
}
}
}
I would remove "max_primary_shard_size" : "5gb", from your heartbeat policy. Or increase that to something a bit larger.
Otherwise there's a lot to dig through there! Can you post the output from _cat/indices?v (or use gist/pastebin/etc)
Many of these are not used at the moment but are sent by default. Our deployment is small so I guess we could disable APM & Fleet logs ... any thoughts on that?
Here's the output:
{
".alerts-ilm-policy" : {
"version" : 5,
"modified_date" : "2022-01-26T15:59:22.375Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
}
}
},
"in_use_by" : {
"indices" : [
".internal.alerts-observability.metrics.alerts-default-000002",
".internal.alerts-observability.metrics.alerts-default-000001"
],
"data_streams" : [ ],
"composable_templates" : [
".alerts-observability.metrics.alerts-default-index-template"
]
}
},
".deprecation-indexing-ilm-policy" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.791Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "10gb",
"max_age" : "14d"
}
}
}
}
},
"in_use_by" : {
"indices" : [
".ds-.logs-deprecation.elasticsearch-default-2022.01.11-000003",
".ds-.logs-deprecation.elasticsearch-default-2021.12.14-000001",
".ds-.logs-deprecation.elasticsearch-default-2021.12.28-000002",
".ds-.logs-deprecation.elasticsearch-default-2022.02.08-000005",
".ds-.logs-deprecation.elasticsearch-default-2022.01.25-000004"
],
"data_streams" : [
".logs-deprecation.elasticsearch-default"
],
"composable_templates" : [
".deprecation-indexing-template"
]
}
},
".fleet-actions-results-ilm-policy" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.815Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "300gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "default policy for fleet action results indices",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
".items-default" : {
"version" : 1,
"modified_date" : "2021-11-23T03:50:00.609Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb"
}
}
}
}
},
"in_use_by" : {
"indices" : [
".items-default-000001"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
".lists-default" : {
"version" : 1,
"modified_date" : "2021-11-23T03:50:00.381Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb"
}
}
}
}
},
"in_use_by" : {
"indices" : [
".lists-default-000001"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
".siem-signals-default" : {
"version" : 1,
"modified_date" : "2021-11-23T03:46:49.893Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
}
}
},
"in_use_by" : {
"indices" : [
".siem-signals-default-000001",
".siem-signals-default-000002",
".siem-signals-default-000003"
],
"data_streams" : [ ],
"composable_templates" : [
".siem-signals-default"
]
}
},
".siem-signals-default-migration-cleanup" : {
"version" : 1,
"modified_date" : "2021-11-23T03:46:49.742Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "30d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"180-days-default" : {
"version" : 1,
"modified_date" : "2021-12-14T06:43:19.674Z",
"policy" : {
"phases" : {
"warm" : {
"min_age" : "2d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"shrink" : {
"number_of_shards" : 1
}
}
},
"cold" : {
"min_age" : "30d",
"actions" : { }
},
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "180d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot, warm, and cold phases with a retention of 180 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"30-days-default" : {
"version" : 1,
"modified_date" : "2021-12-14T06:43:16.995Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "30d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
},
"warm" : {
"min_age" : "2d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"shrink" : {
"number_of_shards" : 1
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot and warm phases with a retention of 30 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"365-days-default" : {
"version" : 1,
"modified_date" : "2021-12-14T06:43:15.964Z",
"policy" : {
"phases" : {
"warm" : {
"min_age" : "2d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"shrink" : {
"number_of_shards" : 1
}
}
},
"cold" : {
"min_age" : "30d",
"actions" : { }
},
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "365d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot, warm, and cold phases with a retention of 365 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"7-days-default" : {
"version" : 1,
"modified_date" : "2021-12-14T06:43:18.265Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "7d"
}
}
},
"delete" : {
"min_age" : "7d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
},
"warm" : {
"min_age" : "2d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"shrink" : {
"number_of_shards" : 1
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot and warm phases with a retention of 7 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"90-days-default" : {
"version" : 1,
"modified_date" : "2021-12-14T06:43:14.466Z",
"policy" : {
"phases" : {
"warm" : {
"min_age" : "2d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"shrink" : {
"number_of_shards" : 1
}
}
},
"cold" : {
"min_age" : "30d",
"actions" : { }
},
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "built-in ILM policy using the hot, warm, and cold phases with a retention of 90 days",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"apm-rollover-30-days" : {
"version" : 1,
"modified_date" : "2021-11-23T02:02:39.061Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
},
"set_priority" : {
"priority" : 100
}
}
},
"warm" : {
"min_age" : "30d",
"actions" : {
"readonly" : { },
"set_priority" : {
"priority" : 50
}
}
}
}
},
"in_use_by" : {
"indices" : [
"apm-7.15.2-transaction-000001",
"apm-7.15.2-transaction-000003",
"apm-7.15.2-transaction-000002",
"apm-7.16.1-metric-000001",
"apm-7.16.1-error-000002",
"apm-7.16.1-error-000001",
"apm-7.15.2-span-000001",
"apm-7.15.2-error-000002",
"apm-7.15.2-error-000003",
"apm-7.15.2-metric-000003",
"apm-7.15.2-span-000003",
"apm-7.15.2-metric-000002",
"apm-7.15.2-error-000001",
"apm-7.15.2-span-000002",
"apm-7.15.2-metric-000001",
"apm-7.16.1-transaction-000001",
"apm-7.16.1-metric-000002",
"apm-7.15.2-profile-000003",
"apm-7.15.2-profile-000001",
"apm-7.16.1-profile-000001",
"apm-7.16.1-profile-000002",
"apm-7.15.2-profile-000002",
"apm-7.16.1-span-000001",
"apm-7.16.1-span-000002",
"apm-7.16.1-transaction-000002"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"elastic-cloud-logs" : {
"version" : 6,
"modified_date" : "2021-12-02T04:11:07.502Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_primary_shard_size" : "10gb",
"max_age" : "2d"
}
}
},
"delete" : {
"min_age" : "2d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
"elastic-cloud-logs-7-2022.02.10-000039",
"elastic-cloud-logs-7-2022.02.08-000038"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"ent-search-api-ecs-ilm-logs" : {
"version" : 1,
"modified_date" : "2021-11-23T02:02:52.679Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"readonly" : { },
"rollover" : {
"max_size" : "10gb",
"max_age" : "7d"
},
"set_priority" : {
"priority" : 50
}
}
},
"delete" : {
"min_age" : "7d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
".ent-search-api-ecs-ilm-logs-production-2022.02.08-000012",
".ent-search-api-ecs-ilm-logs-production-2022.02.01-000011"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"ent-search-app-search-analytics-ecs-ilm-logs" : {
"version" : 1,
"modified_date" : "2021-11-23T02:02:57.665Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"readonly" : { },
"rollover" : {
"max_size" : "3gb",
"max_age" : "30d"
},
"set_priority" : {
"priority" : 100
}
}
},
"delete" : {
"min_age" : "180d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
".ent-search-app-search-analytics-ecs-ilm-logs-production-2021.12.23-000002",
".ent-search-app-search-analytics-ecs-ilm-logs-production-2021.11.23-000001",
".ent-search-app-search-analytics-ecs-ilm-logs-production-2022.01.22-000003"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"ent-search-crawler-ecs-ilm-logs" : {
"version" : 1,
"modified_date" : "2021-11-23T02:02:58.514Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"readonly" : { },
"rollover" : {
"max_size" : "10gb",
"max_age" : "14d"
},
"set_priority" : {
"priority" : 100
}
}
},
"delete" : {
"min_age" : "14d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
".ent-search-crawler-ecs-ilm-logs-production-2022.01.18-000005",
".ent-search-crawler-ecs-ilm-logs-production-2022.02.01-000006"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"ent-search-workplace-search-analytics-ecs-ilm-logs" : {
"version" : 1,
"modified_date" : "2021-11-23T02:03:01.616Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"readonly" : { },
"rollover" : {
"max_size" : "3gb",
"max_age" : "30d"
},
"set_priority" : {
"priority" : 109
}
}
},
"delete" : {
"min_age" : "180d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
".ent-search-workplace-search-analytics-ecs-ilm-logs-production-2022.01.22-000003",
".ent-search-workplace-search-analytics-ecs-ilm-logs-production-2021.11.23-000001",
".ent-search-workplace-search-analytics-ecs-ilm-logs-production-2021.12.23-000002"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"ent-search-workplace-search-content-events-ecs-ilm-logs" : {
"version" : 1,
"modified_date" : "2021-11-23T02:03:04.012Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"readonly" : { },
"rollover" : {
"max_size" : "3gb",
"max_age" : "30d"
},
"set_priority" : {
"priority" : 109
}
}
},
"delete" : {
"min_age" : "180d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
".ent-search-workplace-search-content-events-ecs-ilm-logs-production-2021.12.23-000002",
".ent-search-workplace-search-content-events-ecs-ilm-logs-production-2022.01.22-000003",
".ent-search-workplace-search-content-events-ecs-ilm-logs-production-2021.11.23-000001"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"filebeat" : {
"version" : 1413,
"modified_date" : "2022-02-10T22:10:05.416Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
}
}
},
"in_use_by" : {
"indices" : [
"filebeat-7.15.2-2021.12.23-000002",
"filebeat-7.16.1-2022.01.13-000002",
"filebeat-7.15.2-2021.11.23-000001",
"filebeat-7.15.2-2022.01.22-000003",
"filebeat-7.16.1-2021.12.14-000001"
],
"data_streams" : [ ],
"composable_templates" : [
"filebeat-7.16.1"
]
}
},
"github" : {
"version" : 1,
"modified_date" : "2021-11-23T03:18:36.090Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
},
"set_priority" : {
"priority" : 100
}
}
}
}
},
"in_use_by" : {
"indices" : [
"github-000003",
"github-000002",
"github-000001"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"heartbeat" : {
"version" : 4,
"modified_date" : "2021-12-02T04:59:37.763Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_primary_shard_size" : "5gb",
"max_age" : "2d"
}
}
},
"delete" : {
"min_age" : "2d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
"heartbeat-7.16.2-2022.02.10-000020",
"heartbeat-7.15.2-2022.02.10-000037",
"heartbeat-7.16.1-2022.02.10-000030",
"heartbeat-7.16.2-2022.02.08-000019",
"heartbeat-7.16.1-2022.02.08-000029",
"heartbeat-7.15.2-2022.02.08-000036"
],
"data_streams" : [ ],
"composable_templates" : [
"heartbeat-7.16.1",
"heartbeat-7.16.2"
]
}
},
"ilm-history-ilm-policy" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.741Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "default policy for the ILM history indices",
"managed" : true
}
},
"in_use_by" : {
"indices" : [
".ds-ilm-history-5-2021.12.23-000002",
".ds-ilm-history-5-2021.11.23-000001",
".ds-ilm-history-5-2022.01.22-000003"
],
"data_streams" : [
"ilm-history-5"
],
"composable_templates" : [
"ilm-history"
]
}
},
"jdbc" : {
"version" : 1,
"modified_date" : "2021-11-23T03:05:04.035Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
},
"set_priority" : {
"priority" : 100
}
}
}
}
},
"in_use_by" : {
"indices" : [
"jdbc-prod-db-audit-000002",
"jdbc-test-event-000003",
"jdbc-test-event-000002",
"jdbc-prod-db-audit-000001",
"jdbc-test-event-000001",
"jdbc-test-error-000001",
"jdbc-test-error-000003",
"jdbc-test-error-000002",
"jdbc-test-auth-000002",
"jdbc-test-auth-000003",
"jdbc-test-auth-000001",
"jdbc-prod-db-audit-000003"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"kibana-event-log-policy" : {
"version" : 1,
"modified_date" : "2021-11-23T02:02:21.979Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
".kibana-event-log-7.15.2-000003",
".kibana-event-log-7.15.2-000002",
".kibana-event-log-7.16.1-000002",
".kibana-event-log-7.15.2-000001",
".kibana-event-log-7.16.1-000001"
],
"data_streams" : [ ],
"composable_templates" : [
".kibana-event-log-7.16.1-template"
]
}
},
"kibana-reporting" : {
"version" : 1,
"modified_date" : "2021-11-23T02:02:22.016Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : { }
}
}
},
"in_use_by" : {
"indices" : [
".reporting-2021-12-05"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"logs" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.684Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
}
},
"_meta" : {
"description" : "default policy for the logs index template installed by x-pack",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
".logs-endpoint.actions",
"logs-elastic_agent.metricbeat",
"logs-elastic_agent.fleet_server",
".logs-endpoint.action.responses",
"logs-system.system",
"logs-system.application",
"logs-elastic_agent.osquerybeat",
"logs-endpoint.events.security",
"logs-endpoint.events.registry",
"logs-elastic_agent.auditbeat",
"logs",
"logs-endpoint.events.network",
"logs-endpoint.events.process",
"logs-elastic_agent.apm_server",
"logs-system.security",
"logs-system.auth",
"logs-elastic_agent.filebeat",
"logs-elastic_agent.packetbeat",
"logs-elastic_agent.endpoint_security",
"logs-endpoint.events.library",
"logs-elastic_agent.heartbeat",
"logs-system.syslog",
"logs-endpoint.alerts",
"logs-elastic_agent",
"logs-endpoint.events.file"
]
}
},
"logs-endpoint.collection-diagnostic" : {
"version" : 1,
"modified_date" : "2021-12-14T07:09:23.165Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "1gb",
"max_age" : "7d",
"max_docs" : 10000
}
}
},
"delete" : {
"min_age" : "10m",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
".logs-endpoint.diagnostic.collection"
]
}
},
"logstash-syslog" : {
"version" : 1,
"modified_date" : "2021-11-23T03:06:03.987Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
},
"set_priority" : {
"priority" : 100
}
}
}
}
},
"in_use_by" : {
"indices" : [
"logstash-syslog-000001",
"logstash-syslog-000002",
"logstash-syslog-000003"
],
"data_streams" : [ ],
"composable_templates" : [ ]
}
},
"metricbeat" : {
"version" : 5,
"modified_date" : "2021-12-02T05:00:58.663Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_primary_shard_size" : "5gb",
"max_age" : "7d"
}
}
},
"delete" : {
"min_age" : "2d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
"metricbeat-7.16.1-2022.02.08-000009"
],
"data_streams" : [ ],
"composable_templates" : [
"metricbeat-7.16.1"
]
}
},
"metrics" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.620Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
}
},
"_meta" : {
"description" : "default policy for the metrics index template installed by x-pack",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
"metrics-system.process",
"metrics-elastic_agent.packetbeat",
"metrics-system.fsstat",
"metrics-elastic_agent.osquerybeat",
"metrics-elastic_agent.endpoint_security",
"metrics-elastic_agent.apm_server",
"metrics-system.memory",
"metrics-endpoint.metadata",
"metrics-system.socket_summary",
"metrics-endpoint.metrics",
"metrics-elastic_agent.elastic_agent",
"metrics-elastic_agent.fleet_server",
"metrics-system.load",
"metrics-system.core",
"metrics-elastic_agent.filebeat",
"metrics-system.uptime",
"metrics-system.process.summary",
"metrics-system.cpu",
"metrics-elastic_agent.heartbeat",
"metrics-system.diskio",
"metrics-endpoint.policy",
"metrics-elastic_agent.metricbeat",
"metrics-elastic_agent.auditbeat",
"metrics-system.network",
"metrics-system.filesystem",
"metrics"
]
}
},
"ml-size-based-ilm-policy" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.566Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb"
}
}
}
},
"_meta" : {
"description" : "default policy for machine learning state and stats indices",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
".ml-state",
".ml-stats"
]
}
},
"slm-history-ilm-policy" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.766Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "default policy for the SLM history indices",
"managed" : true
}
},
"in_use_by" : {
"indices" : [
".ds-.slm-history-5-2021.12.23-000002",
".ds-.slm-history-5-2021.11.23-000001",
".ds-.slm-history-5-2022.01.22-000003"
],
"data_streams" : [
".slm-history-5"
],
"composable_templates" : [
".slm-history"
]
}
},
"synthetics" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.651Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
}
}
}
},
"_meta" : {
"description" : "default policy for the synthetics index template installed by x-pack",
"managed" : true
}
},
"in_use_by" : {
"indices" : [ ],
"data_streams" : [ ],
"composable_templates" : [
"synthetics"
]
}
},
"watch-history-ilm-policy" : {
"version" : 1,
"modified_date" : "2021-11-23T02:01:14.711Z",
"policy" : {
"phases" : {
"delete" : {
"min_age" : "7d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
},
"_meta" : {
"description" : "default policy for the watcher history indices",
"managed" : true
}
},
"in_use_by" : {
"indices" : [
".watcher-history-13-2022.02.09",
".watcher-history-13-2022.02.11",
".watcher-history-13-2022.02.10"
],
"data_streams" : [ ],
"composable_templates" : [
".watch-history-13"
]
}
},
"winlogbeat" : {
"version" : 1,
"modified_date" : "2021-11-23T03:59:27.609Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "30d"
}
}
}
}
},
"in_use_by" : {
"indices" : [
"winlogbeat-7.15.2-2021.11.25-000001",
"winlogbeat-7.15.2-2021.12.25-000002",
"winlogbeat-7.15.1-2021.11.23-000001",
"winlogbeat-7.16.1-2021.12.15-000001",
"winlogbeat-7.16.1-2022.01.14-000002",
"winlogbeat-7.15.1-2021.12.23-000002",
"winlogbeat-7.15.2-2022.01.24-000003",
"winlogbeat-7.15.1-2022.01.22-000003"
],
"data_streams" : [ ],
"composable_templates" : [
"winlogbeat-7.16.1"
]
}
}
}
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.
© 2020. All Rights Reserved - Elasticsearch
Apache, Apache Lucene, Apache Hadoop, Hadoop, HDFS and the yellow elephant logo are trademarks of the Apache Software Foundation in the United States and/or other countries.