ILM doesn't start indexes rollover

Elastic Stack Elasticsearch
1

Hi!
We have cluster from 4 nodes, 2 of them are data nodes, 1 only master node and 1 coordinating one. Recently we implemented some ILM for all indexes. One of policies is shown below

PUT _ilm/policy/winlogbeat
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_age": "7d",
            "max_size": "40gb"
          }
        }
      },
      "delete": {
        "min_age": "365d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

One index winlogbeat-7.6.2-000001 is conected to this policy but nowdays it has size of 75.2 Gb. I tried to look through indexes ilm-history but the last notice about this index dated 11 of April.
I checked parameter concerning polling interval and it's 5 minutes.

     {
       "persistent" : {
         "indices" : {
           "lifecycle" : {
             "poll_interval" : "5m"
           }
         },
       },
       "transient" : {
         "cluster" : {
           "routing" : {
             "allocation" : {
               "disk" : {
                 "watermark" : {
                   "low" : "93%",
                   "high" : "95%"
                 }
               }
             }
           }
          }
       }
     }

Here is explain for this index

{
  "indices" : {
    "winlogbeat-7.6.2-000001" : {
      "index" : "winlogbeat-7.6.2-000001",
      "managed" : true,
      "policy" : "winlogbeat",
      "lifecycle_date_millis" : 1586612989324,
      "age" : "2.81d",
      "phase" : "hot",
      "phase_time_millis" : 1586847222825,
      "action" : "rollover",
      "action_time_millis" : 1586847370845,
      "step" : "check-rollover-ready",
      "step_time_millis" : 1586847370845,
      "phase_execution" : {
        "policy" : "winlogbeat",
        "phase_definition" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_size" : "40gb",
              "max_age" : "7d"
            }
          }
        },
        "version" : 5,
        "modified_date_in_millis" : 1586847091405
      }
    }
  }
}

What's wrong with cluster or this index I can not understand.
Any help appreciated.

2

What does the output section of your winlogbeat configuration look like?

3

We send data through Logstash.

#----------------------------- Logstash output --------------------------------
output.logstash:
  # The Logstash hosts
  hosts: ["el.test.local:5044"]

Logstash config

output {
	if [@metadata][beat] {
		if [@metadata][beat] {
			# Use this output only for Beats.
			if [@metadata][pipeline] {
				elasticsearch {
					hosts => "localhost:9200"
					manage_template => false
					index => "%{[@metadata][beat]}-%{[@metadata][version]}"
					pipeline => "%{[@metadata][pipeline]}"
					user => "logstash_user"
					password => "password"
				}
			}
			else {
				elasticsearch {
					hosts => "localhost:9200"
					manage_template => false
					index => "%{[@metadata][beat]}-%{[@metadata][version]}"
					user => "logstash_user"
					password => "password"
				}			
			}
		}
    }
    else {
        elasticsearch {
            hosts    => "localhost:9200"
            index    => "%{[@metadata][beat]}-%{[@metadata][version]}-unknown-%{+YYYY.ww}"
			user => "logstash_user"
			password => "password"
        }
    }
	#stdout { codec => rubydebug { metadata => true }}
}
4

Today at night rollover happened and what I noticed is the size of the problem index is twice as large as indicated in the policy.
We have 2 data nodes and all indexes have 2 shards plus 1 replica. Can it be connected with such a strange behavior?

5

What does the index stats API report for the rolled over index?

6

I says:

  "_shards" : {
    "total" : 4,
    "successful" : 4,
    "failed" : 0
  },
  "_all" : {
    "primaries" : {
      "docs" : {
        "count" : 52937827,
        "deleted" : 0
      },
      "store" : {
        "size_in_bytes" : 43001807390
      },
      "indexing" : {
        "index_total" : 52938244,
        "index_time_in_millis" : 14803088,
        "index_current" : 0,
        "index_failed" : 0,
        "delete_total" : 0,
        "delete_time_in_millis" : 0,
        "delete_current" : 0,
        "noop_update_total" : 0,
        "is_throttled" : false,
        "throttle_time_in_millis" : 0
      },
      "get" : {
        "total" : 0,
        "time_in_millis" : 0,
        "exists_total" : 0,
        "exists_time_in_millis" : 0,
        "missing_total" : 0,
        "missing_time_in_millis" : 0,
        "current" : 0
      },
      "search" : {
        "open_contexts" : 0,
        "query_total" : 315,
        "query_time_in_millis" : 304283,
        "query_current" : 0,
        "fetch_total" : 21,
        "fetch_time_in_millis" : 17187,
        "fetch_current" : 0,
        "scroll_total" : 0,
        "scroll_time_in_millis" : 0,
        "scroll_current" : 0,
        "suggest_total" : 0,
        "suggest_time_in_millis" : 0,
        "suggest_current" : 0
      },
      "merges" : {
        "current" : 0,
        "current_docs" : 0,
        "current_size_in_bytes" : 0,
        "total" : 19161,
        "total_time_in_millis" : 41008029,
        "total_docs" : 250030626,
        "total_size_in_bytes" : 221893493959,
        "total_stopped_time_in_millis" : 71122,
        "total_throttled_time_in_millis" : 25731038,
        "total_auto_throttle_in_bytes" : 10485760
      },
      "refresh" : {
        "total" : 97878,
        "total_time_in_millis" : 9969283,
        "external_total" : 97626,
        "external_total_time_in_millis" : 10134002,
        "listeners" : 0
      },
      "flush" : {
        "total" : 238,
        "periodic" : 236,
        "total_time_in_millis" : 102937
      },
      "warmer" : {
        "current" : 0,
        "total" : 97624,
        "total_time_in_millis" : 23197
      },
      "query_cache" : {
        "memory_size_in_bytes" : 17450096,
        "total_count" : 4370,
        "hit_count" : 246,
        "miss_count" : 4124,
        "cache_size" : 118,
        "cache_count" : 206,
        "evictions" : 88
      },
      "fielddata" : {
        "memory_size_in_bytes" : 0,
        "evictions" : 0
      },
      "completion" : {
        "size_in_bytes" : 0
      },
      "segments" : {
        "count" : 61,
        "memory_in_bytes" : 41707297,
        "terms_memory_in_bytes" : 18546061,
        "stored_fields_memory_in_bytes" : 23059928,
        "term_vectors_memory_in_bytes" : 0,
        "norms_memory_in_bytes" : 0,
        "points_memory_in_bytes" : 0,
        "doc_values_memory_in_bytes" : 101308,
        "index_writer_memory_in_bytes" : 0,
        "version_map_memory_in_bytes" : 0,
        "fixed_bit_set_memory_in_bytes" : 0,
        "max_unsafe_auto_id_timestamp" : 1586796092788,
        "file_sizes" : { }
      },
      "translog" : {
        "operations" : 0,
        "size_in_bytes" : 110,
        "uncommitted_operations" : 0,
        "uncommitted_size_in_bytes" : 110,
        "earliest_last_modified_age" : 0
      },
      "request_cache" : {
        "memory_size_in_bytes" : 0,
        "evictions" : 0,
        "hit_count" : 2,
        "miss_count" : 263
      },
      "recovery" : {
        "current_as_source" : 0,
        "current_as_target" : 0,
        "throttle_time_in_millis" : 0
      }
    },
    "total" : {
      "docs" : {
        "count" : 105875654,
        "deleted" : 0
      },
      "store" : {
        "size_in_bytes" : 86068358320
      },
      "indexing" : {
        "index_total" : 69040371,
        "index_time_in_millis" : 19226195,
        "index_current" : 0,
        "index_failed" : 0,
        "delete_total" : 0,
        "delete_time_in_millis" : 0,
        "delete_current" : 0,
        "noop_update_total" : 0,
        "is_throttled" : false,
        "throttle_time_in_millis" : 0
      },
      "get" : {
        "total" : 0,
        "time_in_millis" : 0,
        "exists_total" : 0,
        "exists_time_in_millis" : 0,
        "missing_total" : 0,
        "missing_time_in_millis" : 0,
        "current" : 0
      },
      "flush" : {
        "total" : 313,
        "periodic" : 307,
        "total_time_in_millis" : 133683
      },
      "warmer" : {
        "current" : 0,
        "total" : 125242,
        "total_time_in_millis" : 29513
      },
      "query_cache" : {
        "memory_size_in_bytes" : 17450096,
        "total_count" : 5349,
        "hit_count" : 246,
        "miss_count" : 5103,
        "cache_size" : 118,
        "cache_count" : 206,
        "evictions" : 88
      },
      "fielddata" : {
        "memory_size_in_bytes" : 0,
        "evictions" : 0
      },
      "completion" : {
        "size_in_bytes" : 0
      },
      "segments" : {
        "count" : 126,
        "memory_in_bytes" : 83107853,
        "terms_memory_in_bytes" : 36754333,
        "stored_fields_memory_in_bytes" : 46131456,
        "term_vectors_memory_in_bytes" : 0,
        "norms_memory_in_bytes" : 0,
        "points_memory_in_bytes" : 0,
        "doc_values_memory_in_bytes" : 222064,
        "index_writer_memory_in_bytes" : 0,
        "version_map_memory_in_bytes" : 0,
        "fixed_bit_set_memory_in_bytes" : 0,
        "max_unsafe_auto_id_timestamp" : 1586796227306,
        "file_sizes" : { }
      }
    }
  },
  "indices" : {
    "winlogbeat-7.6.2-000001" : {
      "uuid" : "I-KWtKo0Tka7YDoZ_kitwA",
      "primaries" : {
        "docs" : {
          "count" : 52937827,
          "deleted" : 0
        },
        "store" : {
          "size_in_bytes" : 43001807390
        },
        "indexing" : {
          "index_total" : 52938244,
          "index_time_in_millis" : 14803088,
          "index_current" : 0,
          "index_failed" : 0,
          "delete_total" : 0,
          "delete_time_in_millis" : 0,
          "delete_current" : 0,
          "noop_update_total" : 0,
          "is_throttled" : false,
          "throttle_time_in_millis" : 0
        },
        "get" : {
          "total" : 0,
          "time_in_millis" : 0,
          "exists_total" : 0,
          "exists_time_in_millis" : 0,
          "missing_total" : 0,
          "missing_time_in_millis" : 0,
          "current" : 0
        },
        "search" : {
          "open_contexts" : 0,
          "query_total" : 315,
          "query_time_in_millis" : 304283,
          "query_current" : 0,
          "fetch_total" : 21,
          "fetch_time_in_millis" : 17187,
          "fetch_current" : 0,
          "scroll_total" : 0,
          "scroll_time_in_millis" : 0,
          "scroll_current" : 0,
          "suggest_total" : 0,
          "suggest_time_in_millis" : 0,
          "suggest_current" : 0
        },
        "merges" : {
          "current" : 0,
          "current_docs" : 0,
          "current_size_in_bytes" : 0,
          "total" : 19161,
          "total_time_in_millis" : 41008029,
          "total_docs" : 250030626,
          "total_size_in_bytes" : 221893493959,
          "total_stopped_time_in_millis" : 71122,
          "total_throttled_time_in_millis" : 25731038,
          "total_auto_throttle_in_bytes" : 10485760
        },
        "refresh" : {
          "total" : 97878,
          "total_time_in_millis" : 9969283,
          "external_total" : 97626,
          "external_total_time_in_millis" : 10134002,
          "listeners" : 0
        },
        "flush" : {
          "total" : 238,
          "periodic" : 236,
          "total_time_in_millis" : 102937
        },
        "warmer" : {
          "current" : 0,
          "total" : 97624,
          "total_time_in_millis" : 23197
        },
        "query_cache" : {
          "memory_size_in_bytes" : 17450096,
          "total_count" : 4370,
          "hit_count" : 246,
          "miss_count" : 4124,
          "cache_size" : 118,
          "cache_count" : 206,
          "evictions" : 88
        },
        "fielddata" : {
          "memory_size_in_bytes" : 0,
          "evictions" : 0
        },
        "completion" : {
          "size_in_bytes" : 0
        },
        "segments" : {
          "count" : 61,
          "memory_in_bytes" : 41707297,
          "terms_memory_in_bytes" : 18546061,
          "stored_fields_memory_in_bytes" : 23059928,
          "term_vectors_memory_in_bytes" : 0,
          "norms_memory_in_bytes" : 0,
          "points_memory_in_bytes" : 0,
          "doc_values_memory_in_bytes" : 101308,
          "index_writer_memory_in_bytes" : 0,
          "version_map_memory_in_bytes" : 0,
          "fixed_bit_set_memory_in_bytes" : 0,
          "max_unsafe_auto_id_timestamp" : 1586796092788,
          "file_sizes" : { }
        },
        "translog" : {
          "operations" : 0,
          "size_in_bytes" : 110,
          "uncommitted_operations" : 0,
          "uncommitted_size_in_bytes" : 110,
          "earliest_last_modified_age" : 0
        },
        "request_cache" : {
          "memory_size_in_bytes" : 0,
          "evictions" : 0,
          "hit_count" : 2,
          "miss_count" : 263
        },
        "recovery" : {
          "current_as_source" : 0,
          "current_as_target" : 0,
          "throttle_time_in_millis" : 0
        }
      },
      "total" : {
        "docs" : {
          "count" : 105875654,
          "deleted" : 0
        },
        "store" : {
          "size_in_bytes" : 86068358320
        },
        "indexing" : {
          "index_total" : 69040371,
          "index_time_in_millis" : 19226195,
          "index_current" : 0,
          "index_failed" : 0,
          "delete_total" : 0,
          "delete_time_in_millis" : 0,
          "delete_current" : 0,
          "noop_update_total" : 0,
          "is_throttled" : false,
          "throttle_time_in_millis" : 0
        },
        "get" : {
          "total" : 0,
          "time_in_millis" : 0,
          "exists_total" : 0,
          "exists_time_in_millis" : 0,
          "missing_total" : 0,
          "missing_time_in_millis" : 0,
          "current" : 0
        },
        "refresh" : {
          "total" : 125577,
          "total_time_in_millis" : 12467145,
          "external_total" : 125246,
          "external_total_time_in_millis" : 12668210,
          "listeners" : 0
        },
        "flush" : {
          "total" : 313,
          "periodic" : 307,
          "total_time_in_millis" : 133683
        },
        "warmer" : {
          "current" : 0,
          "total" : 125242,
          "total_time_in_millis" : 29513
        },
        "query_cache" : {
          "memory_size_in_bytes" : 17450096,
          "total_count" : 5349,
          "hit_count" : 246,
          "miss_count" : 5103,
          "cache_size" : 118,
          "cache_count" : 206,
          "evictions" : 88
        },
        "fielddata" : {
          "memory_size_in_bytes" : 0,
          "evictions" : 0
        },
        "completion" : {
          "size_in_bytes" : 0
        },
        "segments" : {
          "count" : 126,
          "memory_in_bytes" : 83107853,
          "terms_memory_in_bytes" : 36754333,
          "stored_fields_memory_in_bytes" : 46131456,
          "term_vectors_memory_in_bytes" : 0,
          "norms_memory_in_bytes" : 0,
          "points_memory_in_bytes" : 0,
          "doc_values_memory_in_bytes" : 222064,
          "index_writer_memory_in_bytes" : 0,
          "version_map_memory_in_bytes" : 0,
          "fixed_bit_set_memory_in_bytes" : 0,
          "max_unsafe_auto_id_timestamp" : 1586796227306,
          "file_sizes" : { }
        }
      }
    }
  }
}

I deleted a part of content because it was not limited to 13000 symbols

7

The size of the primary shards seem to be just over 40GB in size just as expected, so I do not see any problem.

8

So determining the index size for rollover operation I must consider that it's the size of primary shards only? And it will be multiplied by the number of replicas in index management page? I didn't know that index management shows us full size.

9

Yes. If you based the size on total size you could get some strange results as you can change the number of replicas at any time.