ILM doesn't start indexes rollover

Elastic Stack Elasticsearch
1

Hi!
We have cluster from 4 nodes, 2 of them are data nodes, 1 only master node and 1 coordinating one. Recently we implemented some ILM for all indexes. One of policies is shown below

PUT _ilm/policy/winlogbeat
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_age": "7d",
            "max_size": "40gb"
          }
        }
      },
      "delete": {
        "min_age": "365d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

One index winlogbeat-7.6.2-000001 is conected to this policy but nowdays it has size of 75.2 Gb. I tried to look through indexes ilm-history but the last notice about this index dated 11 of April.
I checked parameter concerning polling interval and it's 5 minutes.

     {
       "persistent" : {
         "indices" : {
           "lifecycle" : {
             "poll_interval" : "5m"
           }
         },
       },
       "transient" : {
         "cluster" : {
           "routing" : {
             "allocation" : {
               "disk" : {
                 "watermark" : {
                   "low" : "93%",
                   "high" : "95%"
                 }
               }
             }
           }
          }
       }
     }

Here is explain for this index

{
  "indices" : {
    "winlogbeat-7.6.2-000001" : {
      "index" : "winlogbeat-7.6.2-000001",
      "managed" : true,
      "policy" : "winlogbeat",
      "lifecycle_date_millis" : 1586612989324,
      "age" : "2.81d",
      "phase" : "hot",
      "phase_time_millis" : 1586847222825,
      "action" : "rollover",
      "action_time_millis" : 1586847370845,
      "step" : "check-rollover-ready",
      "step_time_millis" : 1586847370845,
      "phase_execution" : {
        "policy" : "winlogbeat",
        "phase_definition" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_size" : "40gb",
              "max_age" : "7d"
            }
          }
        },
        "version" : 5,
        "modified_date_in_millis" : 1586847091405
      }
    }
  }
}

What's wrong with cluster or this index I can not understand.
Any help appreciated.

2

What does the output section of your winlogbeat configuration look like?

3

We send data through Logstash.

#----------------------------- Logstash output --------------------------------
output.logstash:
  # The Logstash hosts
  hosts: ["el.test.local:5044"]

Logstash config

output {
	if [@metadata][beat] {
		if [@metadata][beat] {
			# Use this output only for Beats.
			if [@metadata][pipeline] {
				elasticsearch {
					hosts => "localhost:9200"
					manage_template => false
					index => "%{[@metadata][beat]}-%{[@metadata][version]}"
					pipeline => "%{[@metadata][pipeline]}"
					user => "logstash_user"
					password => "password"
				}
			}
			else {
				elasticsearch {
					hosts => "localhost:9200"
					manage_template => false
					index => "%{[@metadata][beat]}-%{[@metadata][version]}"
					user => "logstash_user"
					password => "password"
				}			
			}
		}
    }
    else {
        elasticsearch {
            hosts    => "localhost:9200"
            index    => "%{[@metadata][beat]}-%{[@metadata][version]}-unknown-%{+YYYY.ww}"
			user => "logstash_user"
			password => "password"
        }
    }
	#stdout { codec => rubydebug { metadata => true }}
}
4

Today at night rollover happened and what I noticed is the size of the problem index is twice as large as indicated in the policy.
We have 2 data nodes and all indexes have 2 shards plus 1 replica. Can it be connected with such a strange behavior?

5

What does the index stats API report for the rolled over index?

6

I says:

  "_shards" : {
    "total" : 4,
    "successful" : 4,
    "failed" : 0
  },
  "_all" : {
    "primaries" : {
      "docs" : {
        "count" : 52937827,
        "deleted" : 0
      },
      "store" : {
        "size_in_bytes" : 43001807390
      },
      "indexing" : {
        "index_total" : 52938244,
        "index_time_in_millis" : 14803088,
        "index_current" : 0,
        "index_failed" : 0,
        "delete_total" : 0,
        "delete_time_in_millis" : 0,
        "delete_current" : 0,
        "noop_update_total" : 0,
        "is_throttled" : false,
        "throttle_time_in_millis" : 0
      },
      "get" : {
        "total" : 0,
        "time_in_millis" : 0,
        "exists_total" : 0,
        "exists_time_in_millis" : 0,
        "missing_total" : 0,
        "missing_time_in_millis" : 0,
        "current" : 0
      },
      "search" : {
        "open_contexts" : 0,
        "query_total" : 315,
        "query_time_in_millis" : 304283,
        "query_current" : 0,
        "fetch_total" : 21,
        "fetch_time_in_millis" : 17187,
        "fetch_current" : 0,
        "scroll_total" : 0,
        "scroll_time_in_millis" : 0,
        "scroll_current" : 0,
        "suggest_total" : 0,
        "suggest_time_in_millis" : 0,
        "suggest_current" : 0
      },
      "merges" : {
        "current" : 0,
        "current_docs" : 0,
        "current_size_in_bytes" : 0,
        "total" : 19161,
        "total_time_in_millis" : 41008029,
        "total_docs" : 250030626,
        "total_size_in_bytes" : 221893493959,
        "total_stopped_time_in_millis" : 71122,
        "total_throttled_time_in_millis" : 25731038,
        "total_auto_throttle_in_bytes" : 10485760
      },
      "refresh" : {
        "total" : 97878,
        "total_time_in_millis" : 9969283,
        "external_total" : 97626,
        "external_total_time_in_millis" : 10134002,
        "listeners" : 0
      },
      "flush" : {
        "total" : 238,
        "periodic" : 236,
        "total_time_in_millis" : 102937
      },
      "warmer" : {
        "current" : 0,
        "total" : 97624,
        "total_time_in_millis" : 23197
      },
      "query_cache" : {
        "memory_size_in_bytes" : 17450096,
        "total_count" : 4370,
        "hit_count" : 246,
        "miss_count" : 4124,
        "cache_size" : 118,
        "cache_count" : 206,
        "evictions" : 88
      },
      "fielddata" : {
        "memory_size_in_bytes" : 0,
        "evictions" : 0
      },
      "completion" : {
        "size_in_bytes" : 0
      },
      "segments" : {
        "count" : 61,
        "memory_in_bytes" : 41707297,
        "terms_memory_in_bytes" : 18546061,
        "stored_fields_memory_in_bytes" : 23059928,
        "term_vectors_memory_in_bytes" : 0,
        "norms_memory_in_bytes" : 0,
        "points_memory_in_bytes" : 0,
        "doc_values_memory_in_bytes" : 101308,
        "index_writer_memory_in_bytes" : 0,
        "version_map_memory_in_bytes" : 0,
        "fixed_bit_set_memory_in_bytes" : 0,
        "max_unsafe_auto_id_timestamp" : 1586796092788,
        "file_sizes" : { }
      },
      "translog" : {
        "operations" : 0,
        "size_in_bytes" : 110,
        "uncommitted_operations" : 0,
        "uncommitted_size_in_bytes" : 110,
        "earliest_last_modified_age" : 0
      },
      "request_cache" : {
        "memory_size_in_bytes" : 0,
        "evictions" : 0,
        "hit_count" : 2,
        "miss_count" : 263
      },
      "recovery" : {
        "current_as_source" : 0,
        "current_as_target" : 0,
        "throttle_time_in_millis" : 0
      }
    },
    "total" : {
      "docs" : {
        "count" : 105875654,
        "deleted" : 0
      },
      "store" : {
        "size_in_bytes" : 86068358320
      },
      "indexing" : {
        "index_total" : 69040371,
        "index_time_in_millis" : 19226195,
        "index_current" : 0,
        "index_failed" : 0,
        "delete_total" : 0,
        "delete_time_in_millis" : 0,
        "delete_current" : 0,
        "noop_update_total" : 0,
        "is_throttled" : false,
        "throttle_time_in_millis" : 0
      },
      "get" : {
        "total" : 0,
        "time_in_millis" : 0,
        "exists_total" : 0,
        "exists_time_in_millis" : 0,
        "missing_total" : 0,
        "missing_time_in_millis" : 0,
        "current" : 0
      },
      "flush" : {
        "total" : 313,
        "periodic" : 307,
        "total_time_in_millis" : 133683
      },
      "warmer" : {
        "current" : 0,
        "total" : 125242,
        "total_time_in_millis" : 29513
      },
      "query_cache" : {
        "memory_size_in_bytes" : 17450096,
        "total_count" : 5349,
        "hit_count" : 246,
        "miss_count" : 5103,
        "cache_size" : 118,
        "cache_count" : 206,
        "evictions" : 88
      },
      "fielddata" : {
        "memory_size_in_bytes" : 0,
        "evictions" : 0
      },
      "completion" : {
        "size_in_bytes" : 0
      },
      "segments" : {
        "count" : 126,
        "memory_in_bytes" : 83107853,
        "terms_memory_in_bytes" : 36754333,
        "stored_fields_memory_in_bytes" : 46131456,
        "term_vectors_memory_in_bytes" : 0,
        "norms_memory_in_bytes" : 0,
        "points_memory_in_bytes" : 0,
        "doc_values_memory_in_bytes" : 222064,
        "index_writer_memory_in_bytes" : 0,
        "version_map_memory_in_bytes" : 0,
        "fixed_bit_set_memory_in_bytes" : 0,
        "max_unsafe_auto_id_timestamp" : 1586796227306,
        "file_sizes" : { }
      }
    }
  },
  "indices" : {
    "winlogbeat-7.6.2-000001" : {
      "uuid" : "I-KWtKo0Tka7YDoZ_kitwA",
      "primaries" : {
        "docs" : {
          "count" : 52937827,
          "deleted" : 0
        },
        "store" : {
          "size_in_bytes" : 43001807390
        },
        "indexing" : {
          "index_total" : 52938244,
          "index_time_in_millis" : 14803088,
          "index_current" : 0,
          "index_failed" : 0,
          "delete_total" : 0,
          "delete_time_in_millis" : 0,
          "delete_current" : 0,
          "noop_update_total" : 0,
          "is_throttled" : false,
          "throttle_time_in_millis" : 0
        },
        "get" : {
          "total" : 0,
          "time_in_millis" : 0,
          "exists_total" : 0,
          "exists_time_in_millis" : 0,
          "missing_total" : 0,
          "missing_time_in_millis" : 0,
          "current" : 0
        },
        "search" : {
          "open_contexts" : 0,
          "query_total" : 315,
          "query_time_in_millis" : 304283,
          "query_current" : 0,
          "fetch_total" : 21,
          "fetch_time_in_millis" : 17187,
          "fetch_current" : 0,
          "scroll_total" : 0,
          "scroll_time_in_millis" : 0,
          "scroll_current" : 0,
          "suggest_total" : 0,
          "suggest_time_in_millis" : 0,
          "suggest_current" : 0
        },
        "merges" : {
          "current" : 0,
          "current_docs" : 0,
          "current_size_in_bytes" : 0,
          "total" : 19161,
          "total_time_in_millis" : 41008029,
          "total_docs" : 250030626,
          "total_size_in_bytes" : 221893493959,
          "total_stopped_time_in_millis" : 71122,
          "total_throttled_time_in_millis" : 25731038,
          "total_auto_throttle_in_bytes" : 10485760
        },
        "refresh" : {
          "total" : 97878,
          "total_time_in_millis" : 9969283,
          "external_total" : 97626,
          "external_total_time_in_millis" : 10134002,
          "listeners" : 0
        },
        "flush" : {
          "total" : 238,
          "periodic" : 236,
          "total_time_in_millis" : 102937
        },
        "warmer" : {
          "current" : 0,
          "total" : 97624,
          "total_time_in_millis" : 23197
        },
        "query_cache" : {
          "memory_size_in_bytes" : 17450096,
          "total_count" : 4370,
          "hit_count" : 246,
          "miss_count" : 4124,
          "cache_size" : 118,
          "cache_count" : 206,
          "evictions" : 88
        },
        "fielddata" : {
          "memory_size_in_bytes" : 0,
          "evictions" : 0
        },
        "completion" : {
          "size_in_bytes" : 0
        },
        "segments" : {
          "count" : 61,
          "memory_in_bytes" : 41707297,
          "terms_memory_in_bytes" : 18546061,
          "stored_fields_memory_in_bytes" : 23059928,
          "term_vectors_memory_in_bytes" : 0,
          "norms_memory_in_bytes" : 0,
          "points_memory_in_bytes" : 0,
          "doc_values_memory_in_bytes" : 101308,
          "index_writer_memory_in_bytes" : 0,
          "version_map_memory_in_bytes" : 0,
          "fixed_bit_set_memory_in_bytes" : 0,
          "max_unsafe_auto_id_timestamp" : 1586796092788,
          "file_sizes" : { }
        },
        "translog" : {
          "operations" : 0,
          "size_in_bytes" : 110,
          "uncommitted_operations" : 0,
          "uncommitted_size_in_bytes" : 110,
          "earliest_last_modified_age" : 0
        },
        "request_cache" : {
          "memory_size_in_bytes" : 0,
          "evictions" : 0,
          "hit_count" : 2,
          "miss_count" : 263
        },
        "recovery" : {
          "current_as_source" : 0,
          "current_as_target" : 0,
          "throttle_time_in_millis" : 0
        }
      },
      "total" : {
        "docs" : {
          "count" : 105875654,
          "deleted" : 0
        },
        "store" : {
          "size_in_bytes" : 86068358320
        },
        "indexing" : {
          "index_total" : 69040371,
          "index_time_in_millis" : 19226195,
          "index_current" : 0,
          "index_failed" : 0,
          "delete_total" : 0,
          "delete_time_in_millis" : 0,
          "delete_current" : 0,
          "noop_update_total" : 0,
          "is_throttled" : false,
          "throttle_time_in_millis" : 0
        },
        "get" : {
          "total" : 0,
          "time_in_millis" : 0,
          "exists_total" : 0,
          "exists_time_in_millis" : 0,
          "missing_total" : 0,
          "missing_time_in_millis" : 0,
          "current" : 0
        },
        "refresh" : {
          "total" : 125577,
          "total_time_in_millis" : 12467145,
          "external_total" : 125246,
          "external_total_time_in_millis" : 12668210,
          "listeners" : 0
        },
        "flush" : {
          "total" : 313,
          "periodic" : 307,
          "total_time_in_millis" : 133683
        },
        "warmer" : {
          "current" : 0,
          "total" : 125242,
          "total_time_in_millis" : 29513
        },
        "query_cache" : {
          "memory_size_in_bytes" : 17450096,
          "total_count" : 5349,
          "hit_count" : 246,
          "miss_count" : 5103,
          "cache_size" : 118,
          "cache_count" : 206,
          "evictions" : 88
        },
        "fielddata" : {
          "memory_size_in_bytes" : 0,
          "evictions" : 0
        },
        "completion" : {
          "size_in_bytes" : 0
        },
        "segments" : {
          "count" : 126,
          "memory_in_bytes" : 83107853,
          "terms_memory_in_bytes" : 36754333,
          "stored_fields_memory_in_bytes" : 46131456,
          "term_vectors_memory_in_bytes" : 0,
          "norms_memory_in_bytes" : 0,
          "points_memory_in_bytes" : 0,
          "doc_values_memory_in_bytes" : 222064,
          "index_writer_memory_in_bytes" : 0,
          "version_map_memory_in_bytes" : 0,
          "fixed_bit_set_memory_in_bytes" : 0,
          "max_unsafe_auto_id_timestamp" : 1586796227306,
          "file_sizes" : { }
        }
      }
    }
  }
}

I deleted a part of content because it was not limited to 13000 symbols

7

The size of the primary shards seem to be just over 40GB in size just as expected, so I do not see any problem.

8

So determining the index size for rollover operation I must consider that it's the size of primary shards only? And it will be multiplied by the number of replicas in index management page? I didn't know that index management shows us full size.

9

Yes. If you based the size on total size you could get some strange results as you can change the number of replicas at any time.

10

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.