Rollover error

Elastic Stack Elasticsearch
1

I am totally lost on how Rollover works. I keep ending up with an error. I deleted my indexes and started fresh but still the same error everytime.

I am having following error and could not figure why this is happening. Any help is much appreciated!

"index.lifecycle.rollover_alias [ecom-alias] does not point to index [ecom-app-logs-2020.10.15-ecom-0000001]",

Below are step I followed:

  1. created an index template with pattern ecom* using this _index_template/ecom_app_logs. Below are initial few lines.
   {
    "index_patterns": ["ecom-*"],
      "template": {
      "settings": {
          "number_of_shards": 1,
          "index.lifecycle.name": "ecom_ilm_policy",
          "index.lifecycle.rollover_alias":"ecom-alias"
        },
        "mappings": {
          "properties": {
            "timestamp": {
  1. Created an index aliases
{
       "aliases": {
         "ecom-alias": {
           "is_write_index": true
         }
       }
}

  1. Started logstash and below is the index name used in logstash output
    output {

         if "ecom_app" in [tags]{
               elasticsearch {
                     hosts => ["eslocalhost:9200"]
                     index => "ecom-app-logs-%{+YYYY.MM.dd}-ecom-0000001"
                     #index => "ecom-0000001"
                     # manage_template => true
                     # template_name => "ecom_template"
                     ilm_enabled => true

               }

Below is the complete error.

Also dont understand why ecom-log-rollover is getting created.

    {
    "indices": {
        "ecom-app-logs-2020.10.15-ecom-0000001": {
            "index": "ecom-app-logs-2020.10.15-ecom-0000001",
            "managed": true,
            "policy": "ecom_ilm_policy",
            "lifecycle_date_millis": 1602797100281,
            "age": "15.61m",
            "phase": "hot",
            "phase_time_millis": 1602797100368,
            "action": "rollover",
            "action_time_millis": 1602797359629,
            "step": "ERROR",
            "step_time_millis": 1602797959362,
            "failed_step": "check-rollover-ready",
            "is_auto_retryable_error": true,
            "step_info": {
                "type": "illegal_argument_exception",
                "reason": "index.lifecycle.rollover_alias [ecom-alias] does not point to index [ecom-app-logs-2020.10.15-ecom-0000001]",
                "stack_trace": "java.lang.IllegalArgumentException: index.lifecycle.rollover_alias [ecom-alias] does not point to index [ecom-app-logs-2020.10.15-ecom-0000001]\n\tat org.elasticsearch.xpack.core.ilm.WaitForRolloverReadyStep.evaluateCondition(WaitForRolloverReadyStep.java:104)\n\tat org.elasticsearch.xpack.ilm.IndexLifecycleRunner.runPeriodicStep(IndexLifecycleRunner.java:173)\n\tat org.elasticsearch.xpack.ilm.IndexLifecycleService.triggerPolicies(IndexLifecycleService.java:329)\n\tat org.elasticsearch.xpack.ilm.IndexLifecycleService.triggered(IndexLifecycleService.java:267)\n\tat org.elasticsearch.xpack.core.scheduler.SchedulerEngine.notifyListeners(SchedulerEngine.java:183)\n\tat org.elasticsearch.xpack.core.scheduler.SchedulerEngine$ActiveSchedule.run(SchedulerEngine.java:211)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)\n\tat java.base/java.lang.Thread.run(Thread.java:832)\n"
            },
            "phase_execution": {
                "policy": "ecom_ilm_policy",
                "phase_definition": {
                    "min_age": "0ms",
                    "actions": {
                        "rollover": {
                            "max_size": "5mb",
                            "max_age": "1d"
                        },
                        "set_priority": {
                            "priority": null
                        }
                    }
                },
                "version": 3,
                "modified_date_in_millis": 1602710733239
            }
        },
        "ecom-0000001": {
            "index": "ecom-0000001",
            "managed": true,
            "policy": "ecom_ilm_policy",
            "lifecycle_date_millis": 1602797039893,
            "age": "16.61m",
            "phase": "hot",
            "phase_time_millis": 1602797039975,
            "action": "rollover",
            "action_time_millis": 1602797359661,
            "step": "check-rollover-ready",
            "step_time_millis": 1602797359661,
            "phase_execution": {
                "policy": "ecom_ilm_policy",
                "phase_definition": {
                    "min_age": "0ms",
                    "actions": {
                        "rollover": {
                            "max_size": "5mb",
                            "max_age": "1d"
                        },
                        "set_priority": {
                            "priority": null
                        }
                    }
                },
                "version": 3,
                "modified_date_in_millis": 1602710733239
            }
        },
        "ecom-log-rollover": {
            "index": "ecom-log-rollover",
            "managed": true,
            "policy": "ecom_ilm_policy",
            "lifecycle_date_millis": 1602797744342,
            "age": "4.87m",
            "phase": "hot",
            "phase_time_millis": 1602797744443,
            "action": "rollover",
            "action_time_millis": 1602797959580,
            "step": "check-rollover-ready",
            "step_time_millis": 1602797959580,
            "phase_execution": {
                "policy": "ecom_ilm_policy",
                "phase_definition": {
                    "min_age": "0ms",
                    "actions": {
                        "rollover": {
                            "max_size": "5mb",
                            "max_age": "1d"
                        },
                        "set_priority": {
                            "priority": null
                        }
                    }
                },
                "version": 3,
                "modified_date_in_millis": 1602710733239
            }
        }
    }
}
2

@Badger, is this something you can help me with, please?

3

Please be patient in waiting for responses to your question and refrain from pinging multiple times asking for a response or opening multiple topics for the same question. This is a community forum, it may take time for someone to reply to your question. For more information please refer to the Community Code of Conduct specifically the section "Be patient". Also, please refrain from pinging folks directly, this is a forum and anyone that participates might be able to assist you.

If you are in need of a service with an SLA that covers response times for questions then you may want to consider talking to us about a subscription.

It's fine to answer on your own thread after 2 or 3 days (not including weekends) if you don't have an answer.

4

Sorry, I understand. Will refrain from tagging people directly.

5

How did u creat the alias ???
Can you show the result of

GET _cat/aliases/ecom-alias*

you can see how alias is linked to the index

Alias create command example:

PUT ecom-00001
{
  "aliases": {
    "ecom-alias": {
      "is_write_index": true
    }
  }
}

logstash output should look like

index => "ecom-alias"

the alias will forward the data to appropriate index ecom-00001
in case you want to have date math as part of index name use reference
see https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-rollover-index.html#_using_date_math_with_the_rollover_api

date in index name is not necessary you can see the index creation date by

GET /_cat/indices/ecom*?v&h=index,creation.date.string,pri,rep,docs.count,store.size,pri.store.size&s=cds
1 Like
6

@Petr.Simik, sorry for the delay. Here is the output of GET _cat/aliases/ecom-alias*.

Output:
ecom-alias ecom-0000001 - - - true

I am going to try your suggestions in few mins and will let you know what I see.
Thanks!

7

I was trying to clean up all the indices and aliases I have in my Env to start-up fresh but for some reason this aliases keeps getting created again. I delete and after few seconds this come back. Do you know why this is happening?

GET /ecom*/_ilm/explain

{
    "indices": {
        "ecom-log-rollover": {
            "index": "ecom-log-rollover",
            "managed": true,
            "policy": "ecom_ilm_policy",
            "lifecycle_date_millis": 1602886408278,
            "age": "1.99m",
            "phase": "hot",
            "phase_time_millis": 1602886408359,
            "action": "unfollow",
            "action_time_millis": 1602886408455,
            "step": "wait-for-follow-shard-tasks",
            "step_time_millis": 1602886408506,
            "phase_execution": {
                "policy": "ecom_ilm_policy",
                "phase_definition": {
                    "min_age": "0ms",
                    "actions": {
                        "rollover": {
                            "max_size": "5mb",
                            "max_age": "1d"
                        },
                        "set_priority": {
                            "priority": null
                        }
                    }
                },
                "version": 3,
                "modified_date_in_millis": 1602710733239
            }
        }
    }
}
8

in logstash just use

hosts => ["eslocalhost:9200"]
index => "ecom-alias"

this will write data to ecom-0000001
if you are not familiar with ILM

ecom-alias -> ecom-0000001
when you reach age/size rollover policy defined in your ILM
it will rollover alias to
ecom-alias -> ecom-0000002

this is how it works.

9

I have done that, but it does not rollover. May be I missing something. Below are detailed steps of what I have done so far.

Simple ILM Policy:

_ilm/policy/ecom_ilm_policy

{
    "policy": {
        "phases": {
            "hot": {
                "min_age": "0ms",
                "actions": {
                    "rollover": {
                        "max_age": "1d",
                        "max_size": "5mb"
                    },
                    "set_priority": {
                        "priority": null
                    }
                }
            },
            "delete": {
                "min_age": "5m",
                "actions": {
                    "delete": {}
                }
            }
        }
    }
}

Template Creation:
/_index_template/ecom_app_logs

{
"index_patterns": ["ecom-*"],
  "template": {
  "settings": {
      "number_of_shards": 1,
      "index.lifecycle.name": "ecom_ilm_policy",
      "index.lifecycle.rollover_alias":"ecom-alias"
    },
    "mappings": {
      "properties": {
        "timestamp": {
          "type": "date"
        },
		"level": {
          "type": "text",
		  "fields": {
            "keyword": {
                "type": "keyword",
                 "ignore_above": 256 ....

Index Creation with alias

Input:

{
  "aliases": {
    "ecom-alias": {
      "is_write_index": true
    }
  }
}

Output:

{
    "acknowledged": true,
   "shards_acknowledged": true,
    "index": "ecom-0000001"
}

After all the above steps below is how the indices are looking

image
image848×212 20.5 KB

Logstash output:

output {

        if "ecom_app" in [tags]{
                  elasticsearch {
                        hosts => ["eslocalhost:9200"]
                        index => "ecom-alias"
                        ilm_enabled => true

                  }
        }

Now I have started logstash. After loading data into the index here is what I see.

Questions:

  1. I am not sure what this ecom-log-rollover index is? I dont remember creating this but while experiment I could have done something which I am not able to figure out.
  2. I don't see that ecom-0000001 is getting rollover to new one after 5mb?

image
image1220×471 36.8 KB

Same as above screenshot but this is just all the indices i have in my env

image
image787×248 14.1 KB

Finally here is the output of ilm explain
ecom*/_ilm/explain

It has following error:
"index.lifecycle.rollover_alias [ecom-alias] does not point to index [ecom-log-rollover]",

{
    "indices": {
        "ecom-0000001": {
            "index": "ecom-0000001",
            "managed": true,
            "policy": "ecom_ilm_policy",
            "lifecycle_date_millis": 1602941959343,
            "age": "37.3s",
            "phase": "hot",
            "phase_time_millis": 1602940967142,
            "action": "complete",
            "action_time_millis": 1602941959819,
            "step": "complete",
            "step_time_millis": 1602941959819,
            "phase_execution": {
                "policy": "ecom_ilm_policy",
                "phase_definition": {
                    "min_age": "0ms",
                    "actions": {
                        "rollover": {
                            "max_size": "5mb",
                            "max_age": "1d"
                        },
                        "set_priority": {
                            "priority": null
                        }
                    }
                },
                "version": 3,
                "modified_date_in_millis": 1602710733239
            }
        },
        "ecom-000002": {
            "index": "ecom-000002",
            "managed": true,
            "policy": "ecom_ilm_policy",
            "lifecycle_date_millis": 1602941959452,
            "age": "37.19s",
            "phase": "hot",
            "phase_time_millis": 1602941959658,
            "action": "unfollow",
            "action_time_millis": 1602941959786,
            "step": "wait-for-follow-shard-tasks",
            "step_time_millis": 1602941959850,
            "phase_execution": {
                "policy": "ecom_ilm_policy",
                "phase_definition": {
                    "min_age": "0ms",
                    "actions": {
                        "rollover": {
                            "max_size": "5mb",
                            "max_age": "1d"
                        },
                        "set_priority": {
                            "priority": null
                        }
                    }
                },
                "version": 3,
                "modified_date_in_millis": 1602710733239
            }
        },
        "ecom-log-rollover": {
            "index": "ecom-log-rollover",
            "managed": true,
            "policy": "ecom_ilm_policy",
            "lifecycle_date_millis": 1602941202061,
            "age": "13.24m",
            "phase": "hot",
            "phase_time_millis": 1602941202320,
            "action": "rollover",
            "action_time_millis": 1602941359778,
            "step": "ERROR",
            "step_time_millis": 1602941959372,
            "failed_step": "check-rollover-ready",
            "is_auto_retryable_error": true,
            "step_info": {
                "type": "illegal_argument_exception",
                "reason": "index.lifecycle.rollover_alias [ecom-alias] does not point to index [ecom-log-rollover]",
                "stack_trace": "java.lang.IllegalArgumentException: index.lifecycle.rollover_alias [ecom-alias] does not point to index [ecom-log-rollover]\n\tat org.elasticsearch.xpack.core.ilm.WaitForRolloverReadyStep.evaluateCondition(WaitForRolloverReadyStep.java:104)\n\tat org.elasticsearch.xpack.ilm.IndexLifecycleRunner.runPeriodicStep(IndexLifecycleRunner.java:173)\n\tat org.elasticsearch.xpack.ilm.IndexLifecycleService.triggerPolicies(IndexLifecycleService.java:329)\n\tat org.elasticsearch.xpack.ilm.IndexLifecycleService.triggered(IndexLifecycleService.java:267)\n\tat org.elasticsearch.xpack.core.scheduler.SchedulerEngine.notifyListeners(SchedulerEngine.java:183)\n\tat org.elasticsearch.xpack.core.scheduler.SchedulerEngine$ActiveSchedule.run(SchedulerEngine.java:211)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)\n\tat java.base/java.lang.Thread.run(Thread.java:832)\n"
            },
            "phase_execution": {
                "policy": "ecom_ilm_policy",
                "phase_definition": {
                    "min_age": "0ms",
                    "actions": {
                        "rollover": {
                            "max_size": "5mb",
                            "max_age": "1d"
                        },
                        "set_priority": {
                            "priority": null
                        }
                    }
                },
                "version": 3,
                "modified_date_in_millis": 1602710733239
            }
        }
    }
}
10

After I posted above response,

  1. I see new Index been created after the first one at 77.2mb. I read somewhere that index rollover cut-off will not be exact. But 77.2mb vs 5mb is a lot of difference. Does that mean rollover is working?
    Our production server will be generating lot more traffic. It would be almost 19K request per 10 mins with 100's of log statements. Is there any special care that needs to be taken for the rollover. I am planning to keep at 20GB before I want to rollover. Any insights here, please?

  2. Also it does not have the same number of zeros before the incremented number (ecom-0000001 vs ecom-000002). Is there any standard in ES that it will only take 5 zeros?

image
image922×276 16.2 KB

  1. Interesting that error I saw in ilm explain is not showing up anymore.

     {
     "indices": {
         "ecom-0000001": {
             "index": "ecom-0000001",
             "managed": true,
             "policy": "ecom_ilm_policy",
             "lifecycle_date_millis": 1602941959343,
             "age": "14.29m",
             "phase": "delete",
             "phase_time_millis": 1602942559435,
             "action": "delete",
             "action_time_millis": 1602942559435,
             "step": "wait-for-shard-history-leases",
             "step_time_millis": 1602942559435,
             "phase_execution": {
                 "policy": "ecom_ilm_policy",
                 "phase_definition": {
                     "min_age": "5m",
                     "actions": {
                         "delete": {
                             "delete_searchable_snapshot": true
                         }
                     }
                 },
                 "version": 3,
                 "modified_date_in_millis": 1602710733239
             }
         },
         "ecom-000002": {
             "index": "ecom-000002",
             "managed": true,
             "policy": "ecom_ilm_policy",
             "lifecycle_date_millis": 1602941959452,
             "age": "14.29m",
             "phase": "hot",
             "phase_time_millis": 1602941959658,
             "action": "rollover",
             "action_time_millis": 1602942559628,
             "step": "check-rollover-ready",
             "step_time_millis": 1602942559628,
             "phase_execution": {
                 "policy": "ecom_ilm_policy",
                 "phase_definition": {
                     "min_age": "0ms",
                     "actions": {
                         "rollover": {
                             "max_size": "5mb",
                             "max_age": "1d"
                         },
                         "set_priority": {
                             "priority": null
                         }
                     }
                 },
                 "version": 3,
                 "modified_date_in_millis": 1602710733239
             }
         },
         "ecom-log-rollover": {
             "index": "ecom-log-rollover",
             "managed": true,
             "policy": "ecom_ilm_policy",
             "lifecycle_date_millis": 1602941202061,
             "age": "26.91m",
             "phase": "hot",
             "phase_time_millis": 1602942559403,
             "action": "rollover",
             "action_time_millis": 1602941359778,
             "step": "check-rollover-ready",
             "step_time_millis": 1602942559403,
             "is_auto_retryable_error": true,
             "failed_step_retry_count": 1,
             "phase_execution": {
                 "policy": "ecom_ilm_policy",
                 "phase_definition": {
                     "min_age": "0ms",
                     "actions": {
                         "rollover": {
                             "max_size": "5mb",
                             "max_age": "1d"
                         },
                         "set_priority": {
                             "priority": null
                         }
                     }
                 },
                 "version": 3,
                 "modified_date_in_millis": 1602710733239
             }
         }
     }
 }

  2. After a while I see the delete policy worked as well. ecom-0000001 is deleted.

image
image783×227 15.9 KB

11

Rollover is checked at intervals and I believe the default is around 10 minutes or so. This is perfect for real scenarios where a reasonably large shard size is desired but my cause sizes to fluctuate with small thresholds.

2 Likes
12

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.