Bad dates in names after rolling with ILM

Hi, i am working on Elasticsearch version 7.7.0 with logstash and creating indices with ILM and rolling index aliases. I have defined ILM like:

    {
      "weekly_rollover-4w_retention" : {
        "version" : 2,
        "modified_date" : "2020-10-15T09:22:19.049Z",
        "policy" : {
          "phases" : {
            "hot" : {
              "min_age" : "0ms",
              "actions" : {
                "rollover" : {
                  "max_size" : "50gb",
                  "max_age" : "7d"
                }
              }
            },
            "delete" : {
              "min_age" : "28d",
              "actions" : {
                "delete" : { }
              }
            },
            "warm" : {
              "min_age" : "7d",
              "actions" : {
                "forcemerge" : {
                  "max_num_segments" : 1
                }
              }
            }
          }
        }
      }
    }

And In my index template i define:

    "lifecycle": {
      "name": "weekly_rollover-4w_retention",
      "parse_origination_date": "true",
      "rollover_alias": "public-api"
    },

I am trying to have date in index name corresponding to date when index is rolled. With absolutely same settings most of index is working, but i found that one is rolling but not changing that date, so it's origination_date is obsolete:

    {
      "indices" : {
        "public-api-2020.11.06-000025" : {
          "index" : "public-api-2020.11.06-000025",
          "managed" : true,
          "policy" : "weekly_rollover-4w_retention",
          "lifecycle_date_millis" : 1604620800000,
          "age" : "129.46d",
          "phase" : "hot",
          "phase_time_millis" : 1615560673691,
          "action" : "rollover",
          "action_time_millis" : 1615561272448,
          "step" : "check-rollover-ready",
          "step_time_millis" : 1615561272448,
          "phase_execution" : {
            "policy" : "weekly_rollover-4w_retention",
            "phase_definition" : {
              "min_age" : "0ms",
              "actions" : {
                "rollover" : {
                  "max_size" : "50gb",
                  "max_age" : "7d"
                }
              }
            },
            "version" : 2,
            "modified_date_in_millis" : 1602753739049
          }
        }
      }
    }

From logs i can see that rolling is working, but have no clue why it's creating new index with bad date:

    {"type": "server", "timestamp": "2021-03-12T14:51:11,051Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"check-rollover-ready\"}] to [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"attempt-rollover\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T14:51:11,634Z", "level": "INFO", "component": "o.e.c.m.MetaDataCreateIndexService", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "[public-api-2020.11.06-000025] creating index, cause [rollover_index], templates [public-api], shards [1]/[1], mappings [_doc]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T14:51:12,309Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000025] from [null] to [{\"phase\":\"new\",\"action\":\"complete\",\"name\":\"complete\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T14:51:12,485Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"attempt-rollover\"}] to [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"wait-for-active-shards\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T14:51:13,185Z", "level": "INFO", "component": "o.e.c.r.a.AllocationService", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[public-api-2020.11.06-000025][0]]]).", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T14:51:13,691Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000025] from [{\"phase\":\"new\",\"action\":\"complete\",\"name\":\"complete\"}] to [{\"phase\":\"hot\",\"action\":\"unfollow\",\"name\":\"wait-for-indexing-complete\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T14:51:14,139Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"wait-for-active-shards\"}] to [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"update-rollover-lifecycle-date\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T14:51:14,141Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"update-rollover-lifecycle-date\"}] to [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"set-indexing-complete\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T14:51:14,745Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000025] from [{\"phase\":\"hot\",\"action\":\"unfollow\",\"name\":\"wait-for-indexing-complete\"}] to [{\"phase\":\"hot\",\"action\":\"unfollow\",\"name\":\"wait-for-follow-shard-tasks\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T14:51:14,955Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"set-indexing-complete\"}] to [{\"phase\":\"hot\",\"action\":\"complete\",\"name\":\"complete\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T14:51:15,248Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"hot\",\"action\":\"complete\",\"name\":\"complete\"}] to [{\"phase\":\"warm\",\"action\":\"forcemerge\",\"name\":\"readonly\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T14:51:15,818Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"warm\",\"action\":\"forcemerge\",\"name\":\"readonly\"}] to [{\"phase\":\"warm\",\"action\":\"forcemerge\",\"name\":\"forcemerge\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T14:51:16,044Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"warm\",\"action\":\"forcemerge\",\"name\":\"forcemerge\"}] to [{\"phase\":\"warm\",\"action\":\"forcemerge\",\"name\":\"segment-count\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T15:01:11,063Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"warm\",\"action\":\"forcemerge\",\"name\":\"segment-count\"}] to [{\"phase\":\"warm\",\"action\":\"complete\",\"name\":\"complete\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T15:01:11,286Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000025] from [{\"phase\":\"hot\",\"action\":\"unfollow\",\"name\":\"wait-for-follow-shard-tasks\"}] to [{\"phase\":\"hot\",\"action\":\"unfollow\",\"name\":\"pause-follower-index\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T15:01:11,528Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"warm\",\"action\":\"complete\",\"name\":\"complete\"}] to [{\"phase\":\"delete\",\"action\":\"delete\",\"name\":\"wait-for-shard-history-leases\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
    {"type": "server", "timestamp": "2021-03-12T15:01:11,733Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000025] from [{\"phase\":\"hot\",\"action\":\"unfollow\",\"name\":\"pause-follower-index\"}] to [{\"phase\":\"hot\",\"action\":\"unfollow\",\"name\":\"close-follower-index\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg"  }
....

Data are send via logstash with simple configuration:

    elasticsearch {
      hosts => ["eslog4:9200", "eslog5:9200"]
      timeout => 5
      manage_template => false
      ilm_enabled => true
      ilm_rollover_alias => "sms-admin"
    }

Wierd is that on different index with different name but absolutely same settings it's working without any problems, indices are rolled and have correct date in their name.

Thanks

I have found that my index was created without date math, so in provided_name was full index name with old date.

Found solution to fix that in: https://discuss.elastic.co/t/date-in-index-name-doesnt-change-on-rollover-on-the-next-day/226701

Sorry for creating new topic, but it was looking like some other problem because index creation was logstash job, not sure how that could happened.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.