ILM Issue

Hi,

I am facing an issue with my index management setup. Below is the error i see when getting the policy set for my index and the patterns listed in my index templates.
Steps:

  1. Created LifeCycle policies logging_small_size.
  2. Logstash sends logs in the format log-sbk-000001
  3. Template created with rollover alias as log-sbk

Error:

illegal_argument_exception: index.lifecycle.rollover_alias [log-sbk] does not point to index [log-sbk-000001]

Template settings:

{
  "order": 0,
  "index_patterns": [
    "log-sbk*"
  ],
  "settings": {
    "index": {
      "lifecycle": {
        "name": "logging_medium_size",
        "rollover_alias": "log-sbk"
      },
      "codec": "best_compression",
      "mapping": {
        "total_fields": {
          "limit": "2000"
        }
      },
      "number_of_shards": "1",
      "number_of_replicas": "1"
    }
  },
  "mappings": {
    "_doc": {
      "_routing": {
        "required": false
      },
      "numeric_detection": false,
      "dynamic_date_formats": [
        "strict_date_optional_time",
        "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
      ],
      "_meta": {},
      "_source": {
        "excludes": [],
        "includes": [],
        "enabled": true
      },
      "dynamic": true,
      "dynamic_templates": [],
      "date_detection": true,
      "properties": {}
    }
  }
}```


Settings after index creation:

```{
  "settings": {
    "index": {
      "lifecycle": {
        "name": "logging_small_size",
        "rollover_alias": "log-sbk"
      },
      "codec": "best_compression",
      "mapping": {
        "total_fields": {
          "limit": "2000"
        }
      },
      "number_of_shards": "1",
      "provided_name": "log-sbk-000001",
      "creation_date": "1597935305943",
      "priority": "100",
      "number_of_replicas": "1",
      "uuid": "qzZZWhpbQGadnlDnXFksyw",
      "version": {
        "created": "7060299"
      }
    }
  },
  "defaults": {
    "index": {
      "flush_after_merge": "512mb",
      "final_pipeline": "_none",
      "max_inner_result_window": "100",
      "unassigned": {
        "node_left": {
          "delayed_timeout": "1m"
        }
      },
      "max_terms_count": "65536",
      "lifecycle": {
        "parse_origination_date": "false",
        "indexing_complete": "false",
        "origination_date": "-1"
      },
      "routing_partition_size": "1",
      "force_memory_term_dictionary": "false",
      "max_docvalue_fields_search": "100",
      "merge": {
        "scheduler": {
          "max_thread_count": "2",
          "auto_throttle": "true",
          "max_merge_count": "7"
        },
        "policy": {
          "reclaim_deletes_weight": "2.0",
          "floor_segment": "2mb",
          "max_merge_at_once_explicit": "30",
          "max_merge_at_once": "10",
          "max_merged_segment": "5gb",
          "expunge_deletes_allowed": "10.0",
          "segments_per_tier": "10.0",
          "deletes_pct_allowed": "33.0"
        }
      },
      "max_refresh_listeners": "1000",
      "max_regex_length": "1000",
      "load_fixed_bitset_filters_eagerly": "true",
      "number_of_routing_shards": "1",
      "write": {
        "wait_for_active_shards": "1"
      },
      "verified_before_close": "false",
      "mapping": {
        "coerce": "false",
        "nested_fields": {
          "limit": "50"
        },
        "nested_objects": {
          "limit": "10000"
        },
        "depth": {
          "limit": "20"
        },
        "ignore_malformed": "false",
        "field_name_length": {
          "limit": "9223372036854775807"
        }
      },
      "source_only": "false",
      "soft_deletes": {
        "enabled": "false",
        "retention": {
          "operations": "0"
        },
        "retention_lease": {
          "period": "12h"
        }
      },
      "max_script_fields": "32",
      "query": {
        "default_field": [
          "*"
        ],
        "parse": {
          "allow_unmapped_fields": "true"
        }
      },
      "format": "0",
      "frozen": "false",
      "sort": {
        "missing": [],
        "mode": [],
        "field": [],
        "order": []
      },
      "max_rescore_window": "10000",
      "max_adjacency_matrix_filters": "100",
      "analyze": {
        "max_token_count": "10000"
      },
      "gc_deletes": "60s",
      "optimize_auto_generated_id": "true",
      "max_ngram_diff": "1",
      "translog": {
        "generation_threshold_size": "64mb",
        "flush_threshold_size": "512mb",
        "sync_interval": "5s",
        "retention": {
          "size": "512MB",
          "age": "12h"
        },
        "durability": "REQUEST"
      },
      "auto_expand_replicas": "false",
      "mapper": {
        "dynamic": "true"
      },
      "requests": {
        "cache": {
          "enable": "true"
        }
      },
      "data_path": "",
      "highlight": {
        "max_analyzed_offset": "1000000"
      },
      "routing": {
        "rebalance": {
          "enable": "all"
        },
        "allocation": {
          "enable": "all",
          "total_shards_per_node": "-1"
        }
      },
      "search": {
        "slowlog": {
          "level": "TRACE",
          "threshold": {
            "fetch": {
              "warn": "-1",
              "trace": "-1",
              "debug": "-1",
              "info": "-1"
            },
            "query": {
              "warn": "-1",
              "trace": "-1",
              "debug": "-1",
              "info": "-1"
            }
          }
        },
        "idle": {
          "after": "30s"
        },
        "throttled": "false"
      },
      "fielddata": {
        "cache": "node"
      },
      "default_pipeline": "_none",
      "max_slices_per_scroll": "1024",
      "shard": {
        "check_on_startup": "false"
      },
      "xpack": {
        "watcher": {
          "template": {
            "version": ""
          }
        },
        "version": "",
        "ccr": {
          "following_index": "false"
        }
      },
      "percolator": {
        "map_unmapped_fields_as_text": "false"
      },
      "allocation": {
        "max_retries": "5"
      },
      "refresh_interval": "1s",
      "indexing": {
        "slowlog": {
          "reformat": "true",
          "threshold": {
            "index": {
              "warn": "-1",
              "trace": "-1",
              "debug": "-1",
              "info": "-1"
            }
          },
          "source": "1000",
          "level": "TRACE"
        }
      },
      "compound_format": "0.1",
      "blocks": {
        "metadata": "false",
        "read": "false",
        "read_only_allow_delete": "false",
        "read_only": "false",
        "write": "false"
      },
      "max_result_window": "10000",
      "store": {
        "stats_refresh_interval": "10s",
        "type": "",
        "fs": {
          "fs_lock": "native"
        },
        "preload": []
      },
      "queries": {
        "cache": {
          "enabled": "true"
        }
      },
      "warmer": {
        "enabled": "true"
      },
      "max_shingle_diff": "3",
      "query_string": {
        "lenient": "false"
      }
    }
  }
}

Please let me know if there is anything missing or being done wrong.

{
 "mapping": {
   "_doc": {
     "dynamic": "true",
     "_meta": {},
     "_source": {
       "includes": [],
       "excludes": []
     },
     "dynamic_date_formats": [
       "strict_date_optional_time",
       "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
     ],
     "dynamic_templates": [],
     "date_detection": true,
     "numeric_detection": false,
     "properties": {
       "@timestamp": {
         "type": "date",
         "format": "strict_date_optional_time"
       },
       "@version": {
         "type": "text",
         "fields": {
           "keyword": {
             "type": "keyword",
             "ignore_above": 256
           }
         }
       },
       "agent": {
         "properties": {
           "ephemeral_id": {
             "type": "text",
             "fields": {
               "keyword": {
                 "type": "keyword",
                 "ignore_above": 256
               }
             }
           },
           "hostname": {
             "type": "text",
             "fields": {
               "keyword": {
                 "type": "keyword",
                 "ignore_above": 256
               }
             }
           },
           "id": {
             "type": "text",
             "fields": {
               "keyword": {
                 "type": "keyword",
                 "ignore_above": 256
               }
             }
           },
           "type": {
             "type": "text",
             "fields": {
               "keyword": {
                 "type": "keyword",
                 "ignore_above": 256
               }
             }
           },
           "version": {
             "type": "text",
             "fields": {
               "keyword": {
                 "type": "keyword",
                 "ignore_above": 256
               }
             }
           }
         }
       },
       "cloud": {
         "properties": {
           "availability_zone": {
             "type": "text",
             "fields": {
               "keyword": {
                 "type": "keyword",
                 "ignore_above": 256
               }
             }
           },
           "instance": {
             "properties": {
               "id": {
                 "type": "text",
                 "fields": {
                   "keyword": {
                     "type": "keyword",
                     "ignore_above": 256
                   }
                 }
               },
               "name": {
                 "type": "text",
                 "fields": {
                   "keyword": {
                     "type": "keyword",
                     "ignore_above": 256
                   }
                 }
               }
             }
           },
           "machine": {
             "properties": {
               "type": {
                 "type": "text",
                 "fields": {
                   "keyword": {
                     "type": "keyword",
                     "ignore_above": 256
                   }
                 }
               }
             }
           },
           "project": {
             "properties": {
               "id": {
                 "type": "text",
                 "fields": {
                   "keyword": {
                     "type": "keyword",
                     "ignore_above": 256
                   }
                 }
               }
             }
           },
           "provider": {
             "type": "text",
             "fields": {
               "keyword": {
                 "type": "keyword",
                 "ignore_above": 256
               }
             }
           }
         }
       },
       "ecs": {
         "properties": {
           "version": {
             "type": "text",
             "fields": {
               "keyword": {
                 "type": "keyword",
                 "ignore_above": 256
               }
             }
           }
         }
       },
       "host": {
         "properties": {
           "architecture": {
             "type": "text",
             "fields": {
               "keyword": {
                 "type": "keyword",
                 "ignore_above": 256
               }
             }
           },
           "containerized": {
             "type": "boolean"
           },
           "hostname": {
             "type": "text",
             "fields": {
               "keyword": {
                 "type": "keyword",
                 "ignore_above": 256
               }
             }
           },
           "id": {
             "type": "text",
             "fields": {
               "keyword": {
                 "type": "keyword",
                 "ignore_above": 256
               }
             }
           },
           "name": {
             "type": "text",
             "fields": {
               "keyword": {
                 "type": "keyword",
                 "ignore_above": 256
               }
             }
           },
           "os": {
             "properties": {
               "codename": {
                 "type": "text",
                 "fields": {
                   "keyword": {
                     "type": "keyword",
                     "ignore_above": 256
                   }
                 }
               },
               "family": {
                 "type": "text",
                 "fields": {
                   "keyword": {
                     "type": "keyword",
                     "ignore_above": 256
                   }
                 }
               },
               "kernel": {
                 "type": "text",
                 "fields": {
                   "keyword": {
                     "type": "keyword",
                     "ignore_above": 256
                   }
                 }
               },
               "name": {
                 "type": "text",
                 "fields": {
                   "keyword": {
                     "type": "keyword",
                     "ignore_above": 256
                   }
                 }
               },
               "platform": {
                 "type": "text",
                 "fields": {
                   "keyword": {
                     "type": "keyword",
                     "ignore_above": 256
                   }
                 }
               },
               "version": {
                 "type": "text",
                 "fields": {
                   "keyword": {
                     "type": "keyword",
                     "ignore_above": 256
                   }
                 }
               }
             }
           }
         }
       },
       "input": {
         "properties": {
           "type": {
             "type": "text",
             "fields": {
               "keyword": {
                 "type": "keyword",
                 "ignore_above": 256
               }
             }
           }
         }
       },
       "log": {
         "properties": {
           "file": {
             "properties": {
               "path": {
                 "type": "text",
                 "fields": {
                   "keyword": {
                     "type": "keyword",
                     "ignore_above": 256
                   }
                 }
               }
             }
           },
           "flags": {
             "type": "text",
             "fields": {
               "keyword": {
                 "type": "keyword",
                 "ignore_above": 256
               }
             }
           },
           "offset": {
             "type": "long"
           }
         }
       },
       "log-type": {
         "type": "text",
         "fields": {
           "keyword": {
             "type": "keyword",
             "ignore_above": 256
           }
         }
       },
       "message": {
         "type": "text",
         "fields": {
           "keyword": {
             "type": "keyword",
             "ignore_above": 256
           }
         }
       },
       "service": {
         "type": "text",
         "fields": {
           "keyword": {
             "type": "keyword",
             "ignore_above": 256
           }
         }
       },
       "tags": {
         "type": "text",
         "fields": {
           "keyword": {
             "type": "keyword",
             "ignore_above": 256
           }
         }
       }
     }
   }
 }
}

{
"index.blocks.read_only_allow_delete": "false",
"index.query.default_field": [
"*"

Some settings:

  "index.refresh_interval": "1s",
  "index.write.wait_for_active_shards": "1",
  "index.lifecycle.name": "logging_small_size",
  "index.lifecycle.rollover_alias": "log-sbk",
  "index.mapping.total_fields.limit": "2000",
  "index.priority": "100",
  "index.number_of_replicas": "1"
}

{
"indices" : {
"log-sbk-000001" : {
"index" : "log-sbk-000001",
"managed" : true,
"policy" : "logging_small_size",
"lifecycle_date_millis" : 1597946682393,
"age" : "1.68h",
"phase" : "hot",
"phase_time_millis" : 1597951596721,
"action" : "rollover",
"action_time_millis" : 1597946797223,
"step" : "ERROR",
"step_time_millis" : 1597952196687,
"failed_step" : "check-rollover-ready",
"is_auto_retryable_error" : true,
"failed_step_retry_count" : 4,
"step_info" : {
"type" : "illegal_argument_exception",
"reason" : "index.lifecycle.rollover_alias [log-sbk] does not point to index [log-sbk-000001]",
"stack_trace" : """java.lang.IllegalArgumentException: index.lifecycle.rollover_alias [log-sbk] does not point to index [log-sbk-000001]
at org.elasticsearch.xpack.core.ilm.WaitForRolloverReadyStep.evaluateCondition(WaitForRolloverReadyStep.java:104)
at org.elasticsearch.xpack.ilm.IndexLifecycleRunner.runPeriodicStep(IndexLifecycleRunner.java:152)
at org.elasticsearch.xpack.ilm.IndexLifecycleService.triggerPolicies(IndexLifecycleService.java:328)
at org.elasticsearch.xpack.ilm.IndexLifecycleService.triggered(IndexLifecycleService.java:266)
at org.elasticsearch.xpack.core.scheduler.SchedulerEngine.notifyListeners(SchedulerEngine.java:183)
at org.elasticsearch.xpack.core.scheduler.SchedulerEngine$ActiveSchedule.run(SchedulerEngine.java:211)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:830)
"""
},
"phase_execution" : {
"policy" : "logging_small_size",
"phase_definition" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "1gb",
"max_age" : "7d"
},
"set_priority" : {
"priority" : 100
}
}
},
"version" : 3,
"modified_date_in_millis" : 1597737178353
}
}
}
}

Did you bootstrap the first index - https://www.elastic.co/guide/en/elasticsearch/reference/current/getting-started-index-lifecycle-management.html#ilm-gs-alias-bootstrap?

Hi,

Yes i did bootstrap the index but should that happen before the index gets created?

My logstash is creating the index pattern like *-000001, does my logstash config need to be something like this ? here is what the filter plugin and output plugin look like:

filter {
    if ( [@metadata][beat] == "filebeat" ) {
        if ( [log-type] == "sbk" ) {
	  mutate {
            add_field => {
              "[@metadata][target_index]" => "log-%{[service]}-000001"
            }
          }
output {
  elasticsearch {
    hosts => ["http://sbk-elasticsearch:9200"]
    user => *****
    password => *******
    index => "%{[@metadata][target_index]}"
  }

Though this is the case(not sure if it is the right way to do it) when i have the rollver enabled i see the initial index gets created with -000001 and does roll over but the index rolled over to i.e -000002 doesn't get written anything.

I am bootstrapping the alias to -000001 as write.