Lifecycle error

Elastic Stack Elasticsearch
1

Hi,

Getting a bit desperate of all things that go wrong.

This time I bump into this lifecycle error. I really want to split indexes op since they grow to hundreds of Gb.

But...
Index lifecycle error

illegal_argument_exception: index.lifecycle.rollover_alias [agl-*] does not point to index [agl-api-7.9.1-2020.09.17-00001]

I created that alias agl in the template.

Settings:

{
  "settings": {
    "index": {
      "lifecycle": {
        "name": "AVS-logs",
        "rollover_alias": "agl-*"
      },
      "refresh_interval": "10s",
      "number_of_shards": "5",
      "provided_name": "agl-api-7.9.1-2020.09.17-00001",
      "creation_date": "1600301134279",
      "priority": "100",
      "number_of_replicas": "1",
      "uuid": "z83FudTWQFK2J5zOZ6gaRw",
      "version": {
        "created": "7090199"
      }
    }
  },
  "defaults": {
    "index": {
      "flush_after_merge": "512mb",
      "final_pipeline": "_none",
      "max_inner_result_window": "100",
      "unassigned": {
        "node_left": {
          "delayed_timeout": "1m"
        }
      },
      "max_terms_count": "65536",
      "lifecycle": {
        "parse_origination_date": "false",
        "indexing_complete": "false",
        "origination_date": "-1"
      },
      "routing_partition_size": "1",
      "force_memory_term_dictionary": "false",
      "max_docvalue_fields_search": "100",
      "merge": {
        "scheduler": {
          "max_thread_count": "4",
          "auto_throttle": "true",
          "max_merge_count": "9"
        },
        "policy": {
          "reclaim_deletes_weight": "2.0",
          "floor_segment": "2mb",
          "max_merge_at_once_explicit": "30",
          "max_merge_at_once": "10",
          "max_merged_segment": "5gb",
          "expunge_deletes_allowed": "10.0",
          "segments_per_tier": "10.0",
          "deletes_pct_allowed": "33.0"
        }
      },
      "max_refresh_listeners": "1000",
      "max_regex_length": "1000",
      "load_fixed_bitset_filters_eagerly": "true",
      "number_of_routing_shards": "1",
      "write": {
        "wait_for_active_shards": "1"
      },
      "verified_before_close": "false",
      "mapping": {
        "coerce": "false",
        "nested_fields": {
          "limit": "50"
        },
        "depth": {
          "limit": "20"
        },
        "field_name_length": {
          "limit": "9223372036854775807"
        },
        "total_fields": {
          "limit": "1000"
        },
        "nested_objects": {
          "limit": "10000"
        },
        "ignore_malformed": "false"
      },
      "source_only": "false",
      "soft_deletes": {
        "enabled": "false",
        "retention": {
          "operations": "0"
        },
        "retention_lease": {
          "period": "12h"
        }
      },
      "max_script_fields": "32",
      "query": {
        "default_field": [
          "*"
        ],
        "parse": {
          "allow_unmapped_fields": "true"
        }
      },
      "format": "0",
      "frozen": "false",
      "sort": {
        "missing": [],
        "mode": [],
        "field": [],
        "order": []
      },
      "codec": "default",
      "max_rescore_window": "10000",
      "max_adjacency_matrix_filters": "100",
      "analyze": {
        "max_token_count": "10000"
      },
      "gc_deletes": "60s",
      "top_metrics_max_size": "10",
      "optimize_auto_generated_id": "true",
      "max_ngram_diff": "1",
      "hidden": "false",
      "translog": {
        "generation_threshold_size": "64mb",
        "flush_threshold_size": "512mb",
        "sync_interval": "5s",
        "retention": {
          "size": "512MB",
          "age": "12h"
        },
        "durability": "REQUEST"
      },
      "auto_expand_replicas": "false",
      "mapper": {
        "dynamic": "true"
      },
      "recovery": {
        "type": ""
      },
      "requests": {
        "cache": {
          "enable": "true"
        }
      },
      "data_path": "",
      "highlight": {
        "max_analyzed_offset": "1000000"
      },
      "routing": {
        "rebalance": {
          "enable": "all"
        },
        "allocation": {
          "enable": "all",
          "total_shards_per_node": "-1"
        }
      },
      "search": {
        "slowlog": {
          "level": "TRACE",
          "threshold": {
            "fetch": {
              "warn": "-1",
              "trace": "-1",
              "debug": "-1",
              "info": "-1"
            },
            "query": {
              "warn": "-1",
              "trace": "-1",
              "debug": "-1",
              "info": "-1"
            }
          }
        },
        "idle": {
          "after": "30s"
        },
        "throttled": "false"
      },
      "fielddata": {
        "cache": "node"
      },
      "default_pipeline": "_none",
      "max_slices_per_scroll": "1024",
      "shard": {
        "check_on_startup": "false"
      },
      "xpack": {
        "watcher": {
          "template": {
            "version": ""
          }
        },
        "version": "",
        "ccr": {
          "following_index": "false"
        }
      },
      "percolator": {
        "map_unmapped_fields_as_text": "false"
      },
      "allocation": {
        "max_retries": "5",
        "existing_shards_allocator": "gateway_allocator"
      },
      "indexing": {
        "slowlog": {
          "reformat": "true",
          "threshold": {
            "index": {
              "warn": "-1",
              "trace": "-1",
              "debug": "-1",
              "info": "-1"
            }
          },
          "source": "1000",
          "level": "TRACE"
        }
      },
      "compound_format": "0.1",
      "blocks": {
        "metadata": "false",
        "read": "false",
        "read_only_allow_delete": "false",
        "read_only": "false",
        "write": "false"
      },
      "max_result_window": "10000",
      "store": {
        "stats_refresh_interval": "10s",
        "type": "",
        "fs": {
          "fs_lock": "native"
        },
        "preload": []
      },
      "queries": {
        "cache": {
          "enabled": "true"
        }
      },
      "warmer": {
        "enabled": "true"
      },
      "max_shingle_diff": "3",
      "query_string": {
        "lenient": "false"
      }
    }
  }
}

Lifecycle policy:

PUT _ilm/policy/AVS-logs
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_age": "2d",
            "max_size": "50gb",
            "max_docs": 2000000000
          },
          "set_priority": {
            "priority": 100
          }
        }
      },
      "warm": {
        "actions": {
          "set_priority": {
            "priority": 50
          },
          "shrink": {
            "number_of_shards": 2
          }
        }
      },
      "cold": {
        "min_age": "3d",
        "actions": {
          "set_priority": {
            "priority": 0
          }
        }
      },
      "delete": {
        "min_age": "12d",
        "actions": {
          "delete": {
            "delete_searchable_snapshot": true
          }
        }
      }
    }
  }
}

What am I doing wrong?

2

Nobody?
I think I need to tackle this, so any help is appreciated.

3

Typically, your rollover_alias won't use a wildcard like that and it'd be something like agl-api.
Can you alter that?

Otherwise what you need to do is something like;

PUT /agl-api-7.9.1-2020.09.17-00001/_alias/agl-*

But I am not sure a wildcard there will work.

4

Thnx. this indeed was a thing I needed to alter. I am still not there however.

I create a new index which does not change name per day.And I manually set some aliases and this seems to work.

However... This s not what I need.

My outputfilter in logstash is will be this:

output {
        # Send the events to ElasticSearch, which a different index based on the log file the events came from.
        if "api-log" in [tags] or "access-log" in [tags] or "tls-proxy" in [tags] {
                elasticsearch {
                        hosts => ["pdbs359.grn.prd.itv.local:9200","pdbs358.grn.prd.itv.local:9200"]
                        document_id => "%{[@metadata][fingerprint]}"
                        index => "%{[@metadata][indexbasename]}-{+YYYY.MM.dd}"
                        sniffing => false
                }
}
}

Can I add stuff here to make Elasticsearch add a decent alias based on the indexname and add the appropriate lifecyclepolicy automatically?

5

The thing that needs reconciling here is that ILM will roll an index on a daily basis (if you tell it), but it doesn't change the name of the index to match that date. The date in the index name is the date that the index was created under the policy.

TLDR - change the index directive in the output to point to your ILM write alias.

6

Switched ilm off for now. First need to get the rest up and running normally. Just can't get it to work.

Thnx anyway

7

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.