Elastic-Stack 8.12.1
We are currently trying to use a downsample ILM on Netflow Data using the elastiflow integration. It seems as the setting for the dimension fields is ignored from the index settings as there is the limit of 64 set. I do not understand why this setting is ignored or how this limit can be extended otherwise.
Getting following error on rollover of the data stream:
{
"failed_step": "rollup",
"step_info": {
"type": "illegal_argument_exception",
"reason": "Limit of total dimension fields [21] has been exceeded"
}
}
Current phase definition
{
"policy": "elastiflow",
"phase_definition": {
"min_age": "0ms",
"actions": {
"downsample": {
"fixed_interval": "10m",
"wait_timeout": "1d"
},
"rollover": {
"max_age": "30d",
"max_primary_shard_size": "50gb"
},
"set_priority": {
"priority": 100
}
}
},
"version": 1,
"modified_date_in_millis": 1708358807259
}
Index Settings
{
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": "16384"
},
"dimension_fields": {
"limit": "64"
}
},
"refresh_interval": "10s",
"hidden": "true",
"time_series": {
"end_time": "2024-02-20T19:39:41.000Z",
"start_time": "2024-02-19T16:26:29.000Z"
},
"blocks": {
"write": "true"
},
"provided_name": ".ds-elastiflow-flow-ecs-8.0-2.3-tsds-2024.02.19-000001",
"final_pipeline": "_none",
"creation_date": "1708367189833",
"priority": "100",
"number_of_replicas": "1",
"routing_path": "tsds.routing_path.key",
"uuid": "2nrY07sqSiOnFjrP-nIuuA",
"version": {
"created": "8500010"
},
"lifecycle": {
"name": "elastiflow",
"indexing_complete": "true"
},
"mode": "time_series",
"codec": "best_compression",
"routing": {
"allocation": {
"include": {
"_tier_preference": "data_hot"
}
}
},
"number_of_shards": "1",
"default_pipeline": "_none"
}
},
"defaults": {
"index": {
"flush_after_merge": "512mb",
"time_series": {
"es87tsdb_codec": {
"enabled": "true"
}
},
"max_inner_result_window": "100",
"unassigned": {
"node_left": {
"delayed_timeout": "1m"
}
},
"max_terms_count": "65536",
"rollup": {
"source": {
"name": "",
"uuid": ""
}
},
"lifecycle": {
"parse_origination_date": "false",
"prefer_ilm": "true",
"step": {
"wait_time_threshold": "12h"
},
"rollover_alias": "",
"origination_date": "-1"
},
"routing_partition_size": "1",
"force_memory_term_dictionary": "false",
"max_docvalue_fields_search": "100",
"merge": {
"scheduler": {
"max_thread_count": "4",
"auto_throttle": "true",
"max_merge_count": "9"
},
"policy": {
"merge_factor": "32",
"floor_segment": "2mb",
"max_merge_at_once_explicit": "30",
"max_merge_at_once": "10",
"max_merged_segment": "0b",
"expunge_deletes_allowed": "10.0",
"segments_per_tier": "10.0",
"type": "UNSET",
"deletes_pct_allowed": "20.0"
}
},
"max_refresh_listeners": "1000",
"max_regex_length": "1000",
"load_fixed_bitset_filters_eagerly": "true",
"number_of_routing_shards": "1",
"write": {
"wait_for_active_shards": "1"
},
"verified_before_close": "false",
"mapping": {
"coerce": "false",
"nested_fields": {
"limit": "50"
},
"nested_objects": {
"limit": "10000"
},
"depth": {
"limit": "20"
},
"ignore_malformed": "false",
"field_name_length": {
"limit": "9223372036854775807"
}
},
"source_only": "false",
"soft_deletes": {
"enabled": "true",
"retention": {
"operations": "0"
},
"retention_lease": {
"period": "12h"
}
},
"max_script_fields": "32",
"query": {
"default_field": [
"*"
],
"parse": {
"allow_unmapped_fields": "true"
}
},
"format": "0",
"frozen": "false",
"sort": {
"missing": [],
"mode": [],
"field": [],
"order": []
},
"version": {
"compatibility": "8500010"
},
"max_rescore_window": "10000",
"bloom_filter_for_id_field": {
"enabled": "true"
},
"max_adjacency_matrix_filters": "100",
"analyze": {
"max_token_count": "10000"
},
"gc_deletes": "60s",
"top_metrics_max_size": "10",
"optimize_auto_generated_id": "true",
"max_ngram_diff": "1",
"translog": {
"flush_threshold_age": "1m",
"generation_threshold_size": "64mb",
"flush_threshold_size": "10gb",
"sync_interval": "5s",
"retention": {
"size": "-1",
"age": "-1"
},
"durability": "REQUEST"
},
"auto_expand_replicas": "false",
"fast_refresh": "false",
"recovery": {
"type": ""
},
"requests": {
"cache": {
"enable": "true"
}
},
"data_path": "",
"highlight": {
"max_analyzed_offset": "1000000",
"weight_matches_mode": {
"enabled": "true"
}
},
"look_back_time": "2h",
"routing": {
"rebalance": {
"enable": "all"
},
"allocation": {
"disk": {
"watermark": {
"ignore": "false"
}
},
"enable": "all",
"total_shards_per_node": "-1"
}
},
"search": {
"slowlog": {
"level": "TRACE",
"threshold": {
"fetch": {
"warn": "-1",
"trace": "-1",
"debug": "-1",
"info": "-1"
},
"query": {
"warn": "-1",
"trace": "-1",
"debug": "-1",
"info": "-1"
}
}
},
"idle": {
"after": "30s"
},
"throttled": "false"
},
"fielddata": {
"cache": "node"
},
"look_ahead_time": "2h",
"max_slices_per_scroll": "1024",
"shard": {
"check_on_startup": "false"
},
"xpack": {
"watcher": {
"template": {
"version": ""
}
},
"version": "",
"ccr": {
"following_index": "false"
}
},
"percolator": {
"map_unmapped_fields_as_text": "false"
},
"allocation": {
"max_retries": "5",
"existing_shards_allocator": "gateway_allocator"
},
"indexing": {
"slowlog": {
"reformat": "true",
"threshold": {
"index": {
"warn": "-1",
"trace": "-1",
"debug": "-1",
"info": "-1"
}
},
"source": "1000",
"level": "TRACE"
}
},
"compound_format": "1gb",
"blocks": {
"metadata": "false",
"read": "false",
"read_only_allow_delete": "false",
"read_only": "false"
},
"max_result_window": "10000",
"store": {
"stats_refresh_interval": "10s",
"type": "",
"fs": {
"fs_lock": "native"
},
"preload": [],
"snapshot": {
"snapshot_name": "",
"index_uuid": "",
"cache": {
"prewarm": {
"enabled": "true"
},
"enabled": "true",
"excluded_file_types": []
},
"repository_uuid": "",
"uncached_chunk_size": "-1b",
"delete_searchable_snapshot": "false",
"index_name": "",
"partial": "false",
"blob_cache": {
"metadata_files": {
"max_length": "64kb"
}
},
"repository_name": "",
"snapshot_uuid": ""
}
},
"queries": {
"cache": {
"enabled": "true"
}
},
"shard_limit": {
"group": "normal"
},
"warmer": {
"enabled": "true"
},
"downsample": {
"interval": "",
"source": {
"name": "",
"uuid": ""
},
"origin": {
"name": "",
"uuid": ""
},
"status": "unknown"
},
"override_write_load_forecast": "0.0",
"max_shingle_diff": "3",
"query_string": {
"lenient": "false"
}
}
}
}