Hello, since i deployed an ELK 7.2 stack ( 4 days ago ) the auditbeat shards are failing with 3 out of 4 shards when i try to use the defaulf [[Auditbeat Auditd] Overview ECS] dashboard ( or any of the other auditbeat default dashboards).Which leaves me with accessible events only older than 3 days Everything is green in the cluster.
Incoming events are indexed correctly but when i execute a query the shards fails, from a data node:
{
"type": "server",
"timestamp": "2019-08-08T09:53:03,254+0000",
"level": "DEBUG",
"component": "o.e.a.s.TransportSearchAction",
"cluster.name": "elkelasticsearch_name",
"node.name": "elkelasticsearch-data-1",
"cluster.uuid": "uzybupYsRiuyxlorl0gVfQ",
"node.id": "wkLzxc7pSeuhpJqFwAhnsQ",
"message": "[auditbeat-2019.08.07][0], node[OM0otphDQ1yK3zRlqJhhKA], [R], s[STARTED], a[id=aAg3q3uXSjSS63M2r3VASQ]: Failed to execute [SearchRequest{searchType=QUERY_THEN_FETCH, indices=[auditbeat-*], indice
sOptions=IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_al
iases=false, ignore_throttled=true], types=[], routing='null', preference='null', requestCache=null, scroll=null, maxConcurrentShardRequests=0, batchedReduceSize=512, preFilterShardSize=128, allowPartialSearch
Results=true, localClusterAlias=null, getOrCreateAbsoluteStartMillis=-1, ccsMinimizeRoundtrips=true, source={\"size\":0,\"timeout\":\"30000ms\",\"query\":{\"bool\":{\"must\":[{\"range\":{\"@timestamp\":{\"from
\":\"2019-08-08T09:52:02.844Z\",\"to\":\"2019-08-08T09:53:02.844Z\",\"include_lower\":true,\"include_upper\":true,\"format\":\"strict_date_optional_time\",\"boost\":1.0}}},{\"query_string\":{\"query\":\"event.
module:auditd\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy
_max_expansions\":50,\"phrase_slop\":0,\"analyze_wildcard\":true,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}}],\"filter\":[{\"match_all\":{\"boost
\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\":{\"61ca57f1-469d-11e7-af02-69e470af7417\":{\"meta\":{\"timeField\":\"@timestamp\",\"intervalString\":\"1s\",\"bucketSize\":1,\"seriesId\
":\"61ca57f1-469d-11e7-af02-69e470af7417\"},\"terms\":{\"field\":\"event.action\",\"size\":10,\"min_doc_count\":1,\"shard_min_doc_count\":0,\"show_term_doc_count_error\":false,\"order\":[{\"_count\":\"desc\"},
{\"_key\":\"asc\"}]},\"aggregations\":{\"timeseries\":{\"date_histogram\":{\"field\":\"@timestamp\",\"time_zone\":\"Europe\",\"interval\":\"1s\",\"offset\":0,\"order\":{\"_key\":\"asc\"},\"keyed\":false,\"min_
doc_count\":0,\"extended_bounds\":{\"min\":1565257922844,\"max\":1565257982844}},\"aggregations\":{\"6b9fb2d0-c1bc-11e7-938f-ab0645b6c431\":{\"bucket_script\":{\"buckets_path\":{\"count\":\"_count\"},\"script\
":{\"source\":\"count * 1\",\"lang\":\"expression\"},\"gap_policy\":\"skip\"}}}}}}}}}] lastShard [true]",
"stacktrace": [
"org.elasticsearch.transport.RemoteTransportException: [elkelasticsearch-data-0][10.224.3.247:9300][indices:data/read/search[phase/query]]",
"Caused by: java.lang.IllegalArgumentException: Fielddata is disabled on text fields by default. Set fielddata=true on [event.action] in order to load fielddata in memory by uninverting the inverted index.
Note that this can however use significant memory. Alternatively use a keyword field instead.",
"at org.elasticsearch.index.mapper.TextFieldMapper$TextFieldType.fielddataBuilder(TextFieldMapper.java:711) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.index.fielddata.IndexFieldDataService.getForField(IndexFieldDataService.java:116) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.index.query.QueryShardContext.getForField(QueryShardContext.java:179) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.search.aggregations.support.ValuesSourceConfig.resolve(ValuesSourceConfig.java:95) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder.resolveConfig(ValuesSourceAggregationBuilder.java:321) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder.doBuild(ValuesSourceAggregationBuilder.java:314) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder.doBuild(ValuesSourceAggregationBuilder.java:39) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.search.aggregations.AbstractAggregationBuilder.build(AbstractAggregationBuilder.java:139) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.search.aggregations.AggregatorFactories$Builder.build(AggregatorFactories.java:332) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.search.SearchService.parseSource(SearchService.java:789) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.search.SearchService.createContext(SearchService.java:591) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:550) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:353) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.search.SearchService$2.doRun(SearchService.java:1052) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:758) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.2.0.jar:7.2.0]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
"at java.lang.Thread.run(Thread.java:835) [?:?]"
]
}