We have a bunch of Watchers where the Query includes time calculation using ctx.trigger.triggered_time in the following manner:
"filter": [
{
"range": {
"@timestamp": {
"gte": "{{ctx.trigger.triggered_time}}||-2h",
"lte": "{{ctx.trigger.triggered_time}}"
}
}
}, ...]
The watchers worked fine until we upgraded Elastic to 6.7.0
Now, we have Elastic logs full of following exceptions:
org.elasticsearch.transport.RemoteTransportException: [10.3.214.4-csos-logs][10.3.214.4:9300][indices:data/read/search[phase/query]]
Caused by: org.elasticsearch.ElasticsearchParseException: failed to parse date field [{{ctx.trigger.triggered_time}}] with format [strict_date_optional_time||epoch_millis]
at org.elasticsearch.common.joda.JodaDateMathParser.parseDateTime(JodaDateMathParser.java:215) ~[elasticsearch-6.7.0.jar:6.7.0]
Heres the full exception and it will take sometime for me to get the watcher response, deploying the setup right now... But I should have it in the next couple of hours:
[2019-04-17T20:45:22,131][DEBUG][o.e.a.s.TransportSearchAction] [10.3.214.26-csos-logs] [172.20.105.6-logs-2019.16][3], node[Faa3d0SgT3-IO43lbakRMg], [R], s[STARTED], a[id=DsSm7Jw6TvW2vB82NY5fzg]: Failed to execute [SearchRequest{searchType=QUERY_THEN_FETCH, indices=[<*-logs-{now/w{yyyy.ww}}>], indicesOptions=IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true], types=, routing='null', preference='null', requestCache=null, scroll=null, maxConcurrentShardRequests=10, batchedReduceSize=512, preFilterShardSize=128, allowPartialSearchResults=true, localClusterAlias=null, getOrCreateAbsoluteStartMillis=-1, source={"query":{"bool":{"filter":[{"range":{"@timestamp":{"from":"{{ctx.trigger.triggered_time}}||-2h","to":"{{ctx.trigger.triggered_time}}||","include_lower":true,"include_upper":true,"boost":1.0}}},{"match_phrase":{"message":{"query":"Event","slop":0,"zero_terms_query":"NONE","boost":1.0}}},{"match_phrase":{"message":{"query":"has entered maintenance mode","slop":0,"zero_terms_query":"NONE","boost":1.0}}}],"must_not":[{"exists":{"field":"tagged","boost":1.0}},{"match_phrase":{"message":{"query":"[Unknown user]","slop":0,"zero_terms_query":"NONE","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}}}]
org.elasticsearch.transport.RemoteTransportException: [10.3.214.4-csos-logs][10.3.214.4:9300][indices:data/read/search[phase/query]]
Caused by: org.elasticsearch.ElasticsearchParseException: failed to parse date field [{{ctx.trigger.triggered_time}}] with format [strict_date_optional_time||epoch_millis]
at org.elasticsearch.common.joda.JodaDateMathParser.parseDateTime(JodaDateMathParser.java:215) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.common.joda.JodaDateMathParser.parse(JodaDateMathParser.java:69) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.mapper.DateFieldMapper$DateFieldType.parseToMilliseconds(DateFieldMapper.java:316) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.mapper.DateFieldMapper$DateFieldType.isFieldWithinQuery(DateFieldMapper.java:329) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.RangeQueryBuilder.getRelation(RangeQueryBuilder.java:459) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.RangeQueryBuilder.doRewrite(RangeQueryBuilder.java:476) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.AbstractQueryBuilder.rewrite(AbstractQueryBuilder.java:284) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.BoolQueryBuilder.rewriteClauses(BoolQueryBuilder.java:485) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.BoolQueryBuilder.doRewrite(BoolQueryBuilder.java:452) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.AbstractQueryBuilder.rewrite(AbstractQueryBuilder.java:284) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.builder.SearchSourceBuilder.rewrite(SearchSourceBuilder.java:949) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.builder.SearchSourceBuilder.rewrite(SearchSourceBuilder.java:80) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.Rewriteable.rewrite(Rewriteable.java:68) ~[elasticsearch-6.7.0.jar:6.7.0]
Second half of the exception:
at org.elasticsearch.index.query.Rewriteable.rewrite(Rewriteable.java:51) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.internal.ShardSearchLocalRequest$RequestRewritable.rewrite(ShardSearchLocalRequest.java:307) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.internal.ShardSearchLocalRequest$RequestRewritable.rewrite(ShardSearchLocalRequest.java:297) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.Rewriteable.rewrite(Rewriteable.java:68) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService.createSearchContext(SearchService.java:671) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService.createSearchContext(SearchService.java:651) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService.createContext(SearchService.java:614) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:595) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:386) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService.access$100(SearchService.java:125) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:358) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:354) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1085) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41) [elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.7.0.jar:6.7.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
Caused by: java.lang.IllegalArgumentException: Parse failure at index [0] of [{{ctx.trigger.triggered_time}}]
at org.elasticsearch.common.joda.JodaDateMathParser.parseDateTime(JodaDateMathParser.java:208) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.common.joda.JodaDateMathParser.parse(JodaDateMathParser.java:69) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.mapper.DateFieldMapper$DateFieldType.parseToMilliseconds(DateFieldMapper.java:316) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.mapper.DateFieldMapper$DateFieldType.isFieldWithinQuery(DateFieldMapper.java:329) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.RangeQueryBuilder.getRelation(RangeQueryBuilder.java:459) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.RangeQueryBuilder.doRewrite(RangeQueryBuilder.java:476) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.AbstractQueryBuilder.rewrite(AbstractQueryBuilder.java:284) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.BoolQueryBuilder.rewriteClauses(BoolQueryBuilder.java:485) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.BoolQueryBuilder.doRewrite(BoolQueryBuilder.java:452) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.AbstractQueryBuilder.rewrite(AbstractQueryBuilder.java:284) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.builder.SearchSourceBuilder.rewrite(SearchSourceBuilder.java:949) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.builder.SearchSourceBuilder.rewrite(SearchSourceBuilder.java:80) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.Rewriteable.rewrite(Rewriteable.java:68) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.Rewriteable.rewrite(Rewriteable.java:51) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.internal.ShardSearchLocalRequest$RequestRewritable.rewrite(ShardSearchLocalRequest.java:307) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.internal.ShardSearchLocalRequest$RequestRewritable.rewrite(ShardSearchLocalRequest.java:297) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.index.query.Rewriteable.rewrite(Rewriteable.java:68) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService.createSearchContext(SearchService.java:671) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService.createSearchContext(SearchService.java:651) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService.createContext(SearchService.java:614) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:595) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:386) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService.access$100(SearchService.java:125) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:358) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:354) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1085) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-6.7.0.jar:6.7.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-6.7.0.jar:6.7.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_161]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_161]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_161]
Apache, Apache Lucene, Apache Hadoop, Hadoop, HDFS and the yellow elephant
logo are trademarks of the
Apache Software Foundation
in the United States and/or other countries.