Hi - I'm starting out with Machine Learning by following the video @ Machine learning tutorial. However , I keep getting the error below.
Datafeed is encountering errors extracting data: [test] Search request returned shard failures; first failure: shard [[axxPcITtQcu4h6gHQ2hRQA][flows-2019-01-27][0]], reason [RemoteTransportException[[sin-cpt-sl-dev-flow10g][10.0.4.42:9300][indices:data/read/search[phase/query]]]; nested: QueryShardException[No mapping found for [timestamp] in order to sort on]; ];
This results in the job not processing Any records.
I have an index template for dynamic mapping - which resolved timestamp to
"timestamp" : {
"type" : "date",
"format" : "date_optional_time"
},
when I do a GET flows-*/_mapping.
And when I go into Kibana->index patterns ... it also shows it as type date , searchable , aggregatable .
There definitely IS data in the flows-2019-01-27 index it refers to in the error (although there are gaps in the timeline... i.e. There is data for 01-01-2019 -> today , but some days are missing)
I'm running ES 5.4
Below is the job config:
{
"job_id": "test",
"job_type": "anomaly_detector",
"create_time": 1550657010661,
"finished_time": 1550657011029,
"analysis_config": {
"bucket_span": "5m",
"detectors": [
{
"detector_description": "sum(event_count)",
"function": "sum",
"field_name": "event_count",
"partition_field_name": "from.ip",
"detector_rules": []
}
],
"influencers": [
"from.ip",
"http.site"
]
},
"data_description": {
"time_field": "timestamp",
"time_format": "epoch_ms"
},
"model_snapshot_retention_days": 1,
"results_index_name": "shared",
"data_counts": {
"job_id": "test",
"processed_record_count": 0,
"processed_field_count": 0,
"input_bytes": 0,
"input_field_count": 0,
"invalid_date_count": 0,
"missing_field_count": 0,
"out_of_order_timestamp_count": 0,
"empty_bucket_count": 0,
"sparse_bucket_count": 0,
"bucket_count": 0,
"input_record_count": 0
},
"model_size_stats": {
"job_id": "test",
"result_type": "model_size_stats",
"model_bytes": 0,
"total_by_field_count": 0,
"total_over_field_count": 0,
"total_partition_field_count": 0,
"bucket_allocation_failures_count": 0,
"memory_status": "ok",
"log_time": 1550657011000,
"timestamp": -300000
},
"datafeed_config": {
"datafeed_id": "datafeed-test",
"job_id": "test",
"query_delay": "60s",
"frequency": "150s",
"indexes": [
"flows-*"
],
"types": [
"flow",
"_default_",
"reverse"
],
"query": {
"match_all": {
"boost": 1
}
},
"scroll_size": 1000,
"chunking_config": {
"mode": "auto"
},
"state": "stopped"
},
"state": "closed"
}