Running ELK 7.16.3 version Platinum Subscription.
I am getting error message Datafeed has missed 152 documents due to ingest latency, latest bucket with missing data is [2022-10-10T17:00:00.000Z]. Consider increasing query_delay. I tried to increase query delay value but my job stop after some hours.
Here is the job config
{
"job_id": "new",
"job_type": "anomaly_detector",
"job_version": "7.16.3",
"create_time": 1665421273515,
"model_snapshot_id": "1665421283",
"groups": [
"bib"
],
"description": "",
"analysis_config": {
"bucket_span": "15m",
"summary_count_field_name": "_doc_count",
"detectors": [
{
"detector_description": "max(test)",
"function": "max",
"field_name": "test",
"detector_index": 0
}
],
"influencers": [
"verb.keyword"
]
},
"analysis_limits": {
"model_memory_limit": "11mb",
"categorization_examples_limit": 4
},
"data_description": {
"time_field": "@timestamp",
"time_format": "epoch_ms"
},
"model_plot_config": {
"enabled": true,
"annotations_enabled": true
},
"model_snapshot_retention_days": 10,
"daily_model_snapshot_retention_after_days": 1,
"results_index_name": "custom-new",
"allow_lazy_open": false,
"data_counts": {
"job_id": "new",
"processed_record_count": 14115,
"processed_field_count": 28230,
"input_bytes": 1012935,
"input_field_count": 28230,
"invalid_date_count": 0,
"missing_field_count": 14115,
"out_of_order_timestamp_count": 0,
"empty_bucket_count": 0,
"sparse_bucket_count": 0,
"bucket_count": 741,
"earliest_record_timestamp": 1664755206489,
"latest_record_timestamp": 1665422106741,
"last_data_time": 1665422612397,
"input_record_count": 14115,
"log_time": 1665422612397,
"latest_bucket_timestamp": 1665422100000
},
"model_size_stats": {
"job_id": "new",
"result_type": "model_size_stats",
"model_bytes": 28782,
"peak_model_bytes": 31524,
"model_bytes_exceeded": 0,
"model_bytes_memory_limit": 11534336,
"total_by_field_count": 1,
"total_over_field_count": 0,
"total_partition_field_count": 2,
"bucket_allocation_failures_count": 0,
"memory_status": "ok",
"assignment_memory_basis": "current_model_bytes",
"categorized_doc_count": 0,
"total_category_count": 0,
"frequent_category_count": 0,
"rare_category_count": 0,
"dead_category_count": 0,
"failed_category_count": 0,
"categorization_status": "ok",
"log_time": 1665421281514,
"timestamp": 1664772300000
},
"forecasts_stats": {
"total": 0,
"forecasted_jobs": 0
},
"state": "opened",
"node": {
"id": "p03Tck9FSNeCpT5HpYml8w",
"name": "node5",
"ephemeral_id": "SqZjuNWOTfSVyEGXuLcyUA",
"transport_address": "xx.xx.xx.xx:9300",
"attributes": {
"ml.machine_memory": "168606924800",
"ml.max_open_jobs": "512",
"xpack.installed": "true",
"ml.max_jvm_size": "33285996544",
"transform.node": "false"
}
},
"assignment_explanation": "",
"open_time": "41265s",
"timing_stats": {
"job_id": "new",
"bucket_count": 744,
"total_bucket_processing_time_ms": 54.00000000000002,
"minimum_bucket_processing_time_ms": 0,
"maximum_bucket_processing_time_ms": 2,
"average_bucket_processing_time_ms": 0.07258064516129036,
"exponential_average_bucket_processing_time_ms": 0.06157431014508287,
"exponential_average_bucket_processing_time_per_hour_ms": 0.7218152400879877
},
"datafeed_config": {
"datafeed_id": "datafeed-new",
"job_id": "new",
"query_delay": "62272ms",
"chunking_config": {
"mode": "auto"
},
"indices_options": {
"expand_wildcards": [
"open"
],
"ignore_unavailable": false,
"allow_no_indices": true,
"ignore_throttled": true
},
"query": {
"match_phrase": {
"log.file.path.keyword": "/test.log"
}
},
"indices": [
"profile-*"
],
"scroll_size": 1000,
"delayed_data_check_config": {
"enabled": true
},
"runtime_mappings": {
"test": {
"type": "long",
"script": {
"source": "emit(Integer.parseInt(doc['value.keyword'].value))"
}
}
},
"state": "stopped",
"timing_stats": {
"job_id": "new",
"search_count": 25,
"bucket_count": 741,
"total_search_time_ms": 736,
"average_search_time_per_bucket_ms": 0.9932523616734144,
"exponential_average_search_time_per_hour_ms": 33.03721539834794
}
}
}
