Datafeed has been retrieving no data for a while

Elastic Stack Elasticsearch
1

hi, I've created an anomaly detection job, and after the creation it wont update anymore:
attached the json:

{
  "job_id": "ml-test",
  "job_type": "anomaly_detector",
  "job_version": "7.6.1",
  "groups": [
    "fortified"
  ],
  "description": "",
  "create_time": 1584517838863,
  "analysis_config": {
    "bucket_span": "15m",
    "detectors": [
      {
        "detector_description": "high_mean(value) by servername partitionfield=counter",
        "function": "high_mean",
        "field_name": "value",
        "by_field_name": "servername",
        "partition_field_name": "counter",
        "detector_index": 0
      }
    ],
    "influencers": [
      "servername",
      "counter"
    ]
  },
  "analysis_limits": {
    "model_memory_limit": "100mb",
    "categorization_examples_limit": 4
  },
  "data_description": {
    "time_field": "datetime",
    "time_format": "epoch_ms"
  },
  "model_plot_config": {
    "enabled": true
  },
  "model_snapshot_retention_days": 1,
  "custom_settings": {
    "custom_urls": []
  },
  "model_snapshot_id": "1584524718",
  "results_index_name": "shared",
  "allow_lazy_open": false,
  "data_counts": {
    "job_id": "ml-test",
    "processed_record_count": 7851841,
    "processed_field_count": 23555523,
    "input_bytes": 798422865,
    "input_field_count": 23555523,
    "invalid_date_count": 0,
    "missing_field_count": 0,
    "out_of_order_timestamp_count": 0,
    "empty_bucket_count": 20,
    "sparse_bucket_count": 0,
    "bucket_count": 6029,
    "earliest_record_timestamp": 1579098006000,
    "latest_record_timestamp": 1584523811000,
    "last_data_time": 1584524718070,
    "latest_empty_bucket_timestamp": 1584522900000,
    "input_record_count": 7851841,
    "latest_bucket_timestamp": 1584535500000
  },
  "model_size_stats": {
    "job_id": "ml-test",
    "result_type": "model_size_stats",
    "model_bytes": 4339112,
    "model_bytes_exceeded": 0,
    "model_bytes_memory_limit": 104857600,
    "total_by_field_count": 159,
    "total_over_field_count": 0,
    "total_partition_field_count": 8,
    "bucket_allocation_failures_count": 0,
    "memory_status": "ok",
    "log_time": 1584527820000,
    "timestamp": 1584527400000
  },
  "datafeed_config": {
    "datafeed_id": "datafeed-ml-test",
    "job_id": "ml-test",
    "query_delay": "300000ms",
    "frequency": "60s",
    "indices": [
      "xxx-15*"
    ],
    "query": {
      "bool": {
        "must": [
           
        ]
      }
    },
    "scroll_size": 1000,
    "chunking_config": {
      "mode": "auto"
    },
    "delayed_data_check_config": {
      "enabled": true
    },
    "state": "started",
    "timing_stats": {
      "job_id": "ml-test",
      "search_count": 9434,
      "bucket_count": 6029,
      "total_search_time_ms": 227239,
      "average_search_time_per_bucket_ms": 37.69099353126555,
      "exponential_average_search_time_per_hour_ms": 14440.479880858304
    }
  },
  "state": "opened",
  "node": {
    
  },
  "open_time": "18574s"
}

how can I find the root cause?

also I've noticed that when stopping and starting the datafeed,
the UI shows that:

image
image917×390 22.4 KB

but logs messages shows that:

Datafeed started (from: 2020-03-18T13:15:00.000Z to: real-time) with frequency [60000ms]

also is my date format ok?
"datetime" : "2020-03-18T13:10:03.000Z",

thanks a lot

2

What do you mean by "it won't update anymore"? Do you mean that after the initial creation and learning on historical data, it won't process on-going ("real-time") data?

3

Yes.
That the log says
Datafeed has been retrieving no data for a while
But the index does has newer data

4

The simplest possible answer is that your "new" data that is arriving into the index is either:

a) arriving into the index at a time later than "now-query_delay" (which in your case seems to be 5 mins). In other words, you have an ingest delay that is more than 5 minutes. So, when the datafeed looks for it in the index - it isn't actually there yet.
b) have virtually no ingest delay, but the data gets inserted with a timestamp that is significantly "different" than the timestamp of when the data was ingested (due to incorrect time zone definitions, etc)

(or I suppose a combination of the two is possible).

Usually, in most cases, option a) is the reason and that the remedy is usually to either increase your ingest speed or increase the query_delay of the datafeed.

See other articles that mention this, for example: Machine Learning Real-Time Job stopping after initial run

1 Like
5

thanks @richcollier, indeed it was option a. thanks. very appreciated

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.