I'm trying out the anomaly detection package for the first time, and there is a nice example in the tutorial, whereby elastic learns temporal oscillations.
I've configured a very basic job, but the ML does not pick up the time variation (see image).
Have I missed something?
It might be worth noting the timescales are quite large here (6 month period) - maybe it would get there in a few more years, or maybe there is a maximum timescale?
I could potentially remove some of these effects, in so far as they are well known - but I'm trying to understand what i get though the magic of ML first
Thanks
{
"job_id": "small_bucket_incl",
"job_type": "anomaly_detector",
"job_version": "8.5.0",
"create_time": 1668700217159,
"finished_time": 1668700914767,
"model_snapshot_id": "1668700913",
"custom_settings": {
"custom_urls": []
},
"datafeed_config": {
"datafeed_id": "datafeed-small_bucket_incl",
"job_id": "small_bucket_incl",
"query_delay": "105026ms",
"chunking_config": {
"mode": "auto"
},
"indices_options": {
"expand_wildcards": [
"open"
],
"ignore_unavailable": false,
"allow_no_indices": true,
"ignore_throttled": true
},
"query": {
"range": {
"perigee": {
"lt": 400,
"gte": 300
}
}
},
"indices": [
"orbits_1*"
],
"scroll_size": 1000,
"delayed_data_check_config": {
"enabled": true
},
"state": "stopped",
"timing_stats": {
"job_id": "small_bucket_incl",
"search_count": 1195,
"bucket_count": 3787,
"total_search_time_ms": 7909,
"average_search_time_per_bucket_ms": 2.088460522841299,
"exponential_average_search_time_per_hour_ms": 5.291986729993371
}
},
"description": "",
"analysis_config": {
"bucket_span": "6h",
"detectors": [
{
"detector_description": "metric(inclination) partitionfield=combined_primary_id",
"function": "mean",
"field_name": "inclination",
"partition_field_name": "combined_primary_id",
"detector_index": 0
}
],
"influencers": [],
"model_prune_window": "100d"
},
"analysis_limits": {
"model_memory_limit": "1024mb",
"categorization_examples_limit": 4
},
"data_description": {
"time_field": "orbit_date",
"time_format": "epoch_ms"
},
"model_plot_config": {
"enabled": true,
"annotations_enabled": true
},
"model_snapshot_retention_days": 10,
"daily_model_snapshot_retention_after_days": 1,
"results_index_name": "custom-small_bucket_incl",
"allow_lazy_open": false,
"data_counts": {
"job_id": "small_bucket_incl",
"processed_record_count": 1009598,
"processed_field_count": 2019196,
"input_bytes": 97984196,
"input_field_count": 2019196,
"invalid_date_count": 0,
"missing_field_count": 0,
"out_of_order_timestamp_count": 0,
"empty_bucket_count": 1616,
"sparse_bucket_count": 221,
"bucket_count": 3787,
"earliest_record_timestamp": 1583529553000,
"latest_record_timestamp": 1665325056000,
"last_data_time": 1668700912005,
"latest_empty_bucket_timestamp": 1664409600000,
"latest_sparse_bucket_timestamp": 1664496000000,
"input_record_count": 1009598,
"log_time": 1668700912005,
"latest_bucket_timestamp": 1665316800000
},
"model_size_stats": {
"job_id": "small_bucket_incl",
"result_type": "model_size_stats",
"model_bytes": 94983432,
"peak_model_bytes": 94983432,
"model_bytes_exceeded": 0,
"model_bytes_memory_limit": 1073741824,
"total_by_field_count": 2092,
"total_over_field_count": 0,
"total_partition_field_count": 5302,
"bucket_allocation_failures_count": 0,
"memory_status": "ok",
"assignment_memory_basis": "current_model_bytes",
"categorized_doc_count": 0,
"total_category_count": 0,
"frequent_category_count": 0,
"rare_category_count": 0,
"dead_category_count": 0,
"failed_category_count": 0,
"categorization_status": "ok",
"log_time": 1668700913323,
"timestamp": 1665295200000
},
"forecasts_stats": {
"total": 0,
"forecasted_jobs": 0
},
"state": "closed",
"timing_stats": {
"job_id": "small_bucket_incl",
"bucket_count": 3787,
"total_bucket_processing_time_ms": 548519.9999999988,
"minimum_bucket_processing_time_ms": 0,
"maximum_bucket_processing_time_ms": 440,
"average_bucket_processing_time_ms": 144.84288354898305,
"exponential_average_bucket_processing_time_ms": 317.6560509251809,
"exponential_average_bucket_processing_time_per_hour_ms": 0
}
}