elastic version
elasticsearch version 7.5.0
complate configuration
Here is the complete machine learning anomaly detection configuration
{
"job_id": "egress_q_bytes_ml",
"job_type": "anomaly_detector",
"job_version": "7.5.0",
"create_time": 1583762310082,
"analysis_config": {
"bucket_span": "10m",
"summary_count_field_name": "doc_count",
"detectors": [
{
"detector_description": "mean(egress_q_bytes_derivative)",
"function": "mean",
"field_name": "egress_q_bytes_derivative",
"detector_index": 0
}
],
"influencers": []
},
"analysis_limits": {
"model_memory_limit": "1024mb",
"categorization_examples_limit": 4
},
"data_description": {
"time_field": "time",
"time_format": "epoch_ms"
},
"model_snapshot_retention_days": 1,
"results_index_name": "shared",
"allow_lazy_open": false,
"data_counts": {
"job_id": "egress_q_bytes_ml",
"processed_record_count": 0,
"processed_field_count": 0,
"input_bytes": 0,
"input_field_count": 0,
"invalid_date_count": 0,
"missing_field_count": 0,
"out_of_order_timestamp_count": 0,
"empty_bucket_count": 0,
"sparse_bucket_count": 0,
"bucket_count": 0,
"input_record_count": 0
},
"model_size_stats": {
"job_id": "egress_q_bytes_ml",
"result_type": "model_size_stats",
"model_bytes": 0,
"total_by_field_count": 0,
"total_over_field_count": 0,
"total_partition_field_count": 0,
"bucket_allocation_failures_count": 0,
"memory_status": "ok",
"log_time": 1583763080158
},
"datafeed_config": {
"datafeed_id": "datafeed-egress_q_bytes",
"job_id": "egress_q_bytes_ml",
"query_delay": "95640ms",
"indices": [
"jti_logical_usage"
],
"query": {
"match_all": {}
},
"aggregations": {
"bucket_by_sys_id": {
"terms": {
"field": "system_id.keyword",
"size": 10
},
"aggs": {
"bucket_by_time": {
"date_histogram": {
"field": "@timestamp",
"fixed_interval": "1m"
},
"aggs": {
"time": {
"max": {
"field": "@timestamp"
}
},
"egress_q_bytes": {
"max": {
"field": "enterprise.juniperNetworks.jnprLogicalInterfaceExt.interface_info.egress_queue_info.bytes"
}
},
"egress_q_bytes_derivative": {
"derivative": {
"buckets_path": "egress_q_bytes"
}
}
}
}
}
}
},
"scroll_size": 1000,
"chunking_config": {
"mode": "manual",
"time_span": "60000000ms"
},
"delayed_data_check_config": {
"enabled": true
},
"state": "started",
"timing_stats": {
"job_id": "egress_q_bytes_ml",
"search_count": 9,
"bucket_count": 0,
"total_search_time_ms": 8,
"exponential_average_search_time_per_hour_ms": 8
}
},
"state": "opened",
"node": {
"id": "GObuIi9dQ6OGAcinB8nkYw",
"name": "es03",
"ephemeral_id": "4MX7UId9STaiIro8syQKVg",
"transport_address": "172.18.0.4:9300",
"attributes": {
"ml.machine_memory": "41817739264",
"ml.max_open_jobs": "20",
"xpack.installed": "true"
}
},
"open_time": "2469s"
}
I can proof my aggregation is correct
GET jti_logical_usage/_search
{
"size": 0,
"aggs": {
"bucket_by_sys_id": {
"terms": {
"field": "system_id.keyword",
"size": 10
},
"aggs": {
"bucket_by_time": {
"date_histogram": {
"field": "@timestamp",
"fixed_interval": "1m"
},
"aggs": {
"time": {
"max": {
"field": "@timestamp"
}
},
"egress_q_bytes": {
"max": {
"field": "enterprise.juniperNetworks.jnprLogicalInterfaceExt.interface_info.egress_queue_info.bytes"
}
},
"egress_q_bytes_derivative": {
"derivative": {
"buckets_path": "egress_q_bytes"
}
}
}
}
}
}
}
}
# response
{
"took" : 321,
"timed_out" : false,
"_shards" : {
"total" : 3,
"successful" : 3,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 10000,
"relation" : "gte"
},
"max_score" : null,
"hits" : [ ]
},
"aggregations" : {
"bucket_by_sys_id" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "lab-mx960-3d-02-re0-ES-C-02",
"doc_count" : 1147392,
"bucket_by_time" : {
"buckets" : [
{
"key_as_string" : "2020-03-03T03:12:00.000Z",
"key" : 1583205120000,
"doc_count" : 960,
"egress_q_bytes" : {
"value" : 5.918343E8
},
"time" : {
"value" : 1.583205179609E12,
"value_as_string" : "2020-03-03T03:12:59.609Z"
}
},
{
"key_as_string" : "2020-03-03T03:13:00.000Z",
"key" : 1583205180000,
"doc_count" : 1440,
"egress_q_bytes" : {
"value" : 5.918343E8
},
"time" : {
"value" : 1.583205239829E12,
"value_as_string" : "2020-03-03T03:13:59.829Z"
},
"egress_q_bytes_derivative" : {
"value" : 0.0
}
},
{
"key_as_string" : "2020-03-03T03:14:00.000Z",
"key" : 1583205240000,
"doc_count" : 1416,
"egress_q_bytes" : {
"value" : 5.918343E8
},
"time" : {
"value" : 1.583205299999E12,
"value_as_string" : "2020-03-03T03:14:59.999Z"
},
"egress_q_bytes_derivative" : {
"value" : 0.0
}
},
{
"key_as_string" : "2020-03-03T03:15:00.000Z",
"key" : 1583205300000,
"doc_count" : 1416,
"egress_q_bytes" : {
"value" : 5.918343E8
},
"time" : {
"value" : 1.583205358253E12,
"value_as_string" : "2020-03-03T03:15:58.253Z"
},
"egress_q_bytes_derivative" : {
"value" : 0.0
}
},
{
"key_as_string" : "2020-03-03T03:16:00.000Z",
"key" : 1583205360000,
"doc_count" : 1440,
"egress_q_bytes" : {
"value" : 5.918343E8
},
"time" : {
"value" : 1.583205418445E12,
"value_as_string" : "2020-03-03T03:16:58.445Z"
},
"egress_q_bytes_derivative" : {
"value" : 0.0
}
},
{
"key_as_string" : "2020-03-03T03:17:00.000Z",
"key" : 1583205420000,
"doc_count" : 1440,
"egress_q_bytes" : {
"value" : 5.918343E8
},
"time" : {
"value" : 1.583205478659E12,
"value_as_string" : "2020-03-03T03:17:58.659Z"
},
"egress_q_bytes_derivative" : {
"value" : 0.0
}
},
{
"key_as_string" : "2020-03-03T03:18:00.000Z",
"key" : 1583205480000,
"doc_count" : 1440,
"egress_q_bytes" : {
"value" : 5.918343E8
},
"time" : {
"value" : 1.583205538861E12,
"value_as_string" : "2020-03-03T03:18:58.861Z"
},
"egress_q_bytes_derivative" : {
"value" : 0.0
}
},
{
"key_as_string" : "2020-03-03T03:19:00.000Z",
"key" : 1583205540000,
"doc_count" : 1440,
"egress_q_bytes" : {
"value" : 5.918343E8
},
"time" : {
"value" : 1.583205599054E12,
"value_as_string" : "2020-03-03T03:19:59.054Z"
},
"egress_q_bytes_derivative" : {
"value" : 0.0
}
},
{
"key_as_string" : "2020-03-03T03:20:00.000Z",
"key" : 1583205600000,
"doc_count" : 1440,
"egress_q_bytes" : {
"value" : 5.918343E8
},
"time" : {
"value" : 1.583205659253E12,
"value_as_string" : "2020-03-03T03:20:59.253Z"
},
"egress_q_bytes_derivative" : {
"value" : 0.0
}
},
{
"key_as_string" : "2020-03-03T03:21:00.000Z",
"key" : 1583205660000,
"doc_count" : 1440,
"egress_q_bytes" : {
"value" : 5.918343E8
},
"time" : {
"value" : 1.583205719429E12,
"value_as_string" : "2020-03-03T03:21:59.429Z"
},
"egress_q_bytes_derivative" : {
"value" : 0.0
}
},
{
"key_as_string" : "2020-03-03T03:22:00.000Z",
"key" : 1583205720000,
"doc_count" : 1440,
"egress_q_bytes" : {
"value" : 5.918343E8
},
"time" : {
"value" : 1.583205779623E12,
"value_as_string" : "2020-03-03T03:22:59.623Z"
},
"egress_q_bytes_derivative" : {
"value" : 0.0
}
},
{
"key_as_string" : "2020-03-03T03:23:00.000Z",
"key" : 1583205780000,
"doc_count" : 1440,
"egress_q_bytes" : {
"value" : 5.9727825E8
},
"time" : {
"value" : 1.583205839843E12,
"value_as_string" : "2020-03-03T03:23:59.843Z"
},
"egress_q_bytes_derivative" : {
"value" : 5443950.0
}
}
Reference
I referred to a sample datafeed configuration here: https://www.elastic.co/guide/en/machine-learning/current/ml-configuring-aggregation.html
Result
However, when I do preview
GET _ml/datafeeds/datafeed-egress_q_bytes/_preview
# response
[ ]