Kibana Watcher using monitoring index

Elastic Stack Kibana
,
1

Created a watcher script for getting the statistics details from daily monitoring index. But the match phrase condition is not working and no rows returned ( getting NULL Values in output).
The same script was working in lower environment but the script is not working in Production.

Script Details :

{
  "trigger": {
    "schedule": {
      "interval": "30m"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "<.monitoring-es-6-{now/d{yyyy.MM.dd}}>"
        ],
        "types": [],
        "body": {
          "query": {
            "bool": {
              "must": [
                {
                  "range": {
                    "timestamp": {
                      "from": "now-30m",
                      "to": "now"
                    }
                  }
                },
                {
                  "match_phrase_prefix": {
                    "index_stats.index": "my_index_name-"
                  }
                }
              ]
            }
          },
          "aggs": {
            "indices": {
              "terms": {
                "field": "index_stats.index",
                "size": 20
              },
              "aggs": {
                "statistics": {
                  "stats": {
                    "field": "index_stats.total.docs.count"
                  }
                }
              }
            },
            "total_max": {
              "sum_bucket": {
                "buckets_path": "indices>statistics.max"
              }
            },
            "total_min": {
              "sum_bucket": {
                "buckets_path": "indices>statistics.min"
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.aggregations.total_max.value": {
        "eq": "{{ctx.payload.aggregations.total_min.value}}"
      }
    }
  },
  "actions": {
    "log": {
      "logging": {
        "level": "info",
        "text": "ElasticSearch Index  my_index_name-(todays date) does not receive data for the last 30 minutes"
      }
    }
  }
}

Result :

"condition": {
      "type": "compare",
      "status": "success",
      "met": true,
      "compare": {
        "resolved_values": {
          "ctx.payload.aggregations.total_min.value": null,
          "ctx.payload.aggregations.total_max.value": null
        }
      }
2

Hello @Maries

Your Watch is correct when there is at least one index which matches the requirements.
I suppose you're on Elasticsearch 6.x.

I've made some changes.

Demo data

POST my_index_name-/_doc/1
{
  "test":1
}
POST my_index_name-/_doc/2
{
  "test":1
}
POST my_index_name-123/_doc/1
{
  "test":1
}
POST my_index_name-123/_doc/2
{
  "test":1
}
POST my_index_name-*/_refresh

Watch

The condition takes care to verify:

  • There is at least one index
  • The sum buckets are not null
POST _watcher/watch/_execute
{
  "watch": {
    "trigger": {
      "schedule": {
        "interval": "30m"
      }
    },
    "input": {
      "search": {
        "request": {
          "search_type": "query_then_fetch",
          "indices": [
            "<.monitoring-es-6-{now/d{yyyy.MM.dd}}>"
          ],
          "types": [],
          "body": {
            "size": 0,
            "query": {
              "bool": {
                "filter": [
                  {
                    "range": {
                      "timestamp": {
                        "from": "now-30m",
                        "to": "now"
                      }
                    }
                  },
                  {
                    "prefix": {
                      "index_stats.index": "my_index_name-"
                    }
                  }
                ]
              }
            },
            "aggs": {
              "indices": {
                "terms": {
                  "field": "index_stats.index",
                  "size": 20
                },
                "aggs": {
                  "statistics": {
                    "stats": {
                      "field": "index_stats.total.docs.count"
                    }
                  }
                }
              },
              "total_max": {
                "sum_bucket": {
                  "buckets_path": "indices>statistics.max"
                }
              },
              "total_min": {
                "sum_bucket": {
                  "buckets_path": "indices>statistics.min"
                }
              }
            }
          }
        }
      }
    },
    "condition": {
      "script": {
        "source": "return (ctx.payload.aggregations.indices.buckets.size() > 0 && ctx.payload?.aggregations?.total_min != null && ctx.payload?.aggregations?.total_max != null) ? (ctx.payload.aggregations.total_min.value == ctx.payload.aggregations.total_max.value) : false"
      }
      
    },
    "actions": {
      "log": {
        "logging": {
          "level": "info",
          "text": "ElasticSearch Index  my_index_name-(todays date) does not receive data for the last 30 minutes"
        }
      }
    }
  }
}

Result:

{
  "_id" : "_inlined__ed6bd787-f6b0-4a7f-90f0-ef45267f109f-2020-05-10T23:35:59.058Z",
  "watch_record" : {
    "watch_id" : "_inlined_",
    "node" : "e4eEGy3DQGmks2DBOrpC7g",
    "state" : "execution_not_needed",
    "user" : "elastic",
    "status" : {
      "state" : {
        "active" : true,
        "timestamp" : "2020-05-10T23:35:59.058Z"
      },
      "last_checked" : "2020-05-10T23:35:59.058Z",
      "actions" : {
        "log" : {
          "ack" : {
            "timestamp" : "2020-05-10T23:35:59.058Z",
            "state" : "awaits_successful_execution"
          }
        }
      },
      "execution_state" : "execution_not_needed",
      "version" : -1
    },
    "trigger_event" : {
      "type" : "manual",
      "triggered_time" : "2020-05-10T23:35:59.058Z",
      "manual" : {
        "schedule" : {
          "scheduled_time" : "2020-05-10T23:35:59.058Z"
        }
      }
    },
    "input" : {
      "search" : {
        "request" : {
          "search_type" : "query_then_fetch",
          "indices" : [
            "<.monitoring-es-6-{now/d{yyyy.MM.dd}}>"
          ],
          "types" : [ ],
          "body" : {
            "size" : 0,
            "query" : {
              "bool" : {
                "filter" : [
                  {
                    "range" : {
                      "timestamp" : {
                        "from" : "now-30m",
                        "to" : "now"
                      }
                    }
                  },
                  {
                    "prefix" : {
                      "index_stats.index" : "my_index_name-"
                    }
                  }
                ]
              }
            },
            "aggs" : {
              "indices" : {
                "terms" : {
                  "field" : "index_stats.index",
                  "size" : 20
                },
                "aggs" : {
                  "statistics" : {
                    "stats" : {
                      "field" : "index_stats.total.docs.count"
                    }
                  }
                }
              },
              "total_max" : {
                "sum_bucket" : {
                  "buckets_path" : "indices>statistics.max"
                }
              },
              "total_min" : {
                "sum_bucket" : {
                  "buckets_path" : "indices>statistics.min"
                }
              }
            }
          }
        }
      }
    },
    "condition" : {
      "script" : {
        "source" : "return (ctx.payload.aggregations.indices.buckets.size() > 0 && ctx.payload?.aggregations?.total_min != null && ctx.payload?.aggregations?.total_max != null) ? (ctx.payload.aggregations.total_min.value == ctx.payload.aggregations.total_max.value) : false",
        "lang" : "painless"
      }
    },
    "result" : {
      "execution_time" : "2020-05-10T23:35:59.058Z",
      "execution_duration" : 10,
      "input" : {
        "type" : "search",
        "status" : "success",
        "payload" : {
          "_shards" : {
            "total" : 1,
            "failed" : 0,
            "successful" : 1,
            "skipped" : 0
          },
          "hits" : {
            "hits" : [ ],
            "total" : 239,
            "max_score" : 0.0
          },
          "took" : 6,
          "timed_out" : false,
          "aggregations" : {
            "indices" : {
              "doc_count_error_upper_bound" : 0,
              "sum_other_doc_count" : 0,
              "buckets" : [
                {
                  "doc_count" : 162,
                  "key" : "my_index_name-",
                  "statistics" : {
                    "min" : 0.0,
                    "avg" : 0.14814814814814814,
                    "max" : 2.0,
                    "count" : 162,
                    "sum" : 24.0
                  }
                },
                {
                  "doc_count" : 77,
                  "key" : "my_index_name-123",
                  "statistics" : {
                    "min" : 2.0,
                    "avg" : 2.0,
                    "max" : 2.0,
                    "count" : 77,
                    "sum" : 154.0
                  }
                }
              ]
            },
            "total_max" : {
              "value" : 4.0
            },
            "total_min" : {
              "value" : 2.0
            }
          }
        },
        "search" : {
          "request" : {
            "search_type" : "query_then_fetch",
            "indices" : [
              "<.monitoring-es-6-{now/d{yyyy.MM.dd}}>"
            ],
            "types" : [ ],
            "body" : {
              "size" : 0,
              "query" : {
                "bool" : {
                  "filter" : [
                    {
                      "range" : {
                        "timestamp" : {
                          "from" : "now-30m",
                          "to" : "now"
                        }
                      }
                    },
                    {
                      "prefix" : {
                        "index_stats.index" : "my_index_name-"
                      }
                    }
                  ]
                }
              },
              "aggs" : {
                "indices" : {
                  "terms" : {
                    "field" : "index_stats.index",
                    "size" : 20
                  },
                  "aggs" : {
                    "statistics" : {
                      "stats" : {
                        "field" : "index_stats.total.docs.count"
                      }
                    }
                  }
                },
                "total_max" : {
                  "sum_bucket" : {
                    "buckets_path" : "indices>statistics.max"
                  }
                },
                "total_min" : {
                  "sum_bucket" : {
                    "buckets_path" : "indices>statistics.min"
                  }
                }
              }
            }
          }
        }
      },
      "condition" : {
        "type" : "script",
        "status" : "success",
        "met" : false
      },
      "actions" : [ ]
    },
    "messages" : [ ]
  }
}
1 Like
3

Tried the above script you provided in lower environment and it is working as expected.
But the same script is not working in Production. I'm trying to fetch index stats from monitoring index contains high amount of data in Prod. Is that high amount of data causing this issue.

Note : It should result some value since this index is getting updated every 5 mins.

ERROR :

{
  "watch_id": "_inlined_",
  "node": "3CjqCKe-Qweu9HKychAe5g",
  "state": "failed",
  "user": "",
  "status": {
    "state": {
      "active": true,
      "timestamp": "2020-05-11T07:33:04.759Z"
    },
    "actions": {
      "log": {
        "ack": {
          "timestamp": "2020-05-11T07:33:04.759Z",
          "state": "awaits_successful_execution"
        }
      }
    },
    "execution_state": "failed",
    "version": -1
  },
  "trigger_event": {
    "type": "manual",
    "triggered_time": "2020-05-11T07:33:04.759Z",
    "manual": {
      "schedule": {
        "scheduled_time": "2020-05-11T07:33:04.759Z"
      }
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "<.monitoring-es-6-{now/d{yyyy.MM.dd}}>"
        ],
        "types": [],
        "body": {
          "size": 0,
          "query": {
            "bool": {
              "filter": [
                {
                  "range": {
                    "timestamp": {
                      "from": "now-30m",
                      "to": "now"
                    }
                  }
                },
                {
                  "prefix": {
                    "index_stats.index": "my-index-name-"
                  }
                }
              ]
            }
          },
          "aggs": {
            "indices": {
              "terms": {
                "field": "index_stats.index",
                "size": 20
              },
              "aggs": {
                "statistics": {
                  "stats": {
                    "field": "index_stats.total.docs.count"
                  }
                }
              }
            },
            "total_max": {
              "sum_bucket": {
                "buckets_path": "indices>statistics.max"
              }
            },
            "total_min": {
              "sum_bucket": {
                "buckets_path": "indices>statistics.min"
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "script": {
      "source": "return (ctx.payload.aggregations.indices.buckets.size() > 0 && ctx.payload?.aggregations?.total_min != null && ctx.payload?.aggregations?.total_max != null) ? (ctx.payload.aggregations.total_min.value == ctx.payload.aggregations.total_max.value) : false",
      "lang": "painless"
    }
  },
  "metadata": {
    "xpack": {
      "type": "json"
    }
  },
  "result": {
    "execution_time": "2020-05-11T07:33:04.759Z",
    "execution_duration": 0,
    "input": {
      "type": "search",
      "status": "success",
      "payload": {
        "_shards": {
          "total": 0,
          "failed": 0,
          "successful": 0,
          "skipped": 0
        },
        "hits": {
          "hits": [],
          "total": 0,
          "max_score": 0
        },
        "took": 0,
        "timed_out": false
      },
      "search": {
        "request": {
          "search_type": "query_then_fetch",
          "indices": [
            "<.monitoring-es-6-{now/d{yyyy.MM.dd}}>"
          ],
          "types": [],
          "body": {
            "size": 0,
            "query": {
              "bool": {
                "filter": [
                  {
                    "range": {
                      "timestamp": {
                        "from": "now-30m",
                        "to": "now"
                      }
                    }
                  },
                  {
                    "prefix": {
                      "index_stats.index": "my-index-name-"
                    }
                  }
                ]
              }
            },
            "aggs": {
              "indices": {
                "terms": {
                  "field": "index_stats.index",
                  "size": 20
                },
                "aggs": {
                  "statistics": {
                    "stats": {
                      "field": "index_stats.total.docs.count"
                    }
                  }
                }
              },
              "total_max": {
                "sum_bucket": {
                  "buckets_path": "indices>statistics.max"
                }
              },
              "total_min": {
                "sum_bucket": {
                  "buckets_path": "indices>statistics.min"
                }
              }
            }
          }
        }
      }
    },
    "actions": []
  },
  "exception": {
    "type": "script_exception",
    "reason": "runtime error",
    "script_stack": [
      "return (ctx.payload.aggregations.indices.buckets.size() > 0 && ctx.payload?.aggregations?.total_min != null && ctx.payload?.aggregations?.total_max != null) ? (ctx.payload.aggregations.total_min.value == ctx.payload.aggregations.total_max.value) : false",
      "                                ^---- HERE"
    ],
    "script": "return (ctx.payload.aggregations.indices.buckets.size() > 0 && ctx.payload?.aggregations?.total_min != null && ctx.payload?.aggregations?.total_max != null) ? (ctx.payload.aggregations.total_min.value == ctx.payload.aggregations.total_max.value) : false",
    "lang": "painless",
    "caused_by": {
      "type": "null_pointer_exception",
      "reason": null,
      "stack_trace": "java.lang.NullPointerException\n\tat org.elasticsearch.painless.DefBootstrap$PIC.fallback(DefBootstrap.java:207)\n\tat org.elasticsearch.painless.PainlessScript$Script.execute(return (ctx.payload.aggregations.indices.buckets.size() > 0 && ctx.payload?.aggregations?.total_min != null && ctx.payload?.aggregations?.total_max != null) ? (ctx.payload.aggregations.total_min.value == ctx.payload.aggregations.total_max.value) : false:33)\n\tat org.elasticsearch.xpack.watcher.condition.ScriptCondition.doExecute(ScriptCondition.java:60)\n\tat org.elasticsearch.xpack.watcher.condition.ScriptCondition.execute(ScriptCondition.java:55)\n\tat org.elasticsearch.xpack.watcher.execution.ExecutionService.executeInner(ExecutionService.java:507)\n\tat org.elasticsearch.xpack.watcher.execution.ExecutionService.execute(ExecutionService.java:309)\n\tat org.elasticsearch.xpack.watcher.transport.actions.execute.TransportExecuteWatchAction$1.doRun(TransportExecuteWatchAction.java:164)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)\n\tat org.elasticsearch.xpack.watcher.execution.ExecutionService$WatchExecutionTask.run(ExecutionService.java:617)\n\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n"
    },
    "stack_trace": "ScriptException[runtime error]; nested: NullPointerException;\n\tat org.elasticsearch.painless.PainlessScript.convertToScriptException(PainlessScript.java:94)\n\tat org.elasticsearch.painless.PainlessScript$Script.execute(return (ctx.payload.aggregations.indices.buckets.size() > 0 && ctx.payload?.aggregations?.total_min != null && ctx.payload?.aggregations?.total_max != null) ? (ctx.payload.aggregations.total_min.value == ctx.payload.aggregations.total_max.value) : false:239)\n\tat org.elasticsearch.xpack.watcher.condition.ScriptCondition.doExecute(ScriptCondition.java:60)\n\tat org.elasticsearch.xpack.watcher.condition.ScriptCondition.execute(ScriptCondition.java:55)\n\tat org.elasticsearch.xpack.watcher.execution.ExecutionService.executeInner(ExecutionService.java:507)\n\tat org.elasticsearch.xpack.watcher.execution.ExecutionService.execute(ExecutionService.java:309)\n\tat org.elasticsearch.xpack.watcher.transport.actions.execute.TransportExecuteWatchAction$1.doRun(TransportExecuteWatchAction.java:164)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)\n\tat org.elasticsearch.xpack.watcher.execution.ExecutionService$WatchExecutionTask.run(ExecutionService.java:617)\n\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: java.lang.NullPointerException\n\tat org.elasticsearch.painless.DefBootstrap$PIC.fallback(DefBootstrap.java:207)\n\tat org.elasticsearch.painless.PainlessScript$Script.execute(return (ctx.payload.aggregations.indices.buckets.size() > 0 && ctx.payload?.aggregations?.total_min != null && ctx.payload?.aggregations?.total_max != null) ? (ctx.payload.aggregations.total_min.value == ctx.payload.aggregations.total_max.value) : false:33)\n\t... 11 more\n"
  }
}
4

Hello @Maries

The exception is triggered because there are no matching results (extract from the watcher execution):

      "type": "search",
      "status": "success",
      "payload": {
        "_shards": {
          "total": 0,
          "failed": 0,
          "successful": 0,
          "skipped": 0
        },
        "hits": {
          "hits": [],
          "total": 0,
          "max_score": 0
        },
        "took": 0,
        "timed_out": false
      },

Can you please

  • verify if your Production cluster is maybe sending the monitoring metrics to an external cluster?
  • check the output of GET _cat/indices/.monitoring* to check if you have any index matching the pattern?

At the same time, I am tempted to say you should use the following to cope with queries spanning over multiple days (e.g. at midnight):

          "indices": [
            "<.monitoring-es-6-{now-1d/d{yyyy.MM.dd}}>",
            "<.monitoring-es-6-{now/d{yyyy.MM.dd}}>"
          ],
1 Like
5

Thanks a lot for the suggestions and help.

Production Cluster is sending the monitoring metrics to the external cluster.
Tried the below query and getting the result without any errors in Production. Script is not working when tried with watcher

GET .monitoring-es-6-2020.05.11/_search
{
  "size": 0, 
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "timestamp": {
              "from": "now-30m",
              "to": "now"
            }
          }
        },
        {
          "match_phrase_prefix": {
                    "index_stats.index": "my-index-name-"
                  }
        }
      ]
    
    }
  },
  "aggs": {
    "MaxCount": {
      "stats": {
        "field": "index_stats.total.docs.count"
      }
    }
  }
}

Result

{
  "took" : 2,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : 1440,
    "max_score" : 0.0,
    "hits" : [ ]
  },
  "aggregations" : {
    "MaxCount" : {
      "count" : 1440,
      "min" : 427032.0,
      "max" : 3594315.0,
      "avg" : 2878568.2569444445,
      "sum" : 4.14513829E9
    }
  }
}

Also, please find the output of GET _cat/indices/.monitoring*

green open .monitoring-kibana-6-2020.05.10 WCmyIK_IRmqQyluBlSgtag 1 1   8639    0   4.3mb   2.1mb
green open .monitoring-kibana-6-2020.05.05 4536Ir0PTse1aRYvR5V3aQ 1 1   8639    0   4.2mb   2.1mb
green open .monitoring-es-6-2020.05.10     B_7puAO7QPW5ElkgxXg8kg 1 1 873371 6193   1.2gb 645.6mb
green open .monitoring-kibana-6-2020.05.09 c3sCLAqgRBO7GLvNLf6Gkg 1 1   8640    0   4.4mb   2.2mb
green open .monitoring-kibana-6-2020.05.06 MVjJGUfYQCurzccqGSoPtw 1 1   8640    0   4.2mb   2.1mb
green open .monitoring-kibana-6-2020.05.08 0ff571UYTxCwDrX5H1L-6g 1 1   8639    0   4.4mb   2.2mb
green open .monitoring-kibana-6-2020.05.07 qnbiG9mxR7imG0RjolakGg 1 1   8639    0   4.3mb   2.1mb
green open .monitoring-es-6-2020.05.09     v6oPZjB9QsmyosRVtT0yfA 1 1 874869 4662   1.2gb   630mb
green open .monitoring-es-6-2020.05.11     bs27kreBTlyPDXbH5f3K4w 1 1 349472 3361 598.8mb 304.9mb
green open .monitoring-es-6-2020.05.06     X6oylI_kQb26QzhskJIzLQ 1 1 872444 5181   1.1gb 618.1mb
green open .monitoring-es-6-2020.05.08     JD1eruKcR1qobTafr7k2xg 1 1 868629 5240   1.2gb 621.9mb
green open .monitoring-es-6-2020.05.07     zzQKvz0xRWWqWQrB6335mw 1 1 874405 4613   1.2gb 604.8mb
green open .monitoring-kibana-6-2020.05.11 H9zuK3B4STSJw1Pdg6a04w 1 1   3347    0   1.8mb 970.5kb
green open .monitoring-alerts-6            6IOIqc_TREKsXIMYUnkFIw 1 1    173    2 158.8kb  79.4kb
green open .monitoring-es-6-2020.05.05     nJjSeS9uRIqWUvj0LyuFmg 1 1 871970 5181   1.2gb 619.7mb

Attempted the same with below as you suggested. but No luck. Getting same error.

indices": [
            "<.monitoring-es-6-{now-1d/d{yyyy.MM.dd}}>",
            "<.monitoring-es-6-{now/d{yyyy.MM.dd}}>"
          ],

Thank you for all the help !!!

1 Like
6

If the production cluster sends data to an external cluster, you cannot use the search input as the query is ran against the local cluster.

Where did you run this request? The production cluster?

1 Like
7

YES i ran this GET _cat/indices/.monitoring in Production cluster.
got confused with the production cluster sending monitoring metrics to external cluster or not.

Production Cluster is not sending monitoring metrics to external cluster.

running the below search query in Production Cluster gives result . But when running as a watcher , it does not give any result

GET .monitoring-es-6-2020.05.11/_search
{
  "size": 0, 
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "timestamp": {
              "from": "now-30m",
              "to": "now"
            }
          }
        },
        {
          "match_phrase_prefix": {
                    "index_stats.index": "my-index-name-"
                  }
        }
      ]
    
    }
  },
  "aggs": {
    "MaxCount": {
      "stats": {
        "field": "index_stats.total.docs.count"
      }
    }
  }
}
8

Can you try to run the following?
Please paste the full response.

GET %3C.monitoring-es-6-%7Bnow%2Fd%7Byyyy.MM.dd%7D%7D%3E/_search
{
  "size": 1,
  "query": {
    "bool": {
      "filter": [
        {
          "range": {
            "timestamp": {
              "from": "now-30m",
              "to": "now"
            }
          }
        },
        {
          "prefix": {
            "index_stats.index": "my_index_name-"
          }
        }
      ]
    }
  },
  "aggs": {
    "indices": {
      "terms": {
        "field": "index_stats.index",
        "size": 20
      },
      "aggs": {
        "statistics": {
          "stats": {
            "field": "index_stats.total.docs.count"
          }
        }
      }
    },
    "total_max": {
      "sum_bucket": {
        "buckets_path": "indices>statistics.max"
      }
    },
    "total_min": {
      "sum_bucket": {
        "buckets_path": "indices>statistics.min"
      }
    }
  }
}

If you get no response, please check if the cluster has an index with prefix my_index_name-

9

Cluster is having an index with my-index-name and data is getting updated every 15 mins in Production Cluster. Only when executing with watcher (as mentioned above), i'm not getting either result or getting error. And this watcher script is working fine in lower environments. Not sure what is causing this issue.

Thanks a lot for all your efforts and help :slight_smile: Great . Thank you

Here is the response and getting my-index-name in results


  "took" : 6,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : 1440,
    "max_score" : 0.0,
    "hits" : [
      {
        "_index" : ".monitoring-es-6-2020.05.12",
        "_type" : "doc",
        "_id" : "FQJCB3IBdrgwRM7r2QS4",
        "_score" : 0.0,
        "_source" : {
          "cluster_uuid" : "xb2Aq6BrSpSw3jKwWd5MAg",
          "timestamp" : "2020-05-12T05:02:01.100Z",
          "interval_ms" : 10000,
          "type" : "index_stats",
          "source_node" : {
            "uuid" : "1UtanIgTR_WjL6EF5DZA_w",
            "host" : "ohlvelsmt002dp.oh.prd.dat.aws.vz-connect.net",
            "transport_address" : "10.59.92.108:9300",
            "ip" : "10.59.92.108",
            "name" : "ohlvelsmt002dp.oh.prd.dat",
            "timestamp" : "2020-05-12T05:02:00.940Z"
          },
          "index_stats" : {
            "index" : "my-index-name-20200509",
            "uuid" : "4MHWDxYxQhubxpeH4dTYDg",
            "created" : 1588982401188,
            "status" : "green",
            "shards" : {
              "total" : 3,
              "primaries" : 1,
              "replicas" : 2,
              "active_total" : 3,
              "active_primaries" : 1,
              "active_replicas" : 2,
              "unassigned_total" : 0,
              "unassigned_primaries" : 0,
              "unassigned_replicas" : 0,
              "initializing" : 0,
              "relocating" : 0
            },
            "total" : {
              "docs" : {
                "count" : 3103533
              },
              "store" : {
                "size_in_bytes" : 76247101579
              },
              "indexing" : {
                "index_total" : 3103758,
                "index_time_in_millis" : 8039840,
                "throttle_time_in_millis" : 0
              },
              "search" : {
                "query_total" : 2,
                "query_time_in_millis" : 19
              },
              "merges" : {
                "total_size_in_bytes" : 391866338464
              },
              "refresh" : {
                "total_time_in_millis" : 3817155
              },
              "query_cache" : {
                "memory_size_in_bytes" : 0,
                "hit_count" : 0,
                "miss_count" : 0,
                "evictions" : 0
              },
              "fielddata" : {
                "memory_size_in_bytes" : 0,
                "evictions" : 0
              },
              "segments" : {
                "count" : 74,
                "memory_in_bytes" : 48825703,
                "terms_memory_in_bytes" : 1734313,
                "stored_fields_memory_in_bytes" : 8590536,
                "term_vectors_memory_in_bytes" : 0,
                "norms_memory_in_bytes" : 0,
                "points_memory_in_bytes" : 38270966,
                "doc_values_memory_in_bytes" : 229888,
                "index_writer_memory_in_bytes" : 0,
                "version_map_memory_in_bytes" : 0,
                "fixed_bit_set_memory_in_bytes" : 0
              },
              "request_cache" : {
                "memory_size_in_bytes" : 0,
                "evictions" : 0,
                "hit_count" : 0,
                "miss_count" : 0
              }
            },
            "primaries" : {
              "docs" : {
                "count" : 1034511
              },
              "store" : {
                "size_in_bytes" : 25391493813
              },
              "indexing" : {
                "index_total" : 1034588,
                "index_time_in_millis" : 2694707,
                "throttle_time_in_millis" : 0
              },
              "search" : {
                "query_total" : 2,
                "query_time_in_millis" : 19
              },
              "merges" : {
                "total_size_in_bytes" : 132902877589
              },
              "refresh" : {
                "total_time_in_millis" : 1275398
              },
              "query_cache" : {
                "memory_size_in_bytes" : 0,
                "hit_count" : 0,
                "miss_count" : 0,
                "evictions" : 0
              },
              "fielddata" : {
                "memory_size_in_bytes" : 0,
                "evictions" : 0
              },
              "segments" : {
                "count" : 21,
                "memory_in_bytes" : 16235950,
                "terms_memory_in_bytes" : 575215,
                "stored_fields_memory_in_bytes" : 2861744,
                "term_vectors_memory_in_bytes" : 0,
                "norms_memory_in_bytes" : 0,
                "points_memory_in_bytes" : 12755459,
                "doc_values_memory_in_bytes" : 43532,
                "index_writer_memory_in_bytes" : 0,
                "version_map_memory_in_bytes" : 0,
                "fixed_bit_set_memory_in_bytes" : 0
              },
              "request_cache" : {
                "memory_size_in_bytes" : 0,
                "evictions" : 0,
                "hit_count" : 0,
                "miss_count" : 0
              }
            }
          }
        }
      }
    ]
  },
  "aggregations" : {
    "indices" : {
      "doc_count_error_upper_bound" : 0,
      "sum_other_doc_count" : 0,
      "buckets" : [
        {
          "key" : "my-index-name-20200505",
          "doc_count" : 180,
          "statistics" : {
            "count" : 180,
            "min" : 3221835.0,
            "max" : 3221835.0,
            "avg" : 3221835.0,
            "sum" : 5.799303E8
          }
        },
        {
          "key" : "my-index-name-20200506",
          "doc_count" : 180,
          "statistics" : {
            "count" : 180,
            "min" : 3320862.0,
            "max" : 3320862.0,
            "avg" : 3320862.0,
            "sum" : 5.9775516E8
          }
        },
        {
          "key" : "my-index-name-20200507",
          "doc_count" : 180,
          "statistics" : {
            "count" : 180,
            "min" : 3594315.0,
            "max" : 3594315.0,
            "avg" : 3594315.0,
            "sum" : 6.469767E8
          }
        },
        {
          "key" : "my-index-name-20200508",
          "doc_count" : 180,
          "statistics" : {
            "count" : 180,
            "min" : 3478368.0,
            "max" : 3478368.0,
            "avg" : 3478368.0,
            "sum" : 6.2610624E8
          }
        },
        {
          "key" : "my-index-name-20200509",
          "doc_count" : 180,
          "statistics" : {
            "count" : 180,
            "min" : 3103533.0,
            "max" : 3103533.0,
            "avg" : 3103533.0,
            "sum" : 5.5863594E8
          }
        },
        {
          "key" : "my-index-name-20200510",
          "doc_count" : 180,
          "statistics" : {
            "count" : 180,
            "min" : 2534076.0,
            "max" : 2534076.0,
            "avg" : 2534076.0,
            "sum" : 4.5613368E8
          }
        },
        {
          "key" : "my-index-name-20200511",
          "doc_count" : 180,
          "statistics" : {
            "count" : 180,
            "min" : 3239199.0,
            "max" : 3239199.0,
            "avg" : 3239199.0,
            "sum" : 5.8305582E8
          }
        },
        {
          "key" : "my-index-name-20200512",
          "doc_count" : 180,
          "statistics" : {
            "count" : 180,
            "min" : 387060.0,
            "max" : 397305.0,
            "avg" : 392500.4555555555,
            "sum" : 7.0650082E7
          }
        }
      ]
    },
    "total_max" : {
      "value" : 2.2889493E7
    },
    "total_min" : {
      "value" : 2.2879248E7
    }
  }
}
10

Did you change "index_stats.index": "my_index_name-" to my-index-name-?

11

I'm running for another index which is in production and i cannot share the index due to security reasons. So modified the original index here in input and output to my-index-name.. i have manually updated from original index name to my-index-name after i got output..

Please don't get confuse my-index-name and my_index_name since i have updated manually for your verification.

Getting the result from your query and my index is getting listed. and when trying with watcher only, im not getting any result and getting exception.

12

Is there anyway to query particular watcher name from my watcher history and get only last executed watcher and the corresponding value ( say ctx.payload.hits.total value ) and compare the current watcher execution.

Trying to compare the number of hits (ctx.payload.hits.total) from previous watcher execution with the current watcher execution.

Thanks for all the help again !!!

13

You need to query in the watcher history index. Each watcher has an id and an execution timestamp.
This can be retrieved in a chain input as a search input.

14

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.