Search Machine Learning result and use watcher send message to telegram

Hi elastic community,

I want to send alert message to telegram, when ML got anomalies, like 2 times higher or lower.

now I create ML job and run it frequency.

Then I use watcher search ML job, and send message to telegram.

It's fine when run simulate. But actually execute the watcher job, I got

"type": "s_s_l_handshake_exception",
"reason": "Received fatal alert: handshake_failure"

What causes it happened?

forgive my broken english

my environment is:

Docker version 18.06.0-ce, build 0ffa825
docker image: elasticsearch:7.0.0

my watcher config:

 {
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          ".ml-anomalies-.write-ml4"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "size": 0,
          "query": {
            "bool": {
              "filter": {
                "range": {
                  "timestamp": {
                    "gte": "{{ctx.trigger.scheduled_time}}||-5m",
                    "lte": "{{ctx.trigger.scheduled_time}}",
                    "format": "strict_date_optional_time||epoch_millis"
                  }
                }
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "script": {
      "source": "if (ctx.payload.hits.total > params.threshold) { return true; } return false;",
      "lang": "painless",
      "params": {
        "threshold": 10
      }
    }
  },
  "actions": {
    "my_webhook": {
      "webhook": {
        "scheme": "https",
        "host": "api.telegram.org",
        "port": 443,
        "method": "get",
        "path": "/bot*********:***********************************/sendMessage",
        "params": {
          "text": "ML schedule is higher then threshold {{ctx.payload.hits.total}}, threshold is 10 (this is text message)",
          "chat_id": "-*********"
        },
        "headers": {},
        "proxy": {
          "host": "10.33.66.222",
          "port": 3128
        }
      }
    }
  }
}

Simulation Results

  "watch_id": "_inlined_",
  "node": "57m4ePiOR1W2iQda63yBAg",
  "state": "executed",
  "status": {
    "state": {
      "active": true,
      "timestamp": "2020-04-21T09:31:54.554Z"
    },
    "last_checked": "2020-04-21T09:31:54.554Z",
    "last_met_condition": "2020-04-21T09:31:54.554Z",
    "actions": {
      "my_webhook": {
        "ack": {
          "timestamp": "2020-04-21T09:31:54.554Z",
          "state": "awaits_successful_execution"
        },
        "last_execution": {
          "timestamp": "2020-04-21T09:31:54.554Z",
          "successful": false,
          "reason": ""
        }
      }
    },
    "execution_state": "executed",
    "version": -1
  },
  "trigger_event": {
    "type": "manual",
    "triggered_time": "2020-04-21T09:31:54.554Z",
    "manual": {
      "schedule": {
        "scheduled_time": "2020-04-21T09:31:54.554Z"
      }
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          ".ml-anomalies-.write-ml4"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "size": 0,
          "query": {
            "bool": {
              "filter": {
                "range": {
                  "timestamp": {
                    "gte": "{{ctx.trigger.scheduled_time}}||-5m",
                    "lte": "{{ctx.trigger.scheduled_time}}",
                    "format": "strict_date_optional_time||epoch_millis"
                  }
                }
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "script": {
      "source": "if (ctx.payload.hits.total > params.threshold) { return true; } return false;",
      "lang": "painless",
      "params": {
        "threshold": 10
      }
    }
  },
  "metadata": {
    "name": "getML_toTelegram",
    "xpack": {
      "type": "json"
    }
  },
  "result": {
    "execution_time": "2020-04-21T09:31:54.554Z",
    "execution_duration": 489,
    "input": {
      "type": "search",
      "status": "success",
      "payload": {
        "_shards": {
          "total": 1,
          "failed": 0,
          "successful": 1,
          "skipped": 0
        },
        "hits": {
          "hits": [],
          "total": 22,
          "max_score": null
        },
        "took": 0,
        "timed_out": false
      },
      "search": {
        "request": {
          "search_type": "query_then_fetch",
          "indices": [
            ".ml-anomalies-.write-ml4"
          ],
          "rest_total_hits_as_int": true,
          "body": {
            "size": 0,
            "query": {
              "bool": {
                "filter": {
                  "range": {
                    "timestamp": {
                      "gte": "2020-04-21T09:31:54.55457Z||-5m",
                      "lte": "2020-04-21T09:31:54.55457Z",
                      "format": "strict_date_optional_time||epoch_millis"
                    }
                  }
                }
              }
            }
          }
        }
      }
    },
    "condition": {
      "type": "script",
      "status": "success",
      "met": true
    },
    "actions": [
      {
        "id": "my_webhook",
        "type": "webhook",
        "status": "failure",
        "error": {
          "root_cause": [
            {
              "type": "s_s_l_handshake_exception",
              "reason": "Received fatal alert: handshake_failure"
            }
          ],
          "type": "s_s_l_handshake_exception",
          "reason": "Received fatal alert: handshake_failure"
        }
      }
    ]
  },
  "messages": []
}

Update:

I use newest docker image: elasticsearch:7.6.2

then send message to telegram, it can work...

Watcher is:

{
  "trigger": {
    "schedule": {
      "interval": "30m"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "access-*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "query": {
            "bool": {
              "must_not": [
                {
                  "match": {
                    "status": "200"
                  }
                }
              ],
              "filter": [
                {
                  "range": {
                    "@timestamp": {
                      "gt": "now-30m"
                    }
                  }
                }
              ]
            }
          },
          "aggs": {
            "server_name": {
              "terms": {
                "field": "server_name.keyword"
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gt": 1
      }
    }
  },
  "actions": {
    "my_webhook": {
      "webhook": {
        "scheme": "https",
        "host": "api.telegram.org",
        "port": 443,
        "method": "get",
        "path": "/bot******:****************/sendMessage",
        "params": {
          "text": "There are {{ctx.payload.hits.total}} non 200 status occurred 30 minutes ago, (es7.6.2 test message)",
          "chat_id": "-******"
        },
        "headers": {},
        "proxy": {
          "host": "10.33.66.xxx",
          "port": 3128
        }
      }
    }
  }
}

result

{
  "watch_id": "_inlined_",
  "node": "zU1kr8hxSDCQnApEb81qAw",
  "state": "executed",
  "status": {
    "state": {
      "active": true,
      "timestamp": "2020-04-27T01:28:55.861Z"
    },
    "last_checked": "2020-04-27T01:28:55.862Z",
    "last_met_condition": "2020-04-27T01:28:55.862Z",
    "actions": {
      "my_webhook": {
        "ack": {
          "timestamp": "2020-04-27T01:28:55.862Z",
          "state": "ackable"
        },
        "last_execution": {
          "timestamp": "2020-04-27T01:28:55.862Z",
          "successful": true
        },
        "last_successful_execution": {
          "timestamp": "2020-04-27T01:28:55.862Z",
          "successful": true
        }
      }
    },
    "execution_state": "executed",
    "version": -1
  },
  "trigger_event": {
    "type": "manual",
    "triggered_time": "2020-04-27T01:28:55.862Z",
    "manual": {
      "schedule": {
        "scheduled_time": "2020-04-27T01:28:55.862Z"
      }
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "access-*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "query": {
            "bool": {
              "must_not": [
                {
                  "match": {
                    "status": "200"
                  }
                }
              ],
              "filter": [
                {
                  "range": {
                    "@timestamp": {
                      "gt": "now-30m"
                    }
                  }
                }
              ]
            }
          },
          "aggs": {
            "server_name": {
              "terms": {
                "field": "server_name.keyword"
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gt": 1
      }
    }
  },
  "metadata": {
    "name": "none200",
    "xpack": {
      "type": "json"
    }
  },
  "result": {
    "execution_time": "2020-04-27T01:28:55.862Z",
    "execution_duration": 2042,
    "input": {
      "type": "search",
      "status": "success",
      "payload": {
        "_shards": {
          "total": 1,
          "failed": 0,
          "successful": 1,
          "skipped": 0
        },
        "hits": {
          "hits": [
            {
              "_index": "access-2020.04.24",
              "_type": "_doc",
              "_source": {
                "server_name": "deploy",
                "@timestamp": "2020-04-27T09:27:56.000000000+08:00",
                "status": "404"
              },
              "_id": "4c8_uXEBGPvAam3AfrWd",
              "_score": 0
            },
            {
              "_index": "access-2020.04.24",
              "_type": "_doc",
              "_source": {
                "server_name": "deploy",
                "@timestamp": "2020-04-27T09:27:56.000000000+08:00",
                "status": "404"
              },
              "_id": "4s8_uXEBGPvAam3AgrWt",
              "_score": 0
            },
            {
              "_index": "access-2020.04.24",
              "_type": "_doc",
              "_source": {
                "server_name": "deploy",
                "@timestamp": "2020-04-27T09:27:56.000000000+08:00",
                "status": "404"
              },
              "_id": "488_uXEBGPvAam3Ah7Xd",
              "_score": 0
            },
            {
              "_index": "access-2020.04.24",
              "_type": "_doc",
              "_source": {
                "server_name": "deploy",
                "@timestamp": "2020-04-27T09:27:56.000000000+08:00",
                "status": "404"
              },
              "_id": "5M8_uXEBGPvAam3AjLUN",
              "_score": 0
            },
            {
              "_index": "access-2020.04.24",
              "_type": "_doc",
              "_source": {
                "server_name": "deploy",
                "@timestamp": "2020-04-27T09:27:56.000000000+08:00",
                "status": "404"
              },
              "_id": "5c8_uXEBGPvAam3AkLVp",
              "_score": 0
            },
            {
              "_index": "access-2020.04.24",
              "_type": "_doc",
              "_source": {
                "server_name": "deploy",
                "@timestamp": "2020-04-27T09:27:56.000000000+08:00",
                "status": "404"
              },
              "_id": "5s8_uXEBGPvAam3AkrVs",
              "_score": 0
            },
            {
              "_index": "access-2020.04.24",
              "_type": "_doc",
              "_source": {
                "server_name": "deploy",
                "@timestamp": "2020-04-27T09:27:56.000000000+08:00",
                "status": "404"
              },
              "_id": "588_uXEBGPvAam3Ak7X_",
              "_score": 0
            },
            {
              "_index": "access-2020.04.24",
              "_type": "_doc",
              "_source": {
                "server_name": "deploy",
                "@timestamp": "2020-04-27T09:27:56.000000000+08:00",
                "status": "404"
              },
              "_id": "6M8_uXEBGPvAam3AlbV8",
              "_score": 0
            },
            {
              "_index": "access-2020.04.24",
              "_type": "_doc",
              "_source": {
                "server_name": "deploy",
                "@timestamp": "2020-04-27T09:27:56.000000000+08:00",
                "status": "404"
              },
              "_id": "6c8_uXEBGPvAam3AlrVv",
              "_score": 0
            },
            {
              "_index": "access-2020.04.24",
              "_type": "_doc",
              "_source": {
                "server_name": "deploy",
                "@timestamp": "2020-04-27T09:27:56.000000000+08:00",
                "status": "404"
              },
              "_id": "6s8_uXEBGPvAam3Al7VH",
              "_score": 0
            }
          ],
          "total": 20,
          "max_score": 0
        },
        "took": 12,
        "timed_out": false,
        "aggregations": {
          "server_name": {
            "doc_count_error_upper_bound": 0,
            "sum_other_doc_count": 0,
            "buckets": [
              {
                "doc_count": 20,
                "key": "deploy"
              }
            ]
          }
        }
      },
      "search": {
        "request": {
          "search_type": "query_then_fetch",
          "indices": [
            "access-*"
          ],
          "rest_total_hits_as_int": true,
          "body": {
            "query": {
              "bool": {
                "must_not": [
                  {
                    "match": {
                      "status": "200"
                    }
                  }
                ],
                "filter": [
                  {
                    "range": {
                      "@timestamp": {
                        "gt": "now-30m"
                      }
                    }
                  }
                ]
              }
            },
            "aggs": {
              "server_name": {
                "terms": {
                  "field": "server_name.keyword"
                }
              }
            }
          }
        }
      }
    },
    "condition": {
      "type": "compare",
      "status": "success",
      "met": true,
      "compare": {
        "resolved_values": {
          "ctx.payload.hits.total": 20
        }
      }
    },
    "actions": [
      {
        "id": "my_webhook",
        "type": "webhook",
        "status": "success",
        "webhook": {
          "request": {
            "host": "api.telegram.org",
            "port": 443,
            "scheme": "https",
            "method": "get",
            "path": "/******:************/sendMessage",
            "params": {
              "text": "There are 20 non 200 status occurred 30 minutes ago, (es7.6.2 test message)",
              "chat_id": "-*********"
            },
            "proxy": {
              "host": "10.33.66.xxx",
              "port": 3128
            }
          },
          "response": {
            "status": 200,
            "headers": {
              "access-control-allow-origin": [
                "*"
              ],
              "date": [
                "Mon, 27 Apr 2020 01:28:57 GMT"
              ],
              "server": [
                "nginx/1.16.1"
              ],
              "content-length": [
                "329"
              ],
              "connection": [
                "keep-alive"
              ],
              "content-type": [
                "application/json"
              ],
              "strict-transport-security": [
                "max-age=31536000; includeSubDomains; preload"
              ],
              "access-control-allow-methods": [
                "GET, POST, OPTIONS"
              ],
              "access-control-expose-headers": [
                "Content-Length,Content-Type,Date,Server,Connection"
              ]
            },
            "body": "{\"ok\":true,\"result\":{\"message_id\":2342,\"from\":{\"id\":****,\"is_bot\":true,\"first_name\":\"****\",\"username\":\"****\"},\"chat\":{\"id\":-****,\"title\":\"***\",\"type\":\"group\",\"all_members_are_administrators\":true},\"date\":1587950937,\"text\":\"There are 20 non 200 status occurred 30 minutes ago, (es7.6.2 test message)\"}}"
          }
        }
      }
    ]
  },
  "messages": []
}

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.