'no_http_response_exception' Issue With Watcher

Elastic Stack Elasticsearch
1

Hello!

I'm having an issue with a Watcher that I'm try to create that will call the ES /_cluster/health API endpoint.

I'm using the Watcher example from the documentation here pretty much verbatim and am getting a 'no_http_response_exception' failed to respond error.

I've tried using 'localhost' as well as the IP of the master node, with Basic Auth (and no Basic Auth) as well as base64 encoded uname:pw for authentication to no avail.

This is on version 8.13.4 for Kibana and ES. The error happens immediately in the Watcher 'Simulate' feature and doesn't timeout.

Any assistance would be appreciated.

Here is the Watcher code:

{
  "trigger": {
    "schedule": {
      "interval": "10s"
    }
  },
  "input": {
    "http": {
      "request": {
        "host": "172.25.33.193",
        "port": 9200,
        "method": "get",
        "path": "/_cluster/health",
        "auth": {
          "basic": {
            "username": "service",
            "password": "[REDACTED]"
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.status": {
        "eq": "green"
      }
    }
  },
  "actions": {
    "send_email": {
      "email": {
        "profile": "standard",
        "to": [
          "adam.meindel@blah.com"
        ],
        "subject": "Cluster Status Warning",
        "body": {
          "text": "Cluster status is GREEN"
        }
      }
    }
  }
}

Here is the response:

{
  "watch_id": "_inlined_",
  "node": "y3YX2Ia_TrOYjxpkpRvJ_g",
  "state": "failed",
  "@timestamp": "2024-10-23T13:42:52.100372475Z",
  "user": "Adam Meindel",
  "status": {
    "state": {
      "active": true,
      "timestamp": "2024-10-23T13:42:52.100Z"
    },
    "actions": {
      "send_email": {
        "ack": {
          "timestamp": "2024-10-23T13:42:52.100Z",
          "state": "awaits_successful_execution"
        }
      }
    },
    "execution_state": "failed",
    "version": -1
  },
  "trigger_event": {
    "type": "manual",
    "triggered_time": "2024-10-23T13:42:52.100Z",
    "manual": {
      "schedule": {
        "scheduled_time": "2024-10-23T13:42:52.100Z"
      }
    }
  },
  "input": {
    "http": {
      "request": {
        "scheme": "http",
        "host": "172.25.33.193",
        "port": 9200,
        "method": "get",
        "path": "/_cluster/health",
        "params": {},
        "headers": {},
        "auth": {
          "basic": {
            "username": "service",
            "password": "::es_redacted::"
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.status": {
        "eq": "green"
      }
    }
  },
  "metadata": {
    "name": "Cluster Health",
    "xpack": {
      "type": "json"
    }
  },
  "result": {
    "execution_time": "2024-10-23T13:42:52.100Z",
    "execution_duration": 21,
    "input": {
      "type": "http",
      "status": "failure",
      "error": {
        "root_cause": [
          {
            "type": "no_http_response_exception",
            "reason": "172.25.33.193:9200 failed to respond"
          }
        ],
        "type": "no_http_response_exception",
        "reason": "172.25.33.193:9200 failed to respond"
      },
      "http": {
        "request": {
          "host": "172.25.33.193",
          "port": 9200,
          "scheme": "http",
          "method": "get",
          "path": "/_cluster/health",
          "auth": {
            "basic": {
              "username": "service",
              "password": "::es_redacted::"
            }
          }
        }
      }
    },
    "actions": []
  },
  "messages": [
    "failed to execute watch input"
  ]
}

Please let me know if you need any additional information and thanks in advance!

Adam

2

Additional information - I tried using the ES FQDN with https instead of http and got this error:

{
  "watch_id": "_inlined_",
  "node": "d3ZAQkBRSee0ccAnA527Vg",
  "state": "failed",
  "@timestamp": "2024-10-23T18:44:07.63967047Z",
  "user": "Adam Meindel",
  "status": {
    "state": {
      "active": true,
      "timestamp": "2024-10-23T18:44:07.639Z"
    },
    "actions": {
      "send_email": {
        "ack": {
          "timestamp": "2024-10-23T18:44:07.639Z",
          "state": "awaits_successful_execution"
        }
      }
    },
    "execution_state": "failed",
    "version": -1
  },
  "trigger_event": {
    "type": "manual",
    "triggered_time": "2024-10-23T18:44:07.639Z",
    "manual": {
      "schedule": {
        "scheduled_time": "2024-10-23T18:44:07.639Z"
      }
    }
  },
  "input": {
    "http": {
      "request": {
        "scheme": "https",
        "host": "elasticsearch.cvges01-prod.amwell.systems",
        "port": 9200,
        "method": "get",
        "path": "/_cluster/health",
        "params": {},
        "headers": {},
        "auth": {
          "basic": {
            "username": "service",
            "password": "::es_redacted::"
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.status": {
        "eq": "green"
      }
    }
  },
  "metadata": {
    "name": "Cluster Health",
    "xpack": {
      "type": "json"
    }
  },
  "result": {
    "execution_time": "2024-10-23T18:44:07.639Z",
    "execution_duration": 8,
    "input": {
      "type": "http",
      "status": "failure",
      "error": {
        "root_cause": [
          {
            "type": "s_s_l_handshake_exception",
            "reason": "Remote host terminated the handshake"
          }
        ],
        "type": "s_s_l_handshake_exception",
        "reason": "Remote host terminated the handshake",
        "caused_by": {
          "type": "e_o_f_exception",
          "reason": "SSL peer shut down incorrectly"
        }
      },
      "http": {
        "request": {
          "host": "elasticsearch.fqdn-prod.amwell.systems",
          "port": 9200,
          "scheme": "https",
          "method": "get",
          "path": "/_cluster/health",
          "auth": {
            "basic": {
              "username": "service",
              "password": "::es_redacted::"
            }
          }
        }
      }
    },
    "actions": []
  },
  "messages": [
    "failed to execute watch input"
  ]
}

Then on a whim, I tried 127.0.0.1 (https) for the host and got:

{
  "watch_id": "_inlined_",
  "node": "0JXuoTUSQk6YyLPfbDp4Yw",
  "state": "failed",
  "@timestamp": "2024-10-23T18:51:31.724114298Z",
  "user": "Adam Meindel",
  "status": {
    "state": {
      "active": true,
      "timestamp": "2024-10-23T18:51:31.723Z"
    },
    "actions": {
      "send_email": {
        "ack": {
          "timestamp": "2024-10-23T18:51:31.723Z",
          "state": "awaits_successful_execution"
        }
      }
    },
    "execution_state": "failed",
    "version": -1
  },
  "trigger_event": {
    "type": "manual",
    "triggered_time": "2024-10-23T18:51:31.724Z",
    "manual": {
      "schedule": {
        "scheduled_time": "2024-10-23T18:51:31.724Z"
      }
    }
  },
  "input": {
    "http": {
      "request": {
        "scheme": "https",
        "host": "127.0.0.1",
        "port": 9200,
        "method": "get",
        "path": "/_cluster/health",
        "params": {},
        "headers": {},
        "auth": {
          "basic": {
            "username": "service",
            "password": "::es_redacted::"
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.status": {
        "eq": "green"
      }
    }
  },
  "metadata": {
    "name": "Cluster Health",
    "xpack": {
      "type": "json"
    }
  },
  "result": {
    "execution_time": "2024-10-23T18:51:31.724Z",
    "execution_duration": 28,
    "input": {
      "type": "http",
      "status": "failure",
      "error": {
        "root_cause": [
          {
            "type": "s_s_l_handshake_exception",
            "reason": "PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target"
          }
        ],
        "type": "s_s_l_handshake_exception",
        "reason": "PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target",
        "caused_by": {
          "type": "validator_exception",
          "reason": "PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target",
          "caused_by": {
            "type": "sun_cert_path_builder_exception",
            "reason": "unable to find valid certification path to requested target"
          }
        }
      },
      "http": {
        "request": {
          "host": "127.0.0.1",
          "port": 9200,
          "scheme": "https",
          "method": "get",
          "path": "/_cluster/health",
          "auth": {
            "basic": {
              "username": "service",
              "password": "::es_redacted::"
            }
          }
        }
      }
    },
    "actions": []
  },
  "messages": [
    "failed to execute watch input"
  ]
}

It's frustrating because since Watchers apparently run at the ES level, it shouldn't need to do all of this extraneous authentication just to call a native API endpoint on the very machine it's running on.