Kibana : Unexpected token < in JSON at position 1

[krpatel19 (Krunal Patel)]

We are getting below exceptions while accessing Elasticsearch through GCP https load balancer. It is happening for KIbana endpoint as well as for elasticsearch endpoint made for application configured as internal http LB. Can you suggest what parameter we can tune in Logstash , in Elasticsearch , In Kibana or at LB to solve this exceptions. This actually 502 error message in response which has html code in response having < in starting. How to solve this 502? Where to check?

Cluster health is 100% everytime, i am sure we need to tune some parameter to ger response in time. Not able to find those parameters.

Using Elasticsearch , Kibana, Logstash version 7.6.2

appplication ES LB Logs:

{

insertId: "10wzx1zg2bh9mtv"

jsonPayload: {

@type: "type.googleapis.com/google.cloud.loadbalancing.type.LoadBalancerLogEntry"

}

httpRequest: {

requestMethod: "POST"

requestUrl: "http://<mylb>:443/_bulk"

requestSize: "566178"

userAgent: "Manticore 0.6.4"

remoteIp: "10.127.0.2"

serverIp: "10.127.0.5"

latency: "60.076831619s"

protocol: "HTTP/1.1"

}

resource: {

type: "internal_http_lb_rule"

labels: {

backend_target_name: "es-rbs"

url_map_name: "es-urlmap"

matched_url_path_rule: "UNMATCHED"

backend_type: "INSTANCE_GROUP"

backend_target_type: "BACKEND_SERVICE"

backend_name: "es-rigm"

region: "us-central1"

backend_scope_type: "REGION"

forwarding_rule_name: "es-fwd-rule-https"

network_name: "es-network"

project_id: "es-production"

backend_scope: "us-central1"

target_proxy_name: "es-https-proxy"

}

}

timestamp: "2020-06-25T12:34:49.035917Z"

severity: "ERROR"

logName: "projects/es-production/logs/loadbalancing.googleapis.com%2Frequests"

receiveTimestamp: "2020-06-25T12:35:52.965795Z"

} 

Kibana LB Logs:

{
  "insertId": "nldh7cg1jw0wpe",
  "jsonPayload": {
    "@type": "type.googleapis.com/google.cloud.loadbalancing.type.LoadBalancerLogEntry",
    "statusDetails": "backend_timeout"
  },
  "httpRequest": {
    "requestMethod": "POST",
    "requestUrl": "https://kibana.example.com/elasticsearch/jmx-*/_search?rest_total_hits_as_int=true&ignore_unavailable=true&ignore_throttled=true&preference=1593088376570&timeout=30000ms",
    "requestSize": "2851",
    "status": 502,
    "responsesize": "436",
    "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36",
    "remoteIp": "<REDACTED>",
    "referer": "https://kibana.example.com/app/kibana",
    "serverIp": "10.127.0.24"
  },
  "resource": {
    "type": "http_load_balancer",
    "labels": {
      "forwarding_rule_name": "es-fwd-rule-https",
      "zone": "global",
      "project_id": "es-production",
      "target_proxy_name": "es-https-proxy",
      "backend_service_name": "es-kibana-bs",
      "url_map_name": "es-kibana-urlmap"
    }
  },
  "timestamp": "2020-06-25T12:33:20.160419Z",
  "severity": "WARNING",
  "logName": "projects/es-production/logs/requests",
  "trace": "projects/es-production/traces/a1f1df0ebd63b1a254a0802b081145ef",
  "receiveTimestamp": "2020-06-25T12:33:30.994620280Z",
  "spanId": "1a6f71e354e5b40f"
}