Circuit_breaking_exception - Watcher

Elastic Stack Elasticsearch
1

Hi

I have just started using ElasticSearch and have run into an issue when setting up advanced watchers.

I have written several watchers to check for application errors. However, they are returning a circuit_breaking_exception as per below.

Can anyone advise what i need to do to prevent the error. I have looked at using Params without any luck and would like to avoid increasing the [200/1] value, as this feels like a sticky plaster rather than a fix!

Any advise and help much appreciated.


{
            "type": "circuit_breaking_exception",
            "reason": "[script] Too many dynamic script compilations within, max: [200/1m]; please use indexed, or scripts with parameters instead; this limit can be changed by the [script.max_compilations_rate] setting",
            "bytes_wanted": 0,
            "bytes_limit": 0,
            "durability": "TRANSIENT"
          }

Here is the script:


{
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "development-*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "query": {
            "bool": {
              "must": [
                {
                  "match": {
                    "tags": "sometag"
                  }
                },
                {
                  "match": {
                    "message": "Level: Error"
                  }
                }
              ],
              "filter": {
                "range": {
                  "@timestamp": {
                    "from": "now-5m",
                    "to": "now"
                  }
                }
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gt": 5
      }
    }
  },
  "actions": {
    "log": {
      "logging": {
        "level": "info",
        "text": "{{ctx.payload.hits.total}} Errors have occured:{{#ctx.payload.hits.hits}}{{_id}}:{{/ctx.payload.hits.hits}}"
      }
    },
    "DevWebhook": {
      "throttle_period_in_millis": 300000,
      "webhook": {
        "scheme": "https",
        "host": "outlook.office.com",
        "port": 443,
        "method": "post",
        "path": "/webhook/...",
        "params": {},
        "headers": {},
        "body": """{ "text": "Errors Detected - {{ctx.payload.hits.total}}  errors in the past 5 minutes - Please check logs" }"""
      }
    }
  },
  "throttle_period_in_millis": 300000
}

Here is the full error:


{
  "watch_id": "xxxxx",
  "node": "xxxxx",
  "state": "failed",
  "user": "mike",
  "status": {
    "state": {
      "active": true,
      "timestamp": "2020-07-15T08:47:54.650Z"
    },
    "last_checked": "2020-07-15T09:09:23.569Z",
    "actions": {
      "log": {
        "ack": {
          "timestamp": "2020-07-15T08:47:54.650Z",
          "state": "awaits_successful_execution"
        }
      },
      "DevWebhook": {
        "ack": {
          "timestamp": "2020-07-15T08:47:54.650Z",
          "state": "awaits_successful_execution"
        }
      }
    },
    "execution_state": "failed",
    "version": -1
  },
  "trigger_event": {
    "type": "schedule",
    "triggered_time": "2020-07-15T09:10:23.107Z",
    "schedule": {
      "scheduled_time": "2020-07-15T09:10:23.091Z"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "development-*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "query": {
            "bool": {
              "must": [
                {
                  "match": {
                    "tags": "sometag"
                  }
                },
                {
                  "match": {
                    "message": "Level: Error"
                  }
                }
              ],
              "filter": {
                "range": {
                  "@timestamp": {
                    "from": "now-5m",
                    "to": "now"
                  }
                }
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gt": 5
      }
    }
  },
  "metadata": {
    "name": "Dev - Application - Core - Error volume on - Servers",
    "xpack": {
      "type": "json"
    }
  },
  "result": {
    "execution_time": "2020-07-15T09:10:23.107Z",
    "execution_duration": 0,
    "input": {
      "type": "search",
      "status": "failure",
      "error": {
        "root_cause": [
          {
            "type": "circuit_breaking_exception",
            "reason": "[script] Too many dynamic script compilations within, max: [200/1m]; please use indexed, or scripts with parameters instead; this limit can be changed by the [script.max_compilations_rate] setting",
            "bytes_wanted": 0,
            "bytes_limit": 0,
            "durability": "TRANSIENT"
          }
        ],
        "type": "general_script_exception",
        "reason": "Failed to compile inline script [{\"query\":{\"bool\":{\"must\":[{\"match\":{\"tags\":\"sometag\"}},{\"match\":{\"message\":\"Level: Error\"}}],\"filter\":{\"range\":{\"@timestamp\":{\"from\":\"now-5m\",\"to\":\"now\"}}}}}}] using lang [mustache]",
        "caused_by": {
          "type": "circuit_breaking_exception",
          "reason": "[script] Too many dynamic script compilations within, max: [200/1m]; please use indexed, or scripts with parameters instead; this limit can be changed by the [script.max_compilations_rate] setting",
          "bytes_wanted": 0,
          "bytes_limit": 0,
          "durability": "TRANSIENT"
        }
      }
    },
    "actions": []
  },
  "messages": [
    "failed to execute watch input"
  ]
}
2

can you use {{ctx.payload._id}} instead of {{_id}}
You can also use foreach action for looging

{
      "foreach" : "ctx.payload.hits.hits", 
      "max_iterations" : "{{ctx.payload.hits.total}}",
      "logging" : {
        "text" : "{{ctx.payload.hits.total}} Errors have occured: {{ctx.payload._id}}"
      }
    }
1 Like
3

There is a default script compilation limit in Elasticsearch. That you are exceeding. This does not mean that watcher cuased this (maybe there are search requests or ingest processors executing scripts).

You can increase the setting, see https://www.elastic.co/guide/en/elasticsearch/reference/current/circuit-breaker.html#script-compilation-circuit-breaker

However you should try to figure out where those compilations are coming from as well, and maybe reduce them at the source.

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.