Kibana Watcher action failed: specified foreach object was null

Elastic Stack Kibana
1

I'm using ELK 7.9.
I have configured Watchers to trigger Webhook action. I'm trying to add a foreach function to trigger webhooks for each recorded application errors. I'm getting error when trying to execute.
Watcher file

{
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "hamonitor-testing-*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "query": {
            "bool": {
              "must": [
                {
                  "match": {
                    "message": "ERROR!"
                  }
                },
                {
                  "range": {
                    "@timestamp": {
                      "gte": "now-30m"
                    }
                  }
                }
              ]
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gt": 0
      }
    }
  },
  "actions": {
    "log_hits": {
      "foreach": "ctx.payload.hits.hits",
      "max_iterations": 500,
      "logging": {
        "level": "info",
        "text": "{{#toJson}}ctx.payload{{/toJson}}"
      }
    },
    "webhook_action": {
      "foreach": "ctx.payload.hits.hits",
      "max_iterations": 100,
      "webhook": {
        "scheme": "http",
        "host": "169.62.155.30",
        "port": 10086,
        "method": "post",
        "path": "/probe/webhook/logstash",
        "params": {},
        "headers": {},
        "body": "{{#toJson}}ctx.payload{{/toJson}}"
      }
    }
  },
  "transform": {
    "script": {
      "source": "return [ 'timestamp' : ctx.payload.hits.hits[0]._source['@timestamp'],'node': 'CP8-ELK-HAMONITOR-NFT-PreProd','namespace': ctx.payload.hits.hits.0._source.environment,'container_name' : 'payaas-Interac-CP8DEVAPFDP01-preprod','log': ctx.payload.hits.hits.0._source.message,'source':'kibana','errorcode':'ERROR!','description':ctx.payload.hits.hits.0._id + ':' +ctx.payload.hits.hits.0._source.Description ,'AlertKey': 'ELK:'+ ctx.payload.hits.hits.0._source.Description ]",
      "lang": "painless"
    }
  }
}

Regards,
Pavan

2

Watcher execution status, unable to attach the complete log, so attaching the webhook action part:

                  "input" : {
                    "type" : "log"
                  },
                  "environment" : "PreProd",
                  "log_type" : "hamonitor",
                  "@timestamp" : "2021-05-12T13:38:37.636Z",
                  "ecs" : {
                    "version" : "1.5.0"
                  },
                  "host" : {
                    "hostname" : "CP8SHRTRLOG01",
                    "os" : {
                      "kernel" : "3.10.0-1127.13.1.el7.x86_64",
                      "codename" : "Maipo",
                      "name" : "Red Hat Enterprise Linux Server",
                      "family" : "redhat",
                      "version" : "7.7 (Maipo)",
                      "platform" : "rhel"
                    },
                    "containerized" : false,
                    "ip" : [
                      "192.168.119.22"
                    ],
                    "name" : "CP8SHRTRLOG01",
                    "id" : "3274bae7aeb64a11856549d3e7882e30",
                    "mac" : [
                      "00:50:56:bf:27:44"
                    ],
                    "architecture" : "x86_64"
                  },
                  "@version" : "1"
                },
                "_id" : "oQrNYHkBvNS-A2iVLg5L",
                "_score" : 1.2933977
              },
              {
                "_index" : "hamonitor-testing-2021.05.12",
                "_type" : "_doc",
                "_source" : {
                  "app" : "hamonitor",
                  "product" : "interac",
                  "agent" : {
                    "hostname" : "CP8SHRTRLOG01",
                    "name" : "CP8SHRTRLOG01",
                    "id" : "1e793688-c82e-4936-8023-3118e5d482f9",
                    "ephemeral_id" : "c867a6c9-e02c-4fa8-ac3f-a06a5eafb010",
                    "type" : "filebeat",
                    "version" : "7.9.1"
                  },
                  "Description" : "queue manager MQM.CP8PPDAPFIP.NFT ending immediately",
                  "log" : {
                    "file" : {
                      "path" : "/tmp/hamonitor.log"
                    },
                    "offset" : 0,
                    "flags" : [
                      "multiline"
                    ]
                  },
                  "message" : """12125801@Mon Apr 16 12:58:01 UTC 2021:monitor script starts with version: 2021040701
12125801@Mon Apr 16 12:58:01 UTC 2021:ERROR! queue manager MQM.CP8PPDAPFIP.NFT ending immediately
12125801@Mon Apr 16 12:58:01 UTC 2021:system status: NFSAVAIL = Y, OWNVIP = Y, QMSTATUS = STATUS(Endingimmediately), QMCODE = QE
12125801@Mon Apr 16 12:58:01 UTC 2021:releasing service ip address 192.168.6.215
12125801@Mon Apr 16 12:58:02 UTC 2021:case 16, release service ip
12125801@Mon Apr 16 12:58:02 UTC 2021:Queue manager MQM.CP8PPDAPFIP.NFT is ending
12125801@Mon Apr 16 12:58:02 UTC 2021:broker IIB.CP8PPDAPFIP.NFT is running
12125801@Mon Apr 16 12:58:02 UTC 2021:monitor script stops with version: 2021040701""",
                  "tags" : [
                    "hamonitor",
                    "interac",
                    "beats_input_codec_plain_applied"
                  ],
                  "input" : {
                    "type" : "log"
                  },
                  "environment" : "PreProd",
                  "log_type" : "hamonitor",
                  "@timestamp" : "2021-05-12T13:37:37.631Z",
                  "ecs" : {
                    "version" : "1.5.0"
                  },
                  "host" : {
                    "hostname" : "CP8SHRTRLOG01",
                    "os" : {
                      "kernel" : "3.10.0-1127.13.1.el7.x86_64",
                      "codename" : "Maipo",
                      "name" : "Red Hat Enterprise Linux Server",
                      "family" : "redhat",
                      "version" : "7.7 (Maipo)",
                      "platform" : "rhel"
                    },
                    "containerized" : false,
                    "ip" : [
                      "192.168.119.22"
                    ],
                    "name" : "CP8SHRTRLOG01",
                    "id" : "3274bae7aeb64a11856549d3e7882e30",
                    "mac" : [
                      "00:50:56:bf:27:44"
                    ],
                    "architecture" : "x86_64"
                  },
                  "@version" : "1"
                },
                "_id" : "fAnMYHkBvNS-A2iVQND7",
                "_score" : 1.2563564
              },
              {
                "_index" : "hamonitor-testing-2021.05.12",
                "_type" : "_doc",
                "_source" : {
                  "app" : "hamonitor",
                  "product" : "interac",
                  "agent" : {
                    "hostname" : "CP8SHRTRLOG01",
                    "name" : "CP8SHRTRLOG01",
                    "id" : "1e793688-c82e-4936-8023-3118e5d482f9",
                    "ephemeral_id" : "c867a6c9-e02c-4fa8-ac3f-a06a5eafb010",
                    "type" : "filebeat",
                    "version" : "7.9.1"
                  },
                  "Description" : "case 4, start the queue manager as standby",
                  "log" : {
                    "file" : {
                      "path" : "/tmp/hamonitor.log"
                    },
                    "offset" : 0,
                    "flags" : [
                      "multiline"
                    ]
                  },
                  "message" : """12125431@Mon Apr 16 12:54:31 UTC 2021:monitor script starts with version: 2021040801
12125431@Mon Apr 16 12:54:31 UTC 2021:NFS /HA available
12125431@Mon Apr 16 12:54:32 UTC 2021:system status: NFSAVAIL = Y, OWNVIP = N, QMSTATUS = STATUS(Runningelsewhere), QMCODE = RE BKSTATUS = stopped
12125431@Mon Apr 16 12:54:32 UTC 2021:ERROR! case 4, start the queue manager as standby
12125431@Mon Apr 16 12:54:32 UTC 2021:starting queue manager MQM.CP8PPDAPFIP.NFT
12125431@Mon Apr 16 12:54:33 UTC 2021:case 7, no action
12125431@Mon Apr 16 12:54:33 UTC 2021:WARNING! - Queue manager MQM.CP8PPDAPFIP.NFT is running on the other node
12125431@Mon Apr 16 12:54:33 UTC 2021:WARNING! - IIB.CP8PPDAPFIP.NFT is not running locally, please check on the other node
12125431@Mon Apr 16 12:54:33 UTC 2021:monitor script stops with version: 2021040801""",
                  "tags" : [
                    "hamonitor",
                    "interac",
                    "beats_input_codec_plain_applied"
                  ],
                  "input" : {
                    "type" : "log"
                  },
                  "environment" : "PreProd",
                  "log_type" : "hamonitor",
                  "@timestamp" : "2021-05-12T13:37:17.629Z",
                  "ecs" : {
                    "version" : "1.5.0"
                  },
                  "host" : {
                    "hostname" : "CP8SHRTRLOG01",
                    "os" : {
                      "kernel" : "3.10.0-1127.13.1.el7.x86_64",
                      "codename" : "Maipo",
                      "name" : "Red Hat Enterprise Linux Server",
                      "family" : "redhat",
                      "version" : "7.7 (Maipo)",
                      "platform" : "rhel"
                    },
                    "containerized" : false,
                    "ip" : [
                      "192.168.119.22"
                    ],
                    "name" : "CP8SHRTRLOG01",
                    "id" : "3274bae7aeb64a11856549d3e7882e30",
                    "mac" : [
                      "00:50:56:bf:27:44"
                    ],
                    "architecture" : "x86_64"
                  },
                  "@version" : "1"
                },
                "_id" : "sgnLYHkBvNS-A2iV-ruT",
                "_score" : 1.2341824
              }
            ],
            "total" : 4,
            "max_score" : 1.4009326
          },
          "took" : 4,
          "timed_out" : false
        },
        "search" : {
          "request" : {
            "search_type" : "query_then_fetch",
            "indices" : [
              "hamonitor-testing-*"
            ],
            "rest_total_hits_as_int" : true,
            "body" : {
              "query" : {
                "bool" : {
                  "must" : [
                    {
                      "match" : {
                        "message" : "ERROR!"
                      }
                    },
                    {
                      "range" : {
                        "@timestamp" : {
                          "gte" : "now-30m"
                        }
                      }
                    }
                  ]
                }
              }
            }
          }
        }
      },
      "condition" : {
        "type" : "compare",
        "status" : "success",
        "met" : true,
        "compare" : {
          "resolved_values" : {
            "ctx.payload.hits.total" : 4
          }
        }
      },
      "transform" : {
        "type" : "script",
        "status" : "success",
        "payload" : {
          "AlertKey" : "ELK:stop queue manager command has been issued, exit with error 1",
          "node" : "CP8-ELK-HAMONITOR-NFT-PreProd",
          "container_name" : "payaas-Interac-CP8DEVAPFDP01-preprod",
          "log" : """6205631@Thu May  6 20:56:44 UTC 2021:fetching service ip address failed for the try 2
6205631@Thu May  6 20:56:44 UTC 2021:case 12, not able to acquire the service ip address with 2 retries, stop queue manager
6205631@Thu May  6 20:56:44 UTC 2021:stopping queue manager MQM.CP8PPDAPFIP.OAT
6205631@Thu May  6 20:56:47 UTC 2021:ERROR! stop queue manager command has been issued, exit with error 1
6205631@Thu May  6 20:56:47 UTC 2021:monitor script stops with version: 2021041607""",
          "namespace" : "PreProd",
          "description" : "TgfBYHkBvNS-A2iV1hGH:stop queue manager command has been issued, exit with error 1",
          "source" : "kibana",
          "errorcode" : "ERROR!",
          "timestamp" : "2021-05-12T13:26:17.551Z"
        }
      },
      "actions" : [
        {
          "id" : "log_hits",
          "type" : "logging",
          "status" : "failure",
          "error" : {
            "root_cause" : [
              {
                "type" : "exception",
                "reason" : "specified foreach object was null: [ctx.payload.hits.hits]"
              }
            ],
            "type" : "exception",
            "reason" : "specified foreach object was null: [ctx.payload.hits.hits]"
          }
        },
        {
          "id" : "webhook_action",
          "type" : "webhook",
          "status" : "failure",
          "error" : {
            "root_cause" : [
              {
                "type" : "exception",
                "reason" : "specified foreach object was null: [ctx.payload.hits.hits]"
              }
            ],
            "type" : "exception",
            "reason" : "specified foreach object was null: [ctx.payload.hits.hits]"
          }
        }
      ]
    },
    "messages" : [ ]
  }
}
3

Is this supposed to be like that? shouldn't it just be?

ctx.payload.hits[0]
4

I still am getting the same error for ctx.payload.hits[0]
Would you be able to help me with any document that explains setting the ctx variables?

        "id": "webhook_action",
        "type": "webhook",
        "status": "failure",
        "error": {
          "root_cause": [
            {
              "type": "exception",
              "reason": "specified foreach object was null: [ctx.payload.hits[0]]"
            }
          ],
          "type": "exception",
          "reason": "specified foreach object was null: [ctx.payload.hits[0]]"
        }
      }
    ]
  },
  "messages": []
}
5

please put the full output of this in a gist somewhere, the partial output hinders debugging and the full search response is required.

Thanks!