Elastic webhook output

Elastic Stack Elasticsearch
1

Hello,

I wanna know please if the webhook in the watcher send output in a JSON format ? cause I am getting some errors in my webhook and I don't know if it's because the output is not in JSON format, or cause I didn't know how to create a webhook test.
I have configured a small webhook just to test, and the configuration looks like that:

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from flask import json        
from flask import request    
from flask import Flask       
import re

app = Flask(__name__)


@app.route('/')
def api_root():
    return 'Webhook TEST'

@app.route('/webhook', methods=['POST'])
def api_webhook_messages():
    my_info = json.loads(request.data)
    print(json.dumps(my_info))
    return 'Connection sucessful'

if __name__=='__main__':
    app.run(port=8080, host='10.10.13.135',debug=True)

When I test it from Kibana I get a successful connection as it's shown in the picture bellow:

image
image910Ă—804 23.8 KB

And the result in my webhook is working as expected, and show the result as below:

{"body": "test"}
10.13.81.254 - - [14/Dec/2020 09:10:37] "POST /webhook HTTP/1.1" 200 -

But when I configure the webhook directly in my watcher scipt I am getting an error :

Traceback (most recent call last):
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 2464, in __call__
    return self.wsgi_app(environ, start_response)
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 2450, in wsgi_app
    response = self.handle_exception(e)
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 1867, in handle_exception
    reraise(exc_type, exc_value, tb)
  File "/usr/local/lib/python3.8/dist-packages/flask/_compat.py", line 39, in reraise
    raise value
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 2447, in wsgi_app
    response = self.full_dispatch_request()
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 1952, in full_dispatch_request
    rv = self.handle_user_exception(e)
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 1821, in handle_user_exception
    reraise(exc_type, exc_value, tb)
  File "/usr/local/lib/python3.8/dist-packages/flask/_compat.py", line 39, in reraise
    raise value
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 1950, in full_dispatch_request
    rv = self.dispatch_request()
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 1936, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "/opt/thehive/webhook3.py", line 20, in api_webhook_messages
    my_info = json.loads(request.data)
  File "/usr/local/lib/python3.8/dist-packages/flask/json/__init__.py", line 253, in loads
    return _json.loads(s, **kwargs)
  File "/usr/lib/python3/dist-packages/simplejson/__init__.py", line 535, in loads
    return cls(encoding=encoding, **kw).decode(s)
  File "/usr/lib/python3/dist-packages/simplejson/decoder.py", line 370, in decode
    obj, end = self.raw_decode(s)
  File "/usr/lib/python3/dist-packages/simplejson/decoder.py", line 400, in raw_decode
    return self.scan_once(s, idx=_w(s, idx).end())
simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
2

( Sorry couldn't write everything in the same post, so I continue writing here to give more information )

So, My watcher script is like that:

POST _watcher/watch/_execute
{
  "watch": {
    "trigger": {
      "schedule": {
        "interval": "1h"
      }
    },
    "input": {
      "search": {
        "request": {
          "indices": "firewall-*",
          "body": {
            "size": 0,
            "query": {
              "bool": {
                "filter": {
                  "range": {
                    "@timestamp": {
                      "from": "now-1h",
                      "to": "now"
                    }
                  }
                }
              }
            },
            "aggs": {
              "by_source_ip": {
                "terms": {
                  "size": 100,
                  "field": "source.ip"
                },
                "aggs": {
                  "by_destination_ip": {
                    "terms": {
                      "size": 100,
                      "field": "destination.ip"
                    },
                    "aggs": {
                      "by_port_number": {
                        "terms": {
                          "size": 100,
                          "field": "destination.port",
                          "order": {
                            "_count": "asc"
                          }
                        }
                      }
                    }
                  }
                }
              }
            }
          }
        }
      }
    },
    "condition": {
      "script":
      """

      for (int i=0; i < ctx.payload.aggregations.by_source_ip.buckets.size(); i++)
      {
        for (int j=0; j < ctx.payload.aggregations.by_source_ip.buckets[i].by_destination_ip.buckets.size() ; j++ )
        {
          if (ctx.payload.aggregations.by_source_ip.buckets[i].by_destination_ip.buckets[j].by_port_number.buckets.size() > 15 ) // l'action ne sera exécuté seulement s'il y a un scan de 20 port par @IP source / @IP destination
          {
            return true;
          }
        }
      }

        """
    },
    "transform":
    {
      "script":
      """

     String[] source= new String[5]; // Un vercteur qui contient les addresse IP source des scan
     String[] destination= new String[5]; // Un vercteur qui contient les adresse IP destination qui ont été scanné
     int[] nombre_port= new int[5]; // Un vecteur qui contient le nombre de ports scanné pour  (IP source / IP destination)
     int n=0;
      for (int i=0; i < ctx.payload.aggregations.by_source_ip.buckets.size(); i++)
      {
        for (int j=0; j < ctx.payload.aggregations.by_source_ip.buckets[i].by_destination_ip.buckets.size() ; j++ )
        {
          if (ctx.payload.aggregations.by_source_ip.buckets[i].by_destination_ip.buckets[j].by_port_number.buckets.size() > 15) // Pour entregistrer seulement les addresse IP source/destinations qui ont déclenché l'alarme
          {
            if (n<5) // Pour s'assurer qu'on dépasse pas 5 cases qu'on a alloué
            {
            source[n] = ctx.payload.aggregations.by_source_ip.buckets[i].key;
            destination[n] = ctx.payload.aggregations.by_source_ip.buckets[i].by_destination_ip.buckets[j].key;
            nombre_port[n] = ctx.payload.aggregations.by_source_ip.buckets[i].by_destination_ip.buckets[j].by_port_number.buckets.size();
            n++;
            }
          }
        }
      }
      return [source,destination,nombre_port];
      """
    },
    "actions": {
     "my_webhook": {
       "webhook": {
         "method": "POST",
         "host": "10.10.13.135",
         "path": "/webhook",
         "port": 8080,
         "body": "detected "
       }
     }
    }
  }
}

And when I try to execute it, the body output looks like that :

              "body" : """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
  "http://www.w3.org/TR/html4/loose.dtd">
<html>
  <head>
    <title>simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0) // Werkzeug Debugger</title>
<!--

Traceback (most recent call last):
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 2464, in __call__
    return self.wsgi_app(environ, start_response)
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 2450, in wsgi_app
    response = self.handle_exception(e)
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 1867, in handle_exception
    reraise(exc_type, exc_value, tb)
  File "/usr/local/lib/python3.8/dist-packages/flask/_compat.py", line 39, in reraise
    raise value
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 2447, in wsgi_app
    response = self.full_dispatch_request()
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 1952, in full_dispatch_request
    rv = self.handle_user_exception(e)
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 1821, in handle_user_exception
    reraise(exc_type, exc_value, tb)
  File "/usr/local/lib/python3.8/dist-packages/flask/_compat.py", line 39, in reraise
    raise value
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 1950, in full_dispatch_request
    rv = self.dispatch_request()
  File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 1936, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "/opt/thehive/webhook3.py", line 20, in api_webhook_messages
    my_info = json.loads(request.data)
  File "/usr/local/lib/python3.8/dist-packages/flask/json/__init__.py", line 253, in loads
    return _json.loads(s, **kwargs)
  File "/usr/lib/python3/dist-packages/simplejson/__init__.py", line 535, in loads
    return cls(encoding=encoding, **kw).decode(s)
  File "/usr/lib/python3/dist-packages/simplejson/decoder.py", line 370, in decode
    obj, end = self.raw_decode(s)
  File "/usr/lib/python3/dist-packages/simplejson/decoder.py", line 400, in raw_decode
    return self.scan_once(s, idx=_w(s, idx).end())
simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

-->
"""
            }
          }
        }
      ]
    },
    "messages" : [ ]
  }
}
3

I find out the error,

so the error was occurring cause the output of the webhook wasn't in JSON format, I changed my webhook configuration to show the data without parsing it like below :

def api_webhook_messages():
    data = request.data
    print(data)
3 Likes
4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.