How to parse glastopf honeypot log json using logstash for kibana visualization?

I have a raw data in json format that I want to visualize in Kibana, but I'm having trouble parsing the data in logstash, previously I was able to visualize raw data for honeypot dionaea and cowrie. please help me in doing this task. Here I show the configuration from the file /etc/logstash/conf.d/myconfig.conf and an example of the raw data to be parsed. Thank you

/etc/logstash/conf.d/myconfig.conf

input {
        beats {
                port => 5044
        }

}

filter {
       if [document_type] == "cowrie" {

                json {
                        source => "message"
                      }

                mutate {
                        copy => { "_id" => "[@metadata][_id]"}
                        remove_field => ["_id"]
                        }
                grok {
                       match => {
                          "message" => [
                             ".*credentials\":\[\[\"%{DATA:userfailed}\",\"%{DATA:passfailed}\"\]\]",
                             ".*loggedin\":\[\"%{DATA:usersuccess}\",\"%{DATA:passsuccess}\"\]",
                             ".*loggedin\":\[%{DATA:unpassuccess}\]",
                             ".*credentials\"\:\[\[%{DATA:unpassfailed}\]\]"
                                       ]
                                  }
                     }
                geoip { source => "[payload][peerIP]"
                        target => "geoip"
                      }
               }

       if [document_type] == "dionaea" {
                json {
                        source => "message"
                      }

                mutate {
                        copy => { "_id" => "[@metadata][_id]"}
                        remove_field => ["_id"]
                        }         
       }

       if [document_type] == "glastopf" {
                json {
                        source => "message"
                      }
                grok {
                       match => {
                          "message" => [
                             ".*pattern\"\:\"%{DATA:pattern}\"",
                             ".*request_raw\"\:\"%{DATA:attack_method}\/",
                             ".*User-Agent\:\ \%{DATA:user_agent}\"",
                             ".*source\"\:\[\"%{DATA:source_ip}\"\,\%{DATA:source_port}\]",
                             ".*http_host\"\:\"%{DATA:destination_ip}\"\,"
                             #".*sensor_port\"\:\%{DATA:destination_port},"
                                       ]
                                  }
                     }
                     
                mutate {
                        copy => { "_id" => "[@metadata][_id]"}
                        remove_field => ["_id"]
                        }         
       }

}

output {
       if [document_type] == "cowrie" {
                elasticsearch {
                        hosts => ["http://localhost:9200"]
                        index => "logstash-cowrie-%{+YYYY.MM.dd}"
                        document_id => "%{[@metadata][_id]}"
                }
       }
       if [document_type] == "dionaea" {
                elasticsearch {
                        hosts => ["http://localhost:9200"]
                        index => "logstash-dionaea-%{+YYYY.MM.dd}"
                        document_id => "%{[@metadata][_id]}"
                }
       }
       if [document_type] == "glastopf" {
                elasticsearch {
                        hosts => ["http://localhost:9200"]
                        index => "logstash-glastopf-%{+YYYY.MM.dd}"
                        document_id => "%{[@metadata][_id]}"
                }
       }

}

RAW data Glastopf

{"_id":{"$oid":"631930a04e016a52d1af9008"},"ident":"honeypot3-glast","timestamp":{"$date":"2022-09-08T00:00:32.314Z"},"normalized":true,"payload":{"pattern":"unknown","request_raw":"GET /cgi-bin/voteList.asp?item_ID=%29%2F%2A%2A%2FaS%2F%2A%2A%2FGmeF%2F%2A%2A%2FwHerE%2F%2A%2A%2F7374%3D7374%2F%2A%2A%2FanD%2F%2A%2A%2FELt%286133%2F%2A%2A%2FBeTweEN%2F%2A%2A%2F6133%2F%2A%2A%2FanD%2F%2A%2A%2F6133%2C4322%29--%2F%2A%2A%2FAqHq HTTP/1.1\r\nAccept: */*\r\nAccept-Encoding: gzip,deflate\r\nCache-Control: no-cache\r\nConnection: close\r\nHost: 103.219.114.117\r\nUser-Agent: Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1)","source_ip":"216.131.82.64","filename":null,"known_file":false,"source":["216.131.88.64",57540],"version":"3.1.3-dev","http_host":"103.219.112.117","time":"2022-09-08 07:00:30","sensor_ip":"192.168.81.12","source_port":57540,"sensorid":"6ea7fbee-f343-4db4-9dbd-55b1926ae53e","request_url":"/cgi-bin/voteList.asp?item_ID=%29%2F%2A%2A%2FaS%2F%2A%2A%2FGmeF%2F%2A%2A%2FwHerE%2F%2A%2A%2F7374%3D7374%2F%2A%2A%2FanD%2F%2A%2A%2FELt%286133%2F%2A%2A%2FBeTweEN%2F%2A%2A%2F6133%2F%2A%2A%2FanD%2F%2A%2A%2F6133%2C4322%29--%2F%2A%2A%2FAqHq","file_sha256":null,"sensor_port":80},"channel":"glastopf.events"}
{"_id":{"$oid":"631930be4e016a52d1af911c"},"ident":"honeypot3-glast","timestamp":{"$date":"2022-09-08T00:01:02.762Z"},"normalized":true,"payload":{"pattern":"unknown","request_raw":"GET /cgi-bin/voteList.asp?item_ID=%60%2F%2A%2A%2FWhere%2F%2A%2A%2F8781%3D8781%2F%2A%2A%2FaNd%2F%2A%2A%2FeLt%282062%2F%2A%2A%2FbetWEen%2F%2A%2A%2F6788%2F%2A%2A%2FaNd%2F%2A%2A%2F6788%2C6788%29--%2F%2A%2A%2FXZeM HTTP/1.1\r\nAccept: */*\r\nAccept-Encoding: gzip,deflate\r\nCache-Control: no-cache\r\nConnection: close\r\nHost: 103.219.112.117\r\nUser-Agent: Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1)","source_ip":"216.131.88.64","filename":null,"known_file":false,"source":["216.131.88.64",51719],"version":"3.1.3-dev","http_host":"103.219.112.117","time":"2022-09-08 07:01:00","sensor_ip":"192.168.81.12","source_port":51719,"sensorid":"6ea7fbee-f343-4db4-9dbd-55b1926ae53e","request_url":"/cgi-bin/voteList.asp?item_ID=%60%2F%2A%2A%2FWhere%2F%2A%2A%2F8781%3D8781%2F%2A%2A%2FaNd%2F%2A%2A%2FeLt%282062%2F%2A%2A%2FbetWEen%2F%2A%2A%2F6788%2F%2A%2A%2FaNd%2F%2A%2A%2F6788%2C6788%29--%2F%2A%2A%2FXZeM","file_sha256":null,"sensor_port":80},"channel":"glastopf.events"}
{"_id":{"$oid":"6319310a4e016a52d1af9415"},"ident":"honeypot3-glast","timestamp":{"$date":"2022-09-08T00:02:18.730Z"},"normalized":true,"payload":{"pattern":"unknown","request_raw":"GET /cgi-bin/voteList.asp?item_ID=%60%2F%2A%2A%2FwherE%2F%2A%2A%2F6471%3D6471%2F%2A%2A%2FAnD%2F%2A%2A%2FElT%286133%2F%2A%2A%2FbEtween%2F%2A%2A%2F6133%2F%2A%2A%2FAnD%2F%2A%2A%2F6133%2C4322%29--%2F%2A%2A%2FnpTT HTTP/1.1\r\nAccept: */*\r\nAccept-Encoding: gzip,deflate\r\nCache-Control: no-cache\r\nConnection: close\r\nHost: 103.219.112.119\r\nUser-Agent: Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1)","source_ip":"216.131.88.69","filename":null,"known_file":false,"source":["216.131.88.64",55560],"version":"3.1.3-dev","http_host":"103.219.112.117","time":"2022-09-08 07:02:16","sensor_ip":"192.168.81.12","source_port":55560,"sensorid":"6ea7fbee-f343-4db4-9dbd-55b1926ae53e","request_url":"/cgi-bin/voteList.asp?item_ID=%60%2F%2A%2A%2FwherE%2F%2A%2A%2F6471%3D6471%2F%2A%2A%2FAnD%2F%2A%2A%2FElT%286133%2F%2A%2A%2FbEtween%2F%2A%2A%2F6133%2F%2A%2A%2FAnD%2F%2A%2A%2F6133%2C4322%29--%2F%2A%2A%2FnpTT","file_sha256":null,"sensor_port":80},"channel":"glastopf.events"}

I want to visualize raw json data from glastopf in Kibana.

Do you also know how to combine source IP addresses from Cowrie, Dionaea, and Glastopf to form one Kibana table view?

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.