Single JSON document parsed as individual key pair in logstash

Elastic Stack Logstash
1

Hello,

I hope my message find the community members and their loved ones safe and healthy.

I am trying to ingest ~30,000 documents which are API responses with selective keys stored as a JSON document.

[
    {
        "analysis_date": 1657304659,
        "dns_lookups": [
            {
                "hostname": "mail.zonerouge.net",
                "resolved_ips": [
                    "46.105.38.205"
                ]
            }
        ],
        "http_conversations": [
            {
                "request_headers": {
                    "Content-Length": "0",
                    "User-Agent": "Microsoft Office Protocol Discovery"
                },
                "request_method": "OPTIONS",
                "response_headers": {
                    "Alt-Svc": "h3=\":443\"; ma=2592000, h3-29=\":443\"; ma=2592000, h3-Q050=\":443\"; ma=2592000, h3-Q046=\":443\"; ma=2592000, h3-Q043=\":443\"; ma=2592000, quic=\":443\"; ma=2592000; v=\"43,46\"",
                    "Cache-Control": "no-store, no-cache, must-revalidate",
                    "Connection": "close",
                    "Content-Security-Policy": "upgrade-insecure-requests",
                    "Content-Type": "text/html; charset=UTF-8",
                    "Date": "Fri, 08 Jul 2022 18:18:44 GMT",
                    "Expires": "Thu, 19 Nov 1981 08:52:00 GMT",
                    "Keep-Alive": "timeout=5, max=100",
                    "Pragma": "no-cache",
                    "Server": "LiteSpeed",
                    "Set-Cookie": "PHPSESSID=fd1fcfba0e2fb81ba3ceacdaa5221dcc; path=/; secure",
                    "Transfer-Encoding": "chunked",
                    "Vary": "Accept-Encoding",
                    "X-Powered-By": "PHP/7.4.26"
                },
                "response_status_code": 405,
                "url": "https://a-lynk.com:443/"
            },
            {
                "request_headers": {
                    "Cookie": "PHPSESSID=fd1fcfba0e2fb81ba3ceacdaa5221dcc",
                    "User-Agent": "Microsoft Office Existence Discovery"
                },
                "request_method": "HEAD",
                "response_headers": {
                    "Cache-Control": "no-store, no-cache, must-revalidate",
                    "Connection": "close",
                    "Content-Security-Policy": "upgrade-insecure-requests",
                    "Content-Type": "text/html; charset=UTF-8",
                    "Date": "Fri, 08 Jul 2022 18:18:45 GMT",
                    "Expires": "Thu, 19 Nov 1981 08:52:00 GMT",
                    "Keep-Alive": "timeout=5, max=100",
                    "Location": "http://192.3.247.133/dhl/receipt.doc",
                    "Pragma": "no-cache",
                    "Server": "LiteSpeed",
                    "Transfer-Encoding": "chunked",
                    "X-Powered-By": "PHP/7.4.26"
                },
                "response_status_code": 301,
                "url": "https://a-lynk.com:443/POzVE"
            },
            {
                "request_headers": {
                    "Accept": "*/*",
                    "Cookie": "PHPSESSID=fd1fcfba0e2fb81ba3ceacdaa5221dcc",
                    "User-Agent": "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; ms-office; MSOffice 14)"
                },
                "request_method": "GET",
                "response_headers": {
                    "Cache-Control": "no-store, no-cache, must-revalidate",
                    "Connection": "close",
                    "Content-Security-Policy": "upgrade-insecure-requests",
                    "Content-Type": "text/html; charset=UTF-8",
                    "Date": "Fri, 08 Jul 2022 18:18:49 GMT",
                    "Expires": "Thu, 19 Nov 1981 08:52:00 GMT",
                    "Keep-Alive": "timeout=5, max=100",
                    "Location": "http://192.3.247.133/dhl/receipt.doc",
                    "Pragma": "no-cache",
                    "Server": "LiteSpeed",
                    "Set-Cookie": "short_41=1; expires=Fri, 08-Jul-2022 18:33:49 GMT; Max-Age=900; path=/; HttpOnly; secure",
                    "Transfer-Encoding": "chunked",
                    "X-Powered-By": "PHP/7.4.26"
                },
                "response_status_code": 301,
                "url": "https://a-lynk.com:443/POzVE"
            },
            {
                "request_headers": {
                    "Accept": "*/*",
                    "User-Agent": "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; ms-office; MSOffice 14)"
                },
                "request_method": "GET",
                "response_headers": {
                    "Accept-Ranges": "bytes",
                    "Content-Type": "application/msword",
                    "Date": "Fri, 08 Jul 2022 18:18:50 GMT",
                    "Etag": "\"4e6b-5e34bdeab0ead\"",
                    "Last-Modified": "Fri, 08 Jul 2022 14:18:01 GMT",
                    "Server": "Apache/2.4.53 (Win64) OpenSSL/1.1.1n PHP/7.4.29"
                },
                "response_status_code": 200,
                "url": "http://192.3.247.133/dhl/receipt.doc"
            },
            {
                "request_headers": {
                    "Cache-Control": "no-cache",
                    "Content-Length": "0",
                    "Pragma": "no-cache",
                    "User-Agent": "Microsoft Office Existence Discovery"
                },
                "request_method": "HEAD",
                "response_headers": {
                    "Accept-Ranges": "bytes",
                    "Content-Type": "application/msword",
                    "Date": "Fri, 08 Jul 2022 18:18:51 GMT",
                    "Etag": "\"4e6b-5e34bdeab0ead\"",
                    "Last-Modified": "Fri, 08 Jul 2022 14:18:01 GMT",
                    "Server": "Apache/2.4.53 (Win64) OpenSSL/1.1.1n PHP/7.4.29"
                },
                "response_status_code": 200,
                "url": "http://192.3.247.133/dhl/receipt.doc"
            },
            {
                "request_headers": {
                    "Accept": "*/*",
                    "User-Agent": "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)"
                },
                "request_method": "GET",
                "response_headers": {
                    "Accept-Ranges": "bytes",
                    "Content-Type": "application/x-msdownload",
                    "Date": "Fri, 08 Jul 2022 18:18:51 GMT",
                    "Etag": "\"f9600-5e34cb6529f80\"",
                    "Last-Modified": "Fri, 08 Jul 2022 15:18:19 GMT",
                    "Server": "Apache/2.4.53 (Win64) OpenSSL/1.1.1n PHP/7.4.29"
                },
                "response_status_code": 200,
                "url": "http://192.3.247.133/234/vbc.exe"
            }
        ],
        "id": "043412f382965c8b43850f4acaf2e4bdf253827f8d7ab849a081455f01c0898a_C2AE",
        "memory_pattern_domains": [
            "a-lynk.com"
        ],
        "memory_pattern_ips": [
            "192.3.247.133"
        ],
        "memory_pattern_urls": [
            "https://a-lynk.com/POzVE",
            "http://192.3.247.133/234/vbc.exe"
        ],
        "sandbox_name": "C2AE",
        "verdicts": [
            "MALWARE"
        ]
    },
    {
        "analysis_date": 1657367404,
        "dns_lookups": [
            {
                "hostname": "a-lynk.com",
                "resolved_ips": [
                    "46.17.175.231"
                ]
            },
            {
                "hostname": "time.windows.com",
                "resolved_ips": [
                    "20.101.57.9"
                ]
            }
        ],
        "id": "043412f382965c8b43850f4acaf2e4bdf253827f8d7ab849a081455f01c0898a_SecondWrite",
        "ip_traffic": [
            {
                "destination_ip": "224.0.0.252",
                "destination_port": 5355,
                "transport_layer_protocol": "UDP"
            },
            {
                "destination_ip": "255.255.255.255",
                "destination_port": 67,
                "transport_layer_protocol": "UDP"
            },
            {
                "destination_ip": "46.17.175.231",
                "destination_port": 443,
                "transport_layer_protocol": "TCP"
            }
        ],
        "sandbox_name": "SecondWrite",
        "verdicts": [
            "MALWARE",
            "SPREADER"
        ]
    },
    {
        "analysis_date": 1657337051,
        "dns_lookups": [
            {
                "hostname": "a-lynk.com"
            }
        ],
        "id": "043412f382965c8b43850f4acaf2e4bdf253827f8d7ab849a081455f01c0898a_VenusEye Sandbox",
        "sandbox_name": "VenusEye Sandbox"
    },
    {
        "analysis_date": 1657300929,
        "dns_lookups": [
            {
                "hostname": "a-lynk.com",
                "resolved_ips": [
                    "46.17.175.231"
                ]
            },
            {
                "hostname": "mail.zonerouge.net",
                "resolved_ips": [
                    "46.105.38.205"
                ]
            },
            {
                "hostname": "nexus.officeapps.live.com",
                "resolved_ips": [
                    "52.109.76.33",
                    "52.109.88.37",
                    "52.109.88.39"
                ]
            }
        ],
        "http_conversations": [
            {
                "request_headers": {
                    "Host": "192.3.247.133",
                    "User-Agent": "Microsoft Office Word 2014"
                },
                "request_method": "HEAD",
                "response_headers": {
                    "Content-Type": "application/msword"
                },
                "response_status_code": 200,
                "url": "http://192.3.247.133/dhl/receipt.doc"
            },
            {
                "request_headers": {
                    "Host": "192.3.247.133",
                    "User-Agent": "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.2; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729)"
                },
                "request_method": "GET",
                "response_body_filetype": "PE_EXE",
                "response_headers": {
                    "Content-Length": "1021440",
                    "Content-Type": "application/x-msdownload"
                },
                "response_status_code": 200,
                "url": "http://192.3.247.133/234/vbc.exe"
            }
        ],
        "id": "043412f382965c8b43850f4acaf2e4bdf253827f8d7ab849a081455f01c0898a_Zenbox",
        "ip_traffic": [
            {
                "destination_ip": "46.17.175.231",
                "destination_port": 443,
                "transport_layer_protocol": "TCP"
            },
            {
                "destination_ip": "192.3.247.133",
                "destination_port": 80,
                "transport_layer_protocol": "TCP"
            },
            {
                "destination_ip": "46.105.38.205",
                "destination_port": 587,
                "transport_layer_protocol": "TCP"
            },
            {
                "destination_ip": "8.238.8.126",
                "destination_port": 80,
                "transport_layer_protocol": "TCP"
            },
            {
                "destination_ip": "52.109.76.141",
                "destination_port": 443,
                "transport_layer_protocol": "TCP"
            }
        ],
        "sandbox_name": "Zenbox",
        "verdicts": [
            "MALWARE",
            "STEALER",
            "TROJAN",
            "EVADER"
        ]
    }
]

I am using filebeat to send the document to a logstash ingest pipeline with the following configuration:

input {
  beats {
    port => 5557 #port for filebeat - Oxford  only
    #id => "beats_ingest"
  }
}


filter {
    json {
      source => "message"
    }

        date {
          match => ["analysis_date", "UNIX"]
          target => "analysis_date"
          }

        geoip {
          source => "resolved_ips"
          target => "geoip"
          database => "/opt/logstash/vendor/geoip/GeoLite2-City.mmdb"

  }

}


output {
  elasticsearch {

I have attempted with and without the following in the filebeat configuration
Log input | Filebeat Reference [7.17] | Elastic?

- type: filestream
- ndjson:
     json.expand_keys: true

and

- type: filestream
  json.expand_keys: true

and

- type: filestream
  json.keys_under_root: true

However, the file isn't getting parsed correctly. Kindly help.

2

If your file contains pretty-printed JSON then I would suggest using filebeat to do the multiline processing and JSON parsing. You should ask about that in the filebeat thread, not this one. Make sure you say whether you want the entire file consumed as one item.

If filebeat parses the JSON you will not need a json filter in lgostash.

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.