How to parse my json logs as grok pattern in logstash's ".config" file

Elastic Stack Logstash
1

Hi,

my log file looks like the following:

{"version":"0","id":"37a685ca-2e11-6b2-85d5-f09101abc58cec7","detail-type":"GuardDuty Finding","source":"aws.guardduty","account":"******1326","time":"2018-11-09T12:07:07Z","region":"us-north-1","resources":,"detail":{"schemaVersion":"2.0","accountId":"******1326","region":"us-north-1","partition":"aws","id":"a4b2d70661583abcde8b0785d8c50ad0e01c9","arn":"arn:aws:guardduty:us-north-1:******1326:detector/7ab26e1eabc65c9c011295ae0535b0111a0/finding/a4b2d7066158cde38b0785d8c50ad0e01c9","type":"Recon:EC2/PortProbeUnprotectedPort","resource":{"resourceType":"Instance","instanceDetails":{"instanceId":"i-3e9abcdefgh32dcffshj","instanceType":"x2.large","launchTime":"2015-01-26T16:56:21Z","platform":"windows","productCodes":,"iamInstanceProfile":null,"networkInterfaces":,"tags":[{"key":"Name","value":"Server"}],"instanceState":"running","availabilityZone":"us-north-1b","imageId":"ami-aa686jya2fc2","imageDescription":"Backup "}},"service":{"serviceName":"guardduty","detectorId":"7ab26e1e65c9c01jhsfjs1295ae0535b0111a0","action":{"actionType":"PORT_PROBE","portProbeAction":{"portProbeDetails":[{"localPortDetails":{"port":80,"portName":"HTTP"},"remoteIpDetails":{"ipAddressV4":"196.52.43.89","organization":{"asn":"60781","asnOrg":"LeaseWeb Netherlands B.V.","isp":"LogicWeb Inc","org":"LogicWeb Inc"},"country":{"countryName":"United States"},"city":{"cityName":"Edison"},"geoLocation":{"lat":40.4981,"lon":-74.3194}}}],"blocked":false}},"resourceRole":"TARGET","additionalInfo":{"threatName":"Scanner","threatListName":"ProofPoint"},"eventFirstSeen":"2018-09-06T08:19:08Z","eventLastSeen":"2018-11-09T11:40:18Z","archived":false,"count":159},"severity":2,"createdAt":"2018-09-06T08:43:16.528Z","updatedAt":"2018-11-09T11:59:19.442Z","title":"Unprotected port on EC2 instance i-3e93yjhjkhu2df is being probed.","description":"EC2 instance has an unprotected port which is being probed by a known malicious host."}}

I was able to send these logs to elasticsearch through logstash and was able to search them. But couldn't make visualizations as they are in json format. Can you please let me know how can you parse a json file like this in form of grok patterns?

2

Logstash inputs accept JSON formatted logs if you use the JSON codec. There's no need for grok patterns to parse JSON formatted logs.

3

My .conf file looks like following:

input {
                s3 {
                access_key_id => "XXXXXXXXXXXXXXXXXXX"
                secret_access_key => "XXXXXXXXXXXXXXXXXXXXXXXXXXX"
                bucket => "test01"
                region => "us-east-1"
                prefix => "test/xxxx/"
                type => "s3"
                 codec => json
                }
        }

  output {
        elasticsearch {
                hosts => "XX.XX.XX.XX:9200"
                index => "TestInstance-index"
                             }
           }

This works

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.