Pasring Json file

Elastic Stack Logstash
1

Hi All,
I'm trying to parse json file but I'm getting error and incorrect result

the log file look like this:

{
"Records": [{
"EventVersion": "1.0",
"EventSubscriptionArn": "XXX",
"EventSource": "XXX",
"Sns": {
"SignatureVersion": "1",
"Timestamp": "2019-02-11T11:24:00.000",
"Signature": "XXX",
"SigningCertUrl": "XXX",
"MessageId": "XXX",
"Message": "{"notificationType":"Delivery","mail":{"timestamp":"2019-02-10T11:24:52.986Z","source":"XXX.com","sourceArn":"XXX.com","sourceIp":"212.00.00.100","sendingAccountId":"081469076013","messageId":"4654654-000000","destination":["XXX.com"]},"delivery":{"timestamp":"2019-02-11T11:24:54.199Z","processingTimeMillis":1213,"recipients":["XXX.com"],"smtpResponse":"100 2.6.0 01000168dc4d7f7a-3ae05619-0b9e-4deb-82d2-5555018-000000@email.amazonses.com [InternalId=001, Hostname=Aoutlook.com] 10536 bytes in 0.187, 54.735 KB/sec Queued mail for delivery","remoteMtaIp":"100.00.0.55","reporti":"amazonses.com"}}",
"MessageAttributes": {},
"Type": "Notification",
"UnsubscribeUrl": "XXX",
"TopicArn": "XXX",
"Subject": null
}
}]
}

the conf file look like this:

filter {
if "emails_logs" in [tags] {
json {
source => "message"
target => "message"
}
json {
source => "[message][Records]"
target => "[message][Records]"
}

split { field => "[message][Records][Sns]" }
split { field => "[message][Records][Sns][Message]" }

}
}

Can you help me to parse it?

2

The json posted is invalid. On the line where it is says message there are extra " caracters.

This is the correct json

{
	"Records": [{
		"EventVersion": "1.0",
		"EventSubscriptionArn": "XXX",
		"EventSource": "XXX",
		"Sns": {
			"SignatureVersion": "1",
			"Timestamp": "2019-02-11T11:24:00.000",
			"Signature": "XXX",
			"SigningCertUrl": "XXX",
			"MessageId": "XXX",
			"Message": {
				"notificationType": "Delivery",
				"mail": {
					"timestamp": "2019-02-10T11:24:52.986Z",
					"source": "XXX.com",
					"sourceArn": "XXX.com",
					"sourceIp": "212.00.00.100",
					"sendingAccountId": "081469076013",
					"messageId": "4654654-000000",
					"destination": ["XXX.com"]
				},
				"delivery": {
					"timestamp": "2019-02-11T11:24:54.199Z",
					"processingTimeMillis": 1213,
					"recipients": ["XXX.com"],
					"smtpResponse": "100 2.6.0 01000168dc4d7f7a-3ae05619-0b9e-4deb-82d2-5555018-000000@email.amazonses.com [InternalId=001, Hostname=Aoutlook.com] 10536 bytes in 0.187, 54.735 KB/sec Queued mail for delivery",
					"remoteMtaIp": "100.00.0.55",
					"reporti": "amazonses.com"
				}
			},
			"MessageAttributes": {},
			"Type": "Notification",
			"UnsubscribeUrl": "XXX",
			"TopicArn": "XXX",
			"Subject": null
		}
	}]
}
3

Tnx,
Do you have idea how to parse it?

4

Ok, not sure if this is what you need but this is what I got.

Config:

input {
  file {
    path => '/home/pjanzen/json.txt'
    sincedb_path => '/dev/null'
    start_position => 'beginning'
  }
}

filter {
  json {
    source => "message"
  }
}

output {
  stdout {
    codec => "rubydebug"
  }
}

Output:

{
          "path" => "/home/pjanzen/json.txt",
       "message" => "{\"Records\": [{\"EventVersion\": \"1.0\",\"EventSubscriptionArn\": \"XXX\",\"EventSource\": \"XXX\",\"Sns\": {\"SignatureVersion\": \"1\",\"Timestamp\": \"2019-02-11T11:24:00.000\",\"Signature\": \"XXX\",\"SigningCertUrl\": \"XXX\",\"MessageId\": \"XXX\",\"Message\": {\"notificationType\": \"Delivery\",\"mail\": {\"timestamp\": \"2019-02-10T11:24:52.986Z\",\"source\": \"XXX.com\",\"sourceArn\": \"XXX.com\",\"sourceIp\": \"212.00.00.100\",\"sendingAccountId\": \"081469076013\",\"messageId\": \"4654654-000000\",\"destination\": [\"XXX.com\"]},\"delivery\": {\"timestamp\": \"2019-02-11T11:24:54.199Z\",\"processingTimeMillis\": 1213,\"recipients\": [\"XXX.com\"],\"smtpResponse\": \"100 2.6.0 01000168dc4d7f7a-3ae05619-0b9e-4deb-82d2-5555018-000000@email.amazonses.com [InternalId=001, Hostname=Aoutlook.com] 10536 bytes in 0.187, 54.735 KB/sec Queued mail for delivery\",\"remoteMtaIp\": \"100.00.0.55\",\"reporti\": \"amazonses.com\"}},\"MessageAttributes\": {},\"Type\": \"Notification\",\"UnsubscribeUrl\": \"XXX\",\"TopicArn\": \"XXX\",\"Subject\": null}}]}",
    "@timestamp" => 2019-02-18T13:21:06.977Z,
      "@version" => "1",
       "Records" => [
        [0] {
                    "EventVersion" => "1.0",
            "EventSubscriptionArn" => "XXX",
                             "Sns" => {
                          "Message" => {
                                "mail" => {
                               "timestamp" => "2019-02-10T11:24:52.986Z",
                        "sendingAccountId" => "081469076013",
                               "messageId" => "4654654-000000",
                             "destination" => [
                            [0] "XXX.com"
                        ],
                                "sourceIp" => "212.00.00.100",
                                  "source" => "XXX.com",
                               "sourceArn" => "XXX.com"
                    },
                            "delivery" => {
                        "processingTimeMillis" => 1213,
                                   "timestamp" => "2019-02-11T11:24:54.199Z",
                                "smtpResponse" => "100 2.6.0 01000168dc4d7f7a-3ae05619-0b9e-4deb-82d2-5555018-000000@email.amazonses.com [InternalId=001, Hostname=Aoutlook.com] 10536 bytes in 0.187, 54.735 KB/sec Queued mail for delivery",
                                     "reporti" => "amazonses.com",
                                 "remoteMtaIp" => "100.00.0.55",
                                  "recipients" => [
                            [0] "XXX.com"
                        ]
                    },
                    "notificationType" => "Delivery"
                },
                        "Signature" => "XXX",
                        "Timestamp" => "2019-02-11T11:24:00.000",
                        "MessageId" => "XXX",
                   "UnsubscribeUrl" => "XXX",
                   "SigningCertUrl" => "XXX",
                         "TopicArn" => "XXX",
                          "Subject" => nil,
                 "SignatureVersion" => "1",
                "MessageAttributes" => {},
                             "Type" => "Notification"
            },
                     "EventSource" => "XXX"
        }
    ],
          "host" => "tb-clog-ls1"
}
1 Like
5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.