How do I parse CEF messages comprises of json fields in between?

Elastic Stack Logstash
1

Hi Team,

I am accepting Cisco Ironport messages into elasticsearch using logstash cef plugin. However certain messages comprises of json fields in between and those are not getting parsed. Can someone please help me about the parsing of same?

Here is the test message

{
  "_index": "mx-2022.09.19",
  "_type": "_doc",
  "_id": "t4HkVoMBPvhXb-xCeTjP",
  "_version": 1,
  "_score": 1,
  "_source": {
    "ESAICID": "17509866",
    "deviceCustomString3": "N/A",
    "sourceUserName": "vinutshetti049@gmail.com",
    "port": 60752,
    "ESADMARCVerdict": "pass",
    "message": "'Resume'",
    "deviceCustomString1": "DEFAULT",
    "ESATLSOutCipher": "ECDHE-RSA-AES256-GCM-SHA384",
    "deviceProduct": "C600V Email Security Virtual Appliance",
    "ESAHeloDomain": "mail-lj1-f178.google.com",
    "ESACFVerdict": "NO_MATCH",
    "ESATLSInCipher": "ECDHE-RSA-AES128-GCM-SHA256",
    "deviceVendor": "Cisco",
    "@timestamp": "2022-09-19T17:55:57.416Z",
    "ESAAMPVerdict": "UNKNOWN",
    "sourceAddress": "xxx.xxx.xx",
    "deviceDirection": "0",
    "deviceVersion": "14.2.0-620",
    "host": "10.11.44.10",
    "deviceCustomString2Label": "SenderCountry",
    "ESAASVerdict": "NEGATIVE",
    "cefVersion": "0",
    "sourceHostName": "mail-lj1-f178.google.com",
    "deviceExternalId": "564D8B6B9A6E21D2E801-61F9E57CD134",
    "ESAMID": "19308843",
    "ESAHeloIP": "209.85.208.178",
    "deviceCustomString1Label": "MailPolicy",
    "ESAAttachmentDetails": "{'nkj.docx': {'AMP': {'Verdict': 'FILE UNKNOWN', 'fileHash': '0d2230a2af8053646d899b59dff7dcd673db6398b4f2af9967b4fa57c8c7359c'}, 'BodyScanner': {}}}",
    "ESAAVVerdict": "NEGATIVE",
    "deviceAddress": "103.161.42.240",
    "severity": "5",
    "name": "Consolidated Log Event",
    "ESASenderGroup": "UNKNOWNLIST",
    "syslog": "<14>Sep 19 23:25:57 SIEM_ISN_2:",
    "type": "syslog",
    "deviceCustomString3Label": "SDRThreatCategory",
    "deviceEventClassId": "ESA_CONSOLIDATED_LOG_EVENT",
    "@version": "1",
    "deviceCustomString2": "United States",
    "ESAGMVerdict": "NEGATIVE",
    "ESASPFVerdict": "{'mailfrom': {'result': 'Pass', 'sender': 'vinutshetti049@gmail.com'}, 'helo': {'result': 'None', 'sender': 'postmaster@mail-lj1-f178.google.com'}}",
    "destinationUserName": "xxx@xxx.com",
    "ESADCID": "8198674"
  },
  "fields": {
    "deviceCustomString2Label.keyword": [
      "SenderCountry"
    ],
    "deviceAddress.keyword": [
      "103.161.42.240"
    ],
    "deviceCustomString1.keyword": [
      "DEFAULT"
    ],
    "ESATLSInCipher": [
      "ECDHE-RSA-AES128-GCM-SHA256"
    ],
    "deviceCustomString3Label": [
      "SDRThreatCategory"
    ],
    "deviceDirection.keyword": [
      "0"
    ],
    "deviceEventClassId.keyword": [
      "ESA_CONSOLIDATED_LOG_EVENT"
    ],
    "deviceCustomString3.keyword": [
      "N/A"
    ],
    "type": [
      "syslog"
    ],
    "cefVersion.keyword": [
      "0"
    ],
    "severity.keyword": [
      "5"
    ],
    "ESADMARCVerdict": [
      "pass"
    ],
    "ESAAMPVerdict": [
      "UNKNOWN"
    ],
    "type.keyword": [
      "syslog"
    ],
    "deviceVersion.keyword": [
      "14.2.0-620"
    ],
    "host": [
      "10.11.44.10"
    ],
    "ESAHeloDomain": [
      "mail-lj1-f178.google.com"
    ],
    "ESAMID": [
      "19308843"
    ],
    "host.keyword": [
      "10.11.44.10"
    ],
    "ESAHeloIP": [
      "209.85.208.178"
    ],
    "deviceCustomString2Label": [
      "SenderCountry"
    ],
    "ESASenderGroup": [
      "UNKNOWNLIST"
    ],
    "ESAHeloIP.keyword": [
      "209.85.208.178"
    ],
    "ESAASVerdict.keyword": [
      "NEGATIVE"
    ],
    "ESAAVVerdict": [
      "NEGATIVE"
    ],
    "ESASenderGroup.keyword": [
      "UNKNOWNLIST"
    ],
    "ESAGMVerdict": [
      "NEGATIVE"
    ],
    "name.keyword": [
      "Consolidated Log Event"
    ],
    "@version.keyword": [
      "1"
    ],
    "deviceProduct.keyword": [
      "C600V Email Security Virtual Appliance"
    ],
    "sourceAddress.keyword": [
      "209.85.208.178"
    ],
    "ESACFVerdict.keyword": [
      "NO_MATCH"
    ],
    "deviceEventClassId": [
      "ESA_CONSOLIDATED_LOG_EVENT"
    ],
    "port": [
      60752
    ],
    "message.keyword": [
      "'Resume'"
    ],
    "name": [
      "Consolidated Log Event"
    ],
    "cefVersion": [
      "0"
    ],
    "deviceDirection": [
      "0"
    ],
    "ESAMID.keyword": [
      "19308843"
    ],
    "sourceHostName": [
      "mail-lj1-f178.google.com"
    ],
    "sourceHostName.keyword": [
      "mail-lj1-f178.google.com"
    ],
    "ESAAttachmentDetails": [
      "{'nkj.docx': {'AMP': {'Verdict': 'FILE UNKNOWN', 'fileHash': '0d2230a2af8053646d899b59dff7dcd673db6398b4f2af9967b4fa57c8c7359c'}, 'BodyScanner': {}}}"
    ],
    "ESASPFVerdict": [
      "{'mailfrom': {'result': 'Pass', 'sender': 'vinutshetti049@gmail.com'}, 'helo': {'result': 'None', 'sender': 'postmaster@mail-lj1-f178.google.com'}}"
    ],
    "destinationUserName.keyword": [
      "xxx@xxx.com"
    ],
    "deviceVendor": [
      "Cisco"
    ],
    "deviceCustomString3Label.keyword": [
      "SDRThreatCategory"
    ],
    "ESADCID": [
      "8198674"
    ],
    "ESATLSOutCipher": [
      "ECDHE-RSA-AES256-GCM-SHA384"
    ],
    "syslog": [
      "<14>Sep 19 23:25:57 SIEM_ISN_2:"
    ],
    "ESACFVerdict": [
      "NO_MATCH"
    ],
    "deviceCustomString2.keyword": [
      "United States"
    ],
    "ESADCID.keyword": [
      "8198674"
    ],
    "deviceCustomString1Label.keyword": [
      "MailPolicy"
    ],
    "deviceExternalId": [
      "564D8B6B9A6E21D2E801-61F9E57CD134"
    ],
    "destinationUserName": [
      "xxxx@xxx.com"
    ],
    "ESASPFVerdict.keyword": [
      "{'mailfrom': {'result': 'Pass', 'sender': 'vinutshetti049@gmail.com'}, 'helo': {'result': 'None', 'sender': 'postmaster@mail-lj1-f178.google.com'}}"
    ],
    "deviceAddress": [
      "103.161.42.240"
    ],
    "deviceVendor.keyword": [
      "Cisco"
    ],
    "sourceUserName.keyword": [
      "vinutshetti049@gmail.com"
    ],
    "@version": [
      "1"
    ],
    "deviceProduct": [
      "C600V Email Security Virtual Appliance"
    ],
    "deviceExternalId.keyword": [
      "564D8B6B9A6E21D2E801-61F9E57CD134"
    ],
    "ESADMARCVerdict.keyword": [
      "pass"
    ],
    "ESAICID.keyword": [
      "17509866"
    ],
    "severity": [
      "5"
    ],
    "ESAASVerdict": [
      "NEGATIVE"
    ],
    "ESAAMPVerdict.keyword": [
      "UNKNOWN"
    ],
    "sourceAddress": [
      "209.85.208.178"
    ],
    "ESAGMVerdict.keyword": [
      "NEGATIVE"
    ],
    "sourceUserName": [
      "vinutshetti049@gmail.com"
    ],
    "deviceCustomString1": [
      "DEFAULT"
    ],
    "deviceCustomString3": [
      "N/A"
    ],
    "deviceCustomString2": [
      "United States"
    ],
    "deviceVersion": [
      "14.2.0-620"
    ],
    "deviceCustomString1Label": [
      "MailPolicy"
    ],
    "message": [
      "'Resume'"
    ],
    "ESAAttachmentDetails.keyword": [
      "{'nkj.docx': {'AMP': {'Verdict': 'FILE UNKNOWN', 'fileHash': '0d2230a2af8053646d899b59dff7dcd673db6398b4f2af9967b4fa57c8c7359c'}, 'BodyScanner': {}}}"
    ],
    "ESAICID": [
      "17509866"
    ],
    "ESAAVVerdict.keyword": [
      "NEGATIVE"
    ],
    "@timestamp": [
      "2022-09-19T17:55:57.416Z"
    ],
    "syslog.keyword": [
      "<14>Sep 19 23:25:57 SIEM_ISN_2:"
    ],
    "ESAHeloDomain.keyword": [
      "mail-lj1-f178.google.com"
    ],
    "ESATLSOutCipher.keyword": [
      "ECDHE-RSA-AES256-GCM-SHA384"
    ],
    "ESATLSInCipher.keyword": [
      "ECDHE-RSA-AES128-GCM-SHA256"
    ]
  }
}

If you see here I need to parse a message field

"ESAAttachmentDetails.keyword": [
      "{'nkj.docx': {'AMP': {'Verdict': 'FILE UNKNOWN', 'fileHash': '0d2230a2af8053646d899b59dff7dcd673db6398b4f2af9967b4fa57c8c7359c'}, 'BodyScanner': {}}}"
    ],

"ESAAttachmentDetails": [
      "{'nkj.docx': {'AMP': {'Verdict': 'FILE UNKNOWN', 'fileHash': '0d2230a2af8053646d899b59dff7dcd673db6398b4f2af9967b4fa57c8c7359c'}, 'BodyScanner': {}}}"
2

If your event has fields that are valid JSON you can parse them using a json filter.

3

Would you please give me any examples?

4

The documentation I linked to has examples.

5

This is not working as expected and that does not look like json either but wanted to parse those fields and thinking how do I do it.

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.