Hi there,
I hope this is the right place to ask:
I'm trying to integrate logs from Cisco Secure Email Gateway.
It's the managed version of Cisco (Secure Email Cloud Gateway), but it's basically the same product. The version is 15.0.1-030.
The log files are exported via SCP to a system running Elastic Agent.
I'm mostly struggling with "consolidated_event". A sample log file line looks like this:
Mon Apr 8 08:50:06 2024: CEF:0|Cisco|C100V Secure Email Gateway Virtual|15.0.1-030|ESA_CONSOLIDATED_L OG_EVENT|Consolidated Log Event|5|deviceExternalId=42355C9A2DB8AA27627B-57A67969AEBE ESAMID=1985833 ESAICID=5004737 ESADCID=1226971 ESAAMPVerdict=UNKNOWN ESAASVerdict=NEGATIVE ESAAVVerdict=NOT_EVALUATED ESACFVerdict=MATCH ESADLPVerdict=NO_TRIGGER dvc=23.90.104.113 ESAAttachmentDetails={'IMG_0219a.jpg': {'AMP': {'Verdict': 'FILE UNKNOWN', 'fileHash': '1b4e1cdff762c90a7a7dc39bfe3b3e1229ef7f0c4b2deb4d3f8903fe91b50348'}, 'BodyScanner': {}}, 'IMG_0242.JPG': {'AMP': {'Verdict': 'FILE UNKNOWN', 'fileHash': 'a9778569656fe913a7d538dkf91d700e8b9f89ac193d4d30a7d3ebdd33401a70'}, 'BodyScanner': {}}, 'IMG_0246a.jpg': {'AMP': {'Verdict': 'FILE UNKNOWN', 'fileHash': 'c1fc224c3382f8e1758d277947136ce141a320038d10ec5452362b1c48c84520'}, 'BodyScanner': {}}} ESAFriendlyFrom="Name, Name" <user@domain.com> ESAGMVerdict=NOT_EVALUATED deviceOutboundInterface=IncomingMail deviceDirection=1 ESAMailFlowPolicy=RELAY suser=user@domain.com cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=Germany ESAMFVerdict=NO_MATCH act=DELIVERED cs4Label=ExternalMsgID cs4='<5306186a662f4f8ae156186921b4e389@domain.com>' ESAMsgSize=3003778 ESAOFVerdict=NOT_EVALUATED duser=user@domain.com;user@domain.com;user@domain.com ESAHeloDomain=server.domain.com ESAHeloIP=97.153.92.98 cfp1Label=SBRSScore cfp1=5.2 shost=vpn.domain.com ESASenderGroup=RELAYLIST src=42.153.70.96 msg='\=?utf-8?B?V0c6IFtFWFRFUk46XSBBbm1lbGR1bmcgUFYtQW5sYWdlIG5hY2ggdmVyZWlu?\= \=?utf-8?B?ZmFjaHRlbSBBbm1lbGRldmVyZmFocmVnLCBTdGFuZG9ydDogU2llYmVuZWlj?\= \=?utf-8?B?aGVuZXIgU3RyLiA0MiBpbkAwMTY2MiBNZWnDn2Vu?\=' ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESATLSOutCipher=ECDHE-RSA-AES128-GCM-SHA256 ESATLSOutConnStatus=Success ESATLSOutProtocol=TLSv1.2 end=Mon Apr 8 08:50:03 2024 start=Mon Apr 8 08:49:49 2024
It's a little different from your sample log files, but I can't figure out what exactly isn't working here 
The document in Kibana looks like this after processing:
{
"_index": ".ds-logs-cisco_secure_email_gateway.log-default-2024.04.04-000001",
"_id": "kuwkxI4B7ZwEN1o5m7_D",
"_version": 1,
"_score": 0,
"_ignored": [
"_tmp.timestamp"
],
"_source": {
"agent": {
"name": "vmlog1",
"id": "f3861429-7aeb-4b5d-83ff-a7f7134a4ef9",
"ephemeral_id": "9fe44d98-0c46-4cfb-a7c8-cbf29735ef49",
"type": "filebeat",
"version": "8.13.2"
},
"log": {
"file": {
"path": "/temp/consolidated_event.esa3.name.iphmx.com.@20240409T192001.s"
},
"offset": 160755,
"level": "2024"
},
"elastic_agent": {
"id": "f3861429-7aeb-4b5d-83ff-a7f7134a4ef9",
"version": "8.13.2",
"snapshot": false
},
"_conf": {
"tz_offset": "UTC"
},
"error": {
"message": [
"Text 'Tue Apr 9 19:49:20' could not be parsed at index 8"
]
},
"tags": [
"forwarded",
"cisco_secure_email_gateway-log"
],
"input": {
"type": "log"
},
"@timestamp": "2024-04-09T18:33:06.065Z",
"cisco_secure_email_gateway": {
"log": {
"host": "esa3.name.eu.iphmx.com",
"category": {
"name": "consolidated_event"
},
"message": "CEF:0|Cisco|C100V Email Security Virtual Appliance|15.0.1-030|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=423A914879DE5FF34A15-20293010CA5C ESAMID=11343819 ESAICID=52535995 ESAAMPVerdict=NOT_EVALUATED ESAASVerdict=POSITIVE ESAAVVerdict=NOT_EVALUATED ESACFVerdict=NOT_EVALUATED ESADLPVerdict=NOT_EVALUATED ESADMARCVerdict=Skipped dvc=217.54.87.14 ESAFriendlyFrom=\"First Last\" <hello@acompany.co.uk> ESAGMVerdict=NOT_EVALUATED deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=hello@acompany.co.uk cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=unable to retrieve ESAMFVerdict=MATCH act=DROPPED ESAFinalActionDetails=By CASE cs4Label=ExternalMsgID cs4='<8a19b4$aqfmr@esa3.name.iphmx.com>' ESAMsgSize=1098 ESAOFVerdict=NOT_EVALUATED duser=first.last@othercompany.co.uk ESAHeloDomain=luci.acompany.shop ESAHeloIP=14.119.117.185 ESAReplyTo=example1@placeholder.com cfp1Label=SBRSScore cfp1=unable to retrieve ESASPFVerdict={'mailfrom': {'result': 'None', 'sender': 'hello@acompany.co.uk'}, 'helo': {'result': 'None', 'sender': 'postmaster@luci.acompany.shop'}} shost=luci.acompany.shop ESASenderGroup=SUSPECTLIST src=14.119.117.185 msg='UK based' ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 end=Tue Apr 9 19:49:15 2024 start=Tue Apr 9 19:49:13 2024"
}
},
"ecs": {
"version": "8.11.0"
},
"_tmp": {
"filepath": "/temp/consolidated_event.esa3.name.iphmx.com.@20240409T192001.s",
"tz": "UTC",
"timestamp": "Tue Apr 9 19:49:20"
},
"data_stream": {
"namespace": "default",
"type": "logs",
"dataset": "cisco_secure_email_gateway.log"
},
"event": {
"agent_id_status": "verified",
"ingested": "2024-04-09T18:33:06Z",
"kind": "pipeline_error",
"dataset": "cisco_secure_email_gateway.log"
}
},
"fields": {
"elastic_agent.version": [
"8.13.2"
],
"_tmp.tz": [
"UTC"
],
"cisco_secure_email_gateway.log.message": [
"CEF:0|Cisco|C100V Email Security Virtual Appliance|15.0.1-030|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=423A914879DE5FF34A15-20293010CA5C ESAMID=11343819 ESAICID=52535995 ESAAMPVerdict=NOT_EVALUATED ESAASVerdict=POSITIVE ESAAVVerdict=NOT_EVALUATED ESACFVerdict=NOT_EVALUATED ESADLPVerdict=NOT_EVALUATED ESADMARCVerdict=Skipped dvc=217.54.87.14 ESAFriendlyFrom=\"First Last\" <hello@acompany.co.uk> ESAGMVerdict=NOT_EVALUATED deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=hello@acompany.co.uk cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=unable to retrieve ESAMFVerdict=MATCH act=DROPPED ESAFinalActionDetails=By CASE cs4Label=ExternalMsgID cs4='<8a19b4$aqfmr@esa3.name.iphmx.com>' ESAMsgSize=1098 ESAOFVerdict=NOT_EVALUATED duser=first.last@othercompany.co.uk ESAHeloDomain=luci.acompany.shop ESAHeloIP=14.119.117.185 ESAReplyTo=example1@placeholder.com cfp1Label=SBRSScore cfp1=unable to retrieve ESASPFVerdict={'mailfrom': {'result': 'None', 'sender': 'hello@acompany.co.uk'}, 'helo': {'result': 'None', 'sender': 'postmaster@luci.acompany.shop'}} shost=luci.acompany.shop ESASenderGroup=SUSPECTLIST src=14.119.117.185 msg='UK based' ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 end=Tue Apr 9 19:49:15 2024 start=Tue Apr 9 19:49:13 2024"
],
"agent.type": [
"filebeat"
],
"event.module": [
"cisco_secure_email_gateway"
],
"agent.name.text": [
"vmlog1"
],
"_tmp.filepath": [
"/temp/consolidated_event.esa3.name.iphmx.com.@20240409T192001.s"
],
"log.level": [
"2024"
],
"agent.name": [
"vmlog1"
],
"elastic_agent.snapshot": [
false
],
"event.agent_id_status": [
"verified"
],
"event.kind": [
"pipeline_error"
],
"_conf.tz_offset": [
"UTC"
],
"elastic_agent.id": [
"f3861419-7aeb-4b5d-83ff-a7f6134a4ef9"
],
"data_stream.namespace": [
"default"
],
"input.type": [
"log"
],
"log.offset": [
160755
],
"data_stream.type": [
"logs"
],
"cisco_secure_email_gateway.log.category.name": [
"consolidated_event"
],
"tags": [
"forwarded",
"cisco_secure_email_gateway-log"
],
"event.ingested": [
"2024-04-09T18:33:06.000Z"
],
"@timestamp": [
"2024-04-09T18:33:06.065Z"
],
"agent.id": [
"f3861429-7aeb-4b5d-83ff-a7f7134a4ef9"
],
"ecs.version": [
"8.11.0"
],
"error.message": [
"Text 'Tue Apr 9 19:49:20' could not be parsed at index 8"
],
"data_stream.dataset": [
"cisco_secure_email_gateway.log"
],
"log.file.path": [
"/temp/consolidated_event.esa3.name.iphmx.com.@20240409T192001.s"
],
"agent.ephemeral_id": [
"9fe44d98-0c46-4cfb-a7c8-cbf29735ef49"
],
"agent.version": [
"8.13.2"
],
"cisco_secure_email_gateway.log.host": [
"esa3.name.iphmx.com"
],
"event.dataset": [
"cisco_secure_email_gateway.log"
]
},
"ignored_field_values": {
"_tmp.timestamp": [
"Tue Apr 9 19:49:20"
]
}
}
- Timestamp in
tmp.timestampis missing the year. - Year somehow moved to
log.level? - It doesn't process
cisco_secure_email_gateway.log.message?
I think this is coming from here, but I don't really understand it.
Can anyone point me in the right direction on how to debug or fix this? I'm happy to do more tests or share more examples if that's of any help.