Thank you for your time.
I’m new to the ELK system.
I’m collecting information from a Sophos XG 19.5
The structure I have is:
Firewall XG (19.5) --> Filebeat (7.17.10) -> Elastichsearch (7.17.8)
I have detected that the "vent.original" appears in the records twice.
"event": {
"severity": 6,
"original": "<190>device_name=\"SFW\" timestamp=\"2023-05-29T17:07:55+0200\" device_model=\"XGS3100\" device_serial_id=\"X31001111111111\" log_id=\"010101600001\" log_type=\"Firewall\" log_component=\"Firewall Rule\" log_subtype=\"Allowed\" log_version=1 severity=\"Information\" duration=121 fw_rule_id=\"66\" fw_rule_name=\"DPI Pruebas\" fw_rule_section=\"Local rule\" nat_rule_id=\"7\" nat_rule_name=\"SNAT_11111111\" fw_rule_type=\"BUSINESS\" gw_id_request=3 gw_name_request=\"Port5_GW_FFTH\" sdwan_route_id_request=4 sdwan_route_name_request=\"GW_11111111\" user_name=\"pruebas1@pruebas.es\" user_group=\"OU=Usuarios,DC=interno,DC=pruebas,DC=local\" web_policy_id=9 ips_policy_id=5 app_filter_policy_id=9 app_name=\"HTTP\" app_risk=1 app_technology=\"Browser Based\" app_category=\"General Internet\" ether_type=\"Unknown (0x0000)\" out_interface=\"Port5_ppp\" src_mac=\"00:B0:36:B2:26:2C\" dst_mac=\"D8:5F:96:0C:10:11\" src_ip=\"89.30.210.120\" src_country=\"localP\" dst_ip=\"13.117.14.152\" dst_country=\"USA\" protocol=\"TCP\" src_port=37858 dst_port=80 packets_sent=5 packets_received=5 bytlocal_sent=367 bytlocal_received=732 src_zone_type=\"LAN\" src_zone=\"LAN\" dst_zone_type=\"WAN\" dst_zone=\"WAN\" con_event=\"Stop\" con_id=\"3450279172\" master_con_id=\"353812501\" hb_status=\"No Heartbeat\" app_rlocalolved_by=\"Signature\" app_is_cloud=\"FALSE\" qualifier=\"New\" out_display_interface=\"Port5_ppp\" log_occurrence=\"1\" flags=3",
-----------------------------------------------------------------------------------------------------------------------------
"ignored_field_valulocal": {
"event.original": [
"<190>device_name=\"SFW\" timlocaltamp=\"2023-05-29T17:07:55+0200\" device_model=\"XGS3100\" device_serial_id=\"X31001111111111\" log_id=\"010101600001\" log_type=\"Firewall\" log_component=\"Firewall Rule\" log_subtype=\"Allowed\" log_version=1 severity=\"Information\" duration=121 fw_rule_id=\"66\" fw_rule_name=\"DPI Pruebas\" fw_rule_section=\"Local rule\" nat_rule_id=\"7\" nat_rule_name=\"SNAT_11111111\" fw_rule_type=\"BUSINlocalS\" gw_id_requlocalt=3 gw_name_requlocalt=\"Port5_GW_FFTH\" sdwan_route_id_requlocalt=4 sdwan_route_name_requlocalt=\"GW_11111111\" user_name=\"pruebas1@pruebas.local\" user_group=\"OU=Usuarios,DC=interno,DC=pruebas,DC=local\" web_policy_id=9 ips_policy_id=5 app_filter_policy_id=9 app_name=\"HTTP\" app_risk=1 app_technology=\"Browser Based\" app_category=\"General Internet\" ether_type=\"Unknown (0x0000)\" out_interface=\"Port5_ppp\" src_mac=\"00:B0:36:B2:26:2C\" dst_mac=\"D8:5F:96:0C:10:11\" src_ip=\"89.30.210.120\" src_country=\"localP\" dst_ip=\"13.117.14.152\" dst_country=\"USA\" protocol=\"TCP\" src_port=37858 dst_port=80 packets_sent=5 packets_received=5 bytlocal_sent=367 bytlocal_received=732 src_zone_type=\"LAN\" src_zone=\"LAN\" dst_zone_type=\"WAN\" dst_zone=\"WAN\" con_event=\"Stop\" con_id=\"3450279172\" master_con_id=\"353812501\" hb_status=\"No Heartbeat\" app_rlocalolved_by=\"Signature\" app_is_cloud=\"FALSE\" qualifier=\"New\" out_display_interface=\"Port5_ppp\" log_occurrence=\"1\" flags=3"
]
}
Not understood because the original stored event appears.
because if I test the pipeline (filebeat-7.17.10-sophos-xg-pipeline) it does not show
and the last action of the pipeline is to remove the "vent.original"
that I’m doing wrong.
Example
[
{
"_source": {
"message" : "<190>device_name=\"SFW\" timestamp=\"2023-05-29T17:07:55+0200\" device_model=\"XGS3100\" device_serial_id=\"X31001111111111\" log_id=\"010101600001\" log_type=\"Firewall\" log_component=\"Firewall Rule\" log_subtype=\"Allowed\" log_version=1 severity=\"Information\" duration=121 fw_rule_id=\"66\" fw_rule_name=\"DPI Pruebas\" fw_rule_section=\"Local rule\" nat_rule_id=\"7\" nat_rule_name=\"SNAT_11111111\" fw_rule_type=\"BUSINESS\" gw_id_request=3 gw_name_request=\"Port5_GW_FFTH\" sdwan_route_id_request=4 sdwan_route_name_request=\"GW_11111111\" user_name=\"pruebas1@pruebas.es\" user_group=\"OU=Usuarios,DC=interno,DC=pruebas,DC=local\" web_policy_id=9 ips_policy_id=5 app_filter_policy_id=9 app_name=\"HTTP\" app_risk=1 app_technology=\"Browser Based\" app_category=\"General Internet\" ether_type=\"Unknown (0x0000)\" out_interface=\"Port5_ppp\" src_mac=\"00:B0:36:B2:26:2C\" dst_mac=\"D8:5F:96:0C:10:11\" src_ip=\"89.30.210.120\" src_country=\"localP\" dst_ip=\"13.117.14.152\" dst_country=\"USA\" protocol=\"TCP\" src_port=37858 dst_port=80 packets_sent=5 packets_received=5 bytlocal_sent=367 bytlocal_received=732 src_zone_type=\"LAN\" src_zone=\"LAN\" dst_zone_type=\"WAN\" dst_zone=\"WAN\" con_event=\"Stop\" con_id=\"3450279172\" master_con_id=\"353812501\" hb_status=\"No Heartbeat\" app_rlocalolved_by=\"Signature\" app_is_cloud=\"FALSE\" qualifier=\"New\" out_display_interface=\"Port5_ppp\" log_occurrence=\"1\" flags=3"
}
}
]
This is the result of the above example and is correct
{
"docs": [
{
"doc": {
"_index": "_index",
"_type": "_doc",
"_id": "_id",
"_source": {
"log": {
"level": "Information"
},
"destination": {
"geo": {
"continent_name": "North America",
"country_name": "United States",
"location": {
"lon": -97.822,
"lat": 37.751
},
"country_iso_code": "US"
},
"port": 80,
"mac": "D8-5F-96-0C-10-11",
"packets": 5,
"ip": "13.117.14.152"
},
"rule": {
"id": "66"
},
"source": {
"geo": {
"continent_name": "Europe",
"country_name": "Netherlands",
"location": {
"lon": 4.8995,
"lat": 52.3824
},
"country_iso_code": "NL"
},
"as": {
"number": 8315,
"organization": {
"name": "Accenture B. V."
}
},
"port": 37858,
"ip": "89.30.210.120",
"user": {
"name": "pruebas1@pruebas.es"
},
"mac": "00-B0-36-B2-26-2C",
"packets": 5
},
"network": {
"community_id": "1:8e49WLsQTqb6vgvF3KMju07S8eg=",
"transport": "tcp",
"packets": 10,
"direction": "outbound"
},
"observer": {
"ingress": {
"zone": "LAN"
},
"product": "XG",
"serial_number": "X31001111111111",
"type": "firewall",
"vendor": "Sophos",
"egress": {
"interface": {
"name": "Port5_ppp"
},
"zone": "WAN"
}
},
"@timestamp": "2023-05-29T15:07:55.000Z",
"ecs": {
"version": "8.0.0"
},
"related": {
"user": [
"pruebas1@pruebas.es"
],
"ip": [
"89.30.210.120",
"13.117.14.152"
]
},
"sophos": {
"xg": {
"device_model": "XGS3100",
"web_policy_id": "9",
"con_id": "3450279172",
"flags": "3",
"fw_rule_type": "BUSINESS",
"ips_policy_id": "5",
"fw_rule_section": "Local rule",
"app_is_cloud": "FALSE",
"app_rlocalolved_by": "Signature",
"device_name": "SFW",
"log_type": "Firewall",
"ether_type": "Unknown (0x0000)",
"nat_rule_name": "SNAT_11111111",
"bytlocal_sent": "367",
"app_filter_policy_id": "9",
"log_id": "010101600001",
"user_group": "OU=Usuarios,DC=interno,DC=pruebas,DC=local",
"fw_rule_name": "DPI Pruebas",
"log_component": "Firewall Rule",
"log_subtype": "Allowed",
"dst_zone_type": "WAN",
"master_con_id": "353812501",
"hb_status": "No Heartbeat",
"gw_name_request": "Port5_GW_FFTH",
"app_technology": "Browser Based",
"app_name": "HTTP",
"src_zone_type": "LAN",
"con_event": "Stop",
"sdwan_route_name_request": "GW_11111111",
"app_risk": "1",
"qualifier": "New",
"gw_id_request": "3",
"sdwan_route_id_request": "4",
"app_category": "General Internet",
"log_version": "1",
"bytlocal_received": "732"
}
},
"event": {
"duration": 121000000000,
"severity": 6,
"ingested": "2023-05-29T15:58:04.706413058Z",
"code": "00001",
"kind": "event",
"start": "2023-05-29T15:07:55.000Z",
"action": "allowed",
"end": "2023-05-29T15:09:56.000Z",
"category": [
"network"
],
"outcome": "success"
}
},
"_ingest": {
"timestamp": "2023-05-29T15:58:04.706413058Z"
}
}
}
]
}
Thanks