After a day of tests Here is my output :
Filebeat -> ElasticSearch (same setup as you did, i started all over again to make sure it was ok) :
{
"_index": "filebeat-7.12.0-2021.06.08-000001",
"_type": "_doc",
"_id": "HWCk63kBkbHS6yBoemQi",
"_version": 1,
"_score": null,
"fields": {
"event.category": [
"web",
"network"
],
"host.os.name.text": [
"Windows Server 2016 Standard"
],
"user_agent.original.text": [
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36"
],
"host.hostname": [
"HOSTNAME"
],
"traefik.access.geoip.location": [
{
"coordinates": [
-97.822,
37.751
],
"type": "Point"
}
],
"user_agent.os.version": [
"10"
],
"host.mac": [
"00:00:00:00:00:00:00:e0"
],
"traefik.access.user_agent.name": [
"Chrome"
],
"service.type": [
"iis"
],
"http.request.method": [
"GET"
],
"host.os.version": [
"10.0"
],
"host.os.name": [
"Windows Server 2016 Standard"
],
"traefik.access.user_agent.original": [
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36"
],
"source.ip": [
"8.8.8.8"
],
"destination.address": [
"8.8.8.8"
],
"agent.name": [
"HOSTNAME"
],
"host.name": [
"HOSTNAME"
],
"user_agent.version": [
"78.0.3904.87"
],
"http.response.status_code": [
200
],
"event.kind": [
"event"
],
"iis.access.win32_status": [
64
],
"event.outcome": [
"success"
],
"user_agent.original": [
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36"
],
"host.os.type": [
"windows"
],
"traefik.access.geoip.country_iso_code": [
"US"
],
"fileset.name": [
"access"
],
"input.type": [
"log"
],
"log.offset": [
0
],
"user_agent.name": [
"Chrome"
],
"agent.hostname": [
"HOSTNAME"
],
"related.user": [
"username"
],
"host.architecture": [
"x86_64"
],
"suricata.eve.http.status": [
200
],
"url.path": [
"/WebParam/Question"
],
"agent.id": [
"bb3c0327-eef7-4a11-a471-69678f92cd55"
],
"iis.access.sub_status": [
0
],
"ecs.version": [
"1.8.0"
],
"event.created": [
"2021-06-08T12:41:31.082Z"
],
"agent.version": [
"7.12.0"
],
"host.os.family": [
"windows"
],
"source.as.number": [
15169
],
"suricata.eve.src_ip": [
"8.8.8.8"
],
"destination.port": [
80
],
"user_agent.os.full": [
"Windows 10"
],
"user.name": [
"username"
],
"suricata.eve.http.http_refer": [
"http://domain-name.fr/WebParam/"
],
"source.geo.location": [
{
"coordinates": [
-97.822,
37.751
],
"type": "Point"
}
],
"source.address": [
"8.8.8.8"
],
"user_agent.os.name.text": [
"Windows"
],
"suricata.eve.alert.action": [
"success"
],
"user_agent.os.name": [
"Windows"
],
"host.os.build": [
"14393.4350"
],
"host.ip": [
"HIDDEN IPS",
],
"agent.type": [
"filebeat"
],
"event.module": [
"iis"
],
"related.ip": [
"8.8.8.8",
"8.8.8.8"
],
"host.os.kernel": [
"10.0.14393.4350 (rs1_release.210407-2154)"
],
"source.geo.country_iso_code": [
"US"
],
"host.id": [
"cc18fa3f-e6fe-4b77-bf77-009a0c7373b2"
],
"source.as.organization.name.text": [
"Google LLC"
],
"http.request.referrer": [
"http://domain-name.fr/WebParam/"
],
"traefik.access.user_agent.device": [
"Other"
],
"suricata.eve.http.http_user_agent": [
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36"
],
"source.geo.continent_name": [
"North America"
],
"source.as.organization.name": [
"Google LLC"
],
"traefik.access.geoip.continent_name": [
"North America"
],
"suricata.eve.http.http_method": [
"GET"
],
"destination.ip": [
"8.8.8.8"
],
"suricata.eve.dest_ip": [
"8.8.8.8"
],
"traefik.access.user_agent.os_name": [
"Windows"
],
"event.duration": [
1271000000
],
"user_agent.os.full.text": [
"Windows 10"
],
"event.ingested": [
"2021-06-08T12:41:35.497Z"
],
"@timestamp": [
"2021-06-08T11:00:00.000Z"
],
"host.os.platform": [
"windows"
],
"suricata.eve.dest_port": [
80
],
"event.type": [
"connection"
],
"log.file.path": [
"C:\\FILEPATH\\Test_IIS_2.log"
],
"agent.ephemeral_id": [
"8efeccc3-2006-493c-8a9d-42c3070adb92"
],
"user_agent.device.name": [
"Other"
],
"source.geo.country_name": [
"United States"
],
"event.dataset": [
"iis.access"
],
"user.name.text": [
"username"
]
},
"sort": [
1623150000000
]
}
Here parse seems ok, fields has been added and message
field has been removed
Filebeat -> Logstash -> Elastic
{
"_index": "filebeat-test",
"_type": "_doc",
"_id": "zGD563kBkbHS6yBon2ac",
"_version": 1,
"_score": null,
"fields": {
"agent.version.keyword": [
"7.12.0"
],
"host.architecture.keyword": [
"x86_64"
],
"host.name.keyword": [
"HOSTNAME"
],
"event.dataset.keyword": [
"iis.access"
],
"host.os.build.keyword": [
"14393.4350"
],
"host.hostname": [
"HOSTNAME"
],
"host.mac": [
"HIDDEN"
],
"agent.hostname.keyword": [
"HOSTNAME"
],
"service.type": [
"iis"
],
"ecs.version.keyword": [
"1.8.0"
],
"host.ip.keyword": [
"HIDDEN IPS"
],
"host.os.version": [
"10.0"
],
"host.os.name": [
"Windows Server 2016 Standard"
],
"agent.name": [
"HOSTNAME"
],
"host.id.keyword": [
"cc18fa3f-e6fe-4b77-bf77-009a0c7373b2"
],
"host.name": [
"HOSTNAME"
],
"host.os.version.keyword": [
"10.0"
],
"host.os.type": [
"windows"
],
"agent.id.keyword": [
"1f6b65b8-83bd-4b4f-bcb4-c3892f4cb85c"
],
"fileset.name": [
"access"
],
"input.type": [
"log"
],
"@version.keyword": [
"1"
],
"log.offset": [
4370
],
"agent.hostname": [
"HOSTNAME"
],
"tags": [
"beats_input_codec_plain_applied"
],
"host.architecture": [
"x86_64"
],
"fileset.name.keyword": [
"access"
],
"agent.id": [
"1f6b65b8-83bd-4b4f-bcb4-c3892f4cb85c"
],
"ecs.version": [
"1.8.0"
],
"message.keyword": [
"2021-06-08 11:00:00 8.8.8.8 GET /WebParam/Question - 80 username 8.8.8.8 Mozilla/5.0+(Windows+NT+10.0;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/78.0.3904.87+Safari/537.36 http://domain-name.fr/WebParam/ 200 0 64 1271"
],
"event.module.keyword": [
"iis"
],
"host.hostname.keyword": [
"HOSTNAME"
],
"agent.version": [
"7.12.0"
],
"host.os.family": [
"windows"
],
"service.type.keyword": [
"iis"
],
"input.type.keyword": [
"log"
],
"tags.keyword": [
"beats_input_codec_plain_applied"
],
"host.os.build": [
"14393.4350"
],
"host.ip": [
"HIDDEN IPS"
],
"agent.type": [
"filebeat"
],
"event.module": [
"iis"
],
"host.os.kernel.keyword": [
"10.0.14393.4350 (rs1_release.210407-2154)"
],
"host.os.kernel": [
"10.0.14393.4350 (rs1_release.210407-2154)"
],
"@version": [
"1"
],
"host.os.name.keyword": [
"Windows Server 2016 Standard"
],
"host.id": [
"cc18fa3f-e6fe-4b77-bf77-009a0c7373b2"
],
"log.file.path.keyword": [
"C:\\FILEPATH\\Test_IIS_2.log"
],
"agent.type.keyword": [
"filebeat"
],
"agent.ephemeral_id.keyword": [
"dd809d6e-63eb-4c77-964d-dbc7402f0791"
],
"host.mac.keyword": [
"HIDDEN IPS"
],
"agent.name.keyword": [
"HOSTNAME"
],
"message": [
"2021-06-08 11:00:00 8.8.8.8 GET /WebParam/Question - 80 username 8.8.8.8 Mozilla/5.0+(Windows+NT+10.0;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/78.0.3904.87+Safari/537.36 http://domain-name.fr/WebParam/ 200 0 64 1271"
],
"host.os.family.keyword": [
"windows"
],
"@timestamp": [
"2021-06-08T14:14:33.270Z"
],
"host.os.type.keyword": [
"windows"
],
"host.os.platform": [
"windows"
],
"host.os.platform.keyword": [
"windows"
],
"log.file.path": [
"C:\\FILEPATH\\Test_IIS_2.log"
],
"agent.ephemeral_id": [
"dd809d6e-63eb-4c77-964d-dbc7402f0791"
],
"event.dataset": [
"iis.access"
]
},
"sort": [
1623161673270
]
}
This one is not formatted correctly even if the only thing i touched is filebeat output.
I'm starting to think that we're not able to use module with logstash at all
I'll use the grok patterns and "mutates?" i've found in IIS pipeline.yml and transfer them into logstash to make the same thing...