The issue is that on a Windows Server 2022 version with Winlogbeat v7.15.1 and Sysmon v13.24.
Some how on Windows Server 2022 Winlogbeat doesn't send the fields task and message to Logstash v7.15.2.
If we use the same config on a Windows 2019 server the fields are send.
With the Windows Security events these fields are send on Server 2022.
Is this a compatibility issue?
Could you add include_xml: true to your config and share one of the raw JSON documents sent by Winlogbeat. Is this affecting all events send on Win 2022?
Hoi Andrew,
I will do the inxclude_xml and send you the results.
At the moment alle Sysmon events are affected.
Below a raw event without the include_xml how I found out where the event fields got lost.
{"event":{"kind":"event",
"provider":"Microsoft-Windows-Sysmon",
"code":"3",
"created":"2021-11-19T09:52:32.776Z",
"type":["connection","start","protocol"],
"category":["network"],
"module":"sysmon"},
"process":{"pid":2804,
"executable":"C:\\Windows\\System32\\svchost.exe","entity_id":"{d8a96ee4-b0ee-6174-3b00-000000000400}",
"name":"svchost.exe"},
"host":{"name":"hostname"},
"@timestamp":"2021-11-19T09:52:26.720Z",
"network":{"community_id":"1:qCRjr9xiBSRu+qkHzbvzbfTpwDQ=",
"direction":"egress",
"protocol":"-",
"type":"ipv4","transport":"tcp"},
"rule":{"name":"technique_id=T1043,technique_name=Commonly Used Port"},
"tags":["beats_input_raw_event"],
"user":{"name":"SYSTEM","id":"S-1-5-20","domain":"NT AUTHORITY"},
"source":{"port":53326,"ip":"ip","domain":"-"},
"log":{"level":"information"},
"destination":{"ip":"ip",
"port":8080,
"domain":"-"},
"related":{"user":"SYSTEM",
"ip":["ip","ip"]},
"ecs":{"version":"1.11.0"},
"@version":"1",
"winlog":{
"record_id":1315815,
"event_id":"3",
"process":{"pid":5145,"thread":{"id":7204}},
"event_data":{"SourcePortName":"-"},
"computer_name":"hostname",
"version":5,
"channel":"Microsoft-Windows-Sysmon/Operational",
"provider_guid":"{5770385f-c22a-43e0-bf4c-06f5698ffbd9}",
"api":"wineventlog","user":{"name":"SYSTEM","identifier":"S-1-5-20","type":"Well Known Group","domain":"NT AUTHORITY"},
"provider_name":"Microsoft-Windows-Sysmon"},
"beat":{"name":"name","ip":"ip"},
"agent":{"name":"name",
"ephemeral_id":"03f884d6-508e-4681-b762-893702a319d6",
"type":"winlogbeat","id":"eb576931-18a0-4a71-9826-758b4fd41e6f","version":"7.15.2","hostname":"hostname"}}
now with include_xml:
{"log":{"level":"information"},"ecs":{"version":"1.11.0"},
"process":{"name":"MicrosoftEdgeUpdate.exe",
"pid":3888,"entity_id":"{d8a96ee4-da67-619c-1562-010000000400}",
"executable":"C:\\Program Files (x86)\\Microsoft\\EdgeUpdate\\MicrosoftEdgeUpdate.exe"},
"rule":{"name":"technique_id=T1003,technique_name=Credential Dumping"},
"agent":{"name":"name","type":
"winlogbeat","hostname":"hostename",
"ephemeral_id":"c893e776-0973-4237-ae09-b445e30c938b",
"version":"7.15.2","id":"eb576931-18a0-4a71-9826-758b4fd41e6f"},
"tags":["beats_input_raw_event"],
"beat":{"ip":"ip",
"name":"name"},
"@timestamp":"2021-11-23T12:11:19.711Z",
"host":{"name":"hostename"},
"event":{"provider":"Microsoft-Windows-Sysmon",
"code":"12",
"original":"<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System>
<Provider Name='Microsoft-Windows-Sysmon' Guid='{5770385f-c22a-43e0-bf4c-06f5698ffbd9}'/>
<EventID>12</EventID>
<Version>2</Version>
<Level>4</Level>
<Task>12</Task>
<Opcode>0</Opcode>
<Keywords>0x8000000000000000</Keywords>
<TimeCreated SystemTime='2021-11-23T12:11:19.7180946Z'/>
<EventRecordID>1374534</EventRecordID><Correlation/>
<Execution ProcessID='1236' ThreadID='10084'/>
<Channel>Microsoft-Windows-Sysmon/Operational</Channel>
<Computer>hostename</Computer>
<Security UserID='S-1-5-20'/></System>
<EventData><Data Name='RuleName'>technique_id=T1003,technique_name=Credential Dumping</Data>
<Data Name='EventType'>CreateKey</Data><Data Name='UtcTime'>2021-11-23 12:11:19.711</Data>
<Data Name='ProcessGuid'>{d8a96ee4-da67-619c-1562-010000000400}</Data>
<Data Name='ProcessId'>3888</Data><Data Name='Image'>C:\\Program Files (x86)\\Microsoft\\EdgeUpdate\\MicrosoftEdgeUpdate.exe</Data>
<Data Name='TargetObject'>HKLM\\System\\CurrentControlSet\\Control\\SecurityProviders\\SCHANNEL</Data>
</EventData></Event>","created":"2021-11-23T12:11:20.851Z",
"type":["change"],"kind":"event",
"category":["configuration","registry"],
"module":"sysmon"},"@version":"1",
"registry":{"hive":"HKLM","key":"System\\CurrentControlSet\\Control\\SecurityProviders\\SCHANNEL",
"path":"HKLM\\System\\CurrentControlSet\\Control\\SecurityProviders\\SCHANNEL",
"value":"SCHANNEL"},
"winlog":{"process":{"pid":1236,
"thread":{"id":10084}},
"record_id":1374534,
"event_data":{"EventType":"CreateKey","TargetObject":"HKLM\\System\\CurrentControlSet\\Control\\SecurityProviders\\SCHANNEL"},
"channel":"Microsoft-Windows-Sysmon/Operational",
"version":2,"api":
"wineventlog",
"provider_guid":"{5770385f-c22a-43e0-bf4c-06f5698ffbd9}",
"computer_name":"hostname",
"event_id":"12",
"provider_name":"Microsoft-Windows-Sysmon",
"user":{"name":"SYSTEM",
"domain":"NT AUTHORITY",
"identifier":"S-1-5-20","type":"Well Known Group"}}}
This is is the formatted XML from your event. It does not have a <RenderingInfo><Message> field which is where the message comes from. Can you also post the winlogbeat.yml config you're using.
<?xml version="1.0"?>
<Event xmlns="http://schemas.microsoft.com/win/2004/08/events/event">
<System>
<Provider Name="Microsoft-Windows-Sysmon" Guid="{5770385f-c22a-43e0-bf4c-06f5698ffbd9}"/>
<EventID>12</EventID>
<Version>2</Version>
<Level>4</Level>
<Task>12</Task>
<Opcode>0</Opcode>
<Keywords>0x8000000000000000</Keywords>
<TimeCreated SystemTime="2021-11-23T12:11:19.7180946Z"/>
<EventRecordID>1374534</EventRecordID>
<Correlation/>
<Execution ProcessID="1236" ThreadID="10084"/>
<Channel>Microsoft-Windows-Sysmon/Operational</Channel>
<Computer>hostename</Computer>
<Security UserID="S-1-5-20"/>
</System>
<EventData>
<Data Name="RuleName">technique_id=T1003,technique_name=Credential Dumping</Data>
<Data Name="EventType">CreateKey</Data>
<Data Name="UtcTime">2021-11-23 12:11:19.711</Data>
<Data Name="ProcessGuid">{d8a96ee4-da67-619c-1562-010000000400}</Data>
<Data Name="ProcessId">3888</Data>
<Data Name="Image">C:\\Program Files (x86)\\Microsoft\\EdgeUpdate\\MicrosoftEdgeUpdate.exe</Data>
<Data Name="TargetObject">HKLM\\System\\CurrentControlSet\\Control\\SecurityProviders\\SCHANNEL</Data>
</EventData>
</Event>
Is it only Sysmon events that are affected? Do you get Security log events that contain message?
It is only with Sysmon event Windows Security, System and Application event work fine and send the message field
###################### Winlogbeat Configuration Example ########################
# This file is an example configuration file highlighting only the most common
# options. The winlogbeat.reference.yml file from the same directory contains
# all the supported options with more comments. You can use it as a reference.
#
# You can find the full configuration reference here:
# https://www.elastic.co/guide/en/beats/winlogbeat/index.html
#======================= Winlogbeat specific options ===========================
# event_logs specifies a list of event logs to monitor as well as any
# accompanying options. The YAML data type of event_logs is a list of
# dictionaries.
#
# The supported keys are name (required), tags, fields, fields_under_root,
# forwarded, ignore_older, level, event_id, provider, and include_xml. Please
# visit the documentation for the complete details of each option.
# https://go.es.io/WinlogbeatConfig
winlogbeat.event_logs:
- name: Application
ignore_older: 72h
- name: System
ignore_older: 72h
- name: Security
ignore_older: 72h
processors:
- script:
lang: javascript
id: security
file: ${path.home}/module/security/config/winlogbeat-security.js
- name: Microsoft-Windows-Sysmon/Operational
include_xml: true
ignore_older: 72h
processors:
- script:
lang: javascript
id: sysmon
file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js
#==================== Elasticsearch template settings ==========================
#setup.template.settings:
#index.number_of_shards: 1
#index.codec: best_compression
#_source.enabled: false
#================================ General =====================================
# The name of the shipper that publishes the network data. It can be used to group
# all the transactions sent by a single shipper in the web interface.
name: name
# The tags of the shipper are included in their own field with each
# transaction published.
#tags: ["service-X", "web-tier"]
# Optional fields that you can specify to add additional information to the
# output.
#fields:
# env: staging
winlogbeat.registry_file: Y:\ServApps\winlogbeat\.persist_winlogbeat.yml
#============================== Dashboards =====================================
# These settings control loading the sample dashboards to the Kibana index. Loading
# the dashboards is disabled by default and can be enabled either by setting the
# options here or by using the `setup` command.
#setup.dashboards.enabled: false
# The URL from where to download the dashboards archive. By default this URL
# has a value which is computed based on the Beat name and version. For released
# versions, this URL points to the dashboard archive on the artifacts.elastic.co
# website.
#setup.dashboards.url:
#============================== Kibana =====================================
# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
# This requires a Kibana endpoint configuration.
#setup.kibana:
# Kibana Host
# Scheme and port can be left out and will be set to the default (http and 5601)
# In case you specify and additional path, the scheme is required: http://localhost:5601/path
# IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
#host: "localhost:5601"
# Kibana Space ID
# ID of the Kibana Space into which the dashboards should be loaded. By default,
# the Default Space will be used.
#space.id:
#============================= Elastic Cloud ==================================
# These settings simplify using Winlogbeat with the Elastic Cloud (https://cloud.elastic.co/).
# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
# `setup.kibana.host` options.
# You can find the `cloud.id` in the Elastic Cloud web UI.
#cloud.id:
# The cloud.auth setting overwrites the `output.elasticsearch.username` and
# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
#cloud.auth:
#================================ Outputs =====================================
# Configure what output to use when sending the data collected by the beat.
#-------------------------- Elasticsearch output ------------------------------
#output.elasticsearch:
# Array of hosts to connect to.
#hosts: ["localhost:9200"]
# Protocol - either `http` (default) or `https`.
#protocol: "https"
# Authentication credentials - either API key or username/password.
#api_key: "id:api_key"
#username: "elastic"
#password: "changeme"
#----------------------------- Logstash output --------------------------------
output.logstash:
# The Logstash hosts
hosts: ["host"]
ssl.enabled: true
ssl.verification_mode: none
ssl.supported_protocols: [TLSv1.2]
# Optional SSL. By default is off.
# List of root certificates for HTTPS server verifications
ssl.certificate_authorities: ["Y:\\ServApps\\winlogbeat\\Certificates\\host.crt"]
# Certificate for SSL client authentication
#ssl.certificate: "C:\\Program Files\\Winlogbeat\\Certificates\\<Machine naam>.dnb.nl.crt"
# Client Certificate Key
#ssl.key: "C:\\Program Files\\Winlogbeat\\Certificates\\<Machine naam>.dnb.nl.key"
#================================ Processors =====================================
# Configure processors to enhance or manipulate events generated by the beat.
#processors:
#- add_host_metadata: ~
#- add_cloud_metadata: ~
#- add_docker_metadata: ~
#================================ Logging =====================================
# Sets log level. The default log level is info.
# Available log levels are: error, warning, info, debug
#logging.level: debug
logging.level: info
logging.to_files: true
logging.files:
path: "Y:\\ServApps\\winlogbeat\\logs"
name: winlogbeat
keepfiles: 7
# At debug level, you can selectively enable logging only for some components.
# To enable all selectors use ["*"]. Examples of other selectors are "beat",
# "publish", "service".
#logging.selectors: ["*"]
#============================== X-Pack Monitoring ===============================
# winlogbeat can export internal metrics to a central Elasticsearch monitoring
# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The
# reporting is disabled by default.
# Set to true to enable the monitoring reporter.
#monitoring.enabled: false
# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
# Winlogbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
#monitoring.cluster_uuid:
# Uncomment to send the metrics to Elasticsearch. Most settings from the
# Elasticsearch output are accepted here as well.
# Note that the settings should point to your Elasticsearch *monitoring* cluster.
# Any setting that is not set is automatically inherited from the Elasticsearch
# output configuration, so if you have the Elasticsearch output configured such
# that it is pointing to your Elasticsearch monitoring cluster, you can simply
# uncomment the following line.
#monitoring.elasticsearch:
#================================= Migration ==================================
# This allows to enable 6.7 migration aliases
#migration.6_to_7.enabled: true
Can you take a screen shot of one of these Sysmon events inside the Windows Event Viewer? I'm curious if you have a message there.
Hoi Andrew Kroh,
Is there an update?
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.
© 2020. All Rights Reserved - Elasticsearch
Apache, Apache Lucene, Apache Hadoop, Hadoop, HDFS and the yellow elephant logo are trademarks of the Apache Software Foundation in the United States and/or other countries.
