i have set one alert rule, i see some fields are not populated correctly in the alert document, so in the index makes empty fields.
here is my alert rule:
PUT kbn:/api/alerting/rule/6aeaa6d9-8795-4c63-9623-975a20e61bd9
{
"name": "service_state_watch rule",
"tags": ,
"schedule": {
"interval": "1m"
},
"params": {
"searchConfiguration": {
"query": {
"query": "(system.service.state : "inactive" OR system.service.state : "deactivating" OR system.service.state : "failed" OR system.service.sub_state : "dead" OR system.service.sub_state : "exited" OR system.service.sub_state : "failed") AND \n(system.service.name : "mysqld.service" OR system.service.name : "ussddispatcher.service" OR system.service.name : "sugw.service")\n",
"language": "kuery"
},
"index": "metricbeat-*"
},
"timeField": "@timestamp",
"searchType": "searchSource",
"timeWindowSize": 5,
"timeWindowUnit": "m",
"threshold": [
1
],
"thresholdComparator": ">=",
"size": 100,
"aggType": "count",
"groupBy": "top",
"termField": [
"system.service.name",
"system.service.state",
"system.service.sub_state"
],
"termSize": 1,
"excludeHitsFromPreviousRun": true,
"sourceFields": [
{
"label": "host.hostname",
"searchPath": "host.hostname"
},
{
"label": "host.id",
"searchPath": "host.id"
},
{
"label": "host.name",
"searchPath": "host.name"
}
]
},
"actions": [
{
"group": "query matched",
"id": "df177a7d-936d-4de6-b34a-00d9587a27b4",
"params": {
"documents": [
{
"@timestamp": "{{context.execution_time}}",
"rule_id": "{{rule.id}}",
"rule_name": "{{rule.name}}",
"alert_id": "{{alert.id}}",
"alert_type": "Service State Change",
"service_name": "{{context.system.service.name}}",
"service_state": "{{context.system.service.state}}",
"service_sub_state": "{{context.system.service.sub_state}}",
"message": "Service '{{context.system.service.name}}' changed state to '{{context.system.service.state}}'."
}
]
},
"frequency": {
"notify_when": "onActionGroupChange",
"throttle": null,
"summary": false
},
"uuid": "6845c802-4b41-470f-969d-7a88614c93f7"
},
{
"group": "recovered",
"id": "df177a7d-936d-4de6-b34a-00d9587a27b4",
"params": {
"documents": [
{
"@timestamp": "{{context.execution_time}}",
"rule_id": "{{rule.id}}",
"rule_name": "{{rule.name}}",
"alert_id": "{{alert.id}}",
"alert_type": "Service State Change",
"service_name": "{{context.system.service.name}}",
"service_state": "{{context.system.service.state}}",
"service_sub_state": "{{context.system.service.sub_state}}",
"message": "Service '{{context.system.service.name}}' has recovered and changed state to '{{context.service.state}}'."
}
]
},
"frequency": {
"notify_when": "onActionGroupChange",
"throttle": null,
"summary": false
},
"uuid": "020f6fc4-2cce-4395-94ab-ac8a734aef25"
}
]
}
the index mapping to is also here:
{
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"alert_id": {
"type": "keyword"
},
"alert_type": {
"type": "keyword"
},
"error": {
"properties": {
"message": {
"type": "text"
}
}
},
"host_hostname": {
"type": "keyword"
},
"message": {
"type": "text"
},
"metric_name": {
"type": "keyword"
},
"metric_value": {
"type": "float"
},
"rule_id": {
"type": "keyword"
},
"rule_name": {
"type": "keyword"
},
"service_name": {
"type": "keyword"
},
"service_state": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"service_status": {
"type": "keyword"
},
"service_sub_state": {
"type": "keyword"
},
"threshold": {
"type": "float"
}
}
}
}
if status change from one of these systemctl services metrics in the rule , alert is triggered and perform action and document is written in the index
@timestamp
Mar 10, 2025 @ 13:26:56.624
alert_id
ussddispatcher.service,inactive,dead
alert_type
Service State Change
message
Service '' changed state to ''.
rule_id
6aeaa6d9-8795-4c63-9623-975a20e61bd9
rule_name
service_state_watch rule
service_name
(empty)
service_state
(empty)
service_sub_state
(empty)
_id
PraXf5UBmHmFUE1Z5__x
_ignored
_index
metricindexv1
_score
1
why these empty fields? that is my issue for my case.