Hello,
I am quite new to ELK and I am facing the following issue.
For some reason, the .keyword fields (e.g. hostname.keyword) are not being populated in Kibana and I cannot use them in setting up alerts etc.
Let's take for example the following fields from an expanded document:
They have no field.keyword entry.
However, if you look at the json view of the same event:
{
"_index": "xxxxx",
"_id": "xxxxxx",
"_version": 1,
"_score": 0,
"_ignored": [
"message.keyword",
"event.original.keyword",
"xxxxx.keyword"
],
"_source": {
### shortened...###
"type": "syslog",
"object_type": "File",
"@timestamp": "xxxxx",
"need_restart": false,
"object_uri": "https://xxxxx",
"severity": "Warning",
"threat_type": "Application",
"ipv4": "xxxxx",
"today": "xxxxx",
"username": "xxxxx",
"processname": "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
"circumstances": "xxxxx",
"hash": "xxxxx"
},
"fields": {
"hash.keyword": [
"xxxxx"
],
###shortened...###
"type": [
"syslog"
],
"object_type.keyword": [
"File"
],
"severity.keyword": [
"Warning"
],
"username.keyword": [
"xxxxx"
],
"circumstances": [
"xxxxx"
],
"event.original": [
"xxxxx"
],
"action_taken": [
"xxxxx"
],
"@version.keyword": [
"1"
],
"threat_type.keyword": [
"Application"
],
"processname.keyword": [
"C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe"
],
"circumstances.keyword": [
"xxxxx"
]
}
}
My index mapping is as follows:
{
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"@version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"action_taken": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"circumstances": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"datasource": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"engine_version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"esetraw": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"event": {
"properties": {
"original": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"event_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"hash": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"host": {
"properties": {
"ip": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"hostname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ipv4": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"message": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"need_restart": {
"type": "boolean"
},
"object_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"object_uri": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"occured": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"processname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"scan_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"scanner_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"severity": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"source_uuid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"threat_handled": {
"type": "boolean"
},
"threat_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"threat_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"today": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"username": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
Events are coming from Logstash with the following conf file:
input {
tcp {
port => 5140
type => syslog
}
udp {
port => 5140
type => syslog
}
}
filter {
grok {
match => { "message" => "(?<today>\w{3}\s*\d{1,2}\s\d\d:\d\d:\d\d) (?<datasource>TEST TEST)\[\d{1,4}\]\:\s(?<xxxxx>.*)" }
add_tag => [ "TEST" ]
}
if "TEST" in [tags] {
json {
source => "xxxxx"
}
}
}
output {
elasticsearch {
hosts => ["https://xxxxx"]
index => "xxxxx"
user => "elastic"
password => "xxxxx"
cacert => "xxxxx"
}
}
I have tried refreshing the index, flushing it and even deleting it, but it's still the same.
My index mapping looks fine, as far as i understand...
Tried to also create a new index and use KV filter instead of json - same result.
The .keyword fields are also not available for using them in the Security > Alerts page columns.
I think there was even a moment after the initial setup where I was seeing the .keyword fields, but they suddenly disappeared. I have no idea what has changed...
