Missing .keyword fields

Hello,

I am quite new to ELK and I am facing the following issue.
For some reason, the .keyword fields (e.g. hostname.keyword) are not being populated in Kibana and I cannot use them in setting up alerts etc.

Let's take for example the following fields from an expanded document:
image
They have no field.keyword entry.

However, if you look at the json view of the same event:

{
  "_index": "xxxxx",
  "_id": "xxxxxx",
  "_version": 1,
  "_score": 0,
  "_ignored": [
    "message.keyword",
    "event.original.keyword",
    "xxxxx.keyword"
  ],
  "_source": {
		### shortened...###
    "type": "syslog",
    "object_type": "File",
    "@timestamp": "xxxxx",
    "need_restart": false,
    "object_uri": "https://xxxxx",
    "severity": "Warning",
    "threat_type": "Application",
    "ipv4": "xxxxx",
    "today": "xxxxx",
    "username": "xxxxx",
    "processname": "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
    "circumstances": "xxxxx",
    "hash": "xxxxx"
  },
  "fields": {
    "hash.keyword": [
      "xxxxx"
    ],
###shortened...###
    "type": [
      "syslog"
    ],
    "object_type.keyword": [
      "File"
    ],
    "severity.keyword": [
      "Warning"
    ],
    "username.keyword": [
      "xxxxx"
    ],
    "circumstances": [
      "xxxxx"
    ],
    "event.original": [
      "xxxxx"
    ],
    "action_taken": [
      "xxxxx"
    ],
    "@version.keyword": [
      "1"
    ],
    "threat_type.keyword": [
      "Application"
    ],
    "processname.keyword": [
      "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe"
    ],
    "circumstances.keyword": [
      "xxxxx"
    ]
  }
}

My index mapping is as follows:

{
  "mappings": {
    "properties": {
      "@timestamp": {
        "type": "date"
      },
      "@version": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "action_taken": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "circumstances": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "datasource": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "engine_version": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "esetraw": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "event": {
        "properties": {
          "original": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          }
        }
      },
      "event_type": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "hash": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "host": {
        "properties": {
          "ip": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          }
        }
      },
      "hostname": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "ipv4": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "message": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "need_restart": {
        "type": "boolean"
      },
      "object_type": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "object_uri": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "occured": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "processname": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "scan_id": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "scanner_id": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "severity": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "source_uuid": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "tags": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "threat_handled": {
        "type": "boolean"
      },
      "threat_name": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "threat_type": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "today": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "type": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "username": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      }
    }
  }
}

Events are coming from Logstash with the following conf file:

input {
  tcp {
    port => 5140
    type => syslog
  }
  udp {
    port => 5140
    type => syslog
  }
}


filter {

  grok {
     match => { "message" => "(?<today>\w{3}\s*\d{1,2}\s\d\d:\d\d:\d\d) (?<datasource>TEST TEST)\[\d{1,4}\]\:\s(?<xxxxx>.*)" }
     add_tag => [ "TEST" ]
  }

        if "TEST" in [tags] {
        json {
        source => "xxxxx"
        }


}


}

output {
  elasticsearch {
    hosts => ["https://xxxxx"]
    index => "xxxxx"
    user => "elastic"
    password => "xxxxx"
    cacert => "xxxxx"
  }
}

I have tried refreshing the index, flushing it and even deleting it, but it's still the same.
My index mapping looks fine, as far as i understand...
Tried to also create a new index and use KV filter instead of json - same result.

The .keyword fields are also not available for using them in the Security > Alerts page columns.

I think there was even a moment after the initial setup where I was seeing the .keyword fields, but they suddenly disappeared. I have no idea what has changed...

I dont know if this is helpful, but I just noticed that the .keyword fields exist in the Data View of my index, but they are empty:
image

Hi and welcome to our community!

I think while you have the fields in your mapping, e.g. when I have a look the the JSON of your event there's no "hostname" given. this means this fields has not been set when ingesting the document. So you should have a look into your log stash configuration. I can seen no indicator where hostname would be set as a separate field? Or should hostname be part of the syslog event incoming? then you should have a look if it has been sent to logstash. So I'd recommend debugging your ingest pipeline first.
Hope this helps!
Best,
Matthias

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.