So I've done some more research and it appears that the Suricata dashboards don't work either, because for example Could not locate that index-pattern-field (id: source.geo.country_iso_code)
. The dashboards and SIEM work fine without logstash.
I am sure that my index templates and ingest pipelines are correct and identical on both configurations. I'm checking this in Kibana under Index Management
Index Templates
. However, under under Index Management
Indices
, the two configurations are very different, with the logstash version being called filebeat-7.8.1
, and the elasticsearch version being called filebeat-7.8.1-2020.08.10-000001
, furthermore under the Mappings
tab, this is what the elasticsearch configuration is:
{
"mappings": {
"_doc": {
"_meta": {
"beat": "filebeat",
"version": "7.8.1"
},
"dynamic_templates": [
{
"labels": {
"path_match": "labels.*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
},
{
"container.labels": {
"path_match": "container.labels.*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
},
{
"dns.answers": {
"path_match": "dns.answers.*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
},
{
"log.syslog": {
"path_match": "log.syslog.*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
},
{
"network.inner": {
"path_match": "network.inner.*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
},
{
"observer.egress": {
"path_match": "observer.egress.*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
},
{
"observer.ingress": {
"path_match": "observer.ingress.*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
},
{
"fields": {
"path_match": "fields.*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
},
{
"docker.container.labels": {
"path_match": "docker.container.labels.*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
},
{
...
Whereas the non logstash version gives:
{
"mappings": {
"_doc": {
"properties": {
"@timestamp": {
"type": "date"
},
"@version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"agent": {
"properties": {
"ephemeral_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"hostname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"destination": {
"properties": {
"address": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"as": {
"properties": {
"number": {
"type": "long"
},
"organization": {
"properties": {
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
},
"bytes": {
"type": "long"
},
"domain": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"geo": {
"properties": {
"city_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"continent_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_code2": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_code3": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dma_code": {
"type": "long"
},
"ip": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"latitude": {
"type": "float"
},
"location": {
"properties": {
"lat": {
"type": "float"
},
"lon": {
"type": "float"
}
}
},
"longitude": {
"type": "float"
},
"postal_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"region_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"region_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"timezone": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"ip": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"packets": {
"type": "long"
},
"port": {
"type": "long"
}
}
},
"dns": {
"properties": {
"answers": {
"properties": {
"data": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ttl": {
"type": "long"
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
...
(note that the logstash version does NOT include the source.geo.country_iso_code
field, which I think causes the first dashboard error I mentioned. These outputs have been trimmed since they're extremely long.
I'm not sure what could be causing this, or what this "mapping" really even is, since the template is defined and looks good. Does logstash need to create an index mapping or something that I'm not doing?