Remove raw fields from geoip?

Elastic Stack Logstash
#1

I am using Elasticsearch for log analysis, and I am trying to increase performance, reduce storage as much as possible. I have turned off dynamic mapping, added the not_analyzed strings explicitly where needed, as well as stating in the default mapping that strings are not_analyzed. That part is working great.

With the geoip plugin though, it still adds the raw field equivalents of the geoip string fields. These respect my default mapping and are not analyzed, but then they are redundant with the main field. I also cannot mutate remove_field those raw fields like [geoip][region_name][raw]. How can I get rid of these raw fields while keeping their regular partners?

Or, ideally, can I declare which geoip fields I want, instead of mutate deleting some of them? That seems like wasteful processing.

#2

Hello, please add your logstash configuration and your elasticsearch mapping/settings.

#3

Logstash config:

input {
  file {
    start_position => "beginning"
    path => "/home/loguser/testlog.log"
  }
}

filter {
  # exclude comment in log file
  if [message] =~ "^#" { drop {} }
 
  # parse a log entry
  grok {
    match => ["message", "%{TIMESTAMP_ISO8601:log_timestamp} %{URIPATH:uri-stem} %{IP} %{NUMBER:status:int} %{NUMBER} %{NUMBER:sc-bytes} %{WORD:method} %{NOTSPACE:uri-query} %{IPORHOST} %{NOTSPACE} \"%{DATA}\" \"%{DATA}\" \"%{DATA:useragent}\" (?:-|\"%{DATA:referrer}\") %{NUMBER} %{NUMBER:cs-bytes} (?:-|\"%{DATA}\") \"%{WORD:webserver}\" \"%{DATA:x_forwarded_for}\" %{NUMBER:time-taken:int}"]
    }

  # replace . with _ in querystring and split x_forwarded_for
  mutate {
    gsub => [ "querystring", "\.", "_" ]
    gsub => [ "x_forwarded_for", " ", "" ]
    split => [ "x_forwarded_for", "," ]
  }

  # split the querystring into key value pairs, and save some keys
  kv {
    source => "querystring"
    field_split => "&"
    include_keys => [ "id", "no" ]
  }

  # convert the log timestamp to the event timestamp
  date {
    locale => "en"
    match => ["log_timestamp", "ISO8601"]
    timezone => "UTC"
  }
  
  # add geoip data
  geoip {
    source => "x_forwarded_for"
  }

  # remove unneeded fields
  mutate {
    remove_field => "[geoip][real_region_name]"
    remove_field => "[geoip][continent_code]"
    remove_field => "[geoip][area_code]"
    remove_field => "[geoip][country_code2]"
    remove_field => "[geoip][country_code3]"
    remove_field => "log_timestamp"
    remove_field => "message"
  }
}

output {
  elasticsearch {}
}
#4

(In Sense:) GET _template/my_template_1

{
  "my_template_1": {
    "order": 10,
    "template": "logstash*",
    "settings": {
      "index": {
        "mapper": {
          "dynamic": "false"
        }
      }
    },
    "mappings": {
      "_default_": {
        "dynamic": false,
        "dynamic_templates": [
          {
            "string_fields": {
              "mapping": {
                "index": "not_analyzed",
                "type": "string"
              },
              "match_mapping_type": "string",
              "match": "*"
            }
          }
        ]
      },
      "my_mapping_1": {
        "_all": {
          "enabled": true
        },
        "properties": {
          "method": {
            "index": "not_analyzed",
            "type": "string"
          },
          "uri-stem": {
            "index": "not_analyzed",
            "type": "string"
          },
          "cs-bytes": {
            "type": "integer"
          },
          "useragent": {
            "index": "not_analyzed",
            "type": "string"
          },
          "x_forwarded_for": {
            "index": "not_analyzed",
            "type": "string"
          },
          "time-taken": {
            "type": "integer"
          },
          "path": {
            "index": "not_analyzed",
            "type": "string"
          },
          "referrer": {
            "index": "not_analyzed",
            "type": "string"
          },
          "uri-query": {
            "index": "not_analyzed",
            "type": "string"
          },
          "host": {
            "index": "not_analyzed",
            "type": "string"
          },
          "sc-bytes": {
            "type": "integer"
          },
          "webserver": {
            "index": "not_analyzed",
            "type": "string"
          },
          "status": {
            "type": "integer"
          }
        }
      }
    },
    "aliases": {}
  }
}
#5

I guess I ran the wrong command. The combined mapping includes the rest of _default_ too. But it is unmodified. Any help removing geoip.region_name.raw while keeping geoip.region?

#6