Strange ILM error "Mapping definition for ... has unsupported parameters: [normalizer : my_normalizer]"

Elastic Stack Elasticsearch
1

Hey guys, i've got strange ILM error during an ILM phase:

... policy [p-ilm] for index [p-auditbeat-000009] failed on step [{"phase":"hot","action":"rollover","name":"attempt-rollover"}]. Moving to ERROR step

Mapping definition for [client] has unsupported parameters: [normalizer : my_normalizer]

Elasticsearch version is 7.8.1-1

The final mapping for index p-auditbeat-000009 collected from two templates:

{
  "index_patterns": ["p-auditbeat-*"],
  "order" : 1,
  "settings": {
    "index.lifecycle.rollover_alias": "p-auditbeat",
    "index.lifecycle.name": "p-ilm"
  },
  "mappings": {
    "properties": {
      "@timestamp": {
        "type": "date" 
      },
        "client": { "type": "object" },
        "hash": { "type": "object" }
    }
  }
}

AND

{
  "index_patterns": ["p-*"],
  "settings": {
    "number_of_shards": 1,
    "index.mapping.total_fields.limit": 2000,
    "number_of_replicas": 0,
    "analysis": {
            "normalizer": {
 		"my_normalizer": {
                   		"type": "custom",
              			"char_filter": [],
              			"filter": ["lowercase", "asciifolding"]
            		}
          	}
     }

  },
  "mappings": {
    "_source": {
      "enabled": true
    },
    "properties": {  
      "host": {  "type": "object" },
      "@log_source": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "@log_type": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "@log_host": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "@proto_type": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "source.port": { "type": "integer" },
      "source.nat.port": { "type": "integer" },
      "destination.port": { "type": "integer" },
      "contentType": { "type": "keyword",
                                       "normalizer": "my_normalizer"  }, 
      "category": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "command": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "cntr": { "type": "integer" },
      "client": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "client_id": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "device.name": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "domain.name": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "domain.base": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "domain.prefix": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "disposition": { "type": "integer" },
      "duration": { "type": "integer" },
      "duration_hours": { "type": "byte" },
      "duration_min": { "type": "byte" },
      "duration_sec": { "type": "byte" },
      "drop_total_count": { "type": "integer" },
      "source.bytes": { "type": "long" },
      "destination.bytes": { "type": "long" },
      "source.ip" : {"type": "ip" },
      "destination.ip" : {"type": "ip" },
      "src_mapped_ip" : {"type": "ip" },
       "src_xlated_ip" : {"type": "ip" },
      "dst_mapped_ip" : {"type": "ip" },
      "src_mapped_port" : {"type": "integer" },
      "dst_mapped_port" : {"type": "integer" },
      "interface" : {"type": "keyword" },
      "ioc.description" : {"type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "sinkhole" : {"type": "keyword",
                                       "normalizer": "my_normalizer"  },       
      "tcp_flags" : {"type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "temp1": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "temp2": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "token": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "reputation": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "role_main": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
       "referencedHost": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "role_addons": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "reason": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
       "grant_type": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "hit_count" : {"type": "integer" },
       "hits" : {"type": "integer" },

       "hash": { "type": "keyword"                                     },

      "hashcode1" : {"type": "long" },
      "hashcode2" : {"type": "long" },
      "icmp_type" : {"type": "byte"},
      "icmp_code" : {"type": "integer"},
       "url": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
       "value": { "type": "keyword",
                                       "normalizer": "my_normalizer"  },
      "winlog.event_data.param1" : {"type": "text",
                                  "norms": false
                                   },
      "user.id" : {"type": "text",
                   "norms": false
                  },
      "user.name" : {"type": "keyword",
                      "normalizer": "my_normalizer"},
      "event.action" : {"type": "keyword",
                      "normalizer": "my_normalizer"},
      "is_incident" : { "type": "boolean"},
       "location": {
             "type": "geo_point"
      },
      "created_at": {
        "type": "date",
        "format": "EEE MMM dd HH:mm:ss Z yyyy"
      }
    }
  }
}

The result mapping is:

  "mappings": {
    "_doc": {
      "properties": {
       [cut]
        "client": {
          "type": "object"
        },
       [cut]

If the [client] is object why the error saying about normalizer?

2

The "order" parameter is different in an each template.
It seems there is no workaround.
Looks like a conflict (feature?) in mappings when more then 2 templates exists with the same field.

3

The new template API does not merge anymore templates to avoid that kind of conflicts.
You should switch to it.

1 Like
4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.