Mixing up field types

Elastic Stack Elasticsearch
1

Hello, I am new to ElasticSearch and I've read a few of the threads that track back to '17 but I cant seem to find a working solution as I've attempted some of the solutions so I was wondering if anyone could give some pointers!

[2019-01-18T14:16:15,698][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>"310946f7-bbfa-4de7-8fd3-86d6ee4e9480", :_index=>"logstash-txnsummary-2019.01.18", :_type=>"doc", :_routing=>nil}, #<LogStash::Event:0x17809485>], :response=>{"index"=>{"_index"=>"logstash-txnsummary-2019.01.18", "_type"=>"doc", "_id"=>"310946f7-bbfa-4de7-8fd3-86d6ee4e9480", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse", "caused_by"=>{"type"=>"illegal_state_exception", "reason"=>"Mixing up field types: class org.elasticsearch.index.mapper.TextFieldMapper$TextFieldType != class org.elasticsearch.index.mapper.KeywordFieldMapper$KeywordFieldType on field orderOperationCorrelationID"}}}}}

And this is the output-elasticsearch.config on my machine..

output {
    if (![log]) {
        elasticsearch {
            hosts => [ "${OUTPUT_ELASTICSEARCH_HOSTS}" ]
            index => "${OUTPUT_ELASTICSEARCH_INDEX}"
            action => "${OUTPUT_ELASTICSEARCH_ACTION:index}"
            document_id => "%{logGUID}"
            document_type => "${OUTPUT_ELASTICSEARCH_DOCUMENT_TYPE}"
            retry_on_conflict => 50
            template_name => "logstash-txnsummary"
            manage_template => true
            template_overwrite => true
            template => "${CONFIG_DIR}/_/logs-elasticsearch-template.json"
         }
    }
}

And this is the mapping template below

{
    "template": "logstash-txnsummary-*",
    "settings": {
        "index.refresh_interval": "5s"
    },
    "mappings": {
        "logs": {
            "_all": {
                "enabled": true,
                "omit_norms": true
            },
            "dynamic_templates": [
                {
                    "message_field": {
                        "match": "message",
                        "match_mapping_type": "string",
                        "mapping": {
                            "type": "string",
                            "index": "analyzed",
                            "omit_norms": true
                        }
                    }
                },
                {
                    "string_fields": {
                        "match": "*",
                        "match_mapping_type": "string",
                        "mapping": {
                            "type": "string",
                            "index": "not_analyzed",
                            "ignore_above": 256
                        }
                    }
                }
            ],
            "properties": {
                "orderOperationCorrelationID": {
                    "type": "string",
                    "index": "not_analyzed"
                }
            }
        }
    }
}

I can see that the type are different, is that the offender?

type:string vs type:keyword?

I've tried by making a change to my output-elasticsearch.conf (instead of type string, use type keyword), and the errors are still there?

2

This is the mapping file when I run a get mapping on one of the elastic indexes

{
  "logstash-txnsummary-2019.01.18": {
    "aliases": {},
    "mappings": {
      "logs": {
        "_all": {
          "enabled": true,
          "norms": false
        },
        "dynamic_templates": [
          {
            "message_field": {
              "match": "message",
              "path_match": "message",
              "match_mapping_type": "string",
              "mapping": {
                "index": "analyzed",
                "norms": false,
                "omit_norms": true,
                "type": "string"
              }
            }
          },
          {
            "string_fields": {
              "match": "*",
              "match_mapping_type": "string",
              "mapping": {
                "fields": {
                  "keyword": {
                    "ignore_above": 256,
                    "type": "keyword"
                  }
                },
                "ignore_above": 256,
                "index": "not_analyzed",
                "norms": false,
                "type": "string"
              }
            }
          }
        ],
        "properties": {
          "@timestamp": {
            "type": "date",
            "include_in_all": false
          },
          "@version": {
            "type": "keyword",
            "include_in_all": false
          },
          "geoip": {
            "dynamic": "true",
            "properties": {
              "ip": {
                "type": "ip"
              },
              "latitude": {
                "type": "half_float"
              },
              "location": {
                "type": "geo_point"
              },
              "longitude": {
                "type": "half_float"
              }
            }
          },
          "orderOperationCorrelationID": {
            "type": "keyword"
          }
        }
      },
      "_default_": {
        "_all": {
          "enabled": true,
          "norms": false
        },
        "dynamic_templates": [
          {
            "message_field": {
              "path_match": "message",
              "match_mapping_type": "string",
              "mapping": {
                "norms": false,
                "type": "text"
              }
            }
          },
          {
            "string_fields": {
              "match": "*",
              "match_mapping_type": "string",
              "mapping": {
                "fields": {
                  "keyword": {
                    "ignore_above": 256,
                    "type": "keyword"
                  }
                },
                "norms": false,
                "type": "text"
              }
            }
          }
        ],
        "properties": {
          "@timestamp": {
            "type": "date",
            "include_in_all": false
          },
          "@version": {
            "type": "keyword",
            "include_in_all": false
          },
          "geoip": {
            "dynamic": "true",
            "properties": {
              "ip": {
                "type": "ip"
              },
              "latitude": {
                "type": "half_float"
              },
              "location": {
                "type": "geo_point"
              },
              "longitude": {
                "type": "half_float"
              }
            }
          }
        }
      }
    },
    "settings": {
      "index": {
        "refresh_interval": "5s",
        "number_of_shards": "5",
        "provided_name": "logstash-txnsummary-2019.01.18",
        "creation_date": "1547851024755",
        "number_of_replicas": "1",
        "uuid": "F4kcNQ1XRgCRVPanQDLqow",
        "version": {
          "created": "5060899"
        }
      }
    }
  }
}
3

So when I made a modification on my filter instead of

query => "orderOperationCorrelationID:%{[orderOperationCorrelationID]}"

to

query => "orderOperationCorrelationID:%{orderOperationCorrelationID}"

Now, there are no more output logs on Splunk (no successful parsing nor error messages, not output at all), but mappings on the index now has two mappings
one is "logs" and one is "doc"

Both mappings have orderOperationCorrelationID with matching type: keyword

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.