Different behavior on index shards when apply index mapping

[Fdsm]

Hello,

We watch a curious behavior after apply a index mapping on indices:

Before mapping, it was not notice any docs_deleted on indeces and size of the primary and replica shards were basicaly same:

Before custom mapping apply (None docs.deleted and pri.store basically 50% of total store.size)

health  status   index                        pri rep   docs.count  docs.deleted  store.size  pri.store.size
green  open     index3-2021.08.25 1     1      1473137      0                       82.2gb       41.1gb
green  open     index2-2021.08.24 1     1      1659334      0                       90.5gb       45.3gb
green  open     index1-2021.08.23 1     1      1812586      0                       83.4gb       41.6gb

After custom mapping apply (some docs were deleted and pri.store about 77% of store.size

health  status   index                        pri rep   docs.count  docs.deleted  store.size  pri.store.size
green  open     index4-2021.08.26 1     1      1171672      1325                 6.1gb         4.7gb

[
{
"index" : "index4-2021.08.26",
"shard" : "0",
"prirep" : "r",
"state" : "STARTED",
"docs" : "1124449",
"store" : "1.4gb",
"ip" : "ipsnode1",
"node" : "host1"
},
{
"index" : "index4-2021.08.26",
"shard" : "0",
"prirep" : "p",
"state" : "STARTED",
"docs" : "1171672",
"store" : "4.7gb",
"ip" : "ipsnode2",
"node" : "host2"
}
]

Data were ingested without any change other than mapping. Any clues if this disbalancing shard storage might be an issue and why these documents were deleted?

I know that primary and replica shads are not necessary stored the same way. but the difference is huge

[warkolm (Mark Walkom)]

Welcome to our community!

We'd need to see the mappings to make comments on what might be happening, please share them.

[Fdsm]

Follows the mapping:

PUT _index_template/index
{
  "index_patterns": ["index*"],
  "template": {
    "settings": {
      "number_of_shards": 1,
      "number_of_replicas": 1,
      "index.lifecycle.name": "index",
      "index.lifecycle.rollover_alias": "index"
    },
    "mappings" : {
      "dynamic_templates": [
      {
      	"string": {
      	  "match_mapping_type": "string",
      	  "mapping": {
      	  	"type": "keyword"
      	  }
      	}
      }
      ],
      "properties" : {
        "@timestamp" : {
          "type" : "date"
        },
        "@version" : {
          "type" : "keyword"
        },
        "application" : {
          "properties" : {
            "name" : {
              "type" : "keyword"
            },
            "version" : {
              "type" : "keyword"
            }
          }
        },
        "client" : {
          "type" : "ip"
        },
        "level" : {
          "type" : "keyword"
        },
        "logger_name" : {
          "type" : "text",
          "norms": false,
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "message" : {
          "type" : "text",
          "norms": false
        },
        "method" : {
          "type" : "keyword"
        },
        "mongoDb" : {
          "properties" : {
            "collection" : {
              "type" : "keyword"
            },
            "dataBaseName" : {
              "type" : "keyword"
            },
            "nameMethod" : {
              "type" : "keyword"
            },
            "totalTimeMillis" : {
              "type" : "float"
            }
          }
        },
        "port" : {
          "type" : "keyword"
        },
        "processingTimeMillis" : {
          "type" : "float"
        },
        "request" : {
          "properties" : {
            "body" : {
              "type" : "text",
              "norms": false,
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "headers" : {
              "properties" : {
                "Content-Type" : {
                  "type" : "text",
                  "norms": false,
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "catalogcode" : {
                  "type" : "keyword"
                },
                "channel" : {
                  "type" : "keyword"
                },
                "city" : {
                  "type" : "keyword"
                },
                "customerid" : {
                  "type" : "keyword"
                },
                "fidelization" : {
                  "type" : "keyword"
                },
                "forwarded" : {
                  "type" : "text",
                  "norms": false,
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "headercliente" : {
                  "type" : "keyword"
                },
                "host" : {
                  "type" : "keyword"
                },
                "name" : {
                  "type" : "text",
                  "norms": false,
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "ordernumber" : {
                  "type" : "keyword"
                },
                "paymentmean" : {
                  "type" : "keyword"
                },
                "pdv" : {
                  "type" : "keyword"
                },
                "postman-token" : {
                  "type" : "keyword"
                },
                "source" : {
                  "type" : "keyword"
                },
                "transactionid" : {
                  "type" : "text",
                  "norms": false,
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "uf" : {
                  "type" : "keyword"
                },
                "user-agent" : {
                  "type" : "text",
                  "norms": false,
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "x-forwarded-for" : {
                  "type" : "text",
                  "norms": false,
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "x-forwarded-host" : {
                  "type" : "text",
                  "norms": false,
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "x-forwarded-port" : {
                  "type" : "keyword"
                },
                "x-forwarded-proto" : {
                  "type" : "keyword"
                }
              }
            },
            "method" : {
              "type" : "keyword"
            },
            "url" : {
              "type" : "keyword"
            }
          }
        },
        "requestId" : {
          "type" : "keyword"
        },
        "response" : {
          "properties" : {
            "body" : {
              "type" : "text",
              "norms": false
            }
          }
        },
        "stackTrace" : {
          "type" : "text",
          "norms": false
        },
        "statusCode" : {
          "type" : "keyword"
        },
        "tags" : {
          "type" : "keyword"
        },
        "traceId" : {
          "type" : "keyword"
        }
      }
    },
    "aliases": {
      "index": {}
    }
   },
  "version": 1
}

[warkolm (Mark Walkom)]

And what was the original mapping?

[Fdsm]

Hi, Mark.

Follows original mapping (several fields have been removed because of character limitation, but the original mapping is basically multifields text/keyword and long for numeric fields)

{
  "index-2021.08.25" : {
    "mappings" : {
      "properties" : {
        "@timestamp" : {
          "type" : "date"
        },
        "@version" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "LEVEL" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "LOGSTASH" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "X-B3-ParentSpanId" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "X-B3-SpanId" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "X-B3-TraceId" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "X-Span-Export" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "application" : {
          "properties" : {
            "name" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "version" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "className" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "client" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "clientHost" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "feign" : {
          "properties" : {
            "configKey" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "level" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "processingTimeMillis" : {
              "type" : "long"
            },
            "request" : {
              "properties" : {
                "body" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "charset" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "headers" : {
                  "properties" : {
                    "Accept" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "Authorization" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "Cliente" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "Content-Length" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "Content-Type" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "X-B3-ParentSpanId" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "X-B3-Sampled" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "X-B3-SpanId" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "X-B3-TraceId" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "appId" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "cpf" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "proxyAuth" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "proxyauth" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "scope" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "transactionId" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    }
                  }
                },
                "method" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "url" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                }
              }
            },
            "response" : {
              "properties" : {
                "body" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "headers" : {
                  "properties" : {
                    "cache-control" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "connection" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "content-length" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    }
      }
    }
  }
}