Index size increased in Data Stream, in ES 8.x version

Elastic Stack Elasticsearch
1

Hello,

I have a ES 7.10 version Cluster with normal index, and recently I make a ECK cluster with 8.4.2 version with same data but with data stream.
But I figured out in new cluster with data stream, the same data would have 1.49x much bigger size than normal index.

But I've applied index mapping for new cluster, even though there wasn't an index mapping at an old cluster.

my index template example is here.

  "template": {
    "mappings": {
      "_doc": {
        "_meta": {},
        "properties": {
          "@timestamp": {
            "type": "date"
          },
          "@version": {
            "type": "keyword"
          },
          "agent": {
            "type": "object"
          },
          "cloud": {
            "properties": {
              "instance": {
                "properties": {
                  "id": {
                    "type": "keyword"
                  }
                }
              }
            }
          },
          "ecs": {
            "type": "object"
          },
          "environment": {
            "type": "keyword"
          },
          "fields": {
            "properties": {
              "log_type": {
                "type": "keyword"
              }
            }
          },
          "host": {
            "type": "object"
          },
          "input": {
            "type": "object"
          },
          "level": {
            "type": "keyword"
          },
          "thread": {
            "type": "text"
          },
          "log": {
            "properties": {
              "file": {
                "properties": {
                  "path": {
                    "type": "text"
                  }
                }
              },
              "flags": {
                "type": "keyword"
              }
            }
          },
          "message": {
            "type": "text"
          },
          "msg": {
            "type": "text"
          },
          "kubernetes": {
            "properties": {
              "node": {
                "properties": {
                  "name": {
                    "type": "keyword"
                  }
                }
              },
              "pod": {
                "properties": {
                  "labels": {
                    "properties": {
                      "tags_datadoghq_com/env": {
                        "type": "keyword"
                      }
                    }
                  }
                }
              }
            }
          }
        }
      }
    },

I guessed index size would be smaller after I applied this index mapping, because in previous, there's keyword, and text field for cloud.instance.id field before, but now only keyword field exists.

Is it correct that in Data Stream, compression for normal index is also applied? so it shouldn't matter whether it is index or data stream, the size of index in data stream would be same?

I'm confused.

Thanks.

2

Can you please show the output of the index stats API for the index/indices in both versions?

3

for es 7.10 version,

{
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "_all" : {
    "primaries" : {
      "docs" : {
        "count" : 76178,
        "deleted" : 0
      },
      "store" : {
        "size_in_bytes" : 27258418,
        "reserved_in_bytes" : 0
      },
      "indexing" : {
        "index_total" : 76375,
        "index_time_in_millis" : 11769,
        "index_current" : 0,
        "index_failed" : 0,
        "delete_total" : 0,
        "delete_time_in_millis" : 0,
        "delete_current" : 0,
        "noop_update_total" : 0,
        "is_throttled" : false,
        "throttle_time_in_millis" : 0
      },
      "get" : {
        "total" : 0,
        "time_in_millis" : 0,
        "exists_total" : 0,
        "exists_time_in_millis" : 0,
        "missing_total" : 0,
        "missing_time_in_millis" : 0,
        "current" : 0
      },
      "search" : {
        "open_contexts" : 0,
        "query_total" : 24,
        "query_time_in_millis" : 3,
        "query_current" : 0,
        "fetch_total" : 24,
        "fetch_time_in_millis" : 0,
        "fetch_current" : 0,
        "scroll_total" : 0,
        "scroll_time_in_millis" : 0,
        "scroll_current" : 0,
        "suggest_total" : 0,
        "suggest_time_in_millis" : 0,
        "suggest_current" : 0
      },
      "merges" : {
        "current" : 1,
        "current_docs" : 39381,
        "current_size_in_bytes" : 13896876,
        "total" : 14,
        "total_time_in_millis" : 9589,
        "total_docs" : 91442,
        "total_size_in_bytes" : 45573214,
        "total_stopped_time_in_millis" : 0,
        "total_throttled_time_in_millis" : 0,
        "total_auto_throttle_in_bytes" : 20971520
      },
      "refresh" : {
        "total" : 140,
        "total_time_in_millis" : 6055,
        "external_total" : 140,
        "external_total_time_in_millis" : 6153,
        "listeners" : 0
      },
      "flush" : {
        "total" : 0,
        "periodic" : 0,
        "total_time_in_millis" : 0
      },
      "warmer" : {
        "current" : 0,
        "total" : 139,
        "total_time_in_millis" : 1
      },
      "query_cache" : {
        "memory_size_in_bytes" : 0,
        "total_count" : 0,
        "hit_count" : 0,
        "miss_count" : 0,
        "cache_size" : 0,
        "cache_count" : 0,
        "evictions" : 0
      },
      "fielddata" : {
        "memory_size_in_bytes" : 0,
        "evictions" : 0
      },
      "completion" : {
        "size_in_bytes" : 0
      },
      "segments" : {
        "count" : 12,
        "memory_in_bytes" : 67192,
        "terms_memory_in_bytes" : 50944,
        "stored_fields_memory_in_bytes" : 5952,
        "term_vectors_memory_in_bytes" : 0,
        "norms_memory_in_bytes" : 6784,
        "points_memory_in_bytes" : 0,
        "doc_values_memory_in_bytes" : 3512,
        "index_writer_memory_in_bytes" : 1207072,
        "version_map_memory_in_bytes" : 0,
        "fixed_bit_set_memory_in_bytes" : 0,
        "max_unsafe_auto_id_timestamp" : 1670833726211,
        "file_sizes" : { }
      },
      "translog" : {
        "operations" : 76375,
        "size_in_bytes" : 89402474,
        "uncommitted_operations" : 76375,
        "uncommitted_size_in_bytes" : 89402474,
        "earliest_last_modified_age" : 310553
      },
      "request_cache" : {
        "memory_size_in_bytes" : 663,
        "evictions" : 0,
        "hit_count" : 0,
        "miss_count" : 24
      },
      "recovery" : {
        "current_as_source" : 0,
        "current_as_target" : 0,
        "throttle_time_in_millis" : 0
      }
    },
    "total" : {
      "docs" : {
        "count" : 76178,
        "deleted" : 0
      },
      "store" : {
        "size_in_bytes" : 27258418,
        "reserved_in_bytes" : 0
      },
      "indexing" : {
        "index_total" : 76375,
        "index_time_in_millis" : 11769,
        "index_current" : 0,
        "index_failed" : 0,
        "delete_total" : 0,
        "delete_time_in_millis" : 0,
        "delete_current" : 0,
        "noop_update_total" : 0,
        "is_throttled" : false,
        "throttle_time_in_millis" : 0
      },
      "get" : {
        "total" : 0,
        "time_in_millis" : 0,
        "exists_total" : 0,
        "exists_time_in_millis" : 0,
        "missing_total" : 0,
        "missing_time_in_millis" : 0,
        "current" : 0
      },
      "search" : {
        "open_contexts" : 0,
        "query_total" : 24,
        "query_time_in_millis" : 3,
        "query_current" : 0,
        "fetch_total" : 24,
        "fetch_time_in_millis" : 0,
        "fetch_current" : 0,
        "scroll_total" : 0,
        "scroll_time_in_millis" : 0,
        "scroll_current" : 0,
        "suggest_total" : 0,
        "suggest_time_in_millis" : 0,
        "suggest_current" : 0
      },
      "merges" : {
        "current" : 1,
        "current_docs" : 39381,
        "current_size_in_bytes" : 13896876,
        "total" : 14,
        "total_time_in_millis" : 9589,
        "total_docs" : 91442,
        "total_size_in_bytes" : 45573214,
        "total_stopped_time_in_millis" : 0,
        "total_throttled_time_in_millis" : 0,
        "total_auto_throttle_in_bytes" : 20971520
      },
      "refresh" : {
        "total" : 140,
        "total_time_in_millis" : 6055,
        "external_total" : 140,
        "external_total_time_in_millis" : 6153,
        "listeners" : 0
      },
      "flush" : {
        "total" : 0,
        "periodic" : 0,
        "total_time_in_millis" : 0
      },
      "warmer" : {
        "current" : 0,
        "total" : 139,
        "total_time_in_millis" : 1
      },
      "query_cache" : {
        "memory_size_in_bytes" : 0,
        "total_count" : 0,
        "hit_count" : 0,
        "miss_count" : 0,
        "cache_size" : 0,
        "cache_count" : 0,
        "evictions" : 0
      },
      "fielddata" : {
        "memory_size_in_bytes" : 0,
        "evictions" : 0
      },
      "completion" : {
        "size_in_bytes" : 0
      },
      "segments" : {
        "count" : 12,
        "memory_in_bytes" : 67192,
        "terms_memory_in_bytes" : 50944,
        "stored_fields_memory_in_bytes" : 5952,
        "term_vectors_memory_in_bytes" : 0,
        "norms_memory_in_bytes" : 6784,
        "points_memory_in_bytes" : 0,
        "doc_values_memory_in_bytes" : 3512,
        "index_writer_memory_in_bytes" : 1207072,
        "version_map_memory_in_bytes" : 0,
        "fixed_bit_set_memory_in_bytes" : 0,
        "max_unsafe_auto_id_timestamp" : 1670833726211,
        "file_sizes" : { }
      },
      "translog" : {
        "operations" : 76375,
        "size_in_bytes" : 89402474,
        "uncommitted_operations" : 76375,
        "uncommitted_size_in_bytes" : 89402474,
        "earliest_last_modified_age" : 310553
      },
      "request_cache" : {
        "memory_size_in_bytes" : 663,
        "evictions" : 0,
        "hit_count" : 0,
        "miss_count" : 24
      },
      "recovery" : {
        "current_as_source" : 0,
        "current_as_target" : 0,
        "throttle_time_in_millis" : 0
      }
    }
  },
  "indices" : {
    "search-platform.api-java-2022.12.12" : {
      "uuid" : "V8lUUW21QFqPrg87JcAMXA",
      "primaries" : {
        "docs" : {
          "count" : 76178,
          "deleted" : 0
        },
        "store" : {
          "size_in_bytes" : 27258418,
          "reserved_in_bytes" : 0
        },
        "indexing" : {
          "index_total" : 76375,
          "index_time_in_millis" : 11769,
          "index_current" : 0,
          "index_failed" : 0,
          "delete_total" : 0,
          "delete_time_in_millis" : 0,
          "delete_current" : 0,
          "noop_update_total" : 0,
          "is_throttled" : false,
          "throttle_time_in_millis" : 0
        },
        "get" : {
          "total" : 0,
          "time_in_millis" : 0,
          "exists_total" : 0,
          "exists_time_in_millis" : 0,
          "missing_total" : 0,
          "missing_time_in_millis" : 0,
          "current" : 0
        },
        "search" : {
          "open_contexts" : 0,
          "query_total" : 24,
          "query_time_in_millis" : 3,
          "query_current" : 0,
          "fetch_total" : 24,
          "fetch_time_in_millis" : 0,
          "fetch_current" : 0,
          "scroll_total" : 0,
          "scroll_time_in_millis" : 0,
          "scroll_current" : 0,
          "suggest_total" : 0,
          "suggest_time_in_millis" : 0,
          "suggest_current" : 0
        },
        "merges" : {
          "current" : 1,
          "current_docs" : 39381,
          "current_size_in_bytes" : 13896876,
          "total" : 14,
          "total_time_in_millis" : 9589,
          "total_docs" : 91442,
          "total_size_in_bytes" : 45573214,
          "total_stopped_time_in_millis" : 0,
          "total_throttled_time_in_millis" : 0,
          "total_auto_throttle_in_bytes" : 20971520
        },
        "refresh" : {
          "total" : 140,
          "total_time_in_millis" : 6055,
          "external_total" : 140,
          "external_total_time_in_millis" : 6153,
          "listeners" : 0
        },
        "flush" : {
          "total" : 0,
          "periodic" : 0,
          "total_time_in_millis" : 0
        },
        "warmer" : {
          "current" : 0,
          "total" : 139,
          "total_time_in_millis" : 1
        },
        "query_cache" : {
          "memory_size_in_bytes" : 0,
          "total_count" : 0,
          "hit_count" : 0,
          "miss_count" : 0,
          "cache_size" : 0,
          "cache_count" : 0,
          "evictions" : 0
        },
        "fielddata" : {
          "memory_size_in_bytes" : 0,
          "evictions" : 0
        },
        "completion" : {
          "size_in_bytes" : 0
        },
        "segments" : {
          "count" : 12,
          "memory_in_bytes" : 67192,
          "terms_memory_in_bytes" : 50944,
          "stored_fields_memory_in_bytes" : 5952,
          "term_vectors_memory_in_bytes" : 0,
          "norms_memory_in_bytes" : 6784,
          "points_memory_in_bytes" : 0,
          "doc_values_memory_in_bytes" : 3512,
          "index_writer_memory_in_bytes" : 1207072,
          "version_map_memory_in_bytes" : 0,
          "fixed_bit_set_memory_in_bytes" : 0,
          "max_unsafe_auto_id_timestamp" : 1670833726211,
          "file_sizes" : { }
        },
        "translog" : {
          "operations" : 76375,
          "size_in_bytes" : 89402474,
          "uncommitted_operations" : 76375,
          "uncommitted_size_in_bytes" : 89402474,
          "earliest_last_modified_age" : 310553
        },
        "request_cache" : {
          "memory_size_in_bytes" : 663,
          "evictions" : 0,
          "hit_count" : 0,
          "miss_count" : 24
        },
        "recovery" : {
          "current_as_source" : 0,
          "current_as_target" : 0,
          "throttle_time_in_millis" : 0
        }
      },
      "total" : {
        "docs" : {
          "count" : 76178,
          "deleted" : 0
        },
        "store" : {
          "size_in_bytes" : 27258418,
          "reserved_in_bytes" : 0
        },
        "indexing" : {
          "index_total" : 76375,
          "index_time_in_millis" : 11769,
          "index_current" : 0,
          "index_failed" : 0,
          "delete_total" : 0,
          "delete_time_in_millis" : 0,
          "delete_current" : 0,
          "noop_update_total" : 0,
          "is_throttled" : false,
          "throttle_time_in_millis" : 0
        },
        "get" : {
          "total" : 0,
          "time_in_millis" : 0,
          "exists_total" : 0,
          "exists_time_in_millis" : 0,
          "missing_total" : 0,
          "missing_time_in_millis" : 0,
          "current" : 0
        },
        "search" : {
          "open_contexts" : 0,
          "query_total" : 24,
          "query_time_in_millis" : 3,
          "query_current" : 0,
          "fetch_total" : 24,
          "fetch_time_in_millis" : 0,
          "fetch_current" : 0,
          "scroll_total" : 0,
          "scroll_time_in_millis" : 0,
          "scroll_current" : 0,
          "suggest_total" : 0,
          "suggest_time_in_millis" : 0,
          "suggest_current" : 0
        },
        "merges" : {
          "current" : 1,
          "current_docs" : 39381,
          "current_size_in_bytes" : 13896876,
          "total" : 14,
          "total_time_in_millis" : 9589,
          "total_docs" : 91442,
          "total_size_in_bytes" : 45573214,
          "total_stopped_time_in_millis" : 0,
          "total_throttled_time_in_millis" : 0,
          "total_auto_throttle_in_bytes" : 20971520
        },
        "refresh" : {
          "total" : 140,
          "total_time_in_millis" : 6055,
          "external_total" : 140,
          "external_total_time_in_millis" : 6153,
          "listeners" : 0
        },
        "flush" : {
          "total" : 0,
          "periodic" : 0,
          "total_time_in_millis" : 0
        },
        "warmer" : {
          "current" : 0,
          "total" : 139,
          "total_time_in_millis" : 1
        },
        "query_cache" : {
          "memory_size_in_bytes" : 0,
          "total_count" : 0,
          "hit_count" : 0,
          "miss_count" : 0,
          "cache_size" : 0,
          "cache_count" : 0,
          "evictions" : 0
        },
        "fielddata" : {
          "memory_size_in_bytes" : 0,
          "evictions" : 0
        },
        "completion" : {
          "size_in_bytes" : 0
        },
        "segments" : {
          "count" : 12,
          "memory_in_bytes" : 67192,
          "terms_memory_in_bytes" : 50944,
          "stored_fields_memory_in_bytes" : 5952,
          "term_vectors_memory_in_bytes" : 0,
          "norms_memory_in_bytes" : 6784,
          "points_memory_in_bytes" : 0,
          "doc_values_memory_in_bytes" : 3512,
          "index_writer_memory_in_bytes" : 1207072,
          "version_map_memory_in_bytes" : 0,
          "fixed_bit_set_memory_in_bytes" : 0,
          "max_unsafe_auto_id_timestamp" : 1670833726211,
          "file_sizes" : { }
        },
        "translog" : {
          "operations" : 76375,
          "size_in_bytes" : 89402474,
          "uncommitted_operations" : 76375,
          "uncommitted_size_in_bytes" : 89402474,
          "earliest_last_modified_age" : 310553
        },
        "request_cache" : {
          "memory_size_in_bytes" : 663,
          "evictions" : 0,
          "hit_count" : 0,
          "miss_count" : 24
        },
        "recovery" : {
          "current_as_source" : 0,
          "current_as_target" : 0,
          "throttle_time_in_millis" : 0
        }
      }
    }
  }
}

for es 8.4 version,

{
  "_shards": {
    "total": 1,
    "successful": 1,
    "failed": 0
  },
  "_all": {
    "primaries": {
      "docs": {
        "count": 76178,
        "deleted": 0
      },
      "shard_stats": {
        "total_count": 1
      },
      "store": {
        "size_in_bytes": 58572622,
        "total_data_set_size_in_bytes": 58572622,
        "reserved_in_bytes": 0
      },
      "indexing": {
        "index_total": 76186,
        "index_time_in_millis": 7956,
        "index_current": 0,
        "index_failed": 0,
        "delete_total": 0,
        "delete_time_in_millis": 0,
        "delete_current": 0,
        "noop_update_total": 0,
        "is_throttled": false,
        "throttle_time_in_millis": 0
      },
      "get": {
        "total": 0,
        "time_in_millis": 0,
        "exists_total": 0,
        "exists_time_in_millis": 0,
        "missing_total": 0,
        "missing_time_in_millis": 0,
        "current": 0
      },
      "search": {
        "open_contexts": 0,
        "query_total": 26,
        "query_time_in_millis": 3,
        "query_current": 0,
        "fetch_total": 26,
        "fetch_time_in_millis": 0,
        "fetch_current": 0,
        "scroll_total": 0,
        "scroll_time_in_millis": 0,
        "scroll_current": 0,
        "suggest_total": 0,
        "suggest_time_in_millis": 0,
        "suggest_current": 0
      },
      "merges": {
        "current": 0,
        "current_docs": 0,
        "current_size_in_bytes": 0,
        "total": 22,
        "total_time_in_millis": 1382,
        "total_docs": 51576,
        "total_size_in_bytes": 57863860,
        "total_stopped_time_in_millis": 0,
        "total_throttled_time_in_millis": 0,
        "total_auto_throttle_in_bytes": 20971520
      },
      "refresh": {
        "total": 197,
        "total_time_in_millis": 2062,
        "external_total": 193,
        "external_total_time_in_millis": 2101,
        "listeners": 0
      },
      "flush": {
        "total": 0,
        "periodic": 0,
        "total_time_in_millis": 0
      },
      "warmer": {
        "current": 0,
        "total": 192,
        "total_time_in_millis": 5
      },
      "query_cache": {
        "memory_size_in_bytes": 0,
        "total_count": 0,
        "hit_count": 0,
        "miss_count": 0,
        "cache_size": 0,
        "cache_count": 0,
        "evictions": 0
      },
      "fielddata": {
        "memory_size_in_bytes": 0,
        "evictions": 0
      },
      "completion": {
        "size_in_bytes": 0
      },
      "segments": {
        "count": 11,
        "memory_in_bytes": 0,
        "terms_memory_in_bytes": 0,
        "stored_fields_memory_in_bytes": 0,
        "term_vectors_memory_in_bytes": 0,
        "norms_memory_in_bytes": 0,
        "points_memory_in_bytes": 0,
        "doc_values_memory_in_bytes": 0,
        "index_writer_memory_in_bytes": 173816,
        "version_map_memory_in_bytes": 0,
        "fixed_bit_set_memory_in_bytes": 0,
        "max_unsafe_auto_id_timestamp": -1,
        "file_sizes": {}
      },
      "translog": {
        "operations": 76186,
        "size_in_bytes": 188975985,
        "uncommitted_operations": 76186,
        "uncommitted_size_in_bytes": 188975985,
        "earliest_last_modified_age": 794730
      },
      "request_cache": {
        "memory_size_in_bytes": 592,
        "evictions": 0,
        "hit_count": 0,
        "miss_count": 26
      },
      "recovery": {
        "current_as_source": 0,
        "current_as_target": 0,
        "throttle_time_in_millis": 0
      },
      "bulk": {
        "total_operations": 2118,
        "total_time_in_millis": 8115,
        "total_size_in_bytes": 183945929,
        "avg_time_in_millis": 2,
        "avg_size_in_bytes": 68323
      }
    },
    "total": {
      "docs": {
        "count": 76178,
        "deleted": 0
      },
      "shard_stats": {
        "total_count": 1
      },
      "store": {
        "size_in_bytes": 58572622,
        "total_data_set_size_in_bytes": 58572622,
        "reserved_in_bytes": 0
      },
      "indexing": {
        "index_total": 76186,
        "index_time_in_millis": 7956,
        "index_current": 0,
        "index_failed": 0,
        "delete_total": 0,
        "delete_time_in_millis": 0,
        "delete_current": 0,
        "noop_update_total": 0,
        "is_throttled": false,
        "throttle_time_in_millis": 0
      },
      "get": {
        "total": 0,
        "time_in_millis": 0,
        "exists_total": 0,
        "exists_time_in_millis": 0,
        "missing_total": 0,
        "missing_time_in_millis": 0,
        "current": 0
      },
      "search": {
        "open_contexts": 0,
        "query_total": 26,
        "query_time_in_millis": 3,
        "query_current": 0,
        "fetch_total": 26,
        "fetch_time_in_millis": 0,
        "fetch_current": 0,
        "scroll_total": 0,
        "scroll_time_in_millis": 0,
        "scroll_current": 0,
        "suggest_total": 0,
        "suggest_time_in_millis": 0,
        "suggest_current": 0
      },
      "merges": {
        "current": 0,
        "current_docs": 0,
        "current_size_in_bytes": 0,
        "total": 22,
        "total_time_in_millis": 1382,
        "total_docs": 51576,
        "total_size_in_bytes": 57863860,
        "total_stopped_time_in_millis": 0,
        "total_throttled_time_in_millis": 0,
        "total_auto_throttle_in_bytes": 20971520
      },
      "refresh": {
        "total": 197,
        "total_time_in_millis": 2062,
        "external_total": 193,
        "external_total_time_in_millis": 2101,
        "listeners": 0
      },
      "flush": {
        "total": 0,
        "periodic": 0,
        "total_time_in_millis": 0
      },
      "warmer": {
        "current": 0,
        "total": 192,
        "total_time_in_millis": 5
      },
      "query_cache": {
        "memory_size_in_bytes": 0,
        "total_count": 0,
        "hit_count": 0,
        "miss_count": 0,
        "cache_size": 0,
        "cache_count": 0,
        "evictions": 0
      },
      "fielddata": {
        "memory_size_in_bytes": 0,
        "evictions": 0
      },
      "completion": {
        "size_in_bytes": 0
      },
      "segments": {
        "count": 11,
        "memory_in_bytes": 0,
        "terms_memory_in_bytes": 0,
        "stored_fields_memory_in_bytes": 0,
        "term_vectors_memory_in_bytes": 0,
        "norms_memory_in_bytes": 0,
        "points_memory_in_bytes": 0,
        "doc_values_memory_in_bytes": 0,
        "index_writer_memory_in_bytes": 173816,
        "version_map_memory_in_bytes": 0,
        "fixed_bit_set_memory_in_bytes": 0,
        "max_unsafe_auto_id_timestamp": -1,
        "file_sizes": {}
      },
      "translog": {
        "operations": 76186,
        "size_in_bytes": 188975985,
        "uncommitted_operations": 76186,
        "uncommitted_size_in_bytes": 188975985,
        "earliest_last_modified_age": 794730
      },
      "request_cache": {
        "memory_size_in_bytes": 592,
        "evictions": 0,
        "hit_count": 0,
        "miss_count": 26
      },
      "recovery": {
        "current_as_source": 0,
        "current_as_target": 0,
        "throttle_time_in_millis": 0
      },
      "bulk": {
        "total_operations": 2118,
        "total_time_in_millis": 8115,
        "total_size_in_bytes": 183945929,
        "avg_time_in_millis": 2,
        "avg_size_in_bytes": 68323
      }
    }
  },
  "indices": {
    ".ds-logs-search-platform.api-java-2022.12.12-000010": {
      "uuid": "pcM5lgdLSi6oldytMtwEZw",
      "health": "green",
      "status": "open",
      "primaries": {
        "docs": {
          "count": 76178,
          "deleted": 0
        },
        "shard_stats": {
          "total_count": 1
        },
        "store": {
          "size_in_bytes": 58572622,
          "total_data_set_size_in_bytes": 58572622,
          "reserved_in_bytes": 0
        },
        "indexing": {
          "index_total": 76186,
          "index_time_in_millis": 7956,
          "index_current": 0,
          "index_failed": 0,
          "delete_total": 0,
          "delete_time_in_millis": 0,
          "delete_current": 0,
          "noop_update_total": 0,
          "is_throttled": false,
          "throttle_time_in_millis": 0
        },
        "get": {
          "total": 0,
          "time_in_millis": 0,
          "exists_total": 0,
          "exists_time_in_millis": 0,
          "missing_total": 0,
          "missing_time_in_millis": 0,
          "current": 0
        },
        "search": {
          "open_contexts": 0,
          "query_total": 26,
          "query_time_in_millis": 3,
          "query_current": 0,
          "fetch_total": 26,
          "fetch_time_in_millis": 0,
          "fetch_current": 0,
          "scroll_total": 0,
          "scroll_time_in_millis": 0,
          "scroll_current": 0,
          "suggest_total": 0,
          "suggest_time_in_millis": 0,
          "suggest_current": 0
        },
        "merges": {
          "current": 0,
          "current_docs": 0,
          "current_size_in_bytes": 0,
          "total": 22,
          "total_time_in_millis": 1382,
          "total_docs": 51576,
          "total_size_in_bytes": 57863860,
          "total_stopped_time_in_millis": 0,
          "total_throttled_time_in_millis": 0,
          "total_auto_throttle_in_bytes": 20971520
        },
        "refresh": {
          "total": 197,
          "total_time_in_millis": 2062,
          "external_total": 193,
          "external_total_time_in_millis": 2101,
          "listeners": 0
        },
        "flush": {
          "total": 0,
          "periodic": 0,
          "total_time_in_millis": 0
        },
        "warmer": {
          "current": 0,
          "total": 192,
          "total_time_in_millis": 5
        },
        "query_cache": {
          "memory_size_in_bytes": 0,
          "total_count": 0,
          "hit_count": 0,
          "miss_count": 0,
          "cache_size": 0,
          "cache_count": 0,
          "evictions": 0
        },
        "fielddata": {
          "memory_size_in_bytes": 0,
          "evictions": 0
        },
        "completion": {
          "size_in_bytes": 0
        },
        "segments": {
          "count": 11,
          "memory_in_bytes": 0,
          "terms_memory_in_bytes": 0,
          "stored_fields_memory_in_bytes": 0,
          "term_vectors_memory_in_bytes": 0,
          "norms_memory_in_bytes": 0,
          "points_memory_in_bytes": 0,
          "doc_values_memory_in_bytes": 0,
          "index_writer_memory_in_bytes": 173816,
          "version_map_memory_in_bytes": 0,
          "fixed_bit_set_memory_in_bytes": 0,
          "max_unsafe_auto_id_timestamp": -1,
          "file_sizes": {}
        },
        "translog": {
          "operations": 76186,
          "size_in_bytes": 188975985,
          "uncommitted_operations": 76186,
          "uncommitted_size_in_bytes": 188975985,
          "earliest_last_modified_age": 794730
        },
        "request_cache": {
          "memory_size_in_bytes": 592,
          "evictions": 0,
          "hit_count": 0,
          "miss_count": 26
        },
        "recovery": {
          "current_as_source": 0,
          "current_as_target": 0,
          "throttle_time_in_millis": 0
        },
        "bulk": {
          "total_operations": 2118,
          "total_time_in_millis": 8115,
          "total_size_in_bytes": 183945929,
          "avg_time_in_millis": 2,
          "avg_size_in_bytes": 68323
        }
      },
      "total": {
        "docs": {
          "count": 76178,
          "deleted": 0
        },
        "shard_stats": {
          "total_count": 1
        },
        "store": {
          "size_in_bytes": 58572622,
          "total_data_set_size_in_bytes": 58572622,
          "reserved_in_bytes": 0
        },
        "indexing": {
          "index_total": 76186,
          "index_time_in_millis": 7956,
          "index_current": 0,
          "index_failed": 0,
          "delete_total": 0,
          "delete_time_in_millis": 0,
          "delete_current": 0,
          "noop_update_total": 0,
          "is_throttled": false,
          "throttle_time_in_millis": 0
        },
        "get": {
          "total": 0,
          "time_in_millis": 0,
          "exists_total": 0,
          "exists_time_in_millis": 0,
          "missing_total": 0,
          "missing_time_in_millis": 0,
          "current": 0
        },
        "search": {
          "open_contexts": 0,
          "query_total": 26,
          "query_time_in_millis": 3,
          "query_current": 0,
          "fetch_total": 26,
          "fetch_time_in_millis": 0,
          "fetch_current": 0,
          "scroll_total": 0,
          "scroll_time_in_millis": 0,
          "scroll_current": 0,
          "suggest_total": 0,
          "suggest_time_in_millis": 0,
          "suggest_current": 0
        },
        "merges": {
          "current": 0,
          "current_docs": 0,
          "current_size_in_bytes": 0,
          "total": 22,
          "total_time_in_millis": 1382,
          "total_docs": 51576,
          "total_size_in_bytes": 57863860,
          "total_stopped_time_in_millis": 0,
          "total_throttled_time_in_millis": 0,
          "total_auto_throttle_in_bytes": 20971520
        },
        "refresh": {
          "total": 197,
          "total_time_in_millis": 2062,
          "external_total": 193,
          "external_total_time_in_millis": 2101,
          "listeners": 0
        },
        "flush": {
          "total": 0,
          "periodic": 0,
          "total_time_in_millis": 0
        },
        "warmer": {
          "current": 0,
          "total": 192,
          "total_time_in_millis": 5
        },
        "query_cache": {
          "memory_size_in_bytes": 0,
          "total_count": 0,
          "hit_count": 0,
          "miss_count": 0,
          "cache_size": 0,
          "cache_count": 0,
          "evictions": 0
        },
        "fielddata": {
          "memory_size_in_bytes": 0,
          "evictions": 0
        },
        "completion": {
          "size_in_bytes": 0
        },
        "segments": {
          "count": 11,
          "memory_in_bytes": 0,
          "terms_memory_in_bytes": 0,
          "stored_fields_memory_in_bytes": 0,
          "term_vectors_memory_in_bytes": 0,
          "norms_memory_in_bytes": 0,
          "points_memory_in_bytes": 0,
          "doc_values_memory_in_bytes": 0,
          "index_writer_memory_in_bytes": 173816,
          "version_map_memory_in_bytes": 0,
          "fixed_bit_set_memory_in_bytes": 0,
          "max_unsafe_auto_id_timestamp": -1,
          "file_sizes": {}
        },
        "translog": {
          "operations": 76186,
          "size_in_bytes": 188975985,
          "uncommitted_operations": 76186,
          "uncommitted_size_in_bytes": 188975985,
          "earliest_last_modified_age": 794730
        },
        "request_cache": {
          "memory_size_in_bytes": 592,
          "evictions": 0,
          "hit_count": 0,
          "miss_count": 26
        },
        "recovery": {
          "current_as_source": 0,
          "current_as_target": 0,
          "throttle_time_in_millis": 0
        },
        "bulk": {
          "total_operations": 2118,
          "total_time_in_millis": 8115,
          "total_size_in_bytes": 183945929,
          "avg_time_in_millis": 2,
          "avg_size_in_bytes": 68323
        }
      }
    }
  }
}

Perhaps it's because data_stream has event.original fields as well?

4

The indices do notbhave the same amount of documents, so it is hard to compare. The old index has 1 primary shard, while the new one has 2 for very little data, which will increase storage.

If the events are not identical in the two indices that could have a significant impact as well. It is also worth comparing index settings to check which codec that is used.

5

Oh I gave you wrong information. I updated reply in previous.
They are same data, but there's a little difference in there number. it's because timing of data stream rollover in 8.4.x version cluster and delete previous index in 7.10 cluster for compare.

6

The indices still do not have the same number of primary shards andwith very little data that is likely to make a difference. You need to compare the same set of events (same structure) in the same number of primary shards for it to be a fair comparison. Even then it can differ as the indices may be in different stages of merging.

1 Like
7

ok I re-updated reply in previous, with same document count, and with same shard number!!

8

Are the fields and structure of the documents in the two indices the same? Can you post the mappings so we can see if there are any major differences? Can you post the index settings as well?

9

Mappings in 7.10 cluster - default mapping

{
  "mappings": {
    "_doc": {
      "_meta": {},
      "properties": {
        "@timestamp": {
          "type": "date"
        },
        "@version": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "agent": {
          "type": "object"
        },
        "class": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "ecs": {
          "type": "object"
        },
        "fields": {
          "properties": {
            "log_type": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            }
          }
        },
        "host": {
          "type": "object"
        },
        "input": {
          "type": "object"
        },
        "level": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "line": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "log": {
          "properties": {
            "file": {
              "properties": {
                "path": {
                  "type": "text",
                  "fields": {
                    "keyword": {
                      "type": "keyword",
                      "ignore_above": 256
                    }
                  }
                }
              }
            },
            "flags": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            }
          }
        },
        "msg": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "thread": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        }
      }
    }
  }
}

mapping in 8.4.2 cluster - custom mapping

{
  "mappings": {
    "_data_stream_timestamp": {
      "enabled": true
    },
    "properties": {
      "@timestamp": {
        "type": "date"
      },
      "@version": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "agent": {
        "type": "object"
      },
      "class": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "ecs": {
        "type": "object"
      },
      "event": {
        "properties": {
          "original": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          }
        }
      },
      "fields": {
        "properties": {
          "log_type": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          }
        }
      },
      "host": {
        "type": "object"
      },
      "input": {
        "type": "object"
      },
      "level": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "line": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "log": {
        "properties": {
          "file": {
            "properties": {
              "path": {
                "type": "text",
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              }
            }
          },
          "flags": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          }
        }
      },
      "msg": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "thread": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      }
    }
  }
}

index setting of 7.10 cluster

{
  "settings": {
    "index": {
      "lifecycle": {
        "name": "delete-policy"
      },
      "routing": {
        "allocation": {
          "include": {
            "_tier_preference": "data_content"
          }
        }
      },
      "number_of_shards": "1",
      "provided_name": "search-platform.api-java-2022.12.12",
      "creation_date": "1670833725909",
      "priority": "1",
      "number_of_replicas": "0",
      "uuid": "V8lUUW21QFqPrg87JcAMXA",
      "version": {
        "created": "7100299"
      }
    }
  },
  "defaults": {
    "index": {
      "flush_after_merge": "512mb",
      "final_pipeline": "_none",
      "max_inner_result_window": "100",
      "unassigned": {
        "node_left": {
          "delayed_timeout": "1m"
        }
      },
      "max_terms_count": "65536",
      "lifecycle": {
        "parse_origination_date": "false",
        "indexing_complete": "false",
        "rollover_alias": "",
        "origination_date": "-1"
      },
      "routing_partition_size": "1",
      "force_memory_term_dictionary": "false",
      "max_docvalue_fields_search": "100",
      "merge": {
        "scheduler": {
          "max_thread_count": "1",
          "auto_throttle": "true",
          "max_merge_count": "6"
        },
        "policy": {
          "reclaim_deletes_weight": "2.0",
          "floor_segment": "2mb",
          "max_merge_at_once_explicit": "30",
          "max_merge_at_once": "10",
          "max_merged_segment": "5gb",
          "expunge_deletes_allowed": "10.0",
          "segments_per_tier": "10.0",
          "deletes_pct_allowed": "33.0"
        }
      },
      "max_refresh_listeners": "1000",
      "max_regex_length": "1000",
      "load_fixed_bitset_filters_eagerly": "true",
      "number_of_routing_shards": "1",
      "write": {
        "wait_for_active_shards": "1"
      },
      "verified_before_close": "false",
      "mapping": {
        "coerce": "false",
        "nested_fields": {
          "limit": "50"
        },
        "depth": {
          "limit": "20"
        },
        "field_name_length": {
          "limit": "9223372036854775807"
        },
        "total_fields": {
          "limit": "1000"
        },
        "nested_objects": {
          "limit": "10000"
        },
        "ignore_malformed": "false"
      },
      "source_only": "false",
      "soft_deletes": {
        "enabled": "true",
        "retention": {
          "operations": "0"
        },
        "retention_lease": {
          "period": "12h"
        }
      },
      "max_script_fields": "32",
      "query": {
        "default_field": [
          "*"
        ],
        "parse": {
          "allow_unmapped_fields": "true"
        }
      },
      "format": "0",
      "frozen": "false",
      "sort": {
        "missing": [],
        "mode": [],
        "field": [],
        "order": []
      },
      "codec": "default",
      "max_rescore_window": "10000",
      "max_adjacency_matrix_filters": "100",
      "analyze": {
        "max_token_count": "10000"
      },
      "gc_deletes": "60s",
      "top_metrics_max_size": "10",
      "optimize_auto_generated_id": "true",
      "max_ngram_diff": "1",
      "hidden": "false",
      "translog": {
        "generation_threshold_size": "64mb",
        "flush_threshold_size": "512mb",
        "sync_interval": "5s",
        "retention": {
          "size": "-1",
          "age": "-1"
        },
        "durability": "REQUEST"
      },
      "auto_expand_replicas": "false",
      "mapper": {
        "dynamic": "true"
      },
      "recovery": {
        "type": ""
      },
      "requests": {
        "cache": {
          "enable": "true"
        }
      },
      "data_path": "",
      "highlight": {
        "max_analyzed_offset": "1000000"
      },
      "routing": {
        "rebalance": {
          "enable": "all"
        },
        "allocation": {
          "include": {
            "_tier": ""
          },
          "exclude": {
            "_tier": ""
          },
          "require": {
            "_tier": ""
          },
          "enable": "all",
          "total_shards_per_node": "-1"
        }
      },
      "search": {
        "slowlog": {
          "level": "TRACE",
          "threshold": {
            "fetch": {
              "warn": "-1",
              "trace": "-1",
              "debug": "-1",
              "info": "-1"
            },
            "query": {
              "warn": "-1",
              "trace": "-1",
              "debug": "-1",
              "info": "-1"
            }
          }
        },
        "idle": {
          "after": "30s"
        },
        "throttled": "false"
      },
      "fielddata": {
        "cache": "node"
      },
      "default_pipeline": "_none",
      "max_slices_per_scroll": "1024",
      "shard": {
        "check_on_startup": "false"
      },
      "xpack": {
        "watcher": {
          "template": {
            "version": ""
          }
        },
        "version": "",
        "ccr": {
          "following_index": "false"
        }
      },
      "percolator": {
        "map_unmapped_fields_as_text": "false"
      },
      "allocation": {
        "max_retries": "5",
        "existing_shards_allocator": "gateway_allocator"
      },
      "refresh_interval": "1s",
      "indexing": {
        "slowlog": {
          "reformat": "true",
          "threshold": {
            "index": {
              "warn": "-1",
              "trace": "-1",
              "debug": "-1",
              "info": "-1"
            }
          },
          "source": "1000",
          "level": "TRACE"
        }
      },
      "compound_format": "0.1",
      "blocks": {
        "metadata": "false",
        "read": "false",
        "read_only_allow_delete": "false",
        "read_only": "false",
        "write": "false"
      },
      "max_result_window": "10000",
      "store": {
        "stats_refresh_interval": "10s",
        "type": "",
        "fs": {
          "fs_lock": "native"
        },
        "preload": [],
        "snapshot": {
          "snapshot_name": "",
          "index_uuid": "",
          "cache": {
            "prewarm": {
              "enabled": "true"
            },
            "enabled": "true",
            "excluded_file_types": []
          },
          "uncached_chunk_size": "-1b",
          "index_name": "",
          "repository_name": "",
          "snapshot_uuid": ""
        }
      },
      "queries": {
        "cache": {
          "enabled": "true"
        }
      },
      "warmer": {
        "enabled": "true"
      },
      "max_shingle_diff": "3",
      "query_string": {
        "lenient": "false"
      }
    }
  }
}

index setting of 8.4.2 cluster

{
  "settings": {
    "index": {
      "lifecycle": {
        "name": "search-platform-kafka"
      },
      "routing": {
        "allocation": {
          "include": {
            "_tier_preference": "data_hot"
          }
        }
      },
      "hidden": "true",
      "number_of_shards": "1",
      "provided_name": ".ds-logs-search-platform.api-java-2022.12.12-000010",
      "creation_date": "1670833723525",
      "priority": "100",
      "number_of_replicas": "0",
      "uuid": "pcM5lgdLSi6oldytMtwEZw",
      "version": {
        "created": "8040299"
      }
    }
  },
  "defaults": {
    "index": {
      "flush_after_merge": "512mb",
      "final_pipeline": "_none",
      "max_inner_result_window": "100",
      "unassigned": {
        "node_left": {
          "delayed_timeout": "1m"
        }
      },
      "max_terms_count": "65536",
      "rollup": {
        "source": {
          "name": "",
          "uuid": ""
        },
        "status": "unknown"
      },
      "lifecycle": {
        "parse_origination_date": "false",
        "step": {
          "wait_time_threshold": "12h"
        },
        "indexing_complete": "false",
        "rollover_alias": "",
        "origination_date": "-1"
      },
      "routing_partition_size": "1",
      "max_docvalue_fields_search": "100",
      "merge": {
        "scheduler": {
          "max_thread_count": "4",
          "auto_throttle": "true",
          "max_merge_count": "9"
        },
        "policy": {
          "floor_segment": "2mb",
          "max_merge_at_once_explicit": "30",
          "max_merge_at_once": "10",
          "max_merged_segment": "5gb",
          "expunge_deletes_allowed": "10.0",
          "segments_per_tier": "10.0",
          "deletes_pct_allowed": "33.0"
        }
      },
      "max_refresh_listeners": "1000",
      "max_regex_length": "1000",
      "load_fixed_bitset_filters_eagerly": "true",
      "number_of_routing_shards": "1",
      "write": {
        "wait_for_active_shards": "1"
      },
      "verified_before_close": "false",
      "mapping": {
        "coerce": "false",
        "nested_fields": {
          "limit": "50"
        },
        "depth": {
          "limit": "20"
        },
        "field_name_length": {
          "limit": "9223372036854775807"
        },
        "total_fields": {
          "limit": "1000"
        },
        "nested_objects": {
          "limit": "10000"
        },
        "ignore_malformed": "false",
        "dimension_fields": {
          "limit": "16"
        }
      },
      "source_only": "false",
      "soft_deletes": {
        "enabled": "true",
        "retention": {
          "operations": "0"
        },
        "retention_lease": {
          "period": "12h"
        }
      },
      "max_script_fields": "32",
      "query": {
        "default_field": [
          "*"
        ],
        "parse": {
          "allow_unmapped_fields": "true"
        }
      },
      "format": "0",
      "frozen": "false",
      "sort": {
        "missing": [],
        "mode": [],
        "field": [],
        "order": []
      },
      "version": {
        "compatibility": "8040299"
      },
      "codec": "default",
      "max_rescore_window": "10000",
      "analyze": {
        "max_token_count": "10000"
      },
      "gc_deletes": "60s",
      "top_metrics_max_size": "10",
      "max_ngram_diff": "1",
      "translog": {
        "generation_threshold_size": "64mb",
        "flush_threshold_size": "512mb",
        "sync_interval": "5s",
        "retention": {
          "size": "-1",
          "age": "-1"
        },
        "durability": "REQUEST"
      },
      "auto_expand_replicas": "false",
      "recovery": {
        "type": ""
      },
      "requests": {
        "cache": {
          "enable": "true"
        }
      },
      "data_path": "",
      "highlight": {
        "max_analyzed_offset": "1000000"
      },
      "routing": {
        "rebalance": {
          "enable": "all"
        },
        "allocation": {
          "disk": {
            "watermark": {
              "ignore": "false"
            }
          },
          "enable": "all",
          "total_shards_per_node": "-1"
        }
      },
      "search": {
        "slowlog": {
          "threshold": {
            "fetch": {
              "warn": "-1",
              "trace": "-1",
              "debug": "-1",
              "info": "-1"
            },
            "query": {
              "warn": "-1",
              "trace": "-1",
              "debug": "-1",
              "info": "-1"
            }
          }
        },
        "idle": {
          "after": "30s"
        },
        "throttled": "false"
      },
      "fielddata": {
        "cache": "node"
      },
      "default_pipeline": "_none",
      "max_slices_per_scroll": "1024",
      "shard": {
        "check_on_startup": "false"
      },
      "xpack": {
        "watcher": {
          "template": {
            "version": ""
          }
        },
        "version": "",
        "ccr": {
          "following_index": "false"
        }
      },
      "percolator": {
        "map_unmapped_fields_as_text": "false"
      },
      "allocation": {
        "max_retries": "5",
        "existing_shards_allocator": "gateway_allocator"
      },
      "refresh_interval": "1s",
      "indexing": {
        "slowlog": {
          "reformat": "true",
          "threshold": {
            "index": {
              "warn": "-1",
              "trace": "-1",
              "debug": "-1",
              "info": "-1"
            }
          },
          "source": "1000"
        }
      },
      "compound_format": "0.1",
      "blocks": {
        "metadata": "false",
        "read": "false",
        "read_only_allow_delete": "false",
        "read_only": "false",
        "write": "false"
      },
      "max_result_window": "10000",
      "store": {
        "stats_refresh_interval": "10s",
        "type": "",
        "fs": {
          "fs_lock": "native"
        },
        "preload": [],
        "snapshot": {
          "snapshot_name": "",
          "index_uuid": "",
          "cache": {
            "prewarm": {
              "enabled": "true"
            },
            "enabled": "true",
            "excluded_file_types": []
          },
          "repository_uuid": "",
          "uncached_chunk_size": "-1b",
          "delete_searchable_snapshot": "false",
          "index_name": "",
          "partial": "false",
          "blob_cache": {
            "metadata_files": {
              "max_length": "64kb"
            }
          },
          "repository_name": "",
          "snapshot_uuid": ""
        }
      },
      "queries": {
        "cache": {
          "enabled": "true"
        }
      },
      "shard_limit": {
        "group": "normal"
      },
      "warmer": {
        "enabled": "true"
      },
      "max_shingle_diff": "3",
      "query_string": {
        "lenient": "false"
      }
    }
  }
}

Thanks in advance...

And there's another difference.

In 7.10 cluster, Logstash put data to es in index

        elasticsearch {
            hosts => ["es:9200"]
            index => "%{[@metadata][kafka][topic]}-%{[fields][log_type]}-%{+yyyy.MM.dd}"
        }

and in 8.4.2 cluster, put data as data stream but in this way.

        elasticsearch {
            hosts => ["https://{id}:{pw}@es:443"]
            index => "logs-%{[@metadata][kafka][topic]}-%{[@metadata][target]}"
            action => "create"
        }
10

As you said earlier, the new index has an additional field, event.original, mapped as both key word and text. I would expectthis to be a reasonably large field and take up a fair bit of space.

11

oh it also has a keyword field. yeah of course...

okay I'll try to remove this field and re-compare.

12

okay it decreased... but still 1.14x bigger.

13

In order to get a fair comparison (as they can be in different stages of merging), forcemerge both indices down to a single segment and see how that affects the numbers.

14

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.