After migrating from elastic version 5.2 to 7.16.2 force merge for few indices are taking long time. These indices are around 120GB. In the force merge step we are making total segement as 1. Any help here
Welcome to our community! 
What is the output from the _cluster/stats?pretty&human API?
What sort of hardware do you have?
I am using AWS m6gd.xlarge for master and r6gd.4xlarge for data nodes.
output:
"_nodes" : {
"total" : 9,
"successful" : 9,
"failed" : 0
},
"cluster_name" : "xxxxxxxxxxxxxxxxxxxxxxxxxxx",
"cluster_uuid" : "J5ytFIm0T0q2UclSLZsuuA",
"timestamp" : 1661147292263,
"status" : "green",
"indices" : {
"count" : 80,
"shards" : {
"total" : 300,
"primaries" : 150,
"replication" : 1.0,
"index" : {
"shards" : {
"min" : 2,
"max" : 4,
"avg" : 3.75
},
"primaries" : {
"min" : 1,
"max" : 2,
"avg" : 1.875
},
"replication" : {
"min" : 1.0,
"max" : 1.0,
"avg" : 1.0
}
}
},
"docs" : {
"count" : 184524939,
"deleted" : 304
},
"store" : {
"size_in_bytes" : 1185670527337,
"total_data_set_size_in_bytes" : 1185670527337,
"reserved_in_bytes" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 1704,
"evictions" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 76270587,
"total_count" : 452467,
"hit_count" : 20878,
"miss_count" : 431589,
"cache_size" : 822,
"cache_count" : 822,
"evictions" : 0
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 558,
"memory_in_bytes" : 14699400,
"terms_memory_in_bytes" : 9376112,
"stored_fields_memory_in_bytes" : 1600800,
"term_vectors_memory_in_bytes" : 759136,
"norms_memory_in_bytes" : 1350464,
"points_memory_in_bytes" : 0,
"doc_values_memory_in_bytes" : 1612888,
"index_writer_memory_in_bytes" : 267748644,
"version_map_memory_in_bytes" : 439192,
"fixed_bit_set_memory_in_bytes" : 38867928,
"max_unsafe_auto_id_timestamp" : 1654161791759,
"file_sizes" : { }
},
"mappings" : {
"field_types" : [
{
"name" : "boolean",
"count" : 3,
"index_count" : 3,
"script_count" : 0
},
{
"name" : "constant_keyword",
"count" : 18,
"index_count" : 6,
"script_count" : 0
},
{
"name" : "date",
"count" : 523,
"index_count" : 67,
"script_count" : 0
},
{
"name" : "double",
"count" : 80,
"index_count" : 6,
"script_count" : 0
},
{
"name" : "float",
"count" : 244,
"index_count" : 16,
"script_count" : 0
},
{
"name" : "icu_collation_keyword",
"count" : 214,
"index_count" : 60,
"script_count" : 0
},
{
"name" : "integer",
"count" : 226,
"index_count" : 48,
"script_count" : 0
},
{
"name" : "ip",
"count" : 6,
"index_count" : 6,
"script_count" : 0
},
{
"name" : "keyword",
"count" : 1242,
"index_count" : 79,
"script_count" : 0
},
{
"name" : "long",
"count" : 3,
"index_count" : 3,
"script_count" : 0
},
{
"name" : "nested",
"count" : 62,
"index_count" : 20,
"script_count" : 0
},
{
"name" : "object",
"count" : 524,
"index_count" : 61,
"script_count" : 0
},
{
"name" : "text",
"count" : 7371,
"index_count" : 79,
"script_count" : 0
}
],
"runtime_field_types" : [ ]
},
"analysis" : {
"char_filter_types" : [
{
"name" : "pattern_replace",
"count" : 128,
"index_count" : 66
}
],
"tokenizer_types" : [
{
"name" : "pattern",
"count" : 66,
"index_count" : 66
}
],
"filter_types" : [
{
"name" : "edgeNGram",
"count" : 4,
"index_count" : 4
},
{
"name" : "pattern_replace",
"count" : 4,
"index_count" : 4
},
{
"name" : "synonym_graph",
"count" : 56,
"index_count" : 56
}
],
"analyzer_types" : [
{
"name" : "custom",
"count" : 260,
"index_count" : 70
}
],
"built_in_char_filters" : [ ],
"built_in_tokenizers" : [
{
"name" : "keyword",
"count" : 70,
"index_count" : 66
}
],
"built_in_filters" : [
{
"name" : "lowercase",
"count" : 198,
"index_count" : 70
}
],
"built_in_analyzers" : [ ]
},
"versions" : [
{
"version" : "7.16.2",
"index_count" : 80,
"primary_shard_count" : 150,
"total_primary_bytes" : 592879832797
}
]
},
"nodes" : {
"count" : {
"total" : 9,
"coordinating_only" : 0,
"data" : 6,
"data_cold" : 6,
"data_content" : 6,
"data_frozen" : 6,
"data_hot" : 6,
"data_warm" : 6,
"ingest" : 6,
"master" : 3,
"ml" : 9,
"remote_cluster_client" : 9,
"transform" : 6,
"voting_only" : 0
},
"versions" : [
"7.16.2"
],
"os" : {
"available_processors" : 108,
"allocated_processors" : 108,
"names" : [
{
"name" : "Linux",
"count" : 9
}
],
"pretty_names" : [
{
"pretty_name" : "Ubuntu 20.04.3 LTS",
"count" : 9
}
],
"architectures" : [
{
"arch" : "aarch64",
"count" : 9
}
],
"mem" : {
"total_in_bytes" : 852934926336,
"free_in_bytes" : 119647817728,
"used_in_bytes" : 733287108608,
"free_percent" : 14,
"used_percent" : 86
}
},
"process" : {
"cpu" : {
"percent" : 0
},
"open_file_descriptors" : {
"min" : 584,
"max" : 1139,
"avg" : 923
}
},
"jvm" : {
"max_uptime_in_millis" : 6985534266,
"versions" : [
{
"version" : "17.0.1",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "17.0.1+12",
"vm_vendor" : "Eclipse Adoptium",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 9
}
],
"mem" : {
"heap_used_in_bytes" : 136658363640,
"heap_max_in_bytes" : 335007449088
},
"threads" : 986
},
"fs" : {
"total_in_bytes" : 6407838154752,
"free_in_bytes" : 5205016604672,
"available_in_bytes" : 5205016604672
},
"plugins" : [
{
"name" : "analysis-icu",
"version" : "7.16.2",
"elasticsearch_version" : "7.16.2",
"java_version" : "1.8",
"description" : "The ICU Analysis plugin integrates the Lucene ICU module into Elasticsearch, adding ICU-related analysis components.",
"classname" : "org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
},
{
"name" : "discovery-ec2",
"version" : "7.16.2",
"elasticsearch_version" : "7.16.2",
"java_version" : "1.8",
"description" : "The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.",
"classname" : "org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
},
{
"name" : "analyzed-queries",
"version" : "7162.3.0",
"elasticsearch_version" : "7.16.2",
"java_version" : "1.8",
"description" : "Plugin that adds additional analyzed queries to Elasticsearch",
"classname" : "com.clarivate.es.plugin.AnalyzedQueriesPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
},
{
"name" : "ids-search",
"version" : "7162.3.0",
"elasticsearch_version" : "7.16.2",
"java_version" : "1.8",
"description" : "Plugin that executes a search returning only matching ids",
"classname" : "com.clarivate.es.plugin.IdsSearchPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
},
{
"name" : "repository-s3",
"version" : "7.16.2",
"elasticsearch_version" : "7.16.2",
"java_version" : "1.8",
"description" : "The S3 repository plugin adds S3 repositories",
"classname" : "org.elasticsearch.repositories.s3.S3RepositoryPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
},
{
"name" : "rest-logger",
"version" : "7162.3.0",
"elasticsearch_version" : "7.16.2",
"java_version" : "1.8",
"description" : "Plugin that logs and times http requests.",
"classname" : "com.clarivate.es.plugin.RestLoggerPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"licensed" : false,
"type" : "isolated"
}
],
"network_types" : {
"transport_types" : {
"security4" : 9
},
"http_types" : {
"security4" : 9
}
},
"discovery_types" : {
"zen" : 9
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "docker",
"count" : 9
}
],
"ingest" : {
"number_of_pipelines" : 2,
"processor_stats" : {
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"script" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
}
}
}
}
}
You state that force merge is taking a long time, could you provide more detail here? What were the force merge times in 5.2? What are they now in 7.16.2? Have you tried upgrading to the latest 8.4.x? (There have been some performance/scalability improvements in the 8.x release line).
What kind of storage are you using: EBS, EFS, local NVMe? Force merge is a fairly intensive process, are you seeing the node's CPU max out, is your disk IO being maxed?
Hi Ben,
ES5 took around 100 minutes to complete forcemerge on all indices. But ES7 is taking 180 minutes to complete forcemerge on same indices.
Its using local NVM storage.
We dont have any resource constraints.
Regards.
Prasath S
Only four indices which are around 120GB are utilizing max time in it.
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.