Index rebuild taking much time

Elastic Stack Elasticsearch
1

Hi,

I have only 170GB of Data, each time restarting data node, for rebuilding index taking to much time

active percentage increased 18 to 19 it took almost 24 hr, what could be the issue.
{
"cluster_name" : "elk-stack",
"status" : "red",
"timed_out" : false,
"number_of_nodes" : 8,
"number_of_data_nodes" : 3,
"active_primary_shards" : 201,
"active_shards" : 402,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 1640,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 19.686581782566112

Regards
sateesh

2

If you only have 170GB of data, why do you have so many shards?

3

Not sure what went wrong, i have deleted the index even after that so many shards is there, how to clear this one.

4

Are you using time based indices? How many indices are you creating per day? Are you using the default settings (5 primary shards, 1 replica)?

5

Yes using time stamp
filter {
if [fields][log_type] == "test_access" {
grok {
match => { "message" => "%{GREEDYDATA}" }
add_tag => ["test_access"]
remove_tag => ["_grokparsefailure", "beats_input_codec_plain_applied"]
}
date {
match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
}
}
}

output {
if [fields][log_type] == "test_access" {
elasticsearch { hosts => ["x.x.x.x:9200"]
index => "test-%{+YYYY.MM.dd}"
}
}

6

If you only have one time-based index, you can probably switch to using a monthly index instead given the volumes. You should also be able to reduce the number of primary shards through an index template.

What is the point of this grok expression?

7

One of the data node shows the below error, looks like this is the causing issue.

error logs-[SERVICE_UNAVAILABLE/2/no master

This is _cluster/_status?pretty' ---status
{
"error" : {
"root_cause" : [ {
"type" : "illegal_argument_exception",
"reason" : "No feature for name [_status]"
} ],
"type" : "illegal_argument_exception",
"reason" : "No feature for name [_status]"
},
"status" : 400
}

8

Can you provide the full output of the cluster stats API? How much heap do you data nodes have assigned?

9

{
"timestamp" : 1519280649176,
"cluster_name" : "elk-elastic",
"status" : "red",
"indices" : {
"count" : 43,
"shards" : {
"total" : 422,
"primaries" : 211,
"replication" : 1.0,
"index" : {
"shards" : {
"min" : 2,
"max" : 10,
"avg" : 9.813953488372093
},
"primaries" : {
"min" : 1,
"max" : 5,
"avg" : 4.906976744186046
},
"replication" : {
"min" : 1.0,
"max" : 1.0,
"avg" : 1.0
}
}
},
"docs" : {
"count" : 855052872,
"deleted" : 2
},
"store" : {
"size" : "652.1gb",
"size_in_bytes" : 700226555319,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"fielddata" : {
"memory_size" : "0b",
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "31.3kb",
"memory_size_in_bytes" : 32096,
"total_count" : 1110181,
"hit_count" : 0,
"miss_count" : 1110181,
"cache_size" : 57,
"cache_count" : 128,
"evictions" : 71
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 6015,
"memory" : "2.1gb",
"memory_in_bytes" : 2302303124,
"terms_memory" : "1.9gb",
"terms_memory_in_bytes" : 2046090264,
"stored_fields_memory" : "237.8mb",
"stored_fields_memory_in_bytes" : 249426656,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "4.5mb",
"norms_memory_in_bytes" : 4792384,
"doc_values_memory" : "1.9mb",
"doc_values_memory_in_bytes" : 1993820,
"index_writer_memory" : "9.5mb",
"index_writer_memory_in_bytes" : 10044416,
"index_writer_max_memory" : "9.5gb",
"index_writer_max_memory_in_bytes" : 10292884036,
"version_map_memory" : "969.6kb",
"version_map_memory_in_bytes" : 992940,
"fixed_bit_set" : "0b",
"fixed_bit_set_memory_in_bytes" : 0
},
"percolate" : {
"total" : 0,
"time" : "0s",
"time_in_millis" : 0,
"current" : 0,
"memory_size_in_bytes" : -1,
"memory_size" : "-1b",
"queries" : 0
}
},
"nodes" : {
"count" : {
"total" : 8,
"master_only" : 3,
"data_only" : 3,
"master_data" : 0,
"client" : 0
},
"versions" : [ "2.4.6" ],
"os" : {
"available_processors" : 32,
"allocated_processors" : 32,
"mem" : {
"total" : "53.4gb",
"total_in_bytes" : 57378734080
},
"names" : [ {
"name" : "Linux",
"count" : 8
} ]
},
"process" : {
"cpu" : {
"percent" : 9
},
"open_file_descriptors" : {
"min" : 343,
"max" : 3439,
"avg" : 1485
}
},
"jvm" : {
"max_uptime" : "16.7d",
"max_uptime_in_millis" : 1451058144,
"versions" : [ {
"version" : "1.8.0_151",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "25.151-b12",
"vm_vendor" : "Oracle Corporation",
"count" : 8
} ],
"mem" : {
"heap_used" : "8.4gb",
"heap_used_in_bytes" : 9042331416,
"heap_max" : "31.7gb",
"heap_max_in_bytes" : 34080817152
},
"threads" : 501
},
"fs" : {
"total" : "4.6tb",
"total_in_bytes" : 5073091891200,
"free" : "3.9tb",
"free_in_bytes" : 4366264889344,
"available" : "3.7tb",
"available_in_bytes" : 4150526734336
},
"plugins" : [ {
"name" : "head",
"version" : "master",
"description" : "head - A web front end for an Elasticsearch cluster",
"url" : "/_plugin/head/",
"jvm" : false,
"site" : true
}, {
"name" : "elastic-cloud",
"version" : "2.4.6",
"description" : "The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories.",
"jvm" : true,
"classname" : "org.elasticsearch.plugin.cloud.aws.CloudAwsPlugin",
"isolated" : true,
"site" : false
}, {
"name" : "analysis-kuromoji",
"version" : "2.4.6",
"description" : "The Japanese (kuromoji) Analysis plugin integrates Lucene kuromoji analysis module into elasticsearch.",
"jvm" : true,
"classname" : "org.elasticsearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin",
"isolated" : true,
"site" : false
}, {
"name" : "analysis-icu",
"version" : "2.4.6",
"description" : "The ICU Analysis plugin integrates Lucene ICU module into elasticsearch, adding ICU relates analysis components.",
"jvm" : true,
"classname" : "org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin",
"isolated" : true,
"site" : false
} ]
}
}

10

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.