Thankyou for responding.
I had to stop the indexing process as elasticsearch was throwing curcit breaker exception,
Secondly as i mentioned i need to index 8TB of data out of which 2TB of data has been indexed and the heap usage reached 87 percent, so im concerned if 8TB will be indexed or not
This is the result of _cat/stats api while indexing is running.
{
"_nodes" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"cluster_name" : "elasticsearch",
"cluster_uuid" : "ujl4Pr6vRLGuGd-J-Uytog",
"timestamp" : 1605016732338,
"status" : "yellow",
"indices" : {
"count" : 79,
"shards" : {
"total" : 118,
"primaries" : 118,
"replication" : 0.0,
"index" : {
"shards" : {
"min" : 1,
"max" : 2,
"avg" : 1.4936708860759493
},
"primaries" : {
"min" : 1,
"max" : 2,
"avg" : 1.4936708860759493
},
"replication" : {
"min" : 0.0,
"max" : 0.0,
"avg" : 0.0
}
}
},
"docs" : {
"count" : 35462448,
"deleted" : 234215
},
"store" : {
"size_in_bytes" : 2927978413605,
"reserved_in_bytes" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 0,
"total_count" : 0,
"hit_count" : 0,
"miss_count" : 0,
"cache_size" : 0,
"cache_count" : 0,
"evictions" : 0
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 2752,
"memory_in_bytes" : 1603695164,
"terms_memory_in_bytes" : 1316032320,
"stored_fields_memory_in_bytes" : 2499296,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 193009728,
"points_memory_in_bytes" : 0,
"doc_values_memory_in_bytes" : 92153820,
"index_writer_memory_in_bytes" : 280591928,
"version_map_memory_in_bytes" : 124873,
"fixed_bit_set_memory_in_bytes" : 1528,
"max_unsafe_auto_id_timestamp" : 1604355408208,
"file_sizes" : { }
},
"mappings" : {
"field_types" : [
{
"name" : "binary",
"count" : 25,
"index_count" : 4
},
{
"name" : "boolean",
"count" : 651,
"index_count" : 71
},
{
"name" : "byte",
"count" : 22,
"index_count" : 22
},
{
"name" : "date",
"count" : 2794,
"index_count" : 76
},
{
"name" : "flattened",
"count" : 19,
"index_count" : 3
},
{
"name" : "float",
"count" : 131,
"index_count" : 23
},
{
"name" : "geo_point",
"count" : 154,
"index_count" : 22
},
{
"name" : "geo_shape",
"count" : 2,
"index_count" : 2
},
{
"name" : "histogram",
"count" : 22,
"index_count" : 22
},
{
"name" : "integer",
"count" : 82,
"index_count" : 7
},
{
"name" : "ip",
"count" : 286,
"index_count" : 22
},
{
"name" : "keyword",
"count" : 313423,
"index_count" : 78
},
{
"name" : "long",
"count" : 2081,
"index_count" : 72
},
{
"name" : "nested",
"count" : 32,
"index_count" : 8
},
{
"name" : "object",
"count" : 11809,
"index_count" : 76
},
{
"name" : "scaled_float",
"count" : 45,
"index_count" : 23
},
{
"name" : "short",
"count" : 1,
"index_count" : 1
},
{
"name" : "text",
"count" : 303234,
"index_count" : 77
}
]
},
"analysis" : {
"char_filter_types" : [ ],
"tokenizer_types" : [ ],
"filter_types" : [
{
"name" : "pattern_capture",
"count" : 1,
"index_count" : 1
},
{
"name" : "stemmer",
"count" : 39,
"index_count" : 39
}
],
"analyzer_types" : [
{
"name" : "custom",
"count" : 118,
"index_count" : 40
}
],
"built_in_char_filters" : [
{
"name" : "html_strip",
"count" : 39,
"index_count" : 39
}
],
"built_in_tokenizers" : [
{
"name" : "standard",
"count" : 78,
"index_count" : 39
},
{
"name" : "uax_url_email",
"count" : 40,
"index_count" : 40
}
],
"built_in_filters" : [
{
"name" : "lowercase",
"count" : 118,
"index_count" : 40
},
{
"name" : "unique",
"count" : 1,
"index_count" : 1
}
],
"built_in_analyzers" : [ ]
}
},
"nodes" : {
"count" : {
"total" : 1,
"coordinating_only" : 0,
"data" : 1,
"ingest" : 1,
"master" : 1,
"ml" : 1,
"remote_cluster_client" : 1,
"transform" : 1,
"voting_only" : 0
},
"versions" : [
"7.9.1"
],
"os" : {
"available_processors" : 8,
"allocated_processors" : 8,
"names" : [
{
"name" : "Windows Server 2019",
"count" : 1
}
],
"pretty_names" : [
{
"pretty_name" : "Windows Server 2019",
"count" : 1
}
],
"mem" : {
"total_in_bytes" : 51538526208,
"free_in_bytes" : 17155682304,
"used_in_bytes" : 34382843904,
"free_percent" : 33,
"used_percent" : 67
}
},
"process" : {
"cpu" : {
"percent" : 11
},
"open_file_descriptors" : {
"min" : -1,
"max" : -1,
"avg" : 0
}
},
"jvm" : {
"max_uptime_in_millis" : 20173111,
"versions" : [
{
"version" : "14.0.1",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "14.0.1+7",
"vm_vendor" : "AdoptOpenJDK",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 1
}
],
"mem" : {
"heap_used_in_bytes" : 18695740656,
"heap_max_in_bytes" : 21474836480
},
"threads" : 193
},
"fs" : {
"total_in_bytes" : 6597013139456,
"free_in_bytes" : 3666669338624,
"available_in_bytes" : 3666669338624
},
"plugins" : [ ],
"network_types" : {
"transport_types" : {
"security4" : 1
},
"http_types" : {
"security4" : 1
}
},
"discovery_types" : {
"zen" : 1
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "zip",
"count" : 1
}
],
"ingest" : {
"number_of_pipelines" : 7,
"processor_stats" : {
"conditional" : {
"count" : 23268,
"failed" : 0,
"current" : 0,
"time_in_millis" : 3
},
"geoip" : {
"count" : 23268,
"failed" : 0,
"current" : 0,
"time_in_millis" : 178
},
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"pipeline" : {
"count" : 93072,
"failed" : 0,
"current" : 0,
"time_in_millis" : 207
},
"script" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"unknown" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"user_agent" : {
"count" : 23268,
"failed" : 0,
"current" : 0,
"time_in_millis" : 7
}
}
}
}
}
This is the response of _cat/nodes api
ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
127.0.0.1 89 67 34 dilmrt * ---