I have only one node elasticsearch at my cluster. I'm trying to write into elasticsearch using PutElasticsearchHttp control at my NiFi flow but I get an error:
2023-05-24 07:00:17,347 ERROR [Timer-Driven Process Thread-3] o.a.n.p.e.PutElasticsearchHttp PutElasticsearchHttp[id=237111a5-14ab-143f-4e7b-56c0b4bd3a05] Failed to process StandardFlowFileRecord[uuid=e540a2d7-0783-43e4-a07c-8005f8d086ac,claim=StandardContentClaim [resourceClaim=StandardResourceClaim[id=1684911617322-2727, container=default, section=679], offset=9083, length=633],offset=0,name=d273874a-3273-486f-8b78-9e7d0f923ed2,size=633] due to
Validation Failed: 1: this action would add [2] total shards, but this cluster currently has [999]/[1000] maximum shards open;, transferring to failure
when I checked _cluster/stats, I got only 509 shards while error is 999 shards which I do not have! are there more shards not shown? how to control this to avoid such issues while importing data using different tools?
{
"_nodes" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"cluster_name" : "docker-cluster",
"cluster_uuid" : "0lpL8vheSdeFYpfQmbb9uw",
"timestamp" : 1684922893105,
"status" : "yellow",
"indices" : {
"count" : 509,
"shards" : {
"total" : 509,
"primaries" : 509,
"replication" : 0.0,
"index" : {
"shards" : {
"min" : 1,
"max" : 1,
"avg" : 1.0
},
"primaries" : {
"min" : 1,
"max" : 1,
"avg" : 1.0
},
"replication" : {
"min" : 0.0,
"max" : 0.0,
"avg" : 0.0
}
}
},
"docs" : {
"count" : 619613169,
"deleted" : 684
},
"store" : {
"size_in_bytes" : 855924263210,
"reserved_in_bytes" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 2112,
"evictions" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 0,
"total_count" : 0,
"hit_count" : 0,
"miss_count" : 0,
"cache_size" : 0,
"cache_count" : 0,
"evictions" : 0
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 4474,
"memory_in_bytes" : 45867042,
"terms_memory_in_bytes" : 37245464,
"stored_fields_memory_in_bytes" : 2569120,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 5289792,
"points_memory_in_bytes" : 0,
"doc_values_memory_in_bytes" : 762666,
"index_writer_memory_in_bytes" : 2986552,
"version_map_memory_in_bytes" : 0,
"fixed_bit_set_memory_in_bytes" : 1216,
"max_unsafe_auto_id_timestamp" : 1684912919948,
"file_sizes" : { }
},
"mappings" : {
"field_types" : [
{
"name" : "binary",
"count" : 15,
"index_count" : 4
},
{
"name" : "boolean",
"count" : 51,
"index_count" : 14
},
{
"name" : "byte",
"count" : 4,
"index_count" : 4
},
{
"name" : "date",
"count" : 1003,
"index_count" : 476
},
{
"name" : "date_nanos",
"count" : 1,
"index_count" : 1
},
{
"name" : "date_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "double",
"count" : 1,
"index_count" : 1
},
{
"name" : "double_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "flattened",
"count" : 9,
"index_count" : 1
},
{
"name" : "float",
"count" : 4,
"index_count" : 2
},
{
"name" : "float_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "geo_point",
"count" : 1,
"index_count" : 1
},
{
"name" : "geo_shape",
"count" : 1,
"index_count" : 1
},
{
"name" : "half_float",
"count" : 1,
"index_count" : 1
},
{
"name" : "integer",
"count" : 30,
"index_count" : 4
},
{
"name" : "integer_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "ip",
"count" : 1,
"index_count" : 1
},
{
"name" : "ip_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "keyword",
"count" : 10165,
"index_count" : 507
},
{
"name" : "long",
"count" : 37,
"index_count" : 15
},
{
"name" : "long_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "nested",
"count" : 11,
"index_count" : 6
},
{
"name" : "object",
"count" : 212,
"index_count" : 22
},
{
"name" : "shape",
"count" : 1,
"index_count" : 1
},
{
"name" : "short",
"count" : 7,
"index_count" : 4
},
{
"name" : "text",
"count" : 9884,
"index_count" : 504
}
]
},
"analysis" : {
"char_filter_types" : [ ],
"tokenizer_types" : [ ],
"filter_types" : [
{
"name" : "pattern_capture",
"count" : 1,
"index_count" : 1
}
],
"analyzer_types" : [
{
"name" : "custom",
"count" : 1,
"index_count" : 1
}
],
"built_in_char_filters" : [ ],
"built_in_tokenizers" : [
{
"name" : "uax_url_email",
"count" : 1,
"index_count" : 1
}
],
"built_in_filters" : [
{
"name" : "lowercase",
"count" : 1,
"index_count" : 1
},
{
"name" : "unique",
"count" : 1,
"index_count" : 1
}
],
"built_in_analyzers" : [ ]
}
},
"nodes" : {
"count" : {
"total" : 1,
"coordinating_only" : 0,
"data" : 1,
"data_cold" : 1,
"data_content" : 1,
"data_hot" : 1,
"data_warm" : 1,
"ingest" : 1,
"master" : 1,
"ml" : 1,
"remote_cluster_client" : 1,
"transform" : 1,
"voting_only" : 0
},
"versions" : [
"7.10.1"
],
"os" : {
"available_processors" : 4,
"allocated_processors" : 4,
"names" : [
{
"name" : "Linux",
"count" : 1
}
],
"pretty_names" : [
{
"pretty_name" : "CentOS Linux 8 (Core)",
"count" : 1
}
],
"mem" : {
"total_in_bytes" : 25111343104,
"free_in_bytes" : 252829696,
"used_in_bytes" : 24858513408,
"free_percent" : 1,
"used_percent" : 99
}
},
"process" : {
"cpu" : {
"percent" : 0
},
"open_file_descriptors" : {
"min" : 5727,
"max" : 5727,
"avg" : 5727
}
},
"jvm" : {
"max_uptime_in_millis" : 10005104,
"versions" : [
{
"version" : "15.0.1",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "15.0.1+9",
"vm_vendor" : "AdoptOpenJDK",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 1
}
],
"mem" : {
"heap_used_in_bytes" : 1840046080,
"heap_max_in_bytes" : 4294967296
},
"threads" : 114
},
"fs" : {
"total_in_bytes" : 2413785710592,
"free_in_bytes" : 1421521993728,
"available_in_bytes" : 1323284590592
},
"plugins" : [ ],
"network_types" : {
"transport_types" : {
"security4" : 1
},
"http_types" : {
"security4" : 1
}
},
"discovery_types" : {
"single-node" : 1
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "docker",
"count" : 1
}
],
"ingest" : {
"number_of_pipelines" : 1,
"processor_stats" : {
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"script" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
}
}
}
}
}