Hi everyone, I want to ask why the primary shard indexes for 1 day are only stored on 1 Hot3 node. Causing the Hot3 node to get a high CPU boost and denying the bulk request from the Coordination node that controls my forwarding logs from Logstash to Elasticsearch is congested.
You need to provide more context about your cluster.
How many nodes do you have?
What is the result of running the following request in Dev tools.
GET _cat/nodes?v&h=name,diskTotal,diskUsedPercent,diskAvail
What is the full output of the cluster stats API?
Please do not post images of text. Instead copy and paste and make sure to use the formatting buttons.
this is
"_nodes": {
"total": 16,
"successful": 16,
"failed": 0
},
"cluster_name": "xxx-xxx",
"cluster_uuid": "A8e4NOzFReSMSoE9yxixaQ",
"timestamp": 1685977268903,
"status": "red",
"indices": {
"count": 679,
"shards": {
"total": 1140,
"primaries": 679,
"replication": 0.678939617083947,
"index": {
"shards": {
"min": 1,
"max": 2,
"avg": 1.678939617083947
},
"primaries": {
"min": 1,
"max": 1,
"avg": 1
},
"replication": {
"min": 0,
"max": 1,
"avg": 0.678939617083947
}
}
},
"docs": {
"count": 9100465554,
"deleted": 12377962
},
"store": {
"size_in_bytes": 2845594205070,
"total_data_set_size_in_bytes": 2845594205070,
"reserved_in_bytes": 0
},
"fielddata": {
"memory_size_in_bytes": 301724920,
"evictions": 0
},
"query_cache": {
"memory_size_in_bytes": 2570615780,
"total_count": 1277138509,
"hit_count": 149216308,
"miss_count": 1127922201,
"cache_size": 123384,
"cache_count": 573556,
"evictions": 450172
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 12827,
"memory_in_bytes": 0,
"terms_memory_in_bytes": 0,
"stored_fields_memory_in_bytes": 0,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 0,
"points_memory_in_bytes": 0,
"doc_values_memory_in_bytes": 0,
"index_writer_memory_in_bytes": 531281598,
"version_map_memory_in_bytes": 5352169,
"fixed_bit_set_memory_in_bytes": 151481544,
"max_unsafe_auto_id_timestamp": 1685973448412,
"file_sizes": {}
},
"mappings": {
"total_field_count": 239170,
"total_deduplicated_field_count": 168104,
"total_deduplicated_mapping_size_in_bytes": 1455653,
"field_types": [
{
"name": "alias",
"count": 1264,
"index_count": 27,
"script_count": 0
},
{
"name": "binary",
"count": 2,
"index_count": 2,
"script_count": 0
},
{
"name": "boolean",
"count": 1856,
"index_count": 537,
"script_count": 0
},
{
"name": "byte",
"count": 2,
"index_count": 2,
"script_count": 0
},
{
"name": "constant_keyword",
"count": 2332,
"index_count": 491,
"script_count": 0
},
{
"name": "date",
"count": 5519,
"index_count": 645,
"script_count": 0
},
{
"name": "date_nanos",
"count": 2,
"index_count": 2,
"script_count": 0
},
{
"name": "date_range",
"count": 2,
"index_count": 2,
"script_count": 0
},
{
"name": "double",
"count": 291,
"index_count": 31,
"script_count": 0
},
{
"name": "double_range",
"count": 2,
"index_count": 2,
"script_count": 0
},
{
"name": "flattened",
"count": 521,
"index_count": 211,
"script_count": 0
},
{
"name": "float",
"count": 2592,
"index_count": 288,
"script_count": 0
},
{
"name": "float_range",
"count": 2,
"index_count": 2,
"script_count": 0
},
{
"name": "geo_point",
"count": 662,
"index_count": 283,
"script_count": 0
},
{
"name": "geo_shape",
"count": 2,
"index_count": 2,
"script_count": 0
},
{
"name": "half_float",
"count": 85,
"index_count": 26,
"script_count": 0
},
{
"name": "histogram",
"count": 3,
"index_count": 3,
"script_count": 0
},
{
"name": "integer",
"count": 200,
"index_count": 40,
"script_count": 0
},
{
"name": "integer_range",
"count": 2,
"index_count": 2,
"script_count": 0
},
{
"name": "ip",
"count": 4536,
"index_count": 512,
"script_count": 0
},
{
"name": "ip_range",
"count": 2,
"index_count": 2,
"script_count": 0
},
{
"name": "keyword",
"count": 151367,
"index_count": 646,
"script_count": 0
},
{
"name": "long",
"count": 19516,
"index_count": 593,
"script_count": 0
},
{
"name": "long_range",
"count": 2,
"index_count": 2,
"script_count": 0
},
{
"name": "match_only_text",
"count": 2894,
"index_count": 439,
"script_count": 0
},
{
"name": "nested",
"count": 465,
"index_count": 67,
"script_count": 0
},
{
"name": "object",
"count": 40565,
"index_count": 594,
"script_count": 0
},
{
"name": "scaled_float",
"count": 658,
"index_count": 98,
"script_count": 0
},
{
"name": "shape",
"count": 2,
"index_count": 2,
"script_count": 0
},
{
"name": "short",
"count": 441,
"index_count": 15,
"script_count": 0
},
{
"name": "text",
"count": 2599,
"index_count": 564,
"script_count": 0
},
{
"name": "unsigned_long",
"count": 105,
"index_count": 11,
"script_count": 0
},
{
"name": "version",
"count": 26,
"index_count": 26,
"script_count": 0
},
{
"name": "wildcard",
"count": 651,
"index_count": 245,
"script_count": 0
}
],
"runtime_field_types": []
},
"analysis": {
"char_filter_types": [],
"tokenizer_types": [],
"filter_types": [],
"analyzer_types": [
{
"name": "pattern",
"count": 6,
"index_count": 6
}
],
"built_in_char_filters": [],
"built_in_tokenizers": [],
"built_in_filters": [],
"built_in_analyzers": [
{
"name": "powershell_script_analyzer",
"count": 3,
"index_count": 3
},
{
"name": "whitespace",
"count": 4,
"index_count": 4
},
{
"name": "winlogbeat_powershell_script_analyzer",
"count": 3,
"index_count": 3
}
]
},
"versions": [
{
"version": "8.5.0",
"index_count": 680,
"primary_shard_count": 680,
"total_primary_bytes": 2518783322457
}
]
},
"nodes": {
"count": {
"total": 16,
"coordinating_only": 2,
"data": 0,
"data_cold": 0,
"data_content": 7,
"data_frozen": 0,
"data_hot": 5,
"data_warm": 2,
"ingest": 12,
"master": 3,
"ml": 2,
"remote_cluster_client": 3,
"transform": 12,
"voting_only": 0
},
"versions": [
"8.5.0"
],
"os": {
"available_processors": 108,
"allocated_processors": 108,
"names": [
{
"name": "Linux",
"count": 16
}
],
"pretty_names": [
{
"pretty_name": "Ubuntu 20.04.5 LTS",
"count": 16
}
],
"architectures": [
{
"arch": "amd64",
"count": 16
}
],
"mem": {
"total_in_bytes": 429496729600,
"adjusted_total_in_bytes": 429496729600,
"free_in_bytes": 122409902080,
"used_in_bytes": 307086827520,
"free_percent": 29,
"used_percent": 71
}
},
"process": {
"cpu": {
"percent": 188
},
"open_file_descriptors": {
"min": 733,
"max": 2734,
"avg": 1569
}
},
"jvm": {
"max_uptime_in_millis": 3331527803,
"versions": [
{
"version": "19",
"vm_name": "OpenJDK 64-Bit Server VM",
"vm_version": "19+36-2238",
"vm_vendor": "Oracle Corporation",
"bundled_jdk": true,
"using_bundled_jdk": true,
"count": 16
}
],
"mem": {
"heap_used_in_bytes": 83956118280,
"heap_max_in_bytes": 214748364800
},
"threads": 2627
},
"fs": {
"total_in_bytes": 14390645006336,
"free_in_bytes": 11150733922304,
"available_in_bytes": 10477512306688
},
"plugins": [],
"network_types": {
"transport_types": {
"security4": 16
},
"http_types": {
"security4": 16
}
},
"discovery_types": {
"multi-node": 16
},
"packaging_types": [
{
"flavor": "default",
"type": "docker",
"count": 16
}
],
"ingest": {
"number_of_pipelines": 789,
"processor_stats": {
"append": {
"count": 0,
"failed": 0,
"current": 0,
"time_in_millis": 0
},
"community_id": {
"count": 42,
"failed": 0,
"current": 0,
"time_in_millis": 2
},
"conditional": {
"count": 111385028,
"failed": 456,
"current": 0,
"time_in_millis": 4762130
},
"convert": {
"count": 153613942,
"failed": 319186,
"current": 0,
"time_in_millis": 238784
},
"csv": {
"count": 11686,
"failed": 2280,
"current": 0,
"time_in_millis": 47
},
"date": {
"count": 143678708,
"failed": 0,
"current": 0,
"time_in_millis": 2500973
},
"dissect": {
"count": 2260425,
"failed": 2999,
"current": 0,
"time_in_millis": 7318
},
"dot_expander": {
"count": 0,
"failed": 0,
"current": 0,
"time_in_millis": 0
},
"fingerprint": {
"count": 84,
"failed": 0,
"current": 0,
"time_in_millis": 10
},
"foreach": {
"count": 127772,
"failed": 0,
"current": 0,
"time_in_millis": 880
},
"geoip": {
"count": 104497044,
"failed": 125942,
"current": 0,
"time_in_millis": 244963
},
"grok": {
"count": 980841,
"failed": 127991,
"current": 0,
"time_in_millis": 41469
},
"gsub": {
"count": 49975629,
"failed": 0,
"current": 0,
"time_in_millis": 50579
},
"join": {
"count": 0,
"failed": 0,
"current": 0,
"time_in_millis": 0
},
"json": {
"count": 0,
"failed": 0,
"current": 0,
"time_in_millis": 0
},
"kv": {
"count": 1381363,
"failed": 0,
"current": 0,
"time_in_millis": 5500
},
"lowercase": {
"count": 127772,
"failed": 0,
"current": 0,
"time_in_millis": 270
},
"network_direction": {
"count": 0,
"failed": 0,
"current": 0,
"time_in_millis": 0
},
"pipeline": {
"count": 192434538,
"failed": 0,
"current": 0,
"time_in_millis": 68679
},
"registered_domain": {
"count": 0,
"failed": 0,
"current": 0,
"time_in_millis": 0
},
"remove": {
"count": 339804670,
"failed": 46890045,
"current": 0,
"time_in_millis": 1020197
},
"rename": {
"count": 116123861,
"failed": 14270087,
"current": 0,
"time_in_millis": 215597
},
"script": {
"count": 1343710182,
"failed": 0,
"current": 0,
"time_in_millis": 1889665
},
"set": {
"count": 201593065,
"failed": 24829553,
"current": 0,
"time_in_millis": 406922
},
"set_security_user": {
"count": 143545951,
"failed": 0,
"current": 0,
"time_in_millis": 829420
},
"split": {
"count": 127772,
"failed": 127772,
"current": 0,
"time_in_millis": 609
},
"trim": {
"count": 0,
"failed": 0,
"current": 0,
"time_in_millis": 0
},
"uppercase": {
"count": 84,
"failed": 0,
"current": 0,
"time_in_millis": 0
},
"uri_parts": {
"count": 0,
"failed": 0,
"current": 0,
"time_in_millis": 0
},
"urldecode": {
"count": 0,
"failed": 0,
"current": 0,
"time_in_millis": 0
},
"user_agent": {
"count": 0,
"failed": 0,
"current": 0,
"time_in_millis": 0
}
}
},
"indexing_pressure": {
"memory": {
"current": {
"combined_coordinating_and_primary_in_bytes": 0,
"coordinating_in_bytes": 0,
"primary_in_bytes": 0,
"replica_in_bytes": 0,
"all_in_bytes": 0
},
"total": {
"combined_coordinating_and_primary_in_bytes": 0,
"coordinating_in_bytes": 0,
"primary_in_bytes": 0,
"replica_in_bytes": 0,
"all_in_bytes": 0,
"coordinating_rejections": 0,
"primary_rejections": 0,
"replica_rejections": 0
},
"limit_in_bytes": 0
}
}
}
}```Sorry.
node-name-master1 99.9gb 33.04 66.9gb
node-name-master2 99.9gb 31.48 68.5gb
node-name-hot3 1.7tb 9.12 1.5tb
node-name-warm2 1.7tb 41.89 1tb
node-name-hot1 1.6tb 16.50 1.3tb
node-name-ml2 99.9gb 74.01 25.9gb
node-name-hot2 1.7tb 19.10 1.4tb
node-name-hot5 1.7tb 16.41 1.4tb
node-name-coordination3 140.9gb 4.51 134.5gb
node-name-coordination4 140.9gb 5.33 133.4gb
node-name-coordination2 99.9gb 31.48 68.5gb
node-name-master3 99.9gb 53.57 46.4gb
node-name-warm1 1.7tb 58.60 745.6gb
node-name-hot4 1.7tb 24.64 1.3tb
node-name-coordination1 99.9gb 33.04 66.9gb
node-name-ml1 99.9gb 49.21 50.7gb
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.