Elasticsearch 7.13 unassigned shard problem

Elastic Stack Elasticsearch
1

Hi,
i have i weird problem on elasticsearc 7.13 deployed on kubernetes.
Here my configuration:
3 master node
2 date node with heap 8gb and 100gb storage
2 shard and 1 replica

index size:
eventsched n° doc 317864006 size 24.6gb
dtech n° doc 190332087 size 28gb

cluster state

{
  "cluster_name" : "elastic-bitnami",
  "status" : "yellow",
  "timed_out" : false,
  "number_of_nodes" : 10,
  "number_of_data_nodes" : 2,
  "active_primary_shards" : 12,
  "active_shards" : 19,
  "relocating_shards" : 0,
  "initializing_shards" : 0,
  "unassigned_shards" : 5,
  "delayed_unassigned_shards" : 0,
  "number_of_pending_tasks" : 0,
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 0,
  "active_shards_percent_as_number" : 79.16666666666666
}

shard allocation:

shards disk.indices disk.used disk.avail disk.total disk.percent host         ip           node
     7        9.1mb      85gb     12.9gb     97.9gb           86 10.42.9.79   10.42.9.79   elastic-elasticsearch-data-1
    12       52.7gb    52.8gb       45gb     97.9gb           53 10.42.10.177 10.42.10.177 elastic-elasticsearch-data-0
     5                                                                                     UNASSIGNED

Shard status

dtech                               1 p STARTED    
dtech                               1 r UNASSIGNED NODE_LEFT
dtech                               0 p STARTED    
dtech                               0 r UNASSIGNED PRIMARY_FAILED
eventsched                          1 p STARTED    
eventsched                          1 r UNASSIGNED PRIMARY_FAILED
eventsched                          0 p STARTED    
eventsched                          0 r UNASSIGNED PRIMARY_FAILED

allocation error:

{
  "index" : "dtech",
  "shard" : 1,
  "primary" : false,
  "current_state" : "unassigned",
  "unassigned_info" : {
    "reason" : "NODE_LEFT",
    "at" : "2021-08-21T06:29:36.005Z",
    "details" : "node_left [6moXKkrRRcK8slQ-ZdIcDw]",
    "last_allocation_status" : "no_attempt"
  },
  "can_allocate" : "no",
  "allocate_explanation" : "cannot allocate because allocation is not permitted to any of the nodes",
  "node_allocation_decisions" : [
    {
      "node_id" : "6moXKkrRRcK8slQ-ZdIcDw",
      "node_name" : "elastic-elasticsearch-data-1",
      "transport_address" : "10.42.9.79:9300",
      "node_attributes" : {
        "xpack.installed" : "true",
        "transform.node" : "true"
      },
      "node_decision" : "no",
      "deciders" : [
        {
          "decider" : "disk_threshold",
          "decision" : "NO",
          "explanation" : "the node is above the low watermark cluster setting [cluster.routing.allocation.disk.watermark.low=85%], using more disk space than the maximum allowed [85.0%], actual free: [13.198729444367965%]"
        }
      ]

Why the shard are not distribuited equally beyween the two data node?

Thanks
Cristian

2

What is the full output of the cluster stats API?

3

Here it is

{
  "cluster_name" : "elastic-bitnami",
  "status" : "yellow",
  "timed_out" : false,
  "number_of_nodes" : 10,
  "number_of_data_nodes" : 2,
  "active_primary_shards" : 12,
  "active_shards" : 19,
  "relocating_shards" : 0,
  "initializing_shards" : 0,
  "unassigned_shards" : 5,
  "delayed_unassigned_shards" : 0,
  "number_of_pending_tasks" : 0,
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 0,
  "active_shards_percent_as_number" : 79.16666666666666
}

Thanks
Cristian

4

That does not seem to be the cluster stats API.

5

Sorry, here it is

#! Elasticsearch built-in security features are not enabled. Without authentication, your cluster could be accessible to anyone. See https://www.elastic.co/guide/en/elasticsearch/reference/7.13/security-minimal-setup.html to enable security.
{
  "_nodes" : {
    "total" : 10,
    "successful" : 10,
    "failed" : 0
  },
  "cluster_name" : "elastic-bitnami",
  "cluster_uuid" : "Q7SCvz-zSo22yLIrz6ATZA",
  "timestamp" : 1629714321579,
  "status" : "yellow",
  "indices" : {
    "count" : 10,
    "shards" : {
      "total" : 20,
      "primaries" : 12,
      "replication" : 0.6666666666666666,
      "index" : {
        "shards" : {
          "min" : 2,
          "max" : 2,
          "avg" : 2.0
        },
        "primaries" : {
          "min" : 1,
          "max" : 2,
          "avg" : 1.2
        },
        "replication" : {
          "min" : 0.0,
          "max" : 1.0,
          "avg" : 0.8
        }
      }
    },
    "docs" : {
      "count" : 508244620,
      "deleted" : 20424
    },
    "store" : {
      "size_in_bytes" : 56680358271,
      "total_data_set_size_in_bytes" : 56680358271,
      "reserved_in_bytes" : 0
    },
    "fielddata" : {
      "memory_size_in_bytes" : 0,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size_in_bytes" : 0,
      "total_count" : 202,
      "hit_count" : 0,
      "miss_count" : 202,
      "cache_size" : 0,
      "cache_count" : 0,
      "evictions" : 0
    },
    "completion" : {
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 189,
      "memory_in_bytes" : 487116,
      "terms_memory_in_bytes" : 291872,
      "stored_fields_memory_in_bytes" : 139256,
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory_in_bytes" : 24320,
      "points_memory_in_bytes" : 0,
      "doc_values_memory_in_bytes" : 31668,
      "index_writer_memory_in_bytes" : 4578728,
      "version_map_memory_in_bytes" : 524499,
      "fixed_bit_set_memory_in_bytes" : 2352,
      "max_unsafe_auto_id_timestamp" : 1629713758711,
      "file_sizes" : { }
    },
    "mappings" : {
      "field_types" : [
        {
          "name" : "boolean",
          "count" : 1,
          "index_count" : 1,
          "script_count" : 0
        },
        {
          "name" : "date",
          "count" : 10,
          "index_count" : 2,
          "script_count" : 0
        },
        {
          "name" : "float",
          "count" : 2,
          "index_count" : 1,
          "script_count" : 0
        },
        {
          "name" : "keyword",
          "count" : 57,
          "index_count" : 4,
          "script_count" : 0
        },
        {
          "name" : "long",
          "count" : 4,
          "index_count" : 2,
          "script_count" : 0
        },
        {
          "name" : "nested",
          "count" : 1,
          "index_count" : 1,
          "script_count" : 0
        },
        {
          "name" : "object",
          "count" : 9,
          "index_count" : 2,
          "script_count" : 0
        },
        {
          "name" : "text",
          "count" : 13,
          "index_count" : 4,
          "script_count" : 0
        }
      ],
      "runtime_field_types" : [ ]
    },
    "analysis" : {
      "char_filter_types" : [ ],
      "tokenizer_types" : [ ],
      "filter_types" : [ ],
      "analyzer_types" : [ ],
      "built_in_char_filters" : [ ],
      "built_in_tokenizers" : [ ],
      "built_in_filters" : [ ],
      "built_in_analyzers" : [ ]
    },
    "versions" : [
      {
        "version" : "7.13.2",
        "index_count" : 10,
        "primary_shard_count" : 12,
        "total_primary_bytes" : 56675404317
      }
    ]
  },
  "nodes" : {
    "count" : {
      "total" : 10,
      "coordinating_only" : 0,
      "data" : 2,
      "data_cold" : 2,
      "data_content" : 2,
      "data_frozen" : 2,
      "data_hot" : 2,
      "data_warm" : 2,
      "ingest" : 2,
      "master" : 3,
      "ml" : 0,
      "remote_cluster_client" : 10,
      "transform" : 2,
      "voting_only" : 0
    },
    "versions" : [
      "7.13.2"
    ],
    "os" : {
      "available_processors" : 12,
      "allocated_processors" : 12,
      "names" : [
        {
          "name" : "Linux",
          "count" : 10
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "Debian GNU/Linux 10 (buster)",
          "count" : 10
        }
      ],
      "architectures" : [
        {
          "arch" : "amd64",
          "count" : 10
        }
      ],
      "mem" : {
        "total_in_bytes" : 673862340608,
        "free_in_bytes" : 630162006016,
        "used_in_bytes" : 43700334592,
        "free_percent" : 94,
        "used_percent" : 6
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 0
      },
      "open_file_descriptors" : {
        "min" : 462,
        "max" : 588,
        "avg" : 498
      }
    },
    "jvm" : {
      "max_uptime_in_millis" : 731561,
      "versions" : [
        {
          "version" : "11.0.11",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "11.0.11+9-LTS",
          "vm_vendor" : "BellSoft",
          "bundled_jdk" : true,
          "using_bundled_jdk" : false,
          "count" : 10
        }
      ],
      "mem" : {
        "heap_used_in_bytes" : 1274454304,
        "heap_max_in_bytes" : 27015905280
      },
      "threads" : 257
    },
    "fs" : {
      "total_in_bytes" : 701121863680,
      "free_in_bytes" : 536425914368,
      "available_in_bytes" : 536342028288
    },
    "plugins" : [
      {
        "name" : "repository-s3",
        "version" : "7.13.2",
        "elasticsearch_version" : "7.13.2",
        "java_version" : "1.8",
        "description" : "The S3 repository plugin adds S3 repositories",
        "classname" : "org.elasticsearch.repositories.s3.S3RepositoryPlugin",
        "extended_plugins" : [ ],
        "has_native_controller" : false,
        "licensed" : false,
        "type" : "isolated"
      }
    ],
    "network_types" : {
      "transport_types" : {
        "security4" : 10
      },
      "http_types" : {
        "security4" : 10
      }
    },
    "discovery_types" : {
      "zen" : 10
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "tar",
        "count" : 10
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 2,
      "processor_stats" : {
        "gsub" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "script" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        }
      }
    }
  }
}
6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.