Unassigned Shards when New Index was created: Cluster Yellow

Elastic Stack Elasticsearch
1

New Index Creation led to Unassigned Shards and Yellow Cluster State.
Context :
It happened when relocation for one node was already in progress. Checked explanation for its state. It says :
Relocation Throttled , reached the limit of incoming shard recoveries.
"current_state" : "unassigned",
"unassigned_info" : {
"reason" : "INDEX_CREATED",
"at" : "2023-01-18T16:03:40.435Z",
"last_allocation_status" : "no_attempt"
},

is it expected Behaviour ?

2

Welcome to our community! :smiley:

What is the output from the _cluster/stats?pretty&human API?
Can you also provide the full output of GET _cluster/allocation/explain.

3 
_cluster/stats 

{
  "_nodes" : {
    "total" : 5,
    "successful" : 5,
    "failed" : 0
  },
  "timestamp" : 1674115649027,
  "status" : "green",
  "indices" : {
    "count" : 684,
    "shards" : {
      "total" : 6840,
      "primaries" : 3420,
      "replication" : 1.0,
      "index" : {
        "shards" : {
          "min" : 10,
          "max" : 10,
          "avg" : 10.0
        },
        "primaries" : {
          "min" : 5,
          "max" : 5,
          "avg" : 5.0
        },
        "replication" : {
          "min" : 1.0,
          "max" : 1.0,
          "avg" : 1.0
        }
      }
    },
    "docs" : {
      "count" : 2697983230,
      "deleted" : 230170293
    },
    "store" : {
      "size_in_bytes" : 4739618005659
    },
    "fielddata" : {
      "memory_size_in_bytes" : 0,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size_in_bytes" : 3750347245,
      "total_count" : 18269506365,
      "hit_count" : 6563380406,
      "miss_count" : 11706125959,
      "cache_size" : 1408458,
      "cache_count" : 6881466,
      "evictions" : 5473008
    },
    "completion" : {
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 94583,
      "memory_in_bytes" : 17185568921,
      "terms_memory_in_bytes" : 14031391435,
      "stored_fields_memory_in_bytes" : 1364582832,
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory_in_bytes" : 91063104,
      "points_memory_in_bytes" : 951154810,
      "doc_values_memory_in_bytes" : 747376740,
      "index_writer_memory_in_bytes" : 0,
      "version_map_memory_in_bytes" : 0,
      "fixed_bit_set_memory_in_bytes" : 0,
      "max_unsafe_auto_id_timestamp" : -1,
      "file_sizes" : { }
    }
  },
  "nodes" : {
    "count" : {
      "total" : 5,
      "data" : 5,
      "coordinating_only" : 0,
      "master" : 5,
      "ingest" : 5
    },
    "versions" : [
      "6.8.22"
    ],
    "os" : {
      "available_processors" : 80,
      "allocated_processors" : 80,
      "names" : [
        {
          "name" : "Linux",
          "count" : 5
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "Ubuntu 20.04.3 LTS",
          "count" : 4
        },
        {
          "pretty_name" : "Ubuntu 18.04.2 LTS",
          "count" : 1
        }
      ],
      "mem" : {
        "total_in_bytes" : 166532775936,
        "free_in_bytes" : 16606953472,
        "used_in_bytes" : 149925822464,
        "free_percent" : 10,
        "used_percent" : 90
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 10
      },
      "open_file_descriptors" : {
        "min" : 550,
        "max" : 6102,
        "avg" : 4905
      }
    },
    "jvm" : {
      "max_uptime_in_millis" : 23209843391,
      "versions" : [
        {
          "version" : "1.8.0_312",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "25.312-b07",
          "vm_vendor" : "Private Build",
          "count" : 4
        },
        {
          "version" : "1.8.0_212",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "25.212-b03",
          "vm_vendor" : "Oracle Corporation",
          "count" : 1
        }
      ],
      "mem" : {
        "heap_used_in_bytes" : 45718811192,
        "heap_max_in_bytes" : 82700533760
      },
      "threads" : 999
    },
    "fs" : {
      "total_in_bytes" : 9176508006400,
      "free_in_bytes" : 4369979158528,
      "available_in_bytes" : 4369979158528
    },
    "plugins" : [
      {
        "name" : "discovery-ec2",
        "version" : "6.8.22",
        "elasticsearch_version" : "6.8.22",
        "java_version" : "1.8",
        "description" : "The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.",
        "classname" : "org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin",
        "extended_plugins" : [ ],
        "has_native_controller" : false
      },
      {
        "name" : "repository-s3",
        "version" : "6.8.22",
        "elasticsearch_version" : "6.8.22",
        "java_version" : "1.8",
        "description" : "The S3 repository plugin adds S3 repositories",
        "classname" : "org.elasticsearch.repositories.s3.S3RepositoryPlugin",
        "extended_plugins" : [ ],
        "has_native_controller" : false
      }
    ],
    "network_types" : {
      "transport_types" : {
        "security4" : 5
      },
      "http_types" : {
        "security4" : 5
      }
    }
  }
}
4

A few things;

  1. You're running a very old version. 6.8 is EOL and no longer supported so please upgrade.
  2. You are over sharded, you have 1,368 per node which is way too many and likely the root cause of your issue.
5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.