Yellow index (primary shard seems fine)

Elastic Stack Elasticsearch
1

Hi,

most of my indices are yellow. I am curious about one particular index, which is important to me. Due to confidentiallity I have to rename it in Paint and in codes bellow.

When I use this code I get result whats in the screenshot bellow.

GET /_cat/shards?v&h=n,index,shard,prirep,state,sto,sc,unassigned.reason,unassigned.details&s=sto,index

image
image887×62 3.48 KB

Then when I use this code

GET /_cluster/allocation/explain
{
  "index": "index1",
  "shard": 0,
  "primary": true
  
}

I get this result

{
  "index" : "index1",
  "shard" : 0,
  "primary" : true,
  "current_state" : "started",
  "current_node" : {
    "id" : "YW-3Wh3bSh2xvLbqcro13Q",
    "name" : "elasticsearch-es-default-0",
    "transport_address" : "confidential",
    "attributes" : {
      "k8s_node_name" : "confidential",
      "ml.machine_memory" : "4294967296",
      "xpack.installed" : "true",
      "transform.node" : "false",
      "ml.max_open_jobs" : "20"
    },
    "weight_ranking" : 1
  },
  "can_remain_on_current_node" : "yes",
  "can_rebalance_cluster" : "no",
  "can_rebalance_cluster_decisions" : [
    {
      "decider" : "rebalance_only_when_active",
      "decision" : "NO",
      "explanation" : "rebalancing is not allowed until all replicas in the cluster are active"
    },
    {
      "decider" : "cluster_rebalance",
      "decision" : "NO",
      "explanation" : "the cluster has unassigned shards and cluster setting [cluster.routing.allocation.allow_rebalance] is set to [indices_all_active]"
    }
  ],
  "can_rebalance_to_other_node" : "no",
  "rebalance_explanation" : "rebalancing is not allowed"
}

When I use the same code for non primary shard, it gives me this:

{
  "index" : "index1",
  "shard" : 0,
  "primary" : false,
  "current_state" : "unassigned",
  "unassigned_info" : {
    "reason" : "CLUSTER_RECOVERED",
    "at" : "2022-01-26T12:24:03.490Z",
    "last_allocation_status" : "no_attempt"
  },
  "can_allocate" : "no",
  "allocate_explanation" : "cannot allocate because allocation is not permitted to any of the nodes",
  "node_allocation_decisions" : [
    {
      "node_id" : "YW-3Wh3bSh2xvLbqcro13Q",
      "node_name" : "elasticsearch-es-default-0",
      "transport_address" : "confidential",
      "node_attributes" : {
        "k8s_node_name" : "confidential",
        "ml.machine_memory" : "4294967296",
        "xpack.installed" : "true",
        "transform.node" : "false",
        "ml.max_open_jobs" : "20"
      },
      "node_decision" : "no",
      "deciders" : [
        {
          "decider" : "same_shard",
          "decision" : "NO",
          "explanation" : "a copy of this shard is already allocated to this node [[index1][0], node[YW-3Wh3bSh2xvLbqcro13Q], [P], s[STARTED], a[id=39JTpw6pRFKcXWZIRoqKJA]]"
        }
      ]
    }
  ]
}

Do you have any idea why my index1 is yellow?

Thank you

2

This looks to be an issue where your node(s) are having limits on what they can receive. Leaving only one node to host this index, and as a primary shard and replica cannot be hosted on the same node there is no place left where to store the replica shard.

Basically have a look at the routing settings for the index and see how many nodes match those requirement. I believe you will find only one node to match.

3

What is the output from the _cluster/stats?pretty&human API?

4

Hi @warkolm, it gives me this.

{
  "_nodes" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "Hkfpdo4MScyV1H9hbowcXg",
  "timestamp" : 1643625185702,
  "status" : "yellow",
  "indices" : {
    "count" : 86,
    "shards" : {
      "total" : 88,
      "primaries" : 88,
      "replication" : 0.0,
      "index" : {
        "shards" : {
          "min" : 1,
          "max" : 3,
          "avg" : 1.0232558139534884
        },
        "primaries" : {
          "min" : 1,
          "max" : 3,
          "avg" : 1.0232558139534884
        },
        "replication" : {
          "min" : 0.0,
          "max" : 0.0,
          "avg" : 0.0
        }
      }
    },
    "docs" : {
      "count" : 15839049,
      "deleted" : 2035505
    },
    "store" : {
      "size" : "7.3gb",
      "size_in_bytes" : 7847193251,
      "reserved" : "0b",
      "reserved_in_bytes" : 0
    },
    "fielddata" : {
      "memory_size" : "6.3mb",
      "memory_size_in_bytes" : 6608184,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "961.4kb",
      "memory_size_in_bytes" : 984511,
      "total_count" : 3468,
      "hit_count" : 1512,
      "miss_count" : 1956,
      "cache_size" : 69,
      "cache_count" : 101,
      "evictions" : 32
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 406,
      "memory" : "6.1mb",
      "memory_in_bytes" : 6496058,
      "terms_memory" : "3.4mb",
      "terms_memory_in_bytes" : 3609288,
      "stored_fields_memory" : "201kb",
      "stored_fields_memory_in_bytes" : 205904,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "389.9kb",
      "norms_memory_in_bytes" : 399296,
      "points_memory" : "0b",
      "points_memory_in_bytes" : 0,
      "doc_values_memory" : "2.1mb",
      "doc_values_memory_in_bytes" : 2281570,
      "index_writer_memory" : "30.4mb",
      "index_writer_memory_in_bytes" : 31931480,
      "version_map_memory" : "2.2mb",
      "version_map_memory_in_bytes" : 2385846,
      "fixed_bit_set" : "853.8kb",
      "fixed_bit_set_memory_in_bytes" : 874368,
      "max_unsafe_auto_id_timestamp" : 1643278535472,
      "file_sizes" : { }
    },
    "mappings" : {
      "field_types" : [
        {
          "name" : "alias",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "binary",
          "count" : 15,
          "index_count" : 4
        },
        {
          "name" : "boolean",
          "count" : 297,
          "index_count" : 41
        },
        {
          "name" : "byte",
          "count" : 2,
          "index_count" : 2
        },
        {
          "name" : "date",
          "count" : 479,
          "index_count" : 68
        },
        {
          "name" : "date_nanos",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "date_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "double",
          "count" : 2,
          "index_count" : 2
        },
        {
          "name" : "double_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "flattened",
          "count" : 9,
          "index_count" : 1
        },
        {
          "name" : "float",
          "count" : 256,
          "index_count" : 36
        },
        {
          "name" : "float_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "geo_point",
          "count" : 88,
          "index_count" : 15
        },
        {
          "name" : "geo_shape",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "half_float",
          "count" : 57,
          "index_count" : 15
        },
        {
          "name" : "integer",
          "count" : 223,
          "index_count" : 25
        },
        {
          "name" : "integer_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "ip",
          "count" : 159,
          "index_count" : 14
        },
        {
          "name" : "ip_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "keyword",
          "count" : 7584,
          "index_count" : 79
        },
        {
          "name" : "long",
          "count" : 2098,
          "index_count" : 64
        },
        {
          "name" : "long_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "nested",
          "count" : 32,
          "index_count" : 13
        },
        {
          "name" : "object",
          "count" : 2671,
          "index_count" : 46
        },
        {
          "name" : "shape",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "short",
          "count" : 3,
          "index_count" : 2
        },
        {
          "name" : "text",
          "count" : 1267,
          "index_count" : 67
        }
      ]
    },
    "analysis" : {
      "char_filter_types" : [ ],
      "tokenizer_types" : [ ],
      "filter_types" : [
        {
          "name" : "pattern_capture",
          "count" : 1,
          "index_count" : 1
        }
      ],
      "analyzer_types" : [
        {
          "name" : "custom",
          "count" : 1,
          "index_count" : 1
        }
      ],
      "built_in_char_filters" : [ ],
      "built_in_tokenizers" : [
        {
          "name" : "uax_url_email",
          "count" : 1,
          "index_count" : 1
        }
      ],
      "built_in_filters" : [
        {
          "name" : "lowercase",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "unique",
          "count" : 1,
          "index_count" : 1
        }
      ],
      "built_in_analyzers" : [ ]
    }
  },
  "nodes" : {
    "count" : {
      "total" : 1,
      "coordinating_only" : 0,
      "data" : 1,
      "data_cold" : 0,
      "data_content" : 0,
      "data_hot" : 0,
      "data_warm" : 0,
      "ingest" : 1,
      "master" : 1,
      "ml" : 1,
      "remote_cluster_client" : 0,
      "transform" : 0,
      "voting_only" : 0
    },
    "versions" : [
      "7.10.0"
    ],
    "os" : {
      "available_processors" : 1,
      "allocated_processors" : 1,
      "names" : [
        {
          "name" : "Linux",
          "count" : 1
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "Red Hat Enterprise Linux 8.3 (Ootpa)",
          "count" : 1
        }
      ],
      "mem" : {
        "total" : "4gb",
        "total_in_bytes" : 4294967296,
        "free" : "1.1gb",
        "free_in_bytes" : 1242816512,
        "used" : "2.8gb",
        "used_in_bytes" : 3052150784,
        "free_percent" : 29,
        "used_percent" : 71
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 0
      },
      "open_file_descriptors" : {
        "min" : 1591,
        "max" : 1591,
        "avg" : 1591
      }
    },
    "jvm" : {
      "max_uptime" : "4d",
      "max_uptime_in_millis" : 346842553,
      "versions" : [
        {
          "version" : "15.0.1",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "15.0.1+9",
          "vm_vendor" : "AdoptOpenJDK",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 1
        }
      ],
      "mem" : {
        "heap_used" : "648.1mb",
        "heap_used_in_bytes" : 679669616,
        "heap_max" : "1gb",
        "heap_max_in_bytes" : 1073741824
      },
      "threads" : 54
    },
    "fs" : {
      "total" : "20gb",
      "total_in_bytes" : 21474836480,
      "free" : "10.9gb",
      "free_in_bytes" : 11715739648,
      "available" : "10.9gb",
      "available_in_bytes" : 11715739648
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "security4" : 1
      },
      "http_types" : {
        "security4" : 1
      }
    },
    "discovery_types" : {
      "zen" : 1
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "docker",
        "count" : 1
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 20,
      "processor_stats" : {
        "convert" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        },
        "csv" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        },
        "date" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        },
        "grok" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        },
        "gsub" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        },
        "remove" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        },
        "script" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        }
      }
    }
  }
}
5

You only have one node, you cannot allocate primaries and replicas to a single node as there's no value in doing so.
You either need to set your replica's to 0, or add another node (preferably 2).

Also please consider upgrading, 7.10 is pretty old, 7.16 is latest and 8.0 is not far off being GA.

1 Like
6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.