Replicas allocation failed

Elastic Stack Elasticsearch
1

Hi,

After adjusting "index lifecycle policies" several times for this index, some replicas can't be allocated:

image
image755×748 61.4 KB

1- Is there a way to reallocate those replicas?
2- Is it possible to force all indices to have the same latest policy version applied (moving index to correct phase depending on its age)?

Note: Ver 7.6.2 on-prem three-node cluster, each node running on a RHEL vm.

Thank you.

2

What does GET /_cluster/allocation/explain show for one of those?

3

Hi,

I deleted some indexes from previous list as they were quite old and not needed anymore.
I didn't delete this one and here is the output:

{
  "index" : "dev-oshplt-oca-be-v2-000018",
  "shard" : 0,
  "primary" : true,
  "current_state" : "started",
  "current_node" : {
    "id" : "h2OdWbP6RkCljJ5PEUGk_g",
    "name" : "sl000257",
    "transport_address" : "10.220.220.37:9300",
    "attributes" : {
      "ml.machine_memory" : "25111273472",
      "ml.max_open_jobs" : "20",
      "xpack.installed" : "true"
    }
  },
  "can_remain_on_current_node" : "no",
  "can_remain_decisions" : [
    {
      "decider" : "filter",
      "decision" : "NO",
      "explanation" : """node does not match index setting [index.routing.allocation.require] filters [type:"cold"]"""
    }
  ],
  "can_move_to_other_node" : "no",
  "move_explanation" : "cannot move shard to another node, even though it is not allowed to remain on its current node",
  "node_allocation_decisions" : [
    {
      "node_id" : "DApqAgQUQQOS6lXCCIdeOA",
      "node_name" : "sl000256",
      "transport_address" : "10.220.220.36:9300",
      "node_attributes" : {
        "ml.machine_memory" : "25111273472",
        "ml.max_open_jobs" : "20",
        "xpack.installed" : "true"
      },
      "node_decision" : "no",
      "weight_ranking" : 1,
      "deciders" : [
        {
          "decider" : "filter",
          "decision" : "NO",
          "explanation" : """node does not match index setting [index.routing.allocation.require] filters [type:"cold"]"""
        }
      ]
    },
    {
      "node_id" : "IvLXRq87TfW019epGeM_fw",
      "node_name" : "sl000255",
      "transport_address" : "10.220.220.35:9300",
      "node_attributes" : {
        "ml.machine_memory" : "25111273472",
        "ml.max_open_jobs" : "20",
        "xpack.installed" : "true"
      },
      "node_decision" : "no",
      "weight_ranking" : 2,
      "deciders" : [
        {
          "decider" : "filter",
          "decision" : "NO",
          "explanation" : """node does not match index setting [index.routing.allocation.require] filters [type:"cold"]"""
        }
      ]
    }
  ]
}
4

Did you look into your allocation filtering?

5

Thank you for your reply!

Do you mean this explanation output: "node does not match index setting [index.routing.allocation.require] filters [type:"cold"]"?

I don't understand this explanation. There is no hot, warm, cold nodes configured. All three nodes are identical.

6

What do your Elasticsearch configs look like?

7

Look like this:

{
  "_nodes" : {
    "total" : 3,
    "successful" : 3,
    "failed" : 0
  },
  "cluster_name" : "esearch00-dev",
  "cluster_uuid" : "rPAnVfhSQgmx8w4Dn0dzvQ",
  "timestamp" : 1612194677880,
  "status" : "yellow",
  "indices" : {
    "count" : 85,
    "shards" : {
      "total" : 198,
      "primaries" : 110,
      "replication" : 0.8,
      "index" : {
        "shards" : {
          "min" : 1,
          "max" : 4,
          "avg" : 2.3294117647058825
        },
        "primaries" : {
          "min" : 1,
          "max" : 2,
          "avg" : 1.2941176470588236
        },
        "replication" : {
          "min" : 0.0,
          "max" : 1.0,
          "avg" : 0.788235294117647
        }
      }
    },
    "docs" : {
      "count" : 20275575,
      "deleted" : 1541611
    },
    "store" : {
      "size_in_bytes" : 73265721007
    },
    "fielddata" : {
      "memory_size_in_bytes" : 4408,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size_in_bytes" : 522088,
      "total_count" : 10973,
      "hit_count" : 163,
      "miss_count" : 10810,
      "cache_size" : 28,
      "cache_count" : 28,
      "evictions" : 0
    },
    "completion" : {
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 910,
      "memory_in_bytes" : 31735295,
      "terms_memory_in_bytes" : 14815135,
      "stored_fields_memory_in_bytes" : 14214496,
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory_in_bytes" : 1241024,
      "points_memory_in_bytes" : 0,
      "doc_values_memory_in_bytes" : 1464640,
      "index_writer_memory_in_bytes" : 110154242,
      "version_map_memory_in_bytes" : 315875,
      "fixed_bit_set_memory_in_bytes" : 1820312,
      "max_unsafe_auto_id_timestamp" : 1612181621086,
      "file_sizes" : { }
    }
  },
  "nodes" : {
    "count" : {
      "total" : 3,
      "coordinating_only" : 0,
      "data" : 3,
      "ingest" : 3,
      "master" : 3,
      "ml" : 3,
      "voting_only" : 0
    },
    "versions" : [
      "7.6.2"
    ],
    "os" : {
      "available_processors" : 18,
      "allocated_processors" : 18,
      "names" : [
        {
          "name" : "Linux",
          "count" : 3
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "OpenShift Enterprise",
          "count" : 3
        }
      ],
      "mem" : {
        "total_in_bytes" : 75333820416,
        "free_in_bytes" : 2293506048,
        "used_in_bytes" : 73040314368,
        "free_percent" : 3,
        "used_percent" : 97
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 1
      },
      "open_file_descriptors" : {
        "min" : 684,
        "max" : 802,
        "avg" : 727
      }
    },
    "jvm" : {
      "max_uptime_in_millis" : 2847137899,
      "versions" : [
        {
          "version" : "13.0.2",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "13.0.2+8",
          "vm_vendor" : "AdoptOpenJDK",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 3
        }
      ],
      "mem" : {
        "heap_used_in_bytes" : 15182287184,
        "heap_max_in_bytes" : 38497812480
      },
      "threads" : 321
    },
    "fs" : {
      "total_in_bytes" : 1610273009664,
      "free_in_bytes" : 1535425576960,
      "available_in_bytes" : 1535425576960
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "security4" : 3
      },
      "http_types" : {
        "security4" : 3
      }
    },
    "discovery_types" : {
      "zen" : 3
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "rpm",
        "count" : 3
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 2,
      "processor_stats" : {
        "gsub" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "script" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        }
      }
    }
  }
}
8

I mean your elasticsearch.yml file.

9

Thank you, ok, here it is:

$ ccat elasticsearch.yml
cluster.name: esearch00-dev
node.name: sl000255
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
path.repo: /essnapshots
network.host: sl000255
http.port: 9200
discovery.seed_hosts: ["sl000255", "sl000256", "sl000257"]
cluster.initial_master_nodes: ["sl000255", "sl000256", "sl000257"]
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: certs/sl000255.p12
xpack.security.transport.ssl.keystore.type: pkcs12
xpack.security.transport.ssl.truststore.path: certs/sl000255.p12

10

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.