Increase shard count in elastic search kubernetes

I have deployed an Elastic search in GKE. ECK has 3 master and 2 data nodes. When creating an index with 1 shard and 1 replica getting 400 error response when the shard count reached 1000.

Is it possible to increase the shard count? Please suggest a way to increase the shards count.

Error details

Invalid NEST response built from a unsuccessful (400) low level call on PUT: /t-2af14520-e76d-4091-a51d-1c89cf990e97
Audit trail of this API call:
[1] BadResponse: Node: http://eck.com/ Took: 00:00:00.2833118
OriginalException: Elasticsearch.Net.ElasticsearchClientException: Request failed to execute. Call: Status code 400 from: PUT /t-2af14520-e76d-4091-a51d-1c89cf990e97. ServerError: Type: illegal_argument_exception Reason: "Validation Failed: 1: this action would add [2] total shards, but this cluster currently has [2000]/[2000] maximum shards open;"
Request:
<Request stream not captured or already read to completion by serializer. Set DisableDirectStreaming() on ConnectionSettings to force it to be set on the response.>
Response:
<Response stream not captured or already read to completion by serializer. Set DisableDirectStreaming() on ConnectionSettings to force it to be set on the response.>

Hi @rajkumar_25, thanks for your question.

This is not related to ECK, but rather to the Elasticsearch configuration. I will move this to the ES part of the forum.

But to offer you some guidance right away, please make sure that you don't have too many shards - 2000 for two data nodes looks like this could be the case. If you really need to raise that limit you can check out our docs for appropriate settings.

What is the output from the _cluster/stats?pretty&human API endpoint?

@warkolm
Find the _cluster/stats?pretty&human API endpoint output.

{

  "_nodes": {

    "total": 1,

    "successful": 1,

    "failed": 0

  },

  "cluster_name": "development",

  "cluster_uuid": "9OobJTRJu5XCLfwJirig",

  "timestamp": 1609134229177,

  "status": "yellow",

  "indices": {

    "count": 500,

    "shards": {

      "total": 500,

      "primaries": 500,

      "replication": 0,

      "index": {

        "shards": {

          "min": 1,

          "max": 1,

          "avg": 1

        },

        "primaries": {

          "min": 1,

          "max": 1,

          "avg": 1

        },

        "replication": {

          "min": 0,

          "max": 0,

          "avg": 0

        }

      }

    },

    "docs": {

      "count": 5,

      "deleted": 0

    },

    "store": {

      "size": "11.5mb",

      "size_in_bytes": 12148095,

      "reserved": "0b",

      "reserved_in_bytes": 0

    },

    "fielddata": {

      "memory_size": "0b",

      "memory_size_in_bytes": 0,

      "evictions": 0

    },

    "query_cache": {

      "memory_size": "0b",

      "memory_size_in_bytes": 0,

      "total_count": 0,

      "hit_count": 0,

      "miss_count": 0,

      "cache_size": 0,

      "cache_count": 0,

      "evictions": 0

    },

    "completion": {

      "size": "0b",

      "size_in_bytes": 0

    },

    "segments": {

      "count": 5,

      "memory": "11.8kb",

      "memory_in_bytes": 12100,

      "terms_memory": "5.7kb",

      "terms_memory_in_bytes": 5920,

      "stored_fields_memory": "2.3kb",

      "stored_fields_memory_in_bytes": 2440,

      "term_vectors_memory": "0b",

      "term_vectors_memory_in_bytes": 0,

      "norms_memory": "640b",

      "norms_memory_in_bytes": 640,

      "points_memory": "0b",

      "points_memory_in_bytes": 0,

      "doc_values_memory": "3kb",

      "doc_values_memory_in_bytes": 3100,

      "index_writer_memory": "0b",

      "index_writer_memory_in_bytes": 0,

      "version_map_memory": "0b",

      "version_map_memory_in_bytes": 0,

      "fixed_bit_set": "0b",

      "fixed_bit_set_memory_in_bytes": 0,

      "max_unsafe_auto_id_timestamp": -1,

      "file_sizes": {}

    },

    "mappings": {

      "field_types": [

        {

          "name": "double",

          "count": 500,

          "index_count": 500

        },

        {

          "name": "keyword",

          "count": 1000,

          "index_count": 500

        },

        {

          "name": "long",

          "count": 500,

          "index_count": 500

        },

        {

          "name": "object",

          "count": 500,

          "index_count": 500

        },

        {

          "name": "text",

          "count": 1000,

          "index_count": 500

        }

      ]

    },

    "analysis": {

      "char_filter_types": [],

      "tokenizer_types": [],

      "filter_types": [],

      "analyzer_types": [

        {

          "name": "custom",

          "count": 1000,

          "index_count": 500

        }

      ],

      "built_in_char_filters": [],

      "built_in_tokenizers": [

        {

          "name": "ngram",

          "count": 500,

          "index_count": 500

        },

        {

          "name": "uax_url_email",

          "count": 500,

          "index_count": 500

        }

      ],

      "built_in_filters": [

        {

          "name": "lowercase",

          "count": 500,

          "index_count": 500

        }

      ],

      "built_in_analyzers": []

    }

  },

  "nodes": {

    "count": {

      "total": 1,

      "coordinating_only": 0,

      "data": 1,

      "data_cold": 1,

      "data_content": 1,

      "data_hot": 1,

      "data_warm": 1,

      "ingest": 1,

      "master": 1,

      "ml": 1,

      "remote_cluster_client": 1,

      "transform": 1,

      "voting_only": 0

    },

    "versions": [

      "7.10.1"

    ],

    "os": {

      "available_processors": 1,

      "allocated_processors": 1,

      "names": [

        {

          "name": "Linux",

          "count": 1

        }

      ],

      "pretty_names": [

        {

          "pretty_name": "CentOS Linux 8 (Core)",

          "count": 1

        }

      ],

      "mem": {

        "total": "2gb",

        "total_in_bytes": 2147483648,

        "free": "1.1gb",

        "free_in_bytes": 1230901248,

        "used": "874.1mb",

        "used_in_bytes": 916582400,

        "free_percent": 57,

        "used_percent": 43

      }

    },

    "process": {

      "cpu": {

        "percent": 0

      },

      "open_file_descriptors": {

        "min": 1786,

        "max": 1786,

        "avg": 1786

      }

    },

    "jvm": {

      "max_uptime": "4.7d",

      "max_uptime_in_millis": 411561432,

      "versions": [

        {

          "version": "15.0.1",

          "vm_name": "OpenJDK 64-Bit Server VM",

          "vm_version": "15.0.1+9",

          "vm_vendor": "AdoptOpenJDK",

          "bundled_jdk": true,

          "using_bundled_jdk": true,

          "count": 1

        }

      ],

      "mem": {

        "heap_used": "314.8mb",

        "heap_used_in_bytes": 330119456,

        "heap_max": "512mb",

        "heap_max_in_bytes": 536870912

      },

      "threads": 108

    },

    "fs": {

      "total": "9.7gb",

      "total_in_bytes": 10501771264,

      "free": "9.7gb",

      "free_in_bytes": 10427056128,

      "available": "9.6gb",

      "available_in_bytes": 10410278912

    },

    "plugins": [

      {

        "name": "repository-gcs",

        "version": "7.10.1",

        "elasticsearch_version": "7.10.1",

        "java_version": "1.8",

        "description": "The GCS repository plugin adds Google Cloud Storage support for repositories.",

        "classname": "org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin",

        "extended_plugins": [],

        "has_native_controller": false

      }

    ],

    "network_types": {

      "transport_types": {

        "security4": 1

      },

      "http_types": {

        "security4": 1

      }

    },

    "discovery_types": {

      "zen": 1

    },

    "packaging_types": [

      {

        "flavor": "default",

        "type": "docker",

        "count": 1

      }

    ],

    "ingest": {

      "number_of_pipelines": 1,

      "processor_stats": {

        "gsub": {

          "count": 0,

          "failed": 0,

          "current": 0,

          "time": "0s",

          "time_in_millis": 0

        },

        "script": {

          "count": 0,

          "failed": 0,

          "current": 0,

          "time": "0s",

          "time_in_millis": 0

        }

      }

    }

  }

}

That seems wrong. You are saying you are getting this when trying to add more shards, but it looks like you only have 500;

"shards": {
      "total": 500,
      "primaries": 500,
      "replication": 0,

Mind you, having that many shards for a few MB of data is pretty inefficient.

1 Like

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.