Getting maxConcurrentShardRequests must be >= 1 error

Elastic Stack Elasticsearch
1

Hi I'm getting the above error when I run a GET request to any of my indices.

If I try to view elasticsearch in my browser, I get a Error 503 Service Unavailable: Request Timeout after 30000ms error and nothing loads in properly.

A GET request to _cluster/stats?human&pretty returns the following:

{
  "_nodes": {
    "total": 1,
    "successful": 1,
    "failed": 0
  },
  "cluster_name": "dev-cluster-name",
  "timestamp": 1555010187590,
  "status": "red",
  "indices": {
    "count": 0,
    "shards": {},
    "docs": {
      "count": 0,
      "deleted": 0
    },
    "store": {
      "size": "0b",
      "size_in_bytes": 0
    },
    "fielddata": {
      "memory_size": "0b",
      "memory_size_in_bytes": 0,
      "evictions": 0
    },
    "query_cache": {
      "memory_size": "0b",
      "memory_size_in_bytes": 0,
      "total_count": 0,
      "hit_count": 0,
      "miss_count": 0,
      "cache_size": 0,
      "cache_count": 0,
      "evictions": 0
    },
    "completion": {
      "size": "0b",
      "size_in_bytes": 0
    },
    "segments": {
      "count": 0,
      "memory": "0b",
      "memory_in_bytes": 0,
      "terms_memory": "0b",
      "terms_memory_in_bytes": 0,
      "stored_fields_memory": "0b",
      "stored_fields_memory_in_bytes": 0,
      "term_vectors_memory": "0b",
      "term_vectors_memory_in_bytes": 0,
      "norms_memory": "0b",
      "norms_memory_in_bytes": 0,
      "points_memory": "0b",
      "points_memory_in_bytes": 0,
      "doc_values_memory": "0b",
      "doc_values_memory_in_bytes": 0,
      "index_writer_memory": "0b",
      "index_writer_memory_in_bytes": 0,
      "version_map_memory": "0b",
      "version_map_memory_in_bytes": 0,
      "fixed_bit_set": "0b",
      "fixed_bit_set_memory_in_bytes": 0,
      "max_unsafe_auto_id_timestamp": -9223372036854775808,
      "file_sizes": {}
    }
  },
  "nodes": {
    "count": {
      "total": 1,
      "data": 0,
      "coordinating_only": 0,
      "master": 1,
      "ingest": 0
    },
    "versions": [
      "6.2.2"
    ],
    "os": {
      "available_processors": 4,
      "allocated_processors": 4,
      "names": [
        {
          "name": "Linux",
          "count": 1
        }
      ],
      "mem": {
        "total": "15.5gb",
        "total_in_bytes": 16649089024,
        "free": "9.7gb",
        "free_in_bytes": 10477514752,
        "used": "5.7gb",
        "used_in_bytes": 6171574272,
        "free_percent": 63,
        "used_percent": 37
      }
    },
    "process": {
      "cpu": {
        "percent": 0
      },
      "open_file_descriptors": {
        "min": 309,
        "max": 309,
        "avg": 309
      }
    },
    "jvm": {
      "max_uptime": "13m",
      "max_uptime_in_millis": 785436,
      "versions": [
        {
          "version": "1.8.0_151",
          "vm_name": "OpenJDK 64-Bit Server VM",
          "vm_version": "25.151-b12",
          "vm_vendor": "Oracle Corporation",
          "count": 1
        }
      ],
      "mem": {
        "heap_used": "336.8mb",
        "heap_used_in_bytes": 353173312,
        "heap_max": "990.7mb",
        "heap_max_in_bytes": 1038876672
      },
      "threads": 38
    },
    "fs": {
      "total": "249.8gb",
      "total_in_bytes": 268304384000,
      "free": "249.8gb",
      "free_in_bytes": 268268056576,
      "available": "249.8gb",
      "available_in_bytes": 268268056576
    },
    "plugins": [
      {
        "name": "x-pack-upgrade",
        "version": "6.2.2",
        "description": "Elasticsearch Expanded Pack Plugin - Upgrade",
        "classname": "org.elasticsearch.xpack.upgrade.Upgrade",
        "extended_plugins": [
          "x-pack-core"
        ],
        "has_native_controller": false,
        "requires_keystore": false
      },
      {
        "name": "x-pack-ml",
        "version": "6.2.2",
        "description": "Elasticsearch Expanded Pack Plugin - Machine Learning",
        "classname": "org.elasticsearch.xpack.ml.MachineLearning",
        "extended_plugins": [
          "x-pack-core"
        ],
        "has_native_controller": true,
        "requires_keystore": false
      },
      {
        "name": "x-pack-logstash",
        "version": "6.2.2",
        "description": "Elasticsearch Expanded Pack Plugin - Logstash",
        "classname": "org.elasticsearch.xpack.logstash.Logstash",
        "extended_plugins": [
          "x-pack-core"
        ],
        "has_native_controller": false,
        "requires_keystore": false
      },
      {
        "name": "x-pack-deprecation",
        "version": "6.2.2",
        "description": "Elasticsearch Expanded Pack Plugin - Deprecation",
        "classname": "org.elasticsearch.xpack.deprecation.Deprecation",
        "extended_plugins": [
          "x-pack-core"
        ],
        "has_native_controller": false,
        "requires_keystore": false
      },
      {
        "name": "x-pack-monitoring",
        "version": "6.2.2",
        "description": "Elasticsearch Expanded Pack Plugin - Monitoring",
        "classname": "org.elasticsearch.xpack.monitoring.Monitoring",
        "extended_plugins": [
          "x-pack-core"
        ],
        "has_native_controller": false,
        "requires_keystore": false
      },
      {
        "name": "x-pack-core",
        "version": "6.2.2",
        "description": "Elasticsearch Expanded Pack Plugin - Core",
        "classname": "org.elasticsearch.xpack.core.XPackPlugin",
        "extended_plugins": [],
        "has_native_controller": false,
        "requires_keystore": false
      },
      {
        "name": "x-pack-security",
        "version": "6.2.2",
        "description": "Elasticsearch Expanded Pack Plugin - Security",
        "classname": "org.elasticsearch.xpack.security.Security",
        "extended_plugins": [
          "x-pack-core"
        ],
        "has_native_controller": false,
        "requires_keystore": true
      },
      {
        "name": "x-pack-watcher",
        "version": "6.2.2",
        "description": "Elasticsearch Expanded Pack Plugin - Watcher",
        "classname": "org.elasticsearch.xpack.watcher.Watcher",
        "extended_plugins": [
          "x-pack-core"
        ],
        "has_native_controller": false,
        "requires_keystore": false
      },
      {
        "name": "x-pack-graph",
        "version": "6.2.2",
        "description": "Elasticsearch Expanded Pack Plugin - Graph",
        "classname": "org.elasticsearch.xpack.graph.Graph",
        "extended_plugins": [
          "x-pack-core"
        ],
        "has_native_controller": false,
        "requires_keystore": false
      }
    ],
    "network_types": {
      "transport_types": {
        "netty4": 1
      },
      "http_types": {
        "netty4": 1
      }
    }
  }
}

...

2

and _cluster/allocation/explain returns the following

{
    "index": ".watches",
    "shard": 0,
    "primary": true,
    "current_state": "unassigned",
    "unassigned_info": {
        "reason": "CLUSTER_RECOVERED",
        "at": "2019-04-11T20:13:11.781Z",
        "last_allocation_status": "no_valid_shard_copy"
    },
    "can_allocate": "no_valid_shard_copy",
    "allocate_explanation": "cannot allocate because a previous copy of the primary shard existed but can no longer be found on the nodes in the cluster"
}

Does anyone know what the root cause to this error might be? I've tried restarted kibana and elasticsearch, but the error has persisted.

Thanks.

3

I checked my log and I get the following error stack trace (abridged due to character limits):

org.elasticsearch.action.NoShardAvailableActionException: No shard available for [get [.kibana][doc][config:6.2.2]: routing [null]]
	at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.perform(TransportSingleShardAction.java:209) ~[elasticsearch-6.2.2.jar:6.2.2]
	at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.start(TransportSingleShardAction.java:186) ~[elasticsearch-6.2.2.jar:6.2.2]
	at org.elasticsearch.action.support.single.shard.TransportSingleShardAction.doExecute(TransportSingleShardAction.java:95) ~[elasticsearch-6.2.2.jar:6.2.2]
	at org.elasticsearch.action.support.single.shard.TransportSingleShardAction.doExecute(TransportSingleShardAction.java:59) ~[elasticsearch-6.2.2.jar:6.2.2]
	at org.elasticsearch.action.support.TransportAction.doExecute(TransportAction.java:143) ~[elasticsearch-6.2.2.jar:6.2.2]
	at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:167) ~[elasticsearch-6.2.2.jar:6.2.2]
........
[2019-04-11T14:04:28,328][WARN ][o.e.x.m.MonitoringService] [atl-*****] monitoring execution failed
org.elasticsearch.xpack.monitoring.exporter.ExportException: Exception when closing export bulk
	at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$1$1.<init>(ExportBulk.java:107) ~[?:?]
	at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$1.onFailure(ExportBulk.java:105) ~[?:?]
	at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$Compound$1.onResponse(ExportBulk.java:218) ~[?:?]
........
Caused by: org.elasticsearch.xpack.monitoring.exporter.ExportException: failed to flush export bulks
	at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$Compound.lambda$doFlush$0(ExportBulk.java:168) ~[?:?]
	... 29 more
Caused by: org.elasticsearch.xpack.monitoring.exporter.ExportException: failed to flush export bulk [default_local]
	... 28 more
Caused by: java.lang.IllegalStateException: There are no ingest nodes in this cluster, unable to forward request to an ingest node.
	at org.elasticsearch.action.ingest.IngestActionForwarder.randomIngestNode(IngestActionForwarder.java:58) ~[elasticsearch-6.2.2.jar:6.2.2]
	at org.elasticsearch.action.ingest.IngestActionForwarder.forwardIngestRequest(IngestActionForwarder.java:51) ~[elasticsearch-6.2.2.jar:6.2.2]
	at org.elasticsearch.action.bulk.TransportBulkAction.doExecute(TransportBulkAction.java:137) ~[elasticsearch-6.2.2.jar:6.2.2]
	at org.elasticsearch.action.bulk.TransportBulkAction.doExecute(TransportBulkAction.java:86) ~[elasticsearch-6.2.2.jar:6.2.2]
	at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:167) ~[elasticsearch-6.2.2.jar:6.2.2]
	... 23 more
[2019-04-11T14:04:35,239][WARN ][r.suppressed             ] path: /.kibana/doc/config%3A6.2.2, params: {index=.kibana, id=config:6.2.2, type=doc}
org.elasticsearch.action.NoShardAvailableActionException: No shard available for [get [.kibana][doc][config:6.2.2]: routing [null]]
	at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.perform(TransportSingleShardAction.java:209) ~[elasticsearch-6.2.2.jar:6.2.2]
	at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.start(TransportSingleShardAction.java:186) ~[elasticsearch-6.2.2.jar:6.2.2]
	at org.elasticsearch.action.support.single.shard.TransportSingleShardAction.doExecute(TransportSingleShardAction.java:95) ~[elasticsearch-6.2.2.jar:6.2.2]
4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.