"Rejected execution of coordinating operation" exception after upgrade from 8.2.2 to 8.6.0?

Elastic Stack Elasticsearch
1

Hi!

We just upgraded from 8.2.2 to 8.6.0 and suddenly our bulk inserts fails with this exception:

es_rejected_execution_exception Reason: "rejected execution of coordinating operation [coordinating_and_primary_bytes=207947853, replica_bytes=22389942, all_bytes=230337795,coordinating_operation_bytes=1061, max_coordinating_and_primary_bytes=214748364]"

I think this is a configuration problem caused by the new Java version but I'm not sure? How can I configure ES to accept larger requests or how can I fix this error?

Thanks!

/Kristoffer

2

What is the output from the _cluster/stats?pretty&human API?

3

Hi!

This is the outcome of that command:

{
  "_nodes": {
    "total": 3,
    "successful": 3,
    "failed": 0
  },
  "cluster_name": "ELASTIC-TEST",
  "cluster_uuid": "YCKP6URqRtGbygmuNYszyw",
  "timestamp": 1674124573368,
  "status": "green",
  "indices": {
    "count": 83,
    "shards": {
      "total": 166,
      "primaries": 83,
      "replication": 1,
      "index": {
        "shards": {
          "min": 2,
          "max": 2,
          "avg": 2
        },
        "primaries": {
          "min": 1,
          "max": 1,
          "avg": 1
        },
        "replication": {
          "min": 1,
          "max": 1,
          "avg": 1
        }
      }
    },
    "docs": {
      "count": 675064,
      "deleted": 146347
    },
    "store": {
      "size": "1.9gb",
      "size_in_bytes": 2106552712,
      "total_data_set_size": "1.9gb",
      "total_data_set_size_in_bytes": 2106552712,
      "reserved": "0b",
      "reserved_in_bytes": 0
    },
    "fielddata": {
      "memory_size": "0b",
      "memory_size_in_bytes": 0,
      "evictions": 0
    },
    "query_cache": {
      "memory_size": "55.6mb",
      "memory_size_in_bytes": 58309320,
      "total_count": 1665087,
      "hit_count": 63327,
      "miss_count": 1601760,
      "cache_size": 11706,
      "cache_count": 60985,
      "evictions": 49279
    },
    "completion": {
      "size": "36.1mb",
      "size_in_bytes": 37863299
    },
    "segments": {
      "count": 474,
      "memory": "0b",
      "memory_in_bytes": 0,
      "terms_memory": "0b",
      "terms_memory_in_bytes": 0,
      "stored_fields_memory": "0b",
      "stored_fields_memory_in_bytes": 0,
      "term_vectors_memory": "0b",
      "term_vectors_memory_in_bytes": 0,
      "norms_memory": "0b",
      "norms_memory_in_bytes": 0,
      "points_memory": "0b",
      "points_memory_in_bytes": 0,
      "doc_values_memory": "0b",
      "doc_values_memory_in_bytes": 0,
      "index_writer_memory": "808.7kb",
      "index_writer_memory_in_bytes": 828144,
      "version_map_memory": "750b",
      "version_map_memory_in_bytes": 750,
      "fixed_bit_set": "37.5kb",
      "fixed_bit_set_memory_in_bytes": 38456,
      "max_unsafe_auto_id_timestamp": 1674059600395,
      "file_sizes": {}
    },
    "mappings": {
      "total_field_count": 3079,
      "total_deduplicated_field_count": 1047,
      "total_deduplicated_mapping_size": "48kb",
      "total_deduplicated_mapping_size_in_bytes": 49201,
      "field_types": [
        {
          "name": "boolean",
          "count": 49,
          "index_count": 21,
          "script_count": 0
        },
        {
          "name": "completion",
          "count": 12,
          "index_count": 12,
          "script_count": 0
        },
        {
          "name": "constant_keyword",
          "count": 87,
          "index_count": 29,
          "script_count": 0
        },
        {
          "name": "date",
          "count": 220,
          "index_count": 70,
          "script_count": 0
        },
        {
          "name": "double",
          "count": 24,
          "index_count": 12,
          "script_count": 0
        },
        {
          "name": "float",
          "count": 24,
          "index_count": 12,
          "script_count": 0
        },
        {
          "name": "integer",
          "count": 93,
          "index_count": 17,
          "script_count": 0
        },
        {
          "name": "ip",
          "count": 29,
          "index_count": 29,
          "script_count": 0
        },
        {
          "name": "keyword",
          "count": 1308,
          "index_count": 70,
          "script_count": 0
        },
        {
          "name": "long",
          "count": 135,
          "index_count": 30,
          "script_count": 0
        },
        {
          "name": "nested",
          "count": 16,
          "index_count": 16,
          "script_count": 0
        },
        {
          "name": "object",
          "count": 473,
          "index_count": 61,
          "script_count": 0
        },
        {
          "name": "text",
          "count": 601,
          "index_count": 70,
          "script_count": 0
        },
        {
          "name": "version",
          "count": 8,
          "index_count": 8,
          "script_count": 0
        }
      ],
      "runtime_field_types": []
    },
    "analysis": {
      "char_filter_types": [],
      "tokenizer_types": [
        {
          "name": "edge_ngram",
          "count": 12,
          "index_count": 12
        }
      ],
      "filter_types": [
        {
          "name": "stop",
          "count": 12,
          "index_count": 12
        }
      ],
      "analyzer_types": [
        {
          "name": "custom",
          "count": 36,
          "index_count": 12
        },
        {
          "name": "standard",
          "count": 12,
          "index_count": 12
        }
      ],
      "built_in_char_filters": [],
      "built_in_tokenizers": [
        {
          "name": "standard",
          "count": 24,
          "index_count": 12
        }
      ],
      "built_in_filters": [
        {
          "name": "asciifolding",
          "count": 24,
          "index_count": 12
        },
        {
          "name": "lowercase",
          "count": 36,
          "index_count": 12
        }
      ],
      "built_in_analyzers": [
        {
          "name": "accentEdgeNGramAnalyzer",
          "count": 12,
          "index_count": 12
        },
        {
          "name": "accentQueryAnalyzer",
          "count": 24,
          "index_count": 12
        },
        {
          "name": "defaultIndexAnalyzer",
          "count": 6,
          "index_count": 6
        },
        {
          "name": "simple",
          "count": 6,
          "index_count": 6
        }
      ]
    },
    "versions": [
      {
        "version": "7.11.1",
        "index_count": 6,
        "primary_shard_count": 6,
        "total_primary_size": "14.1mb",
        "total_primary_bytes": 14793280
      },
      {
        "version": "7.14.1",
        "index_count": 2,
        "primary_shard_count": 2,
        "total_primary_size": "1.3mb",
        "total_primary_bytes": 1378431
      },
      {
        "version": "7.16.2",
        "index_count": 13,
        "primary_shard_count": 13,
        "total_primary_size": "11.8mb",
        "total_primary_bytes": 12459494
      },
      {
        "version": "7.17.3",
        "index_count": 3,
        "primary_shard_count": 3,
        "total_primary_size": "155.9kb",
        "total_primary_bytes": 159688
      },
      {
        "version": "8.2.2",
        "index_count": 49,
        "primary_shard_count": 49,
        "total_primary_size": "626mb",
        "total_primary_bytes": 656495866
      },
      {
        "version": "8.6.0",
        "index_count": 10,
        "primary_shard_count": 10,
        "total_primary_size": "350.8mb",
        "total_primary_bytes": 367919595
      }
    ],
    "search": {
      "total": 65516,
      "queries": {
        "bool": 65337,
        "function_score": 8,
        "prefix": 1,
        "match": 775,
        "range": 11967,
        "nested": 1,
        "dis_max": 7,
        "wildcard": 1,
        "multi_match": 7,
        "match_phrase": 2,
        "terms": 48873,
        "match_phrase_prefix": 1,
        "exists": 14452,
        "term": 16065,
        "simple_query_string": 2170
      },
      "sections": {
        "highlight": 7,
        "runtime_mappings": 1,
        "query": 65362,
        "terminate_after": 2,
        "_source": 90,
        "pit": 6,
        "suggest": 37,
        "aggs": 1433
      }
    }
  },
  "nodes": {
    "count": {
      "total": 3,
      "coordinating_only": 0,
      "data": 3,
      "data_cold": 0,
      "data_content": 0,
      "data_frozen": 0,
      "data_hot": 0,
      "data_warm": 0,
      "index": 0,
      "ingest": 3,
      "master": 3,
      "ml": 0,
      "remote_cluster_client": 0,
      "search": 0,
      "transform": 0,
      "voting_only": 0
    },
    "versions": [
      "8.6.0"
    ],
    "os": {
      "available_processors": 6,
      "allocated_processors": 6,
      "names": [
        {
          "name": "Windows Server 2019",
          "count": 3
        }
      ],
      "pretty_names": [
        {
          "pretty_name": "Windows Server 2019",
          "count": 3
        }
      ],
      "architectures": [
        {
          "arch": "amd64",
          "count": 3
        }
      ],
      "mem": {
        "total": "47.9gb",
        "total_in_bytes": 51536400384,
        "adjusted_total": "47.9gb",
        "adjusted_total_in_bytes": 51536400384,
        "free": "24.2gb",
        "free_in_bytes": 26044907520,
        "used": "23.7gb",
        "used_in_bytes": 25491492864,
        "free_percent": 51,
        "used_percent": 49
      }
    },
    "process": {
      "cpu": {
        "percent": 0
      },
      "open_file_descriptors": {
        "min": -1,
        "max": -1,
        "avg": 0
      }
    },
    "jvm": {
      "max_uptime": "18.1h",
      "max_uptime_in_millis": 65277869,
      "versions": [
        {
          "version": "19.0.1",
          "vm_name": "OpenJDK 64-Bit Server VM",
          "vm_version": "19.0.1+10-21",
          "vm_vendor": "Oracle Corporation",
          "bundled_jdk": true,
          "using_bundled_jdk": true,
          "count": 3
        }
      ],
      "mem": {
        "heap_used": "5.5gb",
        "heap_used_in_bytes": 6011100704,
        "heap_max": "12gb",
        "heap_max_in_bytes": 12884901888
      },
      "threads": 152
    },
    "fs": {
      "total": "599.9gb",
      "total_in_bytes": 644188274688,
      "free": "582.6gb",
      "free_in_bytes": 625619959808,
      "available": "582.6gb",
      "available_in_bytes": 625619959808
    },
    "plugins": [],
    "network_types": {
      "transport_types": {
        "netty4": 3
      },
      "http_types": {
        "netty4": 3
      }
    },
    "discovery_types": {
      "multi-node": 3
    },
    "packaging_types": [
      {
        "flavor": "default",
        "type": "zip",
        "count": 3
      }
    ],
    "ingest": {
      "number_of_pipelines": 4,
      "processor_stats": {
        "attachment": {
          "count": 14278,
          "failed": 0,
          "current": 0,
          "time": "8.5m",
          "time_in_millis": 511815
        },
        "gsub": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "remove": {
          "count": 14278,
          "failed": 0,
          "current": 0,
          "time": "70ms",
          "time_in_millis": 70
        },
        "script": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "set": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "set_security_user": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        }
      }
    },
    "indexing_pressure": {
      "memory": {
        "current": {
          "combined_coordinating_and_primary": "0b",
          "combined_coordinating_and_primary_in_bytes": 0,
          "coordinating": "0b",
          "coordinating_in_bytes": 0,
          "primary": "0b",
          "primary_in_bytes": 0,
          "replica": "0b",
          "replica_in_bytes": 0,
          "all": "0b",
          "all_in_bytes": 0
        },
        "total": {
          "combined_coordinating_and_primary": "0b",
          "combined_coordinating_and_primary_in_bytes": 0,
          "coordinating": "0b",
          "coordinating_in_bytes": 0,
          "primary": "0b",
          "primary_in_bytes": 0,
          "replica": "0b",
          "replica_in_bytes": 0,
          "all": "0b",
          "all_in_bytes": 0,
          "coordinating_rejections": 0,
          "primary_rejections": 0,
          "replica_rejections": 0
        },
        "limit": "0b",
        "limit_in_bytes": 0
      }
    }
  }
}

It is mainly the documents that is causing problems. The _ingest pipeline. In 8.2.2 you need a plugin but in 8.6.0 it is included. I get the exception on regular content as well.

Thanks Mark!

4

Hi Mark,

Did you have any suggestions to fix this?
Thanks!

/Kristoffer

5

Thanks, there's nothing there that jumps out at me.

Are you able to share some Elasticsearch logs from around the time?

6

Hi!

If I restart everything and run my reindex job the first time it works just fine, maybe on error. If I run the job again maybe five error and the third time 100 errors. Seems like the number of errors increases every time I reindex. Could that give you some clue?
We are using the ElasticSearch.Net version 7.17.5.

Thanks!

/Kristoffer

7

And it is almost always the attachment pipeline that causes the error:
_bulk?pipeline=attachments.

I'm out of ideas here sadly.
Thanks!

/Kristoffer

8

If I restart all services and reindex it works just fine. Feels like some hang up that needs a restart to get everything working againg.

/Kristoffer

9

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.