One of my 6 Elasticsearch clusters at work is not respecting when i try to disable shard allocation prior to pull a node/server from the cluster.
When I run the curl command below I get the "true" response you would expect, also shown below. However, after I pull a node from the cluster it will immediately start to rebalance shards to make up for the lost node. I've confirmed that my command does make it to each node when looking at the _cluster/settings?include_defaults=true call. I've attached those setting as well.
Please let me know if you happen to see any configuration that may be causing the problem.
curl --insecure -u admin:<PASSWORD> -X PUT "https://contoso:9200/_cluster/settings?pretty" -H 'Content-Type: application/json' -d'
{
"persistent": {
"cluster.routing.allocation.enable": "none"
}
}
'
{
"acknowledged" : true,
"persistent" : { },
"transient" : { }
}
{"persistent":{"cluster":{"routing":{"allocation":{"enable":"none"}}},"xpack":{"monitoring":{"collection":{"enabled":"true"},"history":{"duration":"1d"}}}},"transient":{"cluster":{"routing":{"allocation":{"disk":{"threshold_enabled":"false"},"enable":"all"}}},"logger":{"com":{"floragunn":{"searchguard":{"ssl":{"http":{"netty":{"SearchGuardSSLNettyHttpServerTransport":"fatal"}}}}}}}},"defaults":{"cluster":{"max_voting_config_exclusions":"10","auto_shrink_voting_configuration":"true","election":{"duration":"500ms","initial_timeout":"100ms","max_timeout":"10s","back_off_time":"100ms","strategy":"supports_voting_only"},"no_master_block":"write","persistent_tasks":{"allocation":{"enable":"all","recheck_interval":"30s"}},"blocks":{"read_only_allow_delete":"false","read_only":"false"},"remote":{"node":{"attr":""},"initial_connect_timeout":"30s","connect":"true","connections_per_cluster":"3"},"follower_lag":{"timeout":"90000ms"},"routing":{"use_adaptive_replica_selection":"true","rebalance":{"enable":"all"},"allocation":{"enforce_default_tier_preference":"false","node_concurrent_incoming_recoveries":"2","include":{"_tier":""},"node_initial_primaries_recoveries":"4","same_shard":{"host":"false"},"total_shards_per_node":"-1","require":{"_tier":""},"shard_state":{"reroute":{"priority":"NORMAL"}},"type":"balanced","disk":{"include_relocations":"true","reroute_interval":"60s","watermark":{"flood_stage.frozen.max_headroom":"20GB","flood_stage":"95%","high":"90%","low":"85%","enable_for_single_data_node":"false","flood_stage.frozen":"95%"}},"awareness":{"attributes":["server_id"]},"balance":{"index":"0.55","threshold":"1.0","shard":"0.45"},"node_concurrent_outgoing_recoveries":"2","allow_rebalance":"indices_all_active","cluster_concurrent_rebalance":"2","node_concurrent_recoveries":"4","exclude":{"_tier":""}}},"indices":{"tombstones":{"size":"500"},"close":{"enable":"true"}},"max_shards_per_node.frozen":"3000","nodes":{"reconnect_interval":"10s"},"service":{"master_service_starvation_logging_threshold":"5m","slow_master_task_logging_threshold":"10s","slow_task_logging_threshold":"30s"},"publish":{"timeout":"30000ms","info_timeout":"10000ms"},"name":"search_stage","fault_detection":{"leader_check":{"interval":"1000ms","timeout":"10000ms","retry_count":"3"},"follower_check":{"interval":"1000ms","timeout":"10000ms","retry_count":"3"}},"join":{"timeout":"60000ms"},"max_shards_per_node":"1000","initial_master_nodes":[],"deprecation_indexing":{"enabled":"true","x_opaque_id_used":{"enabled":"true"}},"snapshot":{"info":{"max_concurrent_fetches":"5"}},"info":{"update":{"interval":"30s","timeout":"15s"}}},"stack":{"templates":{"enabled":"true"}},"logger":{"level":"INFO"},"bootstrap":{"memory_lock":"true","system_call_filter":"true","ctrlhandler":"true"},"processors":"1","ingest":{"user_agent":{"cache_size":"1000"},"geoip":{"cache_size":"1000","downloader":{"enabled":"true","endpoint":"https://geoip.elastic.co/v1/database","poll":{"interval":"3d"}}},"grok":{"watchdog":{"max_execution_time":"1s","interval":"1s"}}},"network":{"host":["es-search-stage2a.contoso.com"],"tcp":{"reuse_address":"true","keep_count":"-1","connect_timeout":"30s","keep_interval":"-1","no_delay":"true","keep_alive":"true","receive_buffer_size":"-1b","keep_idle":"-1","send_buffer_size":"-1b"},"bind_host":["es-search-stage2a.contoso.com"],"server":"true","breaker":{"inflight_requests":{"limit":"100%","overhead":"2.0"}},"publish_host":["es-search-stage2a.contoso.com"]},"pidfile":"","searchable_snapshots":{"blob_cache":{"periodic_cleanup":{"interval":"1h","batch_size":"100","pit_keep_alive":"10m","retention_period":"1h"}}},"path":{"data":["/var/data/elasticsearch/a"],"logs":"/var/log/elasticsearch/a","shared_data":"","home":"/usr/share/elasticsearch","repo":[]},"search":{"default_search_timeout":"-1","max_open_scroll_context":"500","max_buckets":"65536","max_async_search_response_size":"-1b","keep_alive_interval":"1m","remote":{"node":{"attr":""},"initial_connect_timeout":"30s","connect":"true","connections_per_cluster":"3"},"max_keep_alive":"24h","highlight":{"term_vector_multi_value":"true"},"default_allow_partial_results":"true","low_level_cancellation":"true","allow_expensive_queries":"true","default_keep_alive":"5m","aggs":{"rewrite_to_filter_by_filter":"true"}},"security":{"manager":{"filter_bad_defaults":"true"}},"ccr":{"wait_for_metadata_timeout":"60s","indices":{"recovery":{"recovery_activity_timeout":"60s","chunk_size":"1mb","internal_action_timeout":"60s","max_bytes_per_sec":"40mb","max_concurrent_file_chunks":"5"}},"auto_follow":{"wait_for_metadata_timeout":"60s"}},"repositories":{"fs":{"compress":"false","chunk_size":"9223372036854775807b","location":""},"url":{"supported_protocols":["http","https","ftp","file","jar"],"allowed_urls":[],"url":"http:"}},"action":{"auto_create_index":"true","search":{"pre_filter_shard_size":{"default":"128"},"shard_count":{"limit":"9223372036854775807"}},"destructive_requires_name":"false"},"client":{"type":"node","transport":{"ignore_cluster_name":"false","nodes_sampler_interval":"5s","sniff":"false","ping_timeout":"5s"}},"enrich":{"max_force_merge_attempts":"3","cleanup_period":"15m","fetch_size":"10000","cache_size":"1000","coordinator_proxy":{"max_concurrent_requests":"8","max_lookups_per_request":"128","queue_capacity":"1024"},"max_concurrent_policy_executions":"50"},"xpack":{"flattened":{"enabled":"true"},"watcher":{"execution":{"scroll":{"size":"0","timeout":""},"default_throttle_period":"5s"},"internal":{"ops":{"bulk":{"default_timeout":""},"index":{"default_timeout":""},"search":{"default_timeout":""}}},"thread_pool":{"queue_size":"1000","size":"5"},"index":{"rest":{"direct_access":""}},"use_ilm_index_management":"true","history":{"cleaner_service":{"enabled":"true"}},"trigger":{"schedule":{"ticker":{"tick_interval":"500ms"}}},"enabled":"true","input":{"search":{"default_timeout":""}},"encrypt_sensitive_data":"false","transform":{"search":{"default_timeout":""}},"stop":{"timeout":"30s"},"watch":{"scroll":{"size":"0"}},"bulk":{"concurrent_requests":"0","flush_interval":"1s","size":"1mb","actions":"1"},"actions":{"bulk":{"default_timeout":""},"index":{"default_timeout":""}}},"eql":{"enabled":"true"},"data_frame":{"enabled":"true"},"ilm":{"enabled":"true"},"monitoring":{"migration":{"decommission_alerts":"false"},"collection":{"cluster":{"stats":{"timeout":"10s"}},"node":{"stats":{"timeout":"10s"}},"indices":[],"ccr":{"stats":{"timeout":"10s"}},"enrich":{"stats":{"timeout":"10s"}},"index":{"stats":{"timeout":"10s"},"recovery":{"active_only":"false","timeout":"10s"}},"interval":"10s","ml":{"job":{"stats":{"timeout":"10s"}}}},"elasticsearch":{"collection":{"enabled":"true"}},"enabled":"true"},"graph":{"enabled":"true"},"searchable":{"snapshot":{"allocate_on_rolling_restart":"false","cache":{"range_size":"32mb","sync":{"max_files":"10000","interval":"60s","shutdown_timeout":"10s"},"recovery_range_size":"128kb"},"shared_cache":{"recovery_range_size":"128kb","region_size":"16mb","size":"0","min_time_delta":"60s","decay":{"interval":"60s"},"size.max_headroom":"-1","range_size":"16mb","max_freq":"100"}}},"rollup":{"enabled":"true","task_thread_pool":{"queue_size":"-1","size":"1"}},"sql":{"enabled":"true"},"searchable_snapshots":{"cache_fetch_async_thread_pool":{"core":"0","max":"3","keep_alive":"30s"},"cache_prewarming_thread_pool":{"core":"0","max":"16","keep_alive":"30s"}},"license":{"upload":{"types":["standard","gold","platinum","enterprise","trial"]},"self_generated":{"type":"basic"}},"logstash":{"enabled":"true"},"notification":{"pagerduty":{"default_account":""},"email":{"account":{"domain_allowlist":["*"]},"default_account":"","html":{"sanitization":{"allow":["body","head","_tables","_links","_blocks","_formatting","img:embedded"],"disallow":[],"enabled":"true"}}},"reporting":{"retries":"40","warning":{"enabled":"true"},"interval":"15s"},"jira":{"default_account":""},"slack":{"default_account":""}},"security":{"dls_fls":{"enabled":"true"},"encryption":{"algorithm":"AES/CTR/NoPadding"},"audit":{"enabled":"false"},"authc":{"password_hashing":{"algorithm":"bcrypt"},"reserved_realm":{"enabled":"true"},"api_key":{"enabled":"false"},"token":{"enabled":"false"}},"fips_mode":{"enabled":"false"},"encryption_key":{"length":"128","algorithm":"AES"},"http":{"ssl":{"enabled":"false"}},"transport":{"ssl":{"enabled":"false"}},"ssl":{"diagnose":{"trust":"true"}},"user":null,"enabled":"false"},"transform":{"num_transform_failure_retries":"10","enabled":"true"},"vectors":{"enabled":"true"},"ccr":{"enabled":"true","ccr_thread_pool":{"queue_size":"100","size":"32"}},"idp":{"privileges":{"application":"","cache":{"size":"100","ttl":"90m"}},"metadata":{"signing":{"keystore":{"alias":""}}},"slo_endpoint":{"post":"https:","redirect":"https:"},"defaults":{"nameid_format":"urn:oasis:names:tc:SAML:2.0:nameid-format:transient","authn_expiry":"5m"},"allowed_nameid_formats":["urn:oasis:names:tc:SAML:2.0:nameid-format:transient"],"contact":{"given_name":"","email":"","surname":""},"organization":{"display_name":"","name":"","url":"http:"},"sso_endpoint":{"post":"https:","redirect":"https:"},"entity_id":"","signing":{"keystore":{"alias":""}},"sp":{"cache":{"size":"1000","ttl":"60m"},"wildcard":{"path":"wildcard_services.json"}},"enabled":"false"},"slm":{"enabled":"true"},"enrich":{"enabled":"true"},"http":{"tcp":{"keep_alive":"true"},"default_connection_timeout":"10s","proxy":{"host":"","scheme":"","port":"0"},"connection_pool_ttl":"-1","max_response_size":"10mb","whitelist":["*"],"default_read_timeout":"10s"},"autoscaling":{"memory":{"monitor":{"timeout":"15s"}}},"ml":{"utility_thread_pool":{"core":"1","max":"2048","keep_alive":"10m"},"max_anomaly_records":"500","enable_config_migration":"true","max_open_jobs":"512","delayed_data_check_freq":"15m","min_disk_space_off_heap":"5gb","use_auto_machine_memory_percent":"false","inference_model":{"cache_size":"40%","time_to_live":"5m"},"nightly_maintenance_requests_per_second":"-1.0","node_concurrent_job_allocations":"2","max_model_memory_limit":"0b","enabled":"true","max_lazy_ml_nodes":"0","max_ml_node_size":"0b","max_machine_memory_percent":"30","persist_results_max_retries":"20","autodetect_process":"true","datafeed_thread_pool":{"core":"1","max":"512","keep_alive":"1m"},"max_inference_processors":"50","process_connect_timeout":"10s","job_comms_thread_pool":{"core":"4","max":"2048","keep_alive":"1m"}}},"rest":{"action":{"multi":{"allow_explicit_index":"true"}}},"cache":{"recycler":{"page":{"limit":{"heap":"10%"},"type":"CONCURRENT","weight":{"longs":"1.0","ints":"1.0","bytes":"1.0","objects":"0.1"}}}},"async_search":{"index_cleanup_interval":"1h"},"reindex":{"remote":{"whitelist":[]}},"resource":{"reload":{"enabled":"true","interval":{"low":"60s","high":"5s","medium":"30s"}}},"thread_pool":{"force_merge":{"queue_size":"-1","size":"1"},"search_coordination":{"queue_size":"1000","size":"1"},"snapshot_meta":{"core":"1","max":"3","keep_alive":"30s"},"fetch_shard_started":{"core":"1","max":"2","keep_alive":"5m"},"listener":{"queue_size":"-1","size":"1"},"estimated_time_interval.warn_threshold":"5s","scheduler":{"warn_threshold":"5s"},"search":{"max_queue_size":"1000","queue_size":"1000","size":"2","auto_queue_frame_size":"2000","target_response_time":"1s","min_queue_size":"1000"},"fetch_shard_store":{"core":"1","max":"2","keep_alive":"5m"},"flush":{"core":"1","max":"1","keep_alive":"5m"},"vectortile":{"queue_size":"-1","size":"1"},"get":{"queue_size":"1000","size":"1"},"system_read":{"queue_size":"2000","size":"1"},"system_critical_read":{"queue_size":"2000","size":"1"},"estimated_time_interval":"200ms","write":{"queue_size":"10000","size":"1"},"system_critical_write":{"queue_size":"1500","size":"1"},"refresh":{"core":"1","max":"1","keep_alive":"5m"},"system_write":{"queue_size":"1000","size":"1"},"generic":{"core":"4","max":"128","keep_alive":"30s"},"warmer":{"core":"1","max":"1","keep_alive":"5m"},"auto_complete":{"queue_size":"100","size":"1"},"management":{"core":"1","max":"1","keep_alive":"5m"},"analyze":{"queue_size":"16","size":"1"},"snapshot":{"core":"1","max":"1","keep_alive":"5m"},"search_throttled":{"max_queue_size":"100","queue_size":"100","size":"1","auto_queue_frame_size":"200","target_response_time":"1s","min_queue_size":"100"}},"index":{"codec":"default","recovery":{"type":""},"store":{"type":"","fs":{"fs_lock":"native"},"preload":[],"snapshot":{"uncached_chunk_size":"-1b","cache":{"excluded_file_types":[]}}}},"monitor":{"jvm":{"gc":{"enabled":"true","overhead":{"warn":"50","debug":"10","info":"25"},"refresh_interval":"1s"},"refresh_interval":"1s"},"process":{"refresh_interval":"1s"},"os":{"refresh_interval":"1s"},"fs":{"health":{"enabled":"true","refresh_interval":"120s","slow_path_logging_threshold":"5s"},"refresh_interval":"1s"}},"runtime_fields":{"grok":{"watchdog":{"max_execution_time":"1s","interval":"1s"}}},"transport":{"tcp":{"reuse_address":"true","keep_count":"-1","connect_timeout":"30s","keep_interval":"-1","compress":"FALSE","port":"9300-9400","no_delay":"true","keep_alive":"true","receive_buffer_size":"-1b","keep_idle":"-1","send_buffer_size":"-1b"},"bind_host":[],"connect_timeout":"30s","compress":"FALSE","ping_schedule":"-1","connections_per_node":{"recovery":"2","state":"1","bulk":"3","reg":"6","ping":"1"},"tracer":{"include":[],"exclude":["internal:discovery/zen/fd*","internal:coordination/fault_detection/*","cluster:monitor/nodes/liveness"]},"type":"com.floragunn.searchguard.ssl.http.netty.SearchGuardSSLNettyTransport","slow_operation_logging_threshold":"5s","type.default":"netty4","features":{"x-pack":"true"},"port":"9300-9400","compression_scheme":"DEFLATE","host":[],"publish_port":"-1","tcp_no_delay":"true","publish_host":[],"netty":{"receive_predictor_size":"64kb","receive_predictor_max":"64kb","worker_count":"2","receive_predictor_min":"64kb","boss_count":"1"}},"deprecation":{"skip_deprecated_settings":[]},"script":{"allowed_contexts":[],"max_compilations_rate":"150/5m","cache":{"max_size":"3000","expire":"0ms"},"painless":{"regex":{"enabled":"limited","limit-factor":"6"}},"max_size_in_bytes":"65535","allowed_types":[],"disable_max_compilations_rate":"false"},"indexing_pressure":{"memory":{"limit":"10%"}},"node":{"data":"true","bandwidth":{"recovery":{"disk":{"write":"-1","read":"-1"},"operator":{"factor.read":"0.4","factor.write":"0.4","factor":"0.4","factor.max_overcommit":"100.0"},"network":"-1"}},"roles":["data_frozen","data_warm","transform","data","remote_cluster_client","data_cold","data_content","data_hot","ingest","master","ml"],"max_local_storage_nodes":"1","processors":"2","store":{"allow_mmap":"true"},"ingest":"true","master":"true","pidfile":"/var/run/elasticsearch/a/elasticsearch.pid","transform":"true","remote_cluster_client":"true","enable_lucene_segment_infos_trace":"false","local_storage":"true","name":"es-search-stage2a","id":{"seed":"0"},"voting_only":"false","attr":{"transform":{"node":"true"},"server_id":"server_two","xpack":{"installed":"true"},"ml":{"max_jvm_size":"4294967296","machine_memory":"16774832128","max_open_jobs":"512"}},"portsfile":"false","ml":"true"},"indices":{"replication":{"retry_timeout":"60s","initial_retry_backoff_bound":"50ms"},"cache":{"cleanup_interval":"1m"},"mapping":{"dynamic_timeout":"30s","max_in_flight_updates":"10"},"memory":{"interval":"5s","max_index_buffer_size":"-1","shard_inactive_time":"5m","index_buffer_size":"10%","min_index_buffer_size":"48mb"},"breaker":{"request":{"limit":"60%","type":"memory","overhead":"1.0"},"total":{"limit":"95%","use_real_memory":"true"},"accounting":{"limit":"100%","overhead":"1.0"},"fielddata":{"limit":"60%","type":"memory","overhead":"1.03"},"type":"hierarchy"},"query":{"bool":{"max_nested_depth":"20","max_clause_count":"1024"},"query_string":{"analyze_wildcard":"false","allowLeadingWildcard":"true"}},"id_field_data":{"enabled":"true"},"recovery":{"internal_action_retry_timeout":"1m","recovery_activity_timeout":"1800000ms","retry_delay_network":"5s","internal_action_timeout":"15m","max_concurrent_snapshot_file_downloads_per_node":"25","retry_delay_state_sync":"500ms","max_concurrent_snapshot_file_downloads":"5","internal_action_long_timeout":"1800000ms","max_concurrent_operations":"1","use_snapshots":"true","max_bytes_per_sec":"500mb","max_concurrent_file_chunks":"2"},"requests":{"cache":{"size":"1%","expire":"0ms"}},"store":{"delete":{"shard":{"timeout":"30s"}}},"analysis":{"hunspell":{"dictionary":{"ignore_case":"false","lazy":"false"}}},"queries":{"cache":{"count":"10000","size":"10%","all_segments":"false"}},"lifecycle":{"history_index_enabled":"true","poll_interval":"10m","step":{"master_timeout":"30s"}},"fielddata":{"cache":{"size":"50%"}}},"signals":{"index_names":{"watches":".signals_watches","watches_state":".signals_watches_state","watches_trigger_state":".signals_watches_trigger_state","accounts":".signals_accounts","log":"<.signals_log_{now/d}>"},"enabled":"true","enterprise":{"enabled":"true"},"watch_log":{"sync_indexing":"false","refresh_policy":""},"worker_threads":{"pool":{"max_size":"3","keep_alive":"100m"},"prio":"5"},"all_tenants_active_by_default":"true"},"plugin":{"mandatory":[]},"slm":{"minimum_interval":"15m","retention_schedule":"0 30 1 * * ?","retention_duration":"1h","history_index_enabled":"true"},"discovery":{"seed_hosts":[],"unconfigured_bootstrap_timeout":"3s","request_peers_timeout":"3000ms","zen":{"commit_timeout":"30s","no_master_block":"write","join_retry_delay":"100ms","join_retry_attempts":"3","ping":{"unicast":{"concurrent_connects":"10","hosts":["es-search-stage2a.contoso.com","es-search-stage2b.contoso.com","es-search-stage1a.contoso.com","es-search-stage1b.contoso.com","es-search-stage3a.contoso.com","es-search-stage3b.contoso.com"],"hosts.resolve_timeout":"5s"}},"master_election":{"ignore_non_master_pings":"false","wait_for_joins_timeout":"30000ms"},"send_leave_request":"true","ping_timeout":"10s","bwc_ping_timeout":"3s","join_timeout":"60000ms","publish_diff":{"enable":"true"},"publish":{"max_pending_cluster_states":"25"},"minimum_master_nodes":"4","unsafe_rolling_upgrades_enabled":"true","hosts_provider":[],"publish_timeout":"30s","fd":{"connect_on_network_disconnect":"false","ping_interval":"1s","ping_retries":"3","register_connection_listener":"true","ping_timeout":"30s"},"max_pings_from_another_master":"3"},"initial_state_timeout":"30s","cluster_formation_warning_timeout":"10000ms","seed_providers":[],"type":"zen","seed_resolver":{"max_concurrent_resolvers":"10","timeout":"5s"},"find_peers_interval":"1000ms","probe":{"connect_timeout":"30s","handshake_timeout":"30s"}},"http":{"cors":{"max-age":"1728000","allow-origin":"","allow-headers":"X-Requested-With,Content-Type,Content-Length,Authorization,Accept,User-Agent,X-Elastic-Client-Meta","allow-credentials":"false","allow-methods":"OPTIONS,HEAD,GET,POST,PUT,DELETE","enabled":"false"},"max_chunk_size":"8kb","compression_level":"3","max_initial_line_length":"4kb","type":"com.floragunn.searchguard.http.SearchGuardHttpServerTransport","pipelining":{"max_events":"10000"},"type.default":"netty4","content_type":{"required":"true"},"host":[],"publish_port":"-1","read_timeout":"0ms","max_content_length":"100mb","netty":{"receive_predictor_size":"64kb","max_composite_buffer_components":"69905","worker_count":"0"},"tcp":{"reuse_address":"true","keep_count":"-1","keep_interval":"-1","no_delay":"true","keep_alive":"true","receive_buffer_size":"-1b","keep_idle":"-1","send_buffer_size":"-1b"},"bind_host":[],"client_stats":{"enabled":"true","closed_channels":{"max_age":"5m","max_count":"10000"}},"reset_cookies":"false","max_warning_header_count":"-1","tracer":{"include":[],"exclude":[]},"max_warning_header_size":"-1b","detailed_errors":{"enabled":"true"},"port":"9200","max_header_size":"8kb","tcp_no_delay":"true","compression":"false","publish_host":[]},"gateway":{"recover_after_master_nodes":"0","expected_nodes":"-1","recover_after_data_nodes":"-1","expected_data_nodes":"-1","write_dangling_indices_info":"true","slow_write_logging_threshold":"10s","recover_after_time":"0ms","expected_master_nodes":"-1","recover_after_nodes":"-1","auto_import_dangling_indices":"false"},"snapshot":{"refresh_repo_uuid_on_restore":"true","max_concurrent_operations":"1000"}}}