I cannot create elastic index after removing two nodes from cassandra. We had two nodes earlier, but for some reason they were unable to communicate with each other. So we removed them and changed necessary configuration. Now with single node when I try to create an index, I am getting an error
{
"error" : {
"root_cause" : [
{
"type" : "unavailable_exception",
"reason" : "Cannot achieve consistency level QUORUM"
}
],
"type" : "unavailable_exception",
"reason" : "Cannot achieve consistency level QUORUM"
},
"status" : 500
}
Not sure what is the problem
My cluster state indicates
curl -XGET 'http://10.0.133.89:9200/_cluster/stats?human&pretty'
{
"_nodes" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"cluster_name" : "I2V2 Cluster",
"cluster_uuid" : "ce32e34c-4f0c-407b-bd18-4b17d4c84355",
"timestamp" : 1678170175858,
"status" : "green",
"indices" : {
"count" : 1009,
"shards" : {
"total" : 1009,
"primaries" : 1009,
"replication" : 0.0,
"index" : {
"shards" : {
"min" : 1,
"max" : 1,
"avg" : 1.0
},
"primaries" : {
"min" : 1,
"max" : 1,
"avg" : 1.0
},
"replication" : {
"min" : 0.0,
"max" : 0.0,
"avg" : 0.0
}
}
},
"docs" : {
"count" : 1746591140,
"deleted" : 17244075
},
"store" : {
"size" : "149.9gb",
"size_in_bytes" : 161021104450
},
"fielddata" : {
"memory_size" : "0b",
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "217.8mb",
"memory_size_in_bytes" : 228414396,
"total_count" : 2669992,
"hit_count" : 748311,
"miss_count" : 1921681,
"cache_size" : 30244,
"cache_count" : 58006,
"evictions" : 27762
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 4876,
"memory" : "290.4mb",
"memory_in_bytes" : 304608840,
"terms_memory" : "233mb",
"terms_memory_in_bytes" : 244377119,
"stored_fields_memory" : "37mb",
"stored_fields_memory_in_bytes" : 38854832,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "2.8kb",
"norms_memory_in_bytes" : 2880,
"points_memory" : "18.7mb",
"points_memory_in_bytes" : 19608969,
"doc_values_memory" : "1.6mb",
"doc_values_memory_in_bytes" : 1765040,
"index_writer_memory" : "1.3mb",
"index_writer_memory_in_bytes" : 1446084,
"version_map_memory" : "0b",
"version_map_memory_in_bytes" : 0,
"fixed_bit_set" : "0b",
"fixed_bit_set_memory_in_bytes" : 0,
"token_ranges_bit_set" : "0b",
"token_ranges_bit_set_memory_in_bytes" : 0,
"max_unsafe_auto_id_timestamp" : -1,
"file_sizes" : { }
}
},
"nodes" : {
"count" : {
"total" : 1,
"data" : 1,
"coordinating_only" : 0,
"master" : 1,
"ingest" : 1
},
"versions" : [
"6.8.4"
],
"os" : {
"available_processors" : 8,
"allocated_processors" : 8,
"names" : [
{
"name" : "Linux",
"count" : 1
}
],
"pretty_names" : [
{
"pretty_name" : "Ubuntu 20.04.4 LTS",
"count" : 1
}
],
"mem" : {
"total" : "31.3gb",
"total_in_bytes" : 33662451712,
"free" : "236.9mb",
"free_in_bytes" : 248483840,
"used" : "31.1gb",
"used_in_bytes" : 33413967872,
"free_percent" : 1,
"used_percent" : 99
}
},
"process" : {
"cpu" : {
"percent" : 8
},
"open_file_descriptors" : {
"min" : 4196,
"max" : 4196,
"avg" : 4196
}
},
"jvm" : {
"max_uptime" : "3.6d",
"max_uptime_in_millis" : 319007254,
"versions" : [
{
"version" : "1.8.0_362",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "25.362-b09",
"vm_vendor" : "Private Build",
"count" : 1
}
],
"mem" : {
"heap_used" : "6.6gb",
"heap_used_in_bytes" : 7185105504,
"heap_max" : "11.9gb",
"heap_max_in_bytes" : 12801015808
},
"threads" : 222
},
"fs" : {
"total" : "339.2gb",
"total_in_bytes" : 364231827456,
"free" : "142.5gb",
"free_in_bytes" : 153021927424,
"available" : "142.4gb",
"available_in_bytes" : 153005150208
},
"plugins" : [
{
"name" : "org.apache.cassandra.service.ElassandraDaemon$ElassandraPlugin",
"version" : "NA",
"elasticsearch_version" : "6.8.4",
"java_version" : "1.8",
"description" : "classpath plugin",
"classname" : "org.apache.cassandra.service.ElassandraDaemon$ElassandraPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false
}
],
"network_types" : {
"transport_types" : {
"netty4" : 1
},
"http_types" : {
"netty4" : 1
}
}
}
}
Seems like it is single node only. But some how it is using QUORUM consitency.
My command to create index is
curl -XPUT -H 'Content-Type: application/json' 'http://10.0.133.89:9200/demo_wms_machine_8ce4ad51_3dc6_476b_925d_1207c45c2035_index' -d '{
"settings": {
"keyspace": "demo_wms"
},
"mappings":{
"machine_8ce4ad51_3dc6_476b_925d_1207c45c2035" : {
"properties":{
"internal_tag": {
"type": "keyword",
"cql_primary_key_order": 0,
"cql_partition_key": true,
"cql_collection": "singleton"
},
"generated_at": {
"type": "date",
"cql_primary_key_order": 1,
"cql_collection": "singleton"
},
"collected_at": {
"type": "date",
"cql_collection": "singleton",
"index": false
},
"recorded_at": {
"type": "date",
"cql_collection": "singleton",
"index": false
},
"data_type": {
"type": "keyword",
"cql_collection": "singleton",
"index": false
},
"metadata": {
"type": "keyword",
"cql_collection": "singleton",
"index": false
},
"value": {
"type": "keyword",
"cql_collection": "singleton",
"index": false
}
}
}
}
}'
my cluster health is
{"cluster_name":"I2V2 Cluster","status":"green","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":1009,"active_shards":1009,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":0,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":100.0}
I have tried setting CONSISTENCY to ONE in cqlsh and tried the command with including "action.write_consistency": "one"
Nothing has worked so far.
