Hello,
Elastic search was running 7.10.1 was running successfully (via helm chart, however recently I got the following error
The configuration is pretty standard (3 nodes 3x masters
I would like to know what the corrective action should be when something like this happens? I have read that I must delete the Kubernetes persistant volumes, but I don't think this is the best solution,
Log
{"type": "server", "timestamp": "2021-08-16T14:36:42,994Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "node name [elasticsearch-master-0], node ID [TQUR9Mh8QhGR_H1VGCTrcw], cluster name [elasticsearch], roles [transform, master, remote_cluster_client, data, ml, data_content, data_hot, data_warm, data_cold, ingest]" }
{"type": "server", "timestamp": "2021-08-16T14:36:50,301Z", "level": "INFO", "component": "o.e.x.m.p.l.CppLogMessageHandler", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "[controller/168] [Main.cc@114] controller (64 bit): Version 7.10.1 (Build 11e1ac84105757) Copyright (c) 2020 Elasticsearch BV" }
{"type": "server", "timestamp": "2021-08-16T14:36:51,838Z", "level": "INFO", "component": "o.e.x.s.a.s.FileRolesStore", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "parsed [0] roles from file [/usr/share/elasticsearch/config/roles.yml]" }
{"type": "server", "timestamp": "2021-08-16T14:36:53,811Z", "level": "INFO", "component": "o.e.t.NettyAllocator", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "creating NettyAllocator with the following configs: [name=unpooled, suggested_max_allocation_size=256kb, factors={es.unsafe.use_unpooled_allocator=null, g1gc_enabled=true, g1gc_region_size=1mb, heap_size=1gb}]" }
{"type": "server", "timestamp": "2021-08-16T14:36:53,925Z", "level": "INFO", "component": "o.e.d.DiscoveryModule", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "using discovery type [zen] and seed hosts providers [settings]" }
{"type": "server", "timestamp": "2021-08-16T14:36:54,927Z", "level": "WARN", "component": "o.e.g.DanglingIndicesState", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "gateway.auto_import_dangling_indices is disabled, dangling indices will not be automatically detected or imported and must be managed manually" }
{"type": "server", "timestamp": "2021-08-16T14:36:55,713Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "initialized" }
{"type": "server", "timestamp": "2021-08-16T14:36:55,715Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "starting ..." }
{"type": "server", "timestamp": "2021-08-16T14:36:55,914Z", "level": "INFO", "component": "o.e.t.TransportService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "publish_address {172.18.129.8:9300}, bound_addresses {[::]:9300}" }
{"type": "server", "timestamp": "2021-08-16T14:36:56,456Z", "level": "INFO", "component": "o.e.b.BootstrapChecks", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "bound or publishing to a non-loopback address, enforcing bootstrap checks" }
{"type": "server", "timestamp": "2021-08-16T14:36:56,534Z", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "cluster UUID [b3zNCNo2QriEp87skwt4wg]" }
{"type": "server", "timestamp": "2021-08-16T14:37:01,596Z", "level": "WARN", "component": "o.e.d.SeedHostsResolver", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "timed out after [5s] resolving host [elasticsearch-master-headless]" }
{"type": "server", "timestamp": "2021-08-16T14:37:06,592Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "master not discovered or elected yet, an election requires at least 2 nodes with ids from [2r7aYWwpQ_eBb5fkFNOE2g, 5ksSOzslS3q6h8gKe0auUg, TQUR9Mh8QhGR_H1VGCTrcw], have discovered [{elasticsearch-master-0}{TQUR9Mh8QhGR_H1VGCTrcw}{xzL8kQ1UTFqaPLD8AqeBKA}{172.18.129.8}{172.18.129.8:9300}{cdhilmrstw}{ml.machine_memory=2147483648, xpack.installed=true, transform.node=true, ml.max_open_jobs=20}] which is not a quorum; discovery will continue using [] from hosts providers and [{elasticsearch-master-0}{TQUR9Mh8QhGR_H1VGCTrcw}{xzL8kQ1UTFqaPLD8AqeBKA}{172.18.129.8}{172.18.129.8:9300}{cdhilmrstw}{ml.machine_memory=2147483648, xpack.installed=true, transform.node=true, ml.max_open_jobs=20}] from last-known cluster state; node term 4, last-accepted version 80 in term 4" }
{"type": "server", "timestamp": "2021-08-16T14:37:07,589Z", "level": "WARN", "component": "o.e.d.SeedHostsResolver", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "timed out after [5s] resolving host [elasticsearch-master-headless]" }
{"type": "server", "timestamp": "2021-08-16T14:37:12,591Z", "level": "WARN", "component": "o.e.d.SeedHostsResolver", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "timed out after [5s] resolving host [elasticsearch-master-headless]" }
{"type": "server", "timestamp": "2021-08-16T14:37:16,594Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "master not discovered or elected yet, an election requires at least 2 nodes with ids from [2r7aYWwpQ_eBb5fkFNOE2g, 5ksSOzslS3q6h8gKe0auUg, TQUR9Mh8QhGR_H1VGCTrcw], have discovered [{elasticsearch-master-0}{TQUR9Mh8QhGR_H1VGCTrcw}{xzL8kQ1UTFqaPLD8AqeBKA}{172.18.129.8}{172.18.129.8:9300}{cdhilmrstw}{ml.machine_memory=2147483648, xpack.installed=true, transform.node=true, ml.max_open_jobs=20}] which is not a quorum; discovery will continue using [] from hosts providers and [{elasticsearch-master-0}{TQUR9Mh8QhGR_H1VGCTrcw}{xzL8kQ1UTFqaPLD8AqeBKA}{172.18.129.8}{172.18.129.8:9300}{cdhilmrstw}{ml.machine_memory=2147483648, xpack.installed=true, transform.node=true, ml.max_open_jobs=20}] from last-known cluster state; node term 4, last-accepted version 80 in term 4" }
{"type": "server", "timestamp": "2021-08-16T14:37:16,622Z", "level": "WARN", "component": "o.e.d.SeedHostsResolver", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "failed to resolve host [elasticsearch-master-headless]",
"stacktrace": ["java.net.UnknownHostException: elasticsearch-master-headless",
"at java.net.InetAddress$CachedAddresses.get(InetAddress.java:800) ~[?:?]",
"at java.net.InetAddress$NameServiceAddresses.get(InetAddress.java:886) ~[?:?]",
"at java.net.InetAddress.getAllByName0(InetAddress.java:1507) ~[?:?]",
"at java.net.InetAddress.getAllByName(InetAddress.java:1366) ~[?:?]",
"at java.net.InetAddress.getAllByName(InetAddress.java:1300) ~[?:?]",
"at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:556) ~[elasticsearch-7.10.1.jar:7.10.1]",
"at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:498) ~[elasticsearch-7.10.1.jar:7.10.1]",
"at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:864) ~[elasticsearch-7.10.1.jar:7.10.1]",
"at org.elasticsearch.discovery.SeedHostsResolver.lambda$resolveHostsLists$0(SeedHostsResolver.java:144) ~[elasticsearch-7.10.1.jar:7.10.1]",
"at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:678) ~[elasticsearch-7.10.1.jar:7.10.1]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:832) [?:?]"] }
{"type": "server", "timestamp": "2021-08-16T14:37:17,595Z", "level": "WARN", "component": "o.e.d.SeedHostsResolver", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "failed to resolve host [elasticsearch-master-headless]",
"stacktrace": ["java.net.UnknownHostException: elasticsearch-master-headless",
"at java.net.InetAddress$CachedAddresses.get(InetAddress.java:800) ~[?:?]",
"at java.net.InetAddress.getAllByName0(InetAddress.java:1507) ~[?:?]",
"at java.net.InetAddress.getAllByName(InetAddress.java:1366) ~[?:?]",
"at java.net.InetAddress.getAllByName(InetAddress.java:1300) ~[?:?]",
"at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:556) ~[elasticsearch-7.10.1.jar:7.10.1]",
"at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:498) ~[elasticsearch-7.10.1.jar:7.10.1]",
"at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:864) ~[elasticsearch-7.10.1.jar:7.10.1]",
"at org.elasticsearch.discovery.SeedHostsResolver.lambda$resolveHostsLists$0(SeedHostsResolver.java:144) ~[elasticsearch-7.10.1.jar:7.10.1]",
"at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:678) ~[elasticsearch-7.10.1.jar:7.10.1]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:832) [?:?]"] }
{"type": "server", "timestamp": "2021-08-16T14:37:18,595Z", "level": "WARN", "component": "o.e.d.SeedHostsResolver", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "failed to resolve host [elasticsearch-master-headless]",
"stacktrace": ["java.net.UnknownHostException: elasticsearch-master-headless",
"at java.net.InetAddress$CachedAddresses.get(InetAddress.java:800) ~[?:?]",
"at java.net.InetAddress.getAllByName0(InetAddress.java:1507) ~[?:?]",
"at java.net.InetAddress.getAllByName(InetAddress.java:1366) ~[?:?]",
"at java.net.InetAddress.getAllByName(InetAddress.java:1300) ~[?:?]",
"at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:556) ~[elasticsearch-7.10.1.jar:7.10.1]",
"at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:498) ~[elasticsearch-7.10.1.jar:7.10.1]",
"at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:864) ~[elasticsearch-7.10.1.jar:7.10.1]",
"at org.elasticsearch.discovery.SeedHostsResolver.lambda$resolveHostsLists$0(SeedHostsResolver.java:144) ~[elasticsearch-7.10.1.jar:7.10.1]",
"at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:678) ~[elasticsearch-7.10.1.jar:7.10.1]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:832) [?:?]"] }
