Hi team, I have installed Elasticsearch in my EKS cluster using helm chart with PVC (gp2)and worked fine and tried to scaling down EKS nodegroup and scale up Nodegroup .
all my pods went to pending state
(base) devops@devops-XPS-15-9570:~/Desktop/Repo$ kubectl get po,pvc -n elastic
NAME READY STATUS RESTARTS AGE
pod/Elasticsearch-master-0 0/1 Running 0 19m
pod/Elasticsearch-master-1 0/1 Pending 0 19m
pod/Elasticsearch-master-2 0/1 Pending 0 19m
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
persistentvolumeclaim/Elasticsearch-master-Elasticsearch-master-0 Bound pvc-5aab34ac-e41b-4323-9f69-581f39f7 10Gi RWO elastic-sc 3d
persistentvolumeclaim/Elasticsearch-master-Elasticsearch-master-1 Bound pvc-51bf9a20-f4ca-4f96-aa660f85d107 10Gi RWO elastic-sc 3d
persistentvolumeclaim/Elasticsearch-master-Elasticsearch-master-2 Bound pvc-1e5a1c52-a12d-474f-a1bd-999e49 10Gi RWO elastic-sc 3d
Looking for how i can make up my cluster
Logs of the running pod (pod/Elasticsearch-master-0)
kubectl logs pod/Elasticsearch-master-0 -n elastic
{"type": "server", "timestamp": "2022-05-09T01:03:18,156Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "version[7.17.3], pid[7], build[default/docker/5ad023604c8d7416c9eb6c0eadb62b14e766caff/2022-04-19T08:11:19.070913226Z], OS[Linux/5.4.181-99.354.amzn2.x86_64/amd64], JVM[Eclipse Adoptium/OpenJDK 64-Bit Server VM/18/18+36]" }
{"type": "server", "timestamp": "2022-05-09T01:03:18,159Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2022-05-09T01:03:18,160Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Dlog4j2.formatMsgNoLookups=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -Djava.security.manager=allow, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-8450783023809497710, -XX:+HeapDumpOnOutOfMemoryError, -XX:+ExitOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xms1024m, -Xmx1024m, -XX:MaxDirectMemorySize=536870912, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,531Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [aggs-matrix-stats]" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,531Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [analysis-common]" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,532Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [constant-keyword]" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,544Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-watcher]" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,544Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded plugin [repository-s3]" }
{"type": "deprecation.elasticsearch", "timestamp": "2022-05-09T01:03:23,547Z", "level": "CRITICAL", "component": "o.e.d.c.s.Settings", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "[node.ml] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.", "key": "node.ml", "category": "settings" }
{"type": "deprecation.elasticsearch", "timestamp": "2022-05-09T01:03:23,629Z", "level": "CRITICAL", "component": "o.e.d.c.s.Settings", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "[node.data] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.", "key": "node.data", "category": "settings" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,644Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/xvdck)]], net usable_space [9.6gb], net total_space [9.7gb], types [ext4]" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,645Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "heap size [1gb], compressed ordinary object pointers [true]" }
{"type": "deprecation.elasticsearch", "timestamp": "2022-05-09T01:03:23,725Z", "level": "CRITICAL", "component": "o.e.d.c.s.Settings", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "[node.master] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.", "key": "node.master", "category": "settings" }
{"type": "deprecation.elasticsearch", "timestamp": "2022-05-09T01:03:23,729Z", "level": "CRITICAL", "component": "o.e.d.c.s.Settings", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "[node.remote_cluster_client] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.", "key": "node.remote_cluster_client", "category": "settings" }
{"type": "deprecation.elasticsearch", "timestamp": "2022-05-09T01:03:23,730Z", "level": "CRITICAL", "component": "o.e.d.c.s.Settings", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "[node.ingest] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.", "key": "node.ingest", "category": "settings" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,846Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "node name [elasticsearch-master-0], node ID [VeyMV5ZNRMW3UyW8taNGkQ], cluster name [elasticsearch], roles [transform, data_frozen, master, remote_cluster_client, data, ml, data_content, data_hot, data_warm, data_cold, ingest]" }
{"type": "deprecation.elasticsearch", "timestamp": "2022-05-09T01:03:23,848Z", "level": "CRITICAL", "component": "o.e.d.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "legacy role settings [node.data, node.remote_cluster_client, node.ingest, node.master, node.ml] are deprecated, use [node.roles=[transform, data_frozen, master, remote_cluster_client, data, ml, data_content, data_hot, data_warm, data_cold, ingest]]", "key": "legacy role settings", "category": "settings" }
{"type": "server", "timestamp": "2022-05-09T01:03:33,947Z", "level": "INFO", "component": "o.e.x.m.p.l.CppLogMessageHandler", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "[controller/180] [Main.cc@122] controller (64 bit): Version 7.17.3 (Build b3cb07de236c11) Copyright (c) 2022 Elasticsearch BV" }
{"type": "server", "timestamp": "2022-05-09T01:03:35,946Z", "level": "INFO", "component": "o.e.x.s.a.s.FileRolesStore", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "parsed [0] roles from file [/usr/share/elasticsearch/config/roles.yml]" }
{"type": "server", "timestamp": "2022-05-09T01:03:37,230Z", "level": "INFO", "component": "o.e.i.g.ConfigDatabases", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "initialized default databases [[GeoLite2-Country.mmdb, GeoLite2-City.mmdb, GeoLite2-ASN.mmdb]], config databases [[]] and watching [/usr/share/elasticsearch/config/ingest-geoip] for changes" }
{"type": "server", "timestamp": "2022-05-09T01:03:37,232Z", "level": "INFO", "component": "o.e.i.g.DatabaseNodeService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "initialized database registry, using geoip-databases directory [/tmp/elasticsearch-8450783023809497710/geoip-databases/VeyMV5ZNRMW3UyW8taNGkQ]" }
{"type": "server", "timestamp": "2022-05-09T01:03:38,421Z", "level": "INFO", "component": "o.e.t.NettyAllocator", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "creating NettyAllocator with the following configs: [name=unpooled, suggested_max_allocation_size=1mb, factors={es.unsafe.use_unpooled_allocator=null, g1gc_enabled=true, g1gc_region_size=4mb, heap_size=1gb}]" }
{"type": "server", "timestamp": "2022-05-09T01:03:38,455Z", "level": "INFO", "component": "o.e.i.r.RecoverySettings", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "using rate limit [40mb] with [default=40mb, read=0b, write=0b, max=0b]" }
{"type": "server", "timestamp": "2022-05-09T01:03:38,556Z", "level": "INFO", "component": "o.e.d.DiscoveryModule", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "using discovery type [zen] and seed hosts providers [settings]" }
{"type": "server", "timestamp": "2022-05-09T01:03:39,663Z", "level": "INFO", "component": "o.e.g.DanglingIndicesState", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "gateway.auto_import_dangling_indices is disabled, dangling indices will not be automatically detected or imported and must be managed manually" }
{"type": "server", "timestamp": "2022-05-09T01:03:41,243Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "initialized" }
{"type": "server", "timestamp": "2022-05-09T01:03:41,243Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "starting ..." }
{"type": "server", "timestamp": "2022-05-09T01:03:41,260Z", "level": "INFO", "component": "o.e.x.s.c.f.PersistentCache", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "persistent cache index loaded" }
{"type": "server", "timestamp": "2022-05-09T01:03:41,261Z", "level": "INFO", "component": "o.e.x.d.l.DeprecationIndexingComponent", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "deprecation component started" }
{"type": "server", "timestamp": "2022-05-09T01:03:41,458Z", "level": "INFO", "component": "o.e.t.TransportService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "publish_address {192.168.131.86:9300}, bound_addresses {0.0.0.0:9300}" }
{"type": "server", "timestamp": "2022-05-09T01:03:42,582Z", "level": "INFO", "component": "o.e.b.BootstrapChecks", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "bound or publishing to a non-loopback address, enforcing bootstrap checks" }
{"type": "server", "timestamp": "2022-05-09T01:03:42,636Z", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "cluster UUID [miG-kHiRS02W-Tf8F-Nqjg]" }
(base) devops@devops-XPS-15-9570:~/Desktop/Repo$ kubectl logs pod/elasticsearch-master-0 -n elastic
{"type": "server", "timestamp": "2022-05-09T01:03:18,156Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "version[7.17.3], pid[7], build[default/docker/5ad023604c8d7416c9eb6c0eadb62b14e766caff/2022-04-19T08:11:19.070913226Z], OS[Linux/5.4.181-99.354.amzn2.x86_64/amd64], JVM[Eclipse Adoptium/OpenJDK 64-Bit Server VM/18/18+36]" }
{"type": "server", "timestamp": "2022-05-09T01:03:18,159Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2022-05-09T01:03:18,160Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Dlog4j2.formatMsgNoLookups=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -Djava.security.manager=allow, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-8450783023809497710, -XX:+HeapDumpOnOutOfMemoryError, -XX:+ExitOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xms1024m, -Xmx1024m, -XX:MaxDirectMemorySize=536870912, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,531Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [aggs-matrix-stats]" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,531Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [analysis-common]" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,532Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [constant-keyword]" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,532Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [frozen-indices]" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,532Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [ingest-common]" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,532Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [ingest-geoip]" }
p": "2022-05-09T01:03:23,544Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-voting-only-node]" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,544Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-watcher]" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,544Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded plugin [repository-s3]" }
{"type": "deprecation.elasticsearch", "timestamp": "2022-05-09T01:03:23,547Z", "level": "CRITICAL", "component": "o.e.d.c.s.Settings", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "[node.ml] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.", "key": "node.ml", "category": "settings" }
{"type": "deprecation.elasticsearch", "timestamp": "2022-05-09T01:03:23,629Z", "level": "CRITICAL", "component": "o.e.d.c.s.Settings", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "[node.data] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.", "key": "node.data", "category": "settings" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,644Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/xvdck)]], net usable_space [9.6gb], net total_space [9.7gb], types [ext4]" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,645Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "heap size [1gb], compressed ordinary object pointers [true]" }
{"type": "deprecation.elasticsearch", "timestamp": "2022-05-09T01:03:23,725Z", "level": "CRITICAL", "component": "o.e.d.c.s.Settings", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "[node.master] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.", "key": "node.master", "category": "settings" }
{"type": "deprecation.elasticsearch", "timestamp": "2022-05-09T01:03:23,729Z", "level": "CRITICAL", "component": "o.e.d.c.s.Settings", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "[node.remote_cluster_client] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.", "key": "node.remote_cluster_client", "category": "settings" }
{"type": "deprecation.elasticsearch", "timestamp": "2022-05-09T01:03:23,730Z", "level": "CRITICAL", "component": "o.e.d.c.s.Settings", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "[node.ingest] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.", "key": "node.ingest", "category": "settings" }
{"type": "server", "timestamp": "2022-05-09T01:03:23,846Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "node name [elasticsearch-master-0], node ID [VeyMV5ZNRMW3UyW8taNGkQ], cluster name [elasticsearch], roles [transform, data_frozen, master, remote_cluster_client, data, ml, data_content, data_hot, data_warm, data_cold, ingest]" }
{"type": "deprecation.elasticsearch", "timestamp": "2022-05-09T01:03:23,848Z", "level": "CRITICAL", "component": "o.e.d.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "legacy role settings [node.data, node.remote_cluster_client, node.ingest, node.master, node.ml] are deprecated, use [node.roles=[transform, data_frozen, master, remote_cluster_client, data, ml, data_content, data_hot, data_warm, data_cold, ingest]]", "key": "legacy role settings", "category": "settings" }
{"type": "server", "timestamp": "2022-05-09T01:03:33,947Z", "level": "INFO", "component": "o.e.x.m.p.l.CppLogMessageHandler", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "[controller/180] [Main.cc@122] controller (64 bit): Version 7.17.3 (Build b3cb07de236c11) Copyright (c) 2022 Elasticsearch BV" }
{"type": "server", "timestamp": "2022-05-09T01:03:35,946Z", "level": "INFO", "component": "o.e.x.s.a.s.FileRolesStore", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "parsed [0] roles from file [/usr/share/elasticsearch/config/roles.yml]" }
{"type": "server", "timestamp": "2022-05-09T01:03:37,230Z", "level": "INFO", "component": "o.e.i.g.ConfigDatabases", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "initialized default databases [[GeoLite2-Country.mmdb, GeoLite2-City.mmdb, GeoLite2-ASN.mmdb]], config databases [[]] and watching [/usr/share/elasticsearch/config/ingest-geoip] for changes" }
{"type": "server", "timestamp": "2022-05-09T01:03:37,232Z", "level": "INFO", "component": "o.e.i.g.DatabaseNodeService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "initialized database registry, using geoip-databases directory [/tmp/elasticsearch-8450783023809497710/geoip-databases/VeyMV5ZNRMW3UyW8taNGkQ]" }
{"type": "server", "timestamp": "2022-05-09T01:03:38,421Z", "level": "INFO", "component": "o.e.t.NettyAllocator", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "creating NettyAllocator with the following configs: [name=unpooled, suggested_max_allocation_size=1mb, factors={es.unsafe.use_unpooled_allocator=null, g1gc_enabled=true, g1gc_region_size=4mb, heap_size=1gb}]" }
{"type": "server", "timestamp": "2022-05-09T01:03:38,455Z", "level": "INFO", "component": "o.e.i.r.RecoverySettings", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "using rate limit [40mb] with [default=40mb, read=0b, write=0b, max=0b]" }
{"type": "server", "timestamp": "2022-05-09T01:03:38,556Z", "level": "INFO", "component": "o.e.d.DiscoveryModule", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "using discovery type [zen] and seed hosts providers [settings]" }
{"type": "server", "timestamp": "2022-05-09T01:03:39,663Z", "level": "INFO", "component": "o.e.g.DanglingIndicesState", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "gateway.auto_import_dangling_indices is disabled, dangling indices will not be automatically detected or imported and must be managed manually" }
{"type": "server", "timestamp": "2022-05-09T01:03:41,243Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "initialized" }
{"type": "server", "timestamp": "2022-05-09T01:03:41,243Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "starting ..." }
{"type": "server", "timestamp": "2022-05-09T01:03:41,260Z", "level": "INFO", "component": "o.e.x.s.c.f.PersistentCache", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "persistent cache index loaded" }
{"type": "server", "timestamp": "2022-05-09T01:03:41,261Z", "level": "INFO", "component": "o.e.x.d.l.DeprecationIndexingComponent", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "deprecation component started" }
{"type": "server", "timestamp": "2022-05-09T01:03:41,458Z", "level": "INFO", "component": "o.e.t.TransportService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "publish_address {192.168.131.86:9300}, bound_addresses {0.0.0.0:9300}" }
{"type": "server", "timestamp": "2022-05-09T01:03:42,582Z", "level": "INFO", "component": "o.e.b.BootstrapChecks", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "bound or publishing to a non-loopback address, enforcing bootstrap checks" }
{"type": "server", "timestamp": "2022-05-09T01:03:42,636Z", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "cluster UUID [miG-kHiRS02W-Tf8F-Nqjg]" }
{"type": "server", "timestamp": "2022-05-09T01:03:52,646Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "master not discovered or elected yet, an election requires 2 nodes with ids [VeyMV5ZNRMW3UyW8taNGkQ, r6usGUfQScG8GBc-39cang], have only discovered non-quorum [{elasticsearch-master-0}{VeyMV5ZNRMW3UyW8taNGkQ}{TogPUQAJRqqipcnACoBtoQ}{192.168.131.86}{192.168.131.86:9300}{cdfhilmrstw}]; discovery will continue using [] from hosts providers and [{elasticsearch-master-0}{VeyMV5ZNRMW3UyW8taNGkQ}{TogPUQAJRqqipcnACoBtoQ}{192.168.131.86}{192.168.131.86:9300}{cdfhilmrstw}] from last-known cluster state; node term 1, last-accepted version 196 in term 1" }
{"type": "server", "timestamp": "2022-05-09T01:04:02,648Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "master not discovered or elected yet, an election requires 2 nodes with ids [VeyMV5ZNRMW3UyW8taNGkQ, r6usGUfQScG8GBc-39cang], have only discovered non-quorum [{elasticsearch-master-0}{VeyMV5ZNRMW3UyW8taNGkQ}{TogPUQAJRqqipcnACoBtoQ}{192.168.131.86}{192.168.131.86:9300}{cdfhilmrstw}]; discovery will continue using [] from hosts providers and [{elasticsearch-master-0}{VeyMV5ZNRMW3UyW8taNGkQ}{TogPUQAJRqqipcnACoBtoQ}{192.168.131.86}{192.168.131.86:9300}{cdfhilmrstw}] from last-known cluster state; node term 1, last-accepted version 196 in term 1" }
{"type": "server", "timestamp": "2022-05-09T01:04:12,649Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "master not discovered or elected yet, an election requires 2 nodes with ids [VeyMV5ZNRMW3UyW8taNGkQ, r6usGUfQScG8GBc-39cang], have only discovered non-quorum [{elasticsearch-master-0}{VeyMV5ZNRMW3UyW8taNGkQ}{TogPUQAJRqqipcnACoBtoQ}{192.168.131.86}{192.168.131.86:9300}{cdfhilmrstw}]; discovery will continue using [] from hosts providers and [{elasticsearch-master-0}{VeyMV5ZNRMW3UyW8taNGkQ}{TogPUQAJRqqipcnACoBtoQ}{192.168.131.86}{192.168.131.86:9300}{cdfhilmrstw}] from last-known cluster state; node term 1, last-accepted version 196 in term 1" }
{"type": "server", "timestamp": "2022-05-09T01:04:12,650Z", "level": "WARN", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "timed out while waiting for initial discovery state - timeout: 30s" }
{"type": "server", "timestamp": "2022-05-09T01:04:12,661Z", "level": "INFO", "component": "o.e.h.AbstractHttpServerTransport", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "publish_address {192.168.131.86:9200}, bound_addresses {0.0.0.0:9200}" }
{"type": "server", "timestamp": "2022-05-09T01:04:12,661Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "started" }
{"type": "server", "timestamp": "2022-05-09T01:04:14,174Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "path: /_cluster/health, params: {wait_for_status=green, timeout=1s}",
"stacktrace": ["org.elasticsearch.discovery.MasterNotDiscoveredException: null",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onTimeout(TransportMasterNodeAction.java:297) [elasticsearch-7.17.3.jar:7.17.3]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:345) [elasticsearch-7.17.3.jar:7.17.3]",
"at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:263) [elasticsearch-7.17.3.jar:7.17.3]",
"at org.elasticsearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:660) [elasticsearch-7.17.3.jar:7.17.3]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718) [elasticsearch-7.17.3.jar:7.17.3]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]",
"at java.lang.Thread.run(Thread.java:833) [?:?]"] }
{"type": "server", "timestamp": "2022-05-09T01:04:22,650Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "master not discovered or elected yet, an election requires 2 nodes with ids [VeyMV5ZNRMW3UyW8taNGkQ, r6usGUfQScG8GBc-39cang], have only discovered non-quorum [{elasticsearch-master-0}{VeyMV5ZNRMW3UyW8taNGkQ}{TogPUQAJRqqipcnACoBtoQ}{192.168.131.86}{192.168.131.86:9300}{cdfhilmrstw}]; discovery will continue using [] from hosts providers and [{elasticsearch-master-0}{VeyMV5ZNRMW3UyW8taNGkQ}{TogPUQAJRqqipcnACoBtoQ}{192.168.131.86}{192.168.131.86:9300}{cdfhilmrstw}] from last-known cluster state; node term 1, last-accepted version 196 in term 1" }
{"type": "server", "timestamp": "2022-05-09T01:04:24,129Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "path: /_cluster/health, params: {wait_for_status=green, timeout=1s}",
"stacktrace": ["org.elasticsearch.discovery.MasterNotDiscoveredException: null",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onTimeout(TransportMasterNodeAction.java:297) [elasticsearch-7.17.3.jar:7.17.3]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:345) [elasticsearch-7.17.3.jar:7.17.3]",
"at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:263) [elasticsearch-7.17.3.jar:7.17.3]",
"at org.elasticsearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:660) [elasticsearch-7.17.3.jar:7.17.3]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718) [elasticsearch-7.17.3.jar:7.17.3]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]",
"at java.lang.Thread.run(Thread.java:833) [?:?]"] }
{"type": "server", "timestamp": "2022-05-09T01:04:32,651Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "master not discovered or elected yet, an election requires 2 nodes with ids [VeyMV5ZNRMW3UyW8taNGkQ, r6usGUfQScG8GBc-39cang], have only discovered non-quorum [{elasticsearch-master-0}{VeyMV5ZNRMW3UyW8taNGkQ}{TogPUQAJRqqipcnACoBtoQ}{192.168.131.86}{192.168.131.86:9300}{cdfhilmrstw}]; discovery will continue using [] from hosts providers and [{elasticsearch-master-0}{VeyMV5ZNRMW3UyW8taNGkQ}{TogPUQAJRqqipcnACoBtoQ}{192.168.131.86}{192.168.131.86:9300}{cdfhilmrstw}] from last-known cluster state; node term 1, last-accepted version 196 in term 1" }
{"type": "server", "timestamp": "2022-05-09T01:04:34,136Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "path: /_cluster/health, params: {wait_for_status=green, timeout=1s}",
"stacktrace": ["org.elasticsearch.discovery.MasterNotDiscoveredException: null",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onTimeout(TransportMasterNodeAction.java:297) [elasticsearch-7.17.3.jar:7.17.3]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:345) [elasticsearch-7.17.3.jar:7.17.3]",
"at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:263) [elasticsearch-7.17.3.jar:7.17.3]",
"at org.elasticsearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:660) [elasticsearch-7.17.3.jar:7.17.3]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718) [elasticsearch-7.17.3.jar:7.17.3]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]",
"at java.lang.Thread.run(Thread.java:833) [?:?]"] }