All,
I started dealing with ES and kubernetes and currently trying to create two master nodes. I am sorry for the long post below, but couldn't find a way to attache the files with the error log message and my k8s yaml. Please, advice where the problem is.
THis is the main error message I am getting:
{"type": "server", "timestamp": "2019-12-13T11:57:12,433Z", "level": "WARN", "component": "o.e.d.SeedHostsResolver", "cluster.name": "mytest.es.cluster", "node.name": "es-master-0", "message": "failed to resolve host [es-master-1]",
"stacktrace": ["java.net.UnknownHostException: es-master-1: Name or service not known",
"at java.net.Inet4AddressImpl.lookupAllHostAddr(Native Method) ~[?:?]",
"at java.net.InetAddress$PlatformNameService.lookupAllHostAddr(InetAddress.java:930) ~[?:?]",
"at java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1499) ~[?:?]",
"at java.net.InetAddress$NameServiceAddresses.get(InetAddress.java:849) ~[?:?]",
"at java.net.InetAddress.getAllByName0(InetAddress.java:1489) ~[?:?]",
"at java.net.InetAddress.getAllByName(InetAddress.java:1348) ~[?:?]",
"at java.net.InetAddress.getAllByName(InetAddress.java:1282) ~[?:?]",
"at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:528) ~[elasticsearch-7.5.0.jar:7.5.0]",
"at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:470) ~[elasticsearch-7.5.0.jar:7.5.0]",
"at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:803) ~[elasticsearch-7.5.0.jar:7.5.0]",
"at org.elasticsearch.discovery.SeedHostsResolver.lambda$resolveHostsLists$0(SeedHostsResolver.java:144) ~[elasticsearch-7.5.0.jar:7.5.0]",
"at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703) ~[elasticsearch-7.5.0.jar:7.5.0]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
"at java.lang.Thread.run(Thread.java:830) [?:?]"] }
{"type": "server", "timestamp": "2019-12-13T11:57:13,426Z", "level": "WARN", "component": "o.e.d.SeedHostsResolver", "cluster.name": "mytest.es.cluster", "node.name": "es-master-0", "message": "failed to resolve host [es-master-1]",
"stacktrace": ["java.net.UnknownHostException: es-master-1",
"at java.net.InetAddress$CachedAddresses.get(InetAddress.java:798) ~[?:?]",
"at java.net.InetAddress.getAllByName0(InetAddress.java:1489) ~[?:?]",
"at java.net.InetAddress.getAllByName(InetAddress.java:1348) ~[?:?]",
"at java.net.InetAddress.getAllByName(InetAddress.java:1282) ~[?:?]",
"at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:528) ~[elasticsearch-7.5.0.jar:7.5.0]",
"at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:470) ~[elasticsearch-7.5.0.jar:7.5.0]",
"at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:803) ~[elasticsearch-7.5.0.jar:7.5.0]",
"at org.elasticsearch.discovery.SeedHostsResolver.lambda$resolveHostsLists$0(SeedHostsResolver.java:144) ~[elasticsearch-7.5.0.jar:7.5.0]",
"at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703) ~[elasticsearch-7.5.0.jar:7.5.0]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
"at java.lang.Thread.run(Thread.java:830) [?:?]"] }
{"type": "server", "timestamp": "2019-12-13T11:57:14,118Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "mytest.es.cluster", "node.name": "es-master-0", "message": "master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and this node must discover master-eligible nodes [es-master-0, es-master-1] to bootstrap a cluster: have discovered [{es-master-0}{ObwZ2HFfRZuf5PaZuHZFoA}{tFKlQ8A3QyamJGqGaBX_0w}{10.174.49.48}{10.174.49.48:9300}{lm}{ml.machine_memory=1073741824, xpack.installed=true, ml.max_open_jobs=20}]; discovery will continue using [] from hosts providers and [{es-master-0}{ObwZ2HFfRZuf5PaZuHZFoA}{tFKlQ8A3QyamJGqGaBX_0w}{10.174.49.48}{10.174.49.48:9300}{lm}{ml.machine_memory=1073741824, xpack.installed=true, ml.max_open_jobs=20}] from last-known cluster state; node term 0, last-accepted version 0 in term 0" }
######################################################
And here is my k8s yamle file:
apiVersion: v1
kind: List
items:
- apiVersion: v1
kind: Namespace
metadata:
name: elasticsearch
- kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: fast-es-masters-storage
namespace: elasticsearch
provisioner: kubernetes.io/azure-disk
parameters:
storageaccounttype: StandardSSD_LRS
kind: managed
- apiVersion: apps/v1
kind: StatefulSet
metadata:
name: es-master
namespace: elasticsearch
labels:
component: elasticsearch
role: master
spec:
serviceName: ediscovery
replicas: 2
selector:
matchLabels:
component: elasticsearch
role: master
template:
metadata:
labels:
component: elasticsearch
role: master
spec:
initContainers:
- name: increase-vm-max-map-count
image: busybox:1.27.2
command:
- sysctl
- -w
- vm.max_map_count=262144
securityContext:
privileged: true
- name: increase-fd-ulimit
image: busybox:1.27.2
command:
- sh
- -c
- ulimit
- -n 65536
- name: allow-fs-permissions
image: busybox:1.27.2
command:
- chown
- -R
- 1000:1000
- /usr/share/elasticsearch/data
volumeMounts:
- name: master-data
mountPath: /usr/share/elasticsearch/data
securityContext:
privileged: true
- name: check-swap
image: busybox:1.27.2
command:
- sh
- -c
- cat
- /proc/swaps
securityContext:
privileged: true
terminationGracePeriodSeconds: 120
containers:
- name: es-master
image: docker.elastic.co/elasticsearch/elasticsearch:7.5.0
imagePullPolicy: Always
ports:
- containerPort: 9300
name: transport
resources:
limits:
cpu: "500m"
memory: 1Gi
requests:
cpu: "500m"
memory: 1Gi
securityContext:
capabilities:
add:
- IPC_LOCK
env:
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: node.name
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: MAX_LOCKED_MEMORY
value: "unlimited"
- name: ES_HEAP_SIZE
value: 512m
- name: ES_JAVA_OPTS
value: -Xms512m -Xmx512m
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: node.master
value: "true"
- name: node.data
value: "false"
- name: node.ingest
value: "false"
- name: cluster.name
value: "mytest.es.cluster"
- name: cluster.no_master_block
value: "write"
- name: cluster.initial_master_nodes
value: "es-master-0,es-master-1"
- name: discovery.seed_hosts
value: "es-master-0,es-master-1"
- name: PROCESSORS
valueFrom:
resourceFieldRef:
resource: limits.cpu
volumeMounts:
- name: master-data
mountPath: /usr/share/elasticsearch/data
volumeClaimTemplates:
- metadata:
name: master-data
labels:
component: elasticsearch
role: master
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: fast-es-masters-storage
resources:
requests:
storage: 1Gi