This is my Statefulset for Elasticsearch
spec:
podManagementPolicy: OrderedReady
replicas: 3
revisionHistoryLimit: 10
selector:
matchLabels:
app: elasticsearch
serviceName: elasticsearch-data
template:
metadata:
creationTimestamp: null
labels:
app: elasticsearch
role: data
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- elasticsearch
topologyKey: kubernetes.io/hostname
containers:
- args:
- source /tmp/host && echo $$NODE_HOST && /usr/local/bin/docker-entrypoint.sh
command:
- /bin/sh
- -c
env:
- name: CLUSTER_NAME
value: elasticsearch
- name: NODE_NAME
value: elasticsearch-data
- name: NODE_LIST
value: elasticsearch-data
- name: MASTER_NODES
value: elasticsearch-master
- name: ES_JAVA_OPTS
value: -Xms2g -Xmx2g
image: docker.elastic.co/elasticsearch/elasticsearch:8.7.0
imagePullPolicy: IfNotPresent
name: elasticsearch-data
ports:
- containerPort: 9300
name: transport
protocol: TCP
resources:
limits:
cpu: 22422m
memory: 59G
requests:
cpu: 309m
memory: 1470M
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
name: config
readOnly: true
subPath: elasticsearch.yml
- mountPath: /data/db
name: elasticsearch-data
- mountPath: /usr/share/elasticsearch/config/certs
name: elastic-certificates
- mountPath: /tmp
name: node-info
dnsPolicy: ClusterFirst
imagePullSecrets:
- name: regcred
initContainers:
- args:
- echo "$$SCRIPT" > /tmp/script && sh /tmp/script
command:
- /bin/sh
- -c
env:
- name: NODENAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
- name: APISERVER
value: https://kubernetes.default.svc.cluster.local
- name: SERVICEACCOUNT
value: /var/run/secrets/kubernetes.io/serviceaccount
- name: SCRIPT
value: |
set -eo pipefail
apk add curl
apk add jq
TOKEN=$(cat ${SERVICEACCOUNT}/token)
CACERT=${SERVICEACCOUNT}/ca.crt
curl --cacert ${CACERT} \
--header "Authorization: Bearer ${TOKEN}" \
-X GET ${APISERVER}/api/v1/nodes/${NODENAME} | jq '.metadata.labels' > /tmp/labels.json
NODE_HOST=$(jq '."server"' -r /tmp/labels.json)
echo "export NODE_HOST=${NODE_HOST}" > /tmp/host
image: docker.payten.com/debug-tools/debug-tools-alpine:0.0.1
imagePullPolicy: Always
name: elasticsearch-label
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /tmp
name: node-info
nodeSelector:
elastic: test
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: elastic-permission
serviceAccountName: elastic-permission
terminationGracePeriodSeconds: 30
volumes:
- configMap:
defaultMode: 420
name: elasticsearch-data-config
name: config
- name: elastic-certificates
secret:
defaultMode: 420
secretName: elastic-certificate-pem
- emptyDir: {}
name: node-info
updateStrategy:
rollingUpdate:
partition: 0
type: RollingUpdate
volumeClaimTemplates:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
creationTimestamp: null
name: elasticsearch-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi
storageClassName: rook-cephfs
volumeMode: Filesystem
status:
phase: Pending
and this is the error I get when I start elasticsearch
{"@timestamp":"2024-02-01T14:02:27.076Z", "log.level":"ERROR", "message":"fatal exception while booting Elasticsearch", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.bootstrap.Elasticsearch","elasticsearch.node.name":"elasticsearch-data","elasticsearch.cluster.name":"elasticsearch","error.type":"java.lang.IllegalStateException","error.message":"Unable to access 'path.data' (/data/db/elasticsearch)","error.stack_trace":"java.lang.IllegalStateException: Unable to access 'path.data' (/data/db/elasticsearch)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.FilePermissionUtils.addDirectoryPath(FilePermissionUtils.java:66)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Security.addFilePermissions(Security.java:236)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Security.createPermissions(Security.java:178)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Security.configure(Security.java:125)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Elasticsearch.initPhase2(Elasticsearch.java:188)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:66)\nCaused by: java.nio.file.AccessDeniedException: /data/db/elasticsearch\n\tat java.base/sun.nio.fs.UnixException.translateToIOException(UnixException.java:90)\n\tat java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106)\n\tat java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111)\n\tat java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:397)\n\tat java.base/java.nio.file.Files.createDirectory(Files.java:700)\n\tat java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:807)\n\tat java.base/java.nio.file.Files.createDirectories(Files.java:793)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Security.ensureDirectoryExists(Security.java:330)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.FilePermissionUtils.addDirectoryPath(FilePermissionUtils.java:64)\n\t... 5 more\n"}
ERROR: Elasticsearch did not exit normally - check the logs at /usr/share/elasticsearch/logs/elasticsearch.log
ERROR: Elasticsearch exited unexpectedly
Before I had Init container I didn't have this problem and that is the only thing I have changed in the Statefulset.
What can be the problem and how can I solve this?