Unable to access 'path.data' (/data/db/elasticsearch)

This is my Statefulset for Elasticsearch

spec:
  podManagementPolicy: OrderedReady
  replicas: 3
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: elasticsearch
  serviceName: elasticsearch-data
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: elasticsearch
        role: data
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - elasticsearch
            topologyKey: kubernetes.io/hostname
      containers:
      - args:
        - source /tmp/host && echo $$NODE_HOST && /usr/local/bin/docker-entrypoint.sh
        command:
        - /bin/sh
        - -c
        env:
        - name: CLUSTER_NAME
          value: elasticsearch
        - name: NODE_NAME
          value: elasticsearch-data
        - name: NODE_LIST
          value: elasticsearch-data
        - name: MASTER_NODES
          value: elasticsearch-master
        - name: ES_JAVA_OPTS
          value: -Xms2g -Xmx2g
        image: docker.elastic.co/elasticsearch/elasticsearch:8.7.0
        imagePullPolicy: IfNotPresent
        name: elasticsearch-data
        ports:
        - containerPort: 9300
          name: transport
          protocol: TCP
        resources:
          limits:
            cpu: 22422m
            memory: 59G
          requests:
            cpu: 309m
            memory: 1470M
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
          name: config
          readOnly: true
          subPath: elasticsearch.yml
        - mountPath: /data/db
          name: elasticsearch-data
        - mountPath: /usr/share/elasticsearch/config/certs
          name: elastic-certificates
        - mountPath: /tmp
          name: node-info
      dnsPolicy: ClusterFirst
      imagePullSecrets:
      - name: regcred
      initContainers:
      - args:
        - echo "$$SCRIPT" > /tmp/script && sh /tmp/script
        command:
        - /bin/sh
        - -c
        env:
        - name: NODENAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        - name: APISERVER
          value: https://kubernetes.default.svc.cluster.local
        - name: SERVICEACCOUNT
          value: /var/run/secrets/kubernetes.io/serviceaccount
        - name: SCRIPT
          value: |
            set -eo pipefail
            apk add curl
            apk add jq
            TOKEN=$(cat ${SERVICEACCOUNT}/token)
            CACERT=${SERVICEACCOUNT}/ca.crt
            curl --cacert ${CACERT} \
                 --header "Authorization: Bearer ${TOKEN}" \
                 -X GET ${APISERVER}/api/v1/nodes/${NODENAME} | jq '.metadata.labels' > /tmp/labels.json
            NODE_HOST=$(jq '."server"' -r /tmp/labels.json)
            echo "export NODE_HOST=${NODE_HOST}" > /tmp/host
        image: docker.payten.com/debug-tools/debug-tools-alpine:0.0.1
        imagePullPolicy: Always
        name: elasticsearch-label
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /tmp
          name: node-info
      nodeSelector:
        elastic: test
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      serviceAccount: elastic-permission
      serviceAccountName: elastic-permission
      terminationGracePeriodSeconds: 30
      volumes:
      - configMap:
          defaultMode: 420
          name: elasticsearch-data-config
        name: config
      - name: elastic-certificates
        secret:
          defaultMode: 420
          secretName: elastic-certificate-pem
      - emptyDir: {}
        name: node-info
  updateStrategy:
    rollingUpdate:
      partition: 0
    type: RollingUpdate
  volumeClaimTemplates:
  - apiVersion: v1
    kind: PersistentVolumeClaim
    metadata:
      creationTimestamp: null
      name: elasticsearch-data
    spec:
      accessModes:
      - ReadWriteOnce
      resources:
        requests:
          storage: 20Gi
      storageClassName: rook-cephfs
      volumeMode: Filesystem
    status:
      phase: Pending

and this is the error I get when I start elasticsearch

{"@timestamp":"2024-02-01T14:02:27.076Z", "log.level":"ERROR", "message":"fatal exception while booting Elasticsearch", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.bootstrap.Elasticsearch","elasticsearch.node.name":"elasticsearch-data","elasticsearch.cluster.name":"elasticsearch","error.type":"java.lang.IllegalStateException","error.message":"Unable to access 'path.data' (/data/db/elasticsearch)","error.stack_trace":"java.lang.IllegalStateException: Unable to access 'path.data' (/data/db/elasticsearch)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.FilePermissionUtils.addDirectoryPath(FilePermissionUtils.java:66)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Security.addFilePermissions(Security.java:236)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Security.createPermissions(Security.java:178)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Security.configure(Security.java:125)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Elasticsearch.initPhase2(Elasticsearch.java:188)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:66)\nCaused by: java.nio.file.AccessDeniedException: /data/db/elasticsearch\n\tat java.base/sun.nio.fs.UnixException.translateToIOException(UnixException.java:90)\n\tat java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106)\n\tat java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111)\n\tat java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:397)\n\tat java.base/java.nio.file.Files.createDirectory(Files.java:700)\n\tat java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:807)\n\tat java.base/java.nio.file.Files.createDirectories(Files.java:793)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Security.ensureDirectoryExists(Security.java:330)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.FilePermissionUtils.addDirectoryPath(FilePermissionUtils.java:64)\n\t... 5 more\n"}
ERROR: Elasticsearch did not exit normally - check the logs at /usr/share/elasticsearch/logs/elasticsearch.log

ERROR: Elasticsearch exited unexpectedly

Before I had Init container I didn't have this problem and that is the only thing I have changed in the Statefulset.
What can be the problem and how can I solve this?

Hi,

you can check the permissions of the /data/db/elasticsearch directory inside the Elasticsearch container. The elasticsearch user (usually elasticsearch or elastic ) needs to have read, write, and execute permissions on this directory.

Regards

When i tried ls -al /data/db I got this

total 4
drwxr-xr-x 2 root root    0 Feb  1 13:19 .
drwxr-xr-x 3 root root 4096 Feb  2 09:05 ..

I tried changing the user with chown but wouldn't let me

What I forgot to mention above is that before I was using GlusterFS and now I am using Ceph, I don't think that should be a problem for elastic but maybe it is?

I have seen issues reported around using both GlusterFS and Ceph in the past so that could very well be an issue. I do not know if this has changed recently, but would recommend switching to something different if you can.

Elasticsearch can not run as root and it looks like the directory you listed is owned by root, so this may be the issue.

1 Like

Now I tried with hostPath so I will make sure that the problem is Ceph, but I get the same error and with using hostPath. I also tried with configuration as it was working last time but I still get the same error.

{"@timestamp":"2024-02-05T13:33:28.590Z", "log.level":"ERROR", "message":"fatal exception while booting Elasticsearch", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.bootstrap.Elasticsearch","elasticsearch.node.name":"elasticsearch-data","elasticsearch.cluster.name":"elasticsearch","error.type":"java.lang.IllegalStateException","error.message":"Unable to access 'path.data' (/data/db/elasticsearch)","error.stack_trace":"java.lang.IllegalStateException: Unable to access 'path.data' (/data/db/elasticsearch)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.FilePermissionUtils.addDirectoryPath(FilePermissionUtils.java:66)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Security.addFilePermissions(Security.java:236)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Security.createPermissions(Security.java:178)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Security.configure(Security.java:125)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Elasticsearch.initPhase2(Elasticsearch.java:188)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:66)\nCaused by: java.nio.file.AccessDeniedException: /data/db/elasticsearch\n\tat java.base/sun.nio.fs.UnixException.translateToIOException(UnixException.java:90)\n\tat java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106)\n\tat java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111)\n\tat java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:397)\n\tat java.base/java.nio.file.Files.createDirectory(Files.java:700)\n\tat java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:807)\n\tat java.base/java.nio.file.Files.createDirectories(Files.java:793)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.Security.ensureDirectoryExists(Security.java:330)\n\tat org.elasticsearch.server@8.7.0/org.elasticsearch.bootstrap.FilePermissionUtils.addDirectoryPath(FilePermissionUtils.java:64)\n\t... 5 more\n"}
ERROR: Elasticsearch did not exit normally - check the logs at /usr/share/elasticsearch/logs/elasticsearch.log

ERROR: Elasticsearch exited unexpectedly

I don't know how to fix this, I didn't have this problem before everything was working fine.

And why the directory is owned by root I haven't change anything?

How can I check if user that Elasticsearch runs as can read/write in that path (/data/db/elasticsearch)?
And how can I check what user that is?