khteh
May 21, 2026, 11:55am
1
opened 08:43AM - 21 May 26 UTC
closed 11:49AM - 21 May 26 UTC
triage
```
$ k get sts -n elasticsearch
NAME READY AGE
elastics… earch-es-cold 1/3 16m
elasticsearch-es-hot 0/3 16m
elasticsearch-es-masters 3/3 16m
elasticsearch-es-warm 0/3 16m
```
```
$ k describe sts elasticsearch-es-hot -n elasticsearch
Name: elasticsearch-es-hot
Namespace: elasticsearch
CreationTimestamp: Thu, 21 May 2026 16:37:49 +0800
Selector: common.k8s.elastic.co/type=elasticsearch,elasticsearch.k8s.elastic.co/cluster-name=elasticsearch,elasticsearch.k8s.elastic.co/statefulset-name=elasticsearch-es-hot
Labels: common.k8s.elastic.co/template-hash=1515445537
common.k8s.elastic.co/type=elasticsearch
elasticsearch.k8s.elastic.co/cluster-name=elasticsearch
elasticsearch.k8s.elastic.co/statefulset-name=elasticsearch-es-hot
Annotations: <none>
Replicas: 3 desired | 3 total
Update Strategy: OnDelete
Pods Status: 0 Running / 3 Waiting / 0 Succeeded / 0 Failed
Pod Template:
Labels: common.k8s.elastic.co/type=elasticsearch
elasticsearch.k8s.elastic.co/cluster-name=elasticsearch
elasticsearch.k8s.elastic.co/http-scheme=https
elasticsearch.k8s.elastic.co/node-data=false
elasticsearch.k8s.elastic.co/node-data_cold=false
elasticsearch.k8s.elastic.co/node-data_content=true
elasticsearch.k8s.elastic.co/node-data_frozen=false
elasticsearch.k8s.elastic.co/node-data_hot=true
elasticsearch.k8s.elastic.co/node-data_warm=false
elasticsearch.k8s.elastic.co/node-ingest=true
elasticsearch.k8s.elastic.co/node-master=false
elasticsearch.k8s.elastic.co/node-ml=false
elasticsearch.k8s.elastic.co/node-remote_cluster_client=false
elasticsearch.k8s.elastic.co/node-transform=false
elasticsearch.k8s.elastic.co/node-voting_only=false
elasticsearch.k8s.elastic.co/statefulset-name=elasticsearch-es-hot
elasticsearch.k8s.elastic.co/version=9.4.1
Annotations: co.elastic.logs/module: elasticsearch
elasticsearch.k8s.elastic.co/config-hash: 101637827
policy.k8s.elastic.co/elasticsearch-config-mounts-hash:
Init Containers:
elastic-internal-init-filesystem:
Image: docker.elastic.co/elasticsearch/elasticsearch:9.4.1
Port: <none>
Host Port: <none>
Command:
bash
-c
/mnt/elastic-internal/scripts/prepare-fs.sh
Limits:
cpu: 100m
memory: 50Mi
Requests:
cpu: 100m
memory: 50Mi
Environment:
POD_IP: (v1:status.podIP)
POD_NAME: (v1:metadata.name)
NODE_NAME: (v1:spec.nodeName)
NAMESPACE: (v1:metadata.namespace)
HEADLESS_SERVICE_NAME: elasticsearch-es-hot
PROBE_PASSWORD_PATH: /mnt/elastic-internal/pod-mounted-users/elastic-internal-probe
PROBE_USERNAME: elastic-internal-probe
READINESS_PROBE_PROTOCOL: https
NSS_SDB_USE_CACHE: no
Mounts:
/mnt/elastic-internal/downward-api from downward-api (ro)
/mnt/elastic-internal/elasticsearch-bin-local from elastic-internal-elasticsearch-bin-local (rw)
/mnt/elastic-internal/elasticsearch-config from elastic-internal-elasticsearch-config (ro)
/mnt/elastic-internal/elasticsearch-config-local from elastic-internal-elasticsearch-config-local (rw)
/mnt/elastic-internal/elasticsearch-plugins-local from elastic-internal-elasticsearch-plugins-local (rw)
/mnt/elastic-internal/pod-mounted-users from elastic-internal-probe-user (ro)
/mnt/elastic-internal/scripts from elastic-internal-scripts (ro)
/mnt/elastic-internal/transport-certificates from elastic-internal-transport-certificates (ro)
/mnt/elastic-internal/unicast-hosts from elastic-internal-unicast-hosts (ro)
/mnt/elastic-internal/xpack-file-realm from elastic-internal-xpack-file-realm (ro)
/tmp from tmp-volume (rw)
/usr/share/elasticsearch/config/http-certs from elastic-internal-http-certificates (ro)
/usr/share/elasticsearch/config/operator from file-settings (ro)
/usr/share/elasticsearch/config/transport-remote-certs/ from elastic-internal-remote-certificate-authorities (ro)
/usr/share/elasticsearch/data from elasticsearch-data (rw)
/usr/share/elasticsearch/logs from elasticsearch-logs (rw)
elastic-internal-suspend:
Image: docker.elastic.co/elasticsearch/elasticsearch:9.4.1
Port: <none>
Host Port: <none>
Command:
bash
-c
/mnt/elastic-internal/scripts/suspend.sh
Limits:
cpu: 4
memory: 16Gi
Environment:
POD_IP: (v1:status.podIP)
POD_NAME: (v1:metadata.name)
NODE_NAME: (v1:spec.nodeName)
NAMESPACE: (v1:metadata.namespace)
HEADLESS_SERVICE_NAME: elasticsearch-es-hot
PROBE_PASSWORD_PATH: /mnt/elastic-internal/pod-mounted-users/elastic-internal-probe
PROBE_USERNAME: elastic-internal-probe
READINESS_PROBE_PROTOCOL: https
NSS_SDB_USE_CACHE: no
Mounts:
/mnt/elastic-internal/downward-api from downward-api (ro)
/mnt/elastic-internal/elasticsearch-config from elastic-internal-elasticsearch-config (ro)
/mnt/elastic-internal/pod-mounted-users from elastic-internal-probe-user (ro)
/mnt/elastic-internal/scripts from elastic-internal-scripts (ro)
/mnt/elastic-internal/unicast-hosts from elastic-internal-unicast-hosts (ro)
/mnt/elastic-internal/xpack-file-realm from elastic-internal-xpack-file-realm (ro)
/tmp from tmp-volume (rw)
/usr/share/elasticsearch/bin from elastic-internal-elasticsearch-bin-local (rw)
/usr/share/elasticsearch/config from elastic-internal-elasticsearch-config-local (rw)
/usr/share/elasticsearch/config/http-certs from elastic-internal-http-certificates (ro)
/usr/share/elasticsearch/config/operator from file-settings (ro)
/usr/share/elasticsearch/config/transport-certs from elastic-internal-transport-certificates (ro)
/usr/share/elasticsearch/config/transport-remote-certs/ from elastic-internal-remote-certificate-authorities (ro)
/usr/share/elasticsearch/data from elasticsearch-data (rw)
/usr/share/elasticsearch/logs from elasticsearch-logs (rw)
/usr/share/elasticsearch/plugins from elastic-internal-elasticsearch-plugins-local (rw)
Containers:
elasticsearch:
Image: docker.elastic.co/elasticsearch/elasticsearch:9.4.1
Ports: 9200/TCP (https), 9300/TCP (transport)
Host Ports: 0/TCP (https), 0/TCP (transport)
Limits:
cpu: 4
memory: 16Gi
Readiness: exec [bash -c /mnt/elastic-internal/scripts/readiness-port-script.sh] delay=10s timeout=5s period=5s #success=1 #failure=3
Environment:
POD_IP: (v1:status.podIP)
POD_NAME: (v1:metadata.name)
NODE_NAME: (v1:spec.nodeName)
NAMESPACE: (v1:metadata.namespace)
HEADLESS_SERVICE_NAME: elasticsearch-es-hot
PROBE_PASSWORD_PATH: /mnt/elastic-internal/pod-mounted-users/elastic-internal-probe
PROBE_USERNAME: elastic-internal-probe
READINESS_PROBE_PROTOCOL: https
NSS_SDB_USE_CACHE: no
Mounts:
/mnt/elastic-internal/downward-api from downward-api (ro)
/mnt/elastic-internal/elasticsearch-config from elastic-internal-elasticsearch-config (ro)
/mnt/elastic-internal/pod-mounted-users from elastic-internal-probe-user (ro)
/mnt/elastic-internal/scripts from elastic-internal-scripts (ro)
/mnt/elastic-internal/unicast-hosts from elastic-internal-unicast-hosts (ro)
/mnt/elastic-internal/xpack-file-realm from elastic-internal-xpack-file-realm (ro)
/tmp from tmp-volume (rw)
/usr/share/elasticsearch/bin from elastic-internal-elasticsearch-bin-local (rw)
/usr/share/elasticsearch/config from elastic-internal-elasticsearch-config-local (rw)
/usr/share/elasticsearch/config/http-certs from elastic-internal-http-certificates (ro)
/usr/share/elasticsearch/config/operator from file-settings (ro)
/usr/share/elasticsearch/config/transport-certs from elastic-internal-transport-certificates (ro)
/usr/share/elasticsearch/config/transport-remote-certs/ from elastic-internal-remote-certificate-authorities (ro)
/usr/share/elasticsearch/data from elasticsearch-data (rw)
/usr/share/elasticsearch/logs from elasticsearch-logs (rw)
/usr/share/elasticsearch/plugins from elastic-internal-elasticsearch-plugins-local (rw)
Volumes:
downward-api:
Type: DownwardAPI (a volume populated by information about the pod)
Items:
metadata.labels -> labels
elastic-internal-elasticsearch-bin-local:
Type: EmptyDir (a temporary directory that shares a pod's lifetime)
Medium:
SizeLimit: <unset>
elastic-internal-elasticsearch-config:
Type: Secret (a volume populated by a Secret)
SecretName: elasticsearch-es-hot-es-config
Optional: false
elastic-internal-elasticsearch-config-local:
Type: EmptyDir (a temporary directory that shares a pod's lifetime)
Medium:
SizeLimit: <unset>
elastic-internal-elasticsearch-plugins-local:
Type: EmptyDir (a temporary directory that shares a pod's lifetime)
Medium:
SizeLimit: <unset>
elastic-internal-http-certificates:
Type: Secret (a volume populated by a Secret)
SecretName: elasticsearch-es-http-certs-internal
Optional: false
elastic-internal-probe-user:
Type: Secret (a volume populated by a Secret)
SecretName: elasticsearch-es-internal-users
Optional: false
elastic-internal-remote-certificate-authorities:
Type: Secret (a volume populated by a Secret)
SecretName: elasticsearch-es-remote-ca
Optional: false
elastic-internal-scripts:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: elasticsearch-es-scripts
Optional: false
elastic-internal-transport-certificates:
Type: Secret (a volume populated by a Secret)
SecretName: elasticsearch-es-hot-es-transport-certs
Optional: false
elastic-internal-unicast-hosts:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: elasticsearch-es-unicast-hosts
Optional: false
elastic-internal-xpack-file-realm:
Type: Secret (a volume populated by a Secret)
SecretName: elasticsearch-es-xpack-file-realm
Optional: false
elasticsearch-data:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: claim-name-placeholder
ReadOnly: false
elasticsearch-logs:
Type: EmptyDir (a temporary directory that shares a pod's lifetime)
Medium:
SizeLimit: <unset>
file-settings:
Type: Secret (a volume populated by a Secret)
SecretName: elasticsearch-es-file-settings
Optional: false
tmp-volume:
Type: EmptyDir (a temporary directory that shares a pod's lifetime)
Medium:
SizeLimit: <unset>
Node-Selectors: <none>
Tolerations: <none>
Volume Claims:
Name: elasticsearch-data
StorageClass:
Labels: <none>
Annotations: <none>
Capacity: 10Gi
Access Modes: [ReadWriteOnce]
Events: <none>
```
```
$ k get pods -n elasticsearch -w
NAME READY STATUS RESTARTS AGE
eck-cluster-eck-apm-server-apm-server-6f94cf9969-jb4kp 0/1 Pending 0 11m
eck-cluster-eck-apm-server-apm-server-6f94cf9969-lxffl 0/1 Pending 0 11m
eck-cluster-eck-apm-server-apm-server-6f94cf9969-zbdv7 0/1 Pending 0 11m
elasticsearch-es-cold-0 1/1 Running 0 11m
elasticsearch-es-cold-1 0/1 Pending 0 11m
elasticsearch-es-cold-2 0/1 Pending 0 11m
elasticsearch-es-hot-0 0/1 Pending 0 11m
elasticsearch-es-hot-1 0/1 Pending 0 11m
elasticsearch-es-hot-2 0/1 Pending 0 11m
elasticsearch-es-masters-0 1/1 Running 0 11m
elasticsearch-es-masters-1 1/1 Running 0 11m
elasticsearch-es-masters-2 1/1 Running 0 11m
elasticsearch-es-warm-0 0/1 Pending 0 11m
elasticsearch-es-warm-1 0/1 Pending 0 11m
elasticsearch-es-warm-2 0/1 Pending 0 11m
kibana-kb-786554c95c-gqj4d 0/1 Pending 0 11m
```
opened 09:53AM - 21 May 26 UTC
closed 11:52AM - 21 May 26 UTC
triage
Initial STS count was 3, then I updated the values file to 1, run `helmfile appl… y`, it shows the diff on console but the STS still shows the initial `Replicas: 3 desired | 3 total`. `k get sts -n elasticsearch` also shows the old `replicas` count value.
$ k logs -n elastic-system -l control-plane=elastic-operator
No resources found in elastic-system namespace.
$ k get all -n elastic-system
NAME READY STATUS RESTARTS AGE
pod/elastic-operator-0 1/1 Running 0 3h14m
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/elastic-operator-webhook ClusterIP 10.152.183.149 <none> 443/TCP 3h14m
NAME READY AGE
statefulset.apps/elastic-operator 1/1 3h14m
---
eck-elasticsearch:
version: 9.4.1
nodeSets:
- name: masters
count: 1
config:
node.roles: ["master"]
# Comment out when setting the vm.max_map_count via initContainer, as these are mutually exclusive.
# For production workloads, it is strongly recommended to increase the kernel setting vm.max_map_count to 262144
# and leave node.store.allow_mmap unset.
# ref: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-virtual-memory.html
#
node.store.allow_mmap: false
script.painless.regex.enabled: true
http.cors.enabled: true
http.cors.allow-origin: "*"
#network.host: _site_ Forbidden: Configuration setting is reserved for internal use
#node.name: ${HOSTNAME} Forbidden: Configuration setting is reserved for internal use
bootstrap.memory_lock: false # https://www.elastic.co/guide/en/elasticsearch/reference/8.17/bootstrap-checks-memory-lock.html
show original
$ k version -o yaml
clientVersion:
buildDate: "2025-12-18T08:37:08Z"
compiler: gc
gitCommit: 66452049f3d692768c39c797b21b793dce80314e
gitTreeState: clean
gitVersion: v1.35.0
goVersion: go1.25.5
major: "1"
minor: "35"
platform: linux/amd64
kustomizeVersion: v5.7.1
serverVersion:
buildDate: "2025-12-18T08:38:08Z"
compiler: gc
emulationMajor: "1"
emulationMinor: "35"
gitCommit: 66452049f3d692768c39c797b21b793dce80314e
gitTreeState: clean
gitVersion: v1.35.0
goVersion: go1.25.5
major: "1"
minCompatibilityMajor: "1"
minCompatibilityMinor: "34"
minor: "35"
platform: linux/amd64