please help me , How to resloved this issue
It seems like you have hit the limits of your heap. Either increase the heap size for Elasticsearch (ensure it has access to 2 times the heap size in RAM) or try to reduce your heap usage. What is the full output of the cluster stats API?
How to increase heap size, I am running Elasticsearch inside kubernetes.
{"@timestamp":"2023-05-26T05:22:42.318Z", "log.level": "WARN", "message":"Authentication using apikey failed - apikey authentication for id 29jPD4UBw3fwLK4sIxpF encountered a failure", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[skill-elasticsearch-dev-es-masters-0][transport_worker][T#1]","log.logger":"org.elasticsearch.xpack.security.authc.ApiKeyAuthenticator","elasticsearch.cluster.uuid":"A0-vC5YVR46paQrjeDAAqg","elasticsearch.node.id":"M-ZXS8_GQJaJpjugSRa9uw","elasticsearch.node.name":"skill-elasticsearch-dev-es-masters-0","elasticsearch.cluster.name":"skill-elasticsearch-dev","error.type":"org.elasticsearch.transport.RemoteTransportException","error.message":"[skill-elasticsearch-dev-es-data-0][172.32.3.164:9300][indices:data/read/get[s]]","error.stack_trace":"org.elasticsearch.transport.RemoteTransportException: [skill-elasticsearch-dev-es-data-0][172.32.3.164:9300][indices:data/read/get[s]]\nCaused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [indices:data/read/get[s]] would be [1060547296/1011.4mb], which is larger than the limit of [805306368/768mb], real usage: [1060547120/1011.4mb], new bytes reserved: [176/176b], usages [inflight_requests=176/176b, model_inference=0/0b, eql_sequence=0/0b, fielddata=12879/12.5kb, request=0/0b]\n\tat org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:414)\n\tat org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:109)\n\tat org.elasticsearch.transport.InboundAggregator.checkBreaker(InboundAggregator.java:215)\n\tat org.elasticsearch.transport.InboundAggregator.finishAggregation(InboundAggregator.java:119)\n\tat org.elasticsearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:147)\n\tat org.elasticsearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:121)\n\tat org.elasticsearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:86)\n\tat org.elasticsearch.transport.netty4.Netty4MessageInboundHandler.channelRead(Netty4MessageInboundHandler.java:63)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)\n\tat io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)\n\tat io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)\n\tat io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1372)\n\tat io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1235)\n\tat io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1284)\n\tat io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:510)\n\tat io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:449)\n\tat io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)\n\tat io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)\n\tat io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat java.lang.Thread.run(Thread.java:833)\n"}
Which version of Elasticsearch are you using?
after try to increaese heap to
esJavaOpts: -Xmx8g -Xms8g
{"@timestamp":"2023-05-26T05:30:22.817Z", "log.level": "INFO", "message":"attempting to trigger G1GC due to high heap usage [1052534608]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[skill-elasticsearch-dev-es-data-0][transport_worker][T#1]","log.logger":"org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService","elasticsearch.cluster.uuid":"A0-vC5YVR46paQrjeDAAqg","elasticsearch.node.id":"2TaYnoSBR9uxFwFmDYzLbw","elasticsearch.node.name":"skill-elasticsearch-dev-es-data-0","elasticsearch.cluster.name":"skill-elasticsearch-dev"}
{"@timestamp":"2023-05-26T05:30:22.838Z", "log.level": "INFO", "message":"GC did bring memory usage down, before [1052534608], after [1049133416], allocations [1], duration [21]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[skill-elasticsearch-dev-es-data-0][transport_worker][T#1]","log.logger":"org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService","elasticsearch.cluster.uuid":"A0-vC5YVR46paQrjeDAAqg","elasticsearch.node.id":"2TaYnoSBR9uxFwFmDYzLbw","elasticsearch.node.name":"skill-elasticsearch-dev-es-data-0","elasticsearch.cluster.name":"skill-elasticsearch-dev"}
version 8.0
Make sure the pod is assigned twice as much RAM as your heap size. Elasticsearch requires off heap and also makes use of the operating system page cache to improve performance.
I recommend upgrading to the latest version, currently 8.7.
we are using elastic-operator
If you're using ECK, by default pods will automatically be assigned half of ram (pod ram assignment) as heap. You can override this setting via env variable
ES_JAVA_OPS
Example of how elastickonductor sets it the heap value. ElasticKonductor/es.yaml at main · sunileman/ElasticKonductor · GitHub
can you share your k8s manifest? I suspect you need to jack up your pod memory which will help increase your heap size.
1 Like
apiVersion: apps/v1
kind: StatefulSet
metadata:
creationTimestamp: "2022-08-31T08:31:23Z"
generation: 33
labels:
common.k8s.elastic.co/template-hash: "3499536010"
common.k8s.elastic.co/type: elasticsearch
elasticsearch.k8s.elastic.co/cluster-name: skill-elasticsearch-dev
elasticsearch.k8s.elastic.co/statefulset-name: skill-elasticsearch-dev-es-data
name: skill-elasticsearch-dev-es-data
namespace: elastic
ownerReferences:
- apiVersion: elasticsearch.k8s.elastic.co/v1
blockOwnerDeletion: true
controller: true
kind: Elasticsearch
name: skill-elasticsearch-dev
uid: f5534fcf-0c57-4d31-88c2-bfa17383e405
resourceVersion: "170392465"
uid: aedfe74f-2e9a-49a4-912d-cfb39ec9d414
spec:
podManagementPolicy: Parallel
replicas: 3
revisionHistoryLimit: 10
selector:
matchLabels:
common.k8s.elastic.co/type: elasticsearch
elasticsearch.k8s.elastic.co/cluster-name: skill-elasticsearch-dev
elasticsearch.k8s.elastic.co/statefulset-name: skill-elasticsearch-dev-es-data
serviceName: skill-elasticsearch-dev-es-data
template:
metadata:
annotations:
co.elastic.logs/module: elasticsearch
elasticsearch.k8s.elastic.co/config-hash: "3145394654"
kubectl.kubernetes.io/restartedAt: "2023-05-30T11:34:25+05:30"
creationTimestamp: null
labels:
common.k8s.elastic.co/type: elasticsearch
elasticsearch.k8s.elastic.co/cluster-name: skill-elasticsearch-dev
elasticsearch.k8s.elastic.co/http-scheme: https
elasticsearch.k8s.elastic.co/node-data: "true"
elasticsearch.k8s.elastic.co/node-data_cold: "false"
elasticsearch.k8s.elastic.co/node-data_content: "false"
elasticsearch.k8s.elastic.co/node-data_frozen: "false"
elasticsearch.k8s.elastic.co/node-data_hot: "false"
elasticsearch.k8s.elastic.co/node-data_warm: "false"
elasticsearch.k8s.elastic.co/node-ingest: "true"
elasticsearch.k8s.elastic.co/node-master: "false"
elasticsearch.k8s.elastic.co/node-ml: "true"
elasticsearch.k8s.elastic.co/node-remote_cluster_client: "false"
elasticsearch.k8s.elastic.co/node-transform: "true"
elasticsearch.k8s.elastic.co/node-voting_only: "false"
elasticsearch.k8s.elastic.co/statefulset-name: skill-elasticsearch-dev-es-data
elasticsearch.k8s.elastic.co/version: 8.8.0
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
elasticsearch.k8s.elastic.co/cluster-name: skill-elasticsearch-dev
topologyKey: kubernetes.io/hostname
weight: 100
automountServiceAccountToken: false
containers:- env:
- name: POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP - name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name - name: NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName - name: NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace - name: PROBE_PASSWORD_PATH
value: /mnt/elastic-internal/probe-user/elastic-internal-probe - name: PROBE_USERNAME
value: elastic-internal-probe - name: READINESS_PROBE_PROTOCOL
value: https - name: HEADLESS_SERVICE_NAME
value: skill-elasticsearch-dev-es-data - name: NSS_SDB_USE_CACHE
value: "no" - name: ES_JAVA_OPTS
value: -Xms4g -Xmx4g
image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- bash
- -c
- /mnt/elastic-internal/scripts/pre-stop-hook-script.sh
name: elasticsearch
ports: - containerPort: 9200
name: https
protocol: TCP - containerPort: 9300
name: transport
protocol: TCP
readinessProbe:
exec:
command:- bash
- -c
- /mnt/elastic-internal/scripts/readiness-probe-script.sh
failureThreshold: 3
initialDelaySeconds: 10
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 5
resources:
limits:
memory: 4Gi
requests:
memory: 4Gi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /mnt/elastic-internal/downward-api
name: downward-api
readOnly: true - mountPath: /usr/share/elasticsearch/bin
name: elastic-internal-elasticsearch-bin-local - mountPath: /mnt/elastic-internal/elasticsearch-config
name: elastic-internal-elasticsearch-config
readOnly: true - mountPath: /usr/share/elasticsearch/config
name: elastic-internal-elasticsearch-config-local - mountPath: /usr/share/elasticsearch/plugins
name: elastic-internal-elasticsearch-plugins-local - mountPath: /usr/share/elasticsearch/config/http-certs
name: elastic-internal-http-certificates
readOnly: true - mountPath: /mnt/elastic-internal/probe-user
name: elastic-internal-probe-user
readOnly: true - mountPath: /usr/share/elasticsearch/config/transport-remote-certs/
name: elastic-internal-remote-certificate-authorities
readOnly: true - mountPath: /mnt/elastic-internal/scripts
name: elastic-internal-scripts
readOnly: true - mountPath: /usr/share/elasticsearch/config/transport-certs
name: elastic-internal-transport-certificates
readOnly: true - mountPath: /mnt/elastic-internal/unicast-hosts
name: elastic-internal-unicast-hosts
readOnly: true - mountPath: /mnt/elastic-internal/xpack-file-realm
name: elastic-internal-xpack-file-realm
readOnly: true - mountPath: /usr/share/elasticsearch/data
name: elasticsearch-data - mountPath: /usr/share/elasticsearch/logs
name: elasticsearch-logs
dnsPolicy: ClusterFirst
initContainers:
- name: POD_IP
- command:
- bash
- -c
- /mnt/elastic-internal/scripts/prepare-fs.sh
env: - name: POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP - name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name - name: NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName - name: NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace - name: HEADLESS_SERVICE_NAME
value: skill-elasticsearch-dev-es-data - name: ES_JAVA_OPTS
value: -Xms4g -Xmx4g
image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0
imagePullPolicy: IfNotPresent
name: elastic-internal-init-filesystem
resources:
limits:
cpu: 100m
memory: 150Mi
requests:
cpu: 100m
memory: 50Mi
securityContext:
privileged: false
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts: - mountPath: /mnt/elastic-internal/downward-api
name: downward-api
readOnly: true - mountPath: /mnt/elastic-internal/elasticsearch-bin-local
name: elastic-internal-elasticsearch-bin-local - mountPath: /mnt/elastic-internal/elasticsearch-config
name: elastic-internal-elasticsearch-config
readOnly: true - mountPath: /mnt/elastic-internal/elasticsearch-config-local
name: elastic-internal-elasticsearch-config-local - mountPath: /mnt/elastic-internal/elasticsearch-plugins-local
name: elastic-internal-elasticsearch-plugins-local - mountPath: /usr/share/elasticsearch/config/http-certs
name: elastic-internal-http-certificates
readOnly: true - mountPath: /mnt/elastic-internal/probe-user
name: elastic-internal-probe-user
readOnly: true - mountPath: /usr/share/elasticsearch/config/transport-remote-certs/
name: elastic-internal-remote-certificate-authorities
readOnly: true - mountPath: /mnt/elastic-internal/scripts
name: elastic-internal-scripts
readOnly: true - mountPath: /mnt/elastic-internal/transport-certificates
name: elastic-internal-transport-certificates
readOnly: true - mountPath: /mnt/elastic-internal/unicast-hosts
name: elastic-internal-unicast-hosts
readOnly: true - mountPath: /mnt/elastic-internal/xpack-file-realm
name: elastic-internal-xpack-file-realm
readOnly: true - mountPath: /usr/share/elasticsearch/data
name: elasticsearch-data - mountPath: /usr/share/elasticsearch/logs
name: elasticsearch-logs
- command:
- bash
- -c
- /mnt/elastic-internal/scripts/suspend.sh
env: - name: POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP - name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name - name: NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName - name: NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace - name: HEADLESS_SERVICE_NAME
value: skill-elasticsearch-dev-es-data - name: ES_JAVA_OPTS
value: -Xms4g -Xmx4g
image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0
imagePullPolicy: IfNotPresent
name: elastic-internal-suspend
resources:
limits:
memory: 4Gi
requests:
memory: 4Gi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts: - mountPath: /mnt/elastic-internal/downward-api
name: downward-api
readOnly: true - mountPath: /usr/share/elasticsearch/bin
name: elastic-internal-elasticsearch-bin-local - mountPath: /mnt/elastic-internal/elasticsearch-config
name: elastic-internal-elasticsearch-config
readOnly: true - mountPath: /usr/share/elasticsearch/config
name: elastic-internal-elasticsearch-config-local - mountPath: /usr/share/elasticsearch/plugins
name: elastic-internal-elasticsearch-plugins-local - mountPath: /usr/share/elasticsearch/config/http-certs
name: elastic-internal-http-certificates
readOnly: true - mountPath: /mnt/elastic-internal/probe-user
name: elastic-internal-probe-user
readOnly: true - mountPath: /usr/share/elasticsearch/config/transport-remote-certs/
name: elastic-internal-remote-certificate-authorities
readOnly: true - mountPath: /mnt/elastic-internal/scripts
name: elastic-internal-scripts
readOnly: true - mountPath: /usr/share/elasticsearch/config/transport-certs
name: elastic-internal-transport-certificates
readOnly: true - mountPath: /mnt/elastic-internal/unicast-hosts
name: elastic-internal-unicast-hosts
readOnly: true - mountPath: /mnt/elastic-internal/xpack-file-realm
name: elastic-internal-xpack-file-realm
readOnly: true - mountPath: /usr/share/elasticsearch/data
name: elasticsearch-data - mountPath: /usr/share/elasticsearch/logs
name: elasticsearch-logs
restartPolicy: Always
schedulerName: default-scheduler
securityContext:
fsGroup: 1000
terminationGracePeriodSeconds: 180
volumes:
- downwardAPI:
defaultMode: 420
items:- fieldRef:
apiVersion: v1
fieldPath: metadata.labels
path: labels
name: downward-api
- fieldRef:
- emptyDir: {}
name: elastic-internal-elasticsearch-bin-local - name: elastic-internal-elasticsearch-config
secret:
defaultMode: 420
optional: false
secretName: skill-elasticsearch-dev-es-data-es-config - emptyDir: {}
name: elastic-internal-elasticsearch-config-local - emptyDir: {}
name: elastic-internal-elasticsearch-plugins-local - name: elastic-internal-http-certificates
secret:
defaultMode: 420
optional: false
secretName: skill-elasticsearch-dev-es-http-certs-internal - name: elastic-internal-probe-user
secret:
defaultMode: 420
items:- key: elastic-internal-probe
path: elastic-internal-probe
optional: false
secretName: skill-elasticsearch-dev-es-internal-users
- key: elastic-internal-probe
- name: elastic-internal-remote-certificate-authorities
secret:
defaultMode: 420
optional: false
secretName: skill-elasticsearch-dev-es-remote-ca - configMap:
defaultMode: 493
name: skill-elasticsearch-dev-es-scripts
optional: false
name: elastic-internal-scripts - name: elastic-internal-transport-certificates
secret:
defaultMode: 420
optional: false
secretName: skill-elasticsearch-dev-es-data-es-transport-certs - configMap:
defaultMode: 420
name: skill-elasticsearch-dev-es-unicast-hosts
optional: false
name: elastic-internal-unicast-hosts - name: elastic-internal-xpack-file-realm
secret:
defaultMode: 420
optional: false
secretName: skill-elasticsearch-dev-es-xpack-file-realm - name: elasticsearch-data
persistentVolumeClaim:
claimName: claim-name-placeholder - emptyDir: {}
name: elasticsearch-logs
updateStrategy:
type: OnDelete
volumeClaimTemplates:
- env:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
creationTimestamp: null
name: elasticsearch-data
spec:
accessModes:- ReadWriteOnce
resources:
requests:
storage: 100Gi
storageClassName: gp2
volumeMode: Filesystem
status:
phase: Pending
status:
availableReplicas: 3
collisionCount: 0
currentRevision: skill-elasticsearch-dev-es-data-6c9667fc98
observedGeneration: 33
readyReplicas: 3
replicas: 3
updateRevision: skill-elasticsearch-dev-es-data-5659cc5c64
- ReadWriteOnce
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.