Kibana version: v8.13.3
Elasticsearch version: v8.13.3
APM Server version: v8.13.3
APM Agent language and version: elastic-apm-agent-1.52.1.jar
Original install method (e.g. download page, yum, deb, from source, etc.) and version: -javaagent
Fresh install or upgraded from other version? Fresh install
Is there anything special in your setup? using Kafka output
apm.properties:
recording=true
service_name=xxxxxx
environment=prod
server_urls=http://xxxx:8200,http://xxxxx:8200,http://xxxxx:8200,http://xxxxx:8200,http://xxxxx:8200
service_node_name=xxxxxxx
log_level=INFO
log_file=/opt/apm/apm-agent.log
log_ecs_reformatting=SHADE
log_ecs_reformatting_dir=/opt/apm
capture_body=off
transaction_max_spans=500
transaction_sample_rate=0.3
circuit_breaker_enabled=true
apm server config:
apm-server:
host: "xxxxxxxxxxxxxxxxxxxxx"
max_event_size: 807200
data_streams.namespace: xxxxxxxxxxxxx
data_streams.wait_for_integration: false
agent.config:
cache.expiration: 30s
elasticsearch:
hosts: [xxxxxxxxxxxxxxxxxxxxxxxxx]
username: "xxxxxxxxxxxxxxxx"
password: "xxxxxxxxxxxxx"
output.kafka:
enabled: true
hosts: [xxxxxxxxxxxxxxxxxx]
topic: "apm-%{[data_stream.type]}"
partition.round_robin:
reachable_only: false
worker: 30
bulk_max_size: 102400
channel_buffer_size: 2048
logging.level: info
logging.metrics.enabled: true
logging.metrics.period: 30s
logging.to_files: true
logging.files:
path: /var/log/apm-server/yinhai
name: apm-server
keepfiles: 7
monitoring.enabled: true
monitoring.elasticsearch:
cluster_uuid: xxxxxxxxxxxxxxxxxxx
username: "xxxxxxxxxxxxxxxxxx"
password: "xxxxxxxxxxxxxxxx"
hosts: [xxxxxxxxxxxxxxxxxxxxx]
error log:
2025-03-12 11:08:01,993 [elastic-apm-server-reporter] WARN co.elastic.apm.agent.report.AbstractIntakeApiHandler - Response body: {
"accepted": 30,
"errors": [
{
"message": "cannot handle stream: cannot process batch: queue is full"
}
]
}
2025-03-12 11:08:01,994 [elastic-apm-server-reporter] INFO co.elastic.apm.agent.report.AbstractIntakeApiHandler - Backing off for 0 seconds (+/-10%)
2025-03-12 11:08:01,994 [elastic-apm-server-reporter] ERROR co.elastic.apm.agent.report.AbstractIntakeApiHandler - Error sending data to APM server: Server returned HTTP response code: 503 for URL: http://10.109.97.34:8200/intake/v2/events, response code is 503
2025-03-12 15:36:43,016 [http-nio-8080-exec-5] WARN co.elastic.apm.agent.impl.transaction.SpanImpl - Max spans (500) for transaction 'TarunqianresourcePrintRestController#printAsPdfByUrl' 00-9c16b9b3579c0a06fd9caee57aa39768-cead3d4e7f073b0e-01 (1dad4bf) has been reached. For this transaction and possibly others, further spans will be dropped. See config param 'transaction_max_spans'.
2025-03-12 15:48:59,491 [http-nio-8080-exec-13] WARN co.elastic.apm.agent.impl.transaction.SpanImpl - Max spans (500) for transaction 'CommonQueryController#getPsnByConditionAllot' 00-ba78a1f0eec8ea4b490de66de0d7ac93-6b10cea1a2198fbc-00 (75282f4a) has been reached. For this transaction and possibly others, further spans will be dropped. See config param 'transaction_max_spans'.
2025-03-12 17:11:54,769 [http-nio-8080-exec-3] WARN co.elastic.apm.agent.impl.transaction.SpanImpl - Max spans (500) for transaction 'PublicStaffInsuRegServiceImpl#saveStaffInsuRegJingBan' 00-ccad12bd37b0930511824b174e74a915-7d16523dbaddace3-01 (7f5352a4) has been reached. For this transaction and possibly others, further spans will be dropped. See config param 'transaction_max_spans'.
2025-03-12 17:11:58,793 [elastic-apm-server-reporter] ERROR co.elastic.apm.agent.report.IntakeV2ReportingEventHandler - Failed to handle event of type TRANSACTION with this error: null
2025-03-12 17:11:58,840 [elastic-apm-server-reporter] WARN co.elastic.apm.agent.report.AbstractIntakeApiHandler - Response body: {
"accepted": 10,
"errors": [
{
"message": "decode error: data read error: v2.transactionRoot.Transaction: v2.transaction.DroppedSpanStats: []v2.transactionDroppedSpanStats: v2.transactionDroppedSpanStats.readFieldHash: expect \",",
"document": "{\"transaction\":{\"timestamp\":1741770713031000,\"name\":\"PublicStaffInsuRegServiceImpl#saveStaffInsuRegJingBan\",\"id\":\"7d16523dbaddace3\",\"trace_id\":\"ccad12bd37b0930511824b174e74a915\",\"type\":\"request\",\"duration\":5759.455,\"result\":\"HTTP 2xx\",\"outcome\":\"success\",\"context\":{\"service\":{\"framework\":{\"version\":\"5.0.13.RELEASE\",\"name\":\"Spring Web MVC\"},\"version\":null},\"request\":{\"method\":\"POST\",\"headers\":{\"x-tif-paasid\":\"mbs_gg_dev\",\"x-tif-signature\":\"8C7DDD414EAEB274663571C69EEEA30B5DC9FF1A57E7909EA15A603979778C19\",\"x-tif-timestamp\":\"1741770713\",\"x-tif-nonce\":\"iyuk20EWQbEkT7QCbdmOhyXjMTnNfEIQ\",\"eagleeye-userdata\":\"userData=rO0ABXNyACtjbi5oc2EuaHNhZi5jb3JlLmZyYW1ld29yay51dGlsLkN1cnJlbnRVc2VyAAAAAAAAAAECAA1MABNJbnN1VHlwZVBvb2xBcmVhTWFwdAAPTGphdmEvdXRpbC9NYXA7TAAGYWRtRHZzdAASTGphdmEvbGFuZy9TdHJpbmc7TAAGZGVwdElEcQB+AAJMAAhkZXB0TmFtZXEAfgACTAAEbmFtZXEAfgACTAAHb3JnQ29kZ3EAfgACTAAHb3JnTmFtZXEAfgACTAAIb3JnVW50SURxAH4AAkwADHBvb2xBcmVhQ29kZ3EAfgACTAAJcHJudE9yZ0lEcQB+AAJMAAZ1YWN0SURxAH4AAkwACHVzZXJBY2N0cQB+AAJMAAp1c2VyQWNjdElEcQB+AAJ4cHNyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAB3CAAAABAAAAAAeHBwcHBwcHBwcHBwcA==\u0026aid=1741770716009\",\"content-length\":\"3336\",\"content-type\":\"text/plain; charset=UTF-8\",\"user-agent\":\"Apache-HttpClient/4.5.9 (Java/1.8.0_252)\",\"x-forwarded-for\":\"xx.xx.xx.xx,xx.xx.xx.xx,127.0.0.1\",\"x-real-ip\":\"xx.xx.xx.xx\",\"x-proxy-by\":\"Tif-APIGate\",\"x-forwarded-proto\":\"http\",\"x-rio-nonce\":\"iyuk20EWQbEkT7QCbdmOhyXjMTnNfEIQ\",\"x-rio-paasid\":\"mbs_gg_dev\",\"host\":\"xx.xx.xx.xx:8080\",\"x-rio-timestamp\":\"1741770713\",\"x-rio-signature\":\"8C7DDD414EAEB274663571C69EEEA30B5DC9FF1A57E7909EA15A603979778C19\",\"connection\":\"keep-alive\",\"accept-encoding\":\"gzip\"},\"cookies\":{\"BIGipServerrs_yinhai_arc_8080\":\"1966319808.36895.0000\",\"BIGipServerrs_yinhai_bic_8080\":\"4281903296.36895.0000\",\"BIGipServerrs_yinhai_mbs-pub-svc_8080\":\"3157829824.36895.0000\",\"BIGipServerrs_yinhai_pic_8080\":\"3090655424.36895.0000\",\"BIGipServerrs_yinhai_usc_8080\":\"121219264.36895.0000\",\"XSRF-TOKEN\":\"[REDACTED]\",\"x_host_key\":\"[REDACTED]\"},\"body\":\"[REDACTED]\",\"url\":{\"full\":\"http://xx.xx.xx.xx:8080/mbs/api/publicStaffInsuReg/saveStaffInsuRegJingBan?\",\"hostname\":\"xx.xx.xx.xx\",\"port\":8080,\"pathname\":\"/mbs/api/publicStaffInsuReg/saveStaffInsuRegJingBan\",\"search\":\"\",\"protocol\":\"http\"},\"socket\":{\"remote_address\":\"xx.xx.xx.xx\"},\"http_version\":\"1.1\"},\"response\":{\"headers\":{\"Cache-Control\":\"no-store\",\"X-Content-Type-Options\":\"nosniff\",\"X-XSS-Protection\":\"1; mode=block\",\"Content-Type\":\"application/json;charset=UTF-8\",\"Content-Language\":\"en-US\",\"Transfer-Encoding\":\"chunked\",\"Date\":\"Wed, 12 Mar 2025 09:11:57 GMT\"},\"finished\":true,\"headers_sent\":true,\"status_code\":200},\"tags\":{}},\"span_count\":{\"dropped\":521,\"started\":500},\"dropped_spans_stats\":[{\"destination_service_resource\":\"xx.xx.xx.xx:8080\",\"service_target_type\":\"http\",\"service_target_name\":\"192.168.55.176:8080\",\"outcome\":\"success\",\"duration\":{\"count\":2,\"sum\":{\"us\":11752}}},{\"destination_service_resource\":\"\","
}
]
}
2025-03-12 17:11:58,841 [elastic-apm-server-reporter] INFO co.elastic.apm.agent.report.AbstractIntakeApiHandler - Backing off for 0 seconds (+/-10%)
2025-03-12 17:11:58,841 [elastic-apm-server-reporter] ERROR co.elastic.apm.agent.report.AbstractIntakeApiHandler - Error sending data to APM server: Server returned HTTP response code: 400 for URL: http://xx.xx.xx.xx:8200/intake/v2/events, response code is 400
{"log.level":"error","@timestamp":"2025-03-12T17:11:58.839+0800","log.logger":"request","log.origin":{"function":"github.com/elastic/apm-server/internal/beater/api.apmMiddleware.LogMiddleware.func1.1","file.name":"middleware/log_middleware.go","file.line":59},"message":"data validation error","service.name":"apm-server","url.original":"/intake/v2/events","http.request.method":"POST","user_agent.original":"apm-agent-java/1.52.1 (xx.xx.xx.xx 2.0.0-SNAPSHOT)","source.address":"xx.xx.xx.xx","http.request.id":"620dfbf5-d565-4def-bb3b-d4b787eaf99b","event.duration":941481635,"http.request.body.bytes":5194,"http.response.status_code":400,"error.message":"decode error: data read error: v2.transactionRoot.Transaction: v2.transaction.DroppedSpanStats: []v2.transactionDroppedSpanStats: v2.transactionDroppedSpanStats.readFieldHash: expect \",","ecs.version":"1.6.0"}
