Kibana version: 7.15.0
Elasticsearch version: 7.15.0
APM Server version: 7.15.0
APM Agent language and version: Java 1.27.0 (Java 8)
Description of the problem including expected versus actual behavior. Please include screenshots (if relevant):
agent.version has unknown value after upgrading to Elastic APM Java Agent 1.27.0.
Hi @hendry.lim ,
I also noticed that host.hostname is missing from the document sent by agent v1.27.0, while we had that in 1.26.0. This affects the APM UI which is displaying Host N/A on the JVMs tab.
Sample doc from another app running Java 11 (some fields were removed) - 1.27.0
{
"container": {
"id": "9328fd947c04ddc0e17437f42408aeaa42bcf45da557d6688c65c60c2fe4986d"
},
"agent": {
"name": "java",
"ephemeral_id": "e4ec45b1-ed97-4b6e-bc5a-5b51ec989022",
"version": "unknown"
},
"process": {
"pid": 1,
"title": "/opt/java/openjdk/bin/java"
},
"processor": {
"name": "transaction",
"event": "transaction"
},
"url": {
"path": "/index.html",
"scheme": "https",
"port": 443,
"domain": "xxx",
"full": "xxx"
},
"observer": {
"hostname": "xxx",
"id": "c4e5c03a-3714-4b33-bf8e-e7dd958d7bbe",
"ephemeral_id": "17c9f1f3-5e4b-4898-8f14-d1bc715bc0de",
"type": "apm-server",
"version": "7.15.0",
"version_major": 7
},
"trace": {
"id": "b52dba2843c6af3abf08e9a3a60ffaa1"
},
"@timestamp": "2021-11-24T14:13:22.032Z",
"ecs": {
"version": "1.11.0"
},
"service": {
"node": {
"name": "xxx"
},
"runtime": {
"name": "Java",
"version": "11.0.11"
},
"language": {
"name": "Java",
"version": "11.0.11"
}
},
"host": {
"os": {
"platform": "Linux"
},
"ip": "127.0.0.1",
"name": "xxx",
"architecture": "amd64"
},
"event": {
"ingested": "2021-11-24T14:13:23.433948504Z",
"outcome": "success"
},
"timestamp": {
"us": 1637763202032015
}
}
Comparison with 1.26.0
{
"container": {
"id": "9a1f43470e789214ffae06ba0a29fe3193d6fe80f5e6079e201f8882ed3a18f2"
},
"agent": {
"name": "java",
"ephemeral_id": "46e72620-6668-486f-af71-54c303ce3f12",
"version": "1.26.0"
},
"observer": {
"hostname": "xxx",
"id": "c4e5c03a-3714-4b33-bf8e-e7dd958d7bbe",
"ephemeral_id": "17c9f1f3-5e4b-4898-8f14-d1bc715bc0de",
"type": "apm-server",
"version": "7.15.0",
"version_major": 7
},
"trace": {
"id": "77d3c6093d678e43ad14bca138a9cffb"
},
"ecs": {
"version": "1.11.0"
},
"host": {
"hostname": "xxx",
"os": {
"platform": "Linux"
},
"ip": "127.0.0.1",
"name": "xxx",
"architecture": "amd64"
},
"event": {
"ingested": "2021-11-24T12:35:13.845309666Z",
"outcome": "success"
},
"process": {
"pid": 1,
"title": "/opt/java/openjdk/bin/java"
},
"processor": {
"name": "transaction",
"event": "transaction"
},
"url": {
"path": "xxx",
"scheme": "https",
"port": 443,
"domain": "xxx",
"full": "xxx"
},
"@timestamp": "2021-11-24T12:35:09.445Z",
"service": {
"node": {
"name": "xxx"
},
"name": "xxx",
"runtime": {
"name": "Java",
"version": "11.0.11"
},
"language": {
"name": "Java",
"version": "11.0.11"
}
}
}
Logs
2021-11-24 14:09:40,575 [main] INFO co.elastic.apm.agent.util.JmxUtils - Found JVM-specific OperatingSystemMXBean interface: com.sun.management.OperatingSystemMXBean
2021-11-24 14:09:40,651 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - Starting Elastic APM unknown as xxx (null) on Java 11.0.11 Runtime version: 11.0.11+9 VM version: 11.0.11+9 (AdoptOpenJDK) Linux 3.10.0-1160.42.2.el7.x86_64
2021-11-24 14:09:40,652 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - service_name: 'xxx' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,652 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - service_node_name: 'xxx' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,652 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - hostname: 'xxx' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,652 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - environment: 'production' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,652 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - transaction_max_spans: '2000' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,653 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - capture_body: 'ALL' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,654 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - trace_methods: '*ServiceImpl#*' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,654 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - config_file: '/usr/local/tomcat/conf/elastic-apm-agent.properties' (source: Java System Properties)
2021-11-24 14:09:40,654 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - server_url: 'http://127.0.0.1:8200' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,654 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - verify_server_cert: 'false' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,654 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - application_packages: 'xxx' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,654 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - enable_log_correlation: 'true' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:43,185 [main] INFO co.elastic.apm.agent.impl.ElasticApmTracer - Tracer switched to RUNNING state
2021-11-24 14:09:43,258 [elastic-apm-server-healthcheck] INFO co.elastic.apm.agent.report.ApmServerHealthChecker - Elastic APM server is available: { "build_date": "2021-09-16T02:05:39Z", "build_sha": "a183f675ecd03fca4a897cbe85fda3511bc3ca43", "version": "7.15.0"}
Thanks !
I'm investigating this today, I'll keep you posted when we have a PR to fix that.
Thanks for the update.
There are two issues:
host.hostname appears to be missing in 1.27.0, which might be related to this issue.
Can you provide an obfuscated version of the value you had with 1.26.0 and the provided value of hostname agent configuration ? Replacing all alphabetic characters with x should be fine.
Also, service.name appears to is missing in 1.27.0, thus I'll have to investigate this too.
The service.name is available in 1.27.0. No issue there, but yes the host.hostname is missing. The service.name was missing because I removed it from the sample doc.
You referred to hostname agent configuration, were you referring to service_node_name agent configuration?
Here is another sample doc for 1.27.0. I have kept most of the fields intact.
{
"container": {
"id": "9328fd947c04ddc0e17437f42408aeaa42bcf45da557d6688c65c60c2fe4986d"
},
"parent": {
"id": "676d853ebfccf1e1"
},
"agent": {
"name": "java",
"ephemeral_id": "e4ec45b1-ed97-4b6e-bc5a-5b51ec989022",
"version": "unknown"
},
"source": {
"ip": "1.1.1.1"
},
"observer": {
"hostname": "xxx",
"id": "c4e5c03a-3714-4b33-bf8e-e7dd958d7bbe",
"ephemeral_id": "17c9f1f3-5e4b-4898-8f14-d1bc715bc0de",
"type": "apm-server",
"version": "7.15.0",
"version_major": 7
},
"trace": {
"id": "e516efe8b31e9395f839f152a56e9fe0"
},
"ecs": {
"version": "1.11.0"
},
"host": {
"os": {
"platform": "Linux"
},
"ip": "127.0.0.1",
"name": "offsite.domain.com",
"architecture": "amd64"
},
"@version": "1",
"client": {
"geo": {
...
},
"ip": "1.1.1.1"
},
"event": {
"ingested": "2021-11-25T10:59:35.860366371Z",
"outcome": "success"
},
"user_agent": {
"original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36",
"os": {
"name": "Windows",
"version": "10",
"full": "Windows 10"
},
"name": "Chrome",
"device": {
"name": "Other"
},
"version": "96.0.4664.45"
},
"timestamp": {
"us": 1637837971346000
},
"process": {
"pid": 1,
"title": "/opt/java/openjdk/bin/java"
},
"processor": {
"name": "transaction",
"event": "transaction"
},
"url": {
"path": "/api/xxx",
"scheme": "https",
"port": 443,
"domain": "crm.domain.com",
"full": "https://crm.domain.com/api/xxx"
},
"@timestamp": "2021-11-25T10:59:31.346Z",
"service": {
"node": {
"name": "offsite.domain.com"
},
"environment": "production",
"framework": {
"name": "Spring Web MVC",
"version": "xxx"
},
"name": "somename",
"runtime": {
"name": "Java",
"version": "11.0.11"
},
"language": {
"name": "Java",
"version": "11.0.11"
}
},
"http": {
"request": {
"headers": {
...
},
"method": "GET",
"socket": {
"remote_address": "1.1.1.1"
}
},
"response": {
"headers": {
...
},
"status_code": 200,
"finished": true,
"headers_sent": true
},
"version": "1.0"
},
"user": {
"name": "xxx"
},
"transaction": {
"duration": {
"us": 71422
},
"result": "HTTP 2xx",
"name": "controller#getList",
"span_count": {
"dropped": 0,
"started": 1
},
"id": "3c3eaf0ea6c3cd5c",
"type": "request",
"sampled": true
}
}
1.26.0
{
"container": {
"id": "9a1f43470e789214ffae06ba0a29fe3193d6fe80f5e6079e201f8882ed3a18f2"
},
"parent": {
"id": "5a6a1b27673b681e"
},
"agent": {
"name": "java",
"ephemeral_id": "46e72620-6668-486f-af71-54c303ce3f12",
"version": "1.26.0"
},
"source": {
"ip": "1.1.1.1"
},
"observer": {
"hostname": "xxx",
"id": "c4e5c03a-3714-4b33-bf8e-e7dd958d7bbe",
"ephemeral_id": "17c9f1f3-5e4b-4898-8f14-d1bc715bc0de",
"type": "apm-server",
"version": "7.15.0",
"version_major": 7
},
"trace": {
"id": "72e2c582d5f4a7a29951d8631b618065"
},
"ecs": {
"version": "1.11.0"
},
"host": {
"hostname": "offsite.domain.com",
"os": {
"platform": "Linux"
},
"ip": "127.0.0.1",
"name": "offsite.domain.com",
"architecture": "amd64"
},
"@version": "1",
"client": {
"geo": {
...
},
"ip": "1.1.1.1"
},
"event": {
"ingested": "2021-11-24T11:04:50.303760566Z",
"outcome": "success"
},
"user_agent": {
"original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36",
"os": {
"name": "Windows",
"version": "10",
"full": "Windows 10"
},
"name": "Chrome",
"device": {
"name": "Other"
},
"version": "96.0.4664.45"
},
"timestamp": {
"us": 1637751879654000
},
"process": {
"pid": 1,
"title": "/opt/java/openjdk/bin/java"
},
"processor": {
"name": "transaction",
"event": "transaction"
},
"url": {
"path": "/api/xxx",
"scheme": "https",
"port": 443,
"domain": "crm.domain.com",
"full": "https://crm.domain.com/api/xxx"
},
"@timestamp": "2021-11-24T11:04:39.654Z",
"service": {
"node": {
"name": "offsite.domain.com"
},
"environment": "production",
"framework": {
"name": "Spring Web MVC",
"version": "xxx"
},
"name": "somename",
"runtime": {
"name": "Java",
"version": "11.0.11"
},
"language": {
"name": "Java",
"version": "11.0.11"
}
},
"http": {
"request": {
"headers": {
...
},
"method": "GET",
"socket": {
"encrypted": true,
"remote_address": "1.1.1.1"
}
},
"response": {
"headers": {
...
},
"status_code": 200,
"finished": true,
"headers_sent": true
},
"version": "1.0"
},
"user": {
"name": "xxx"
},
"transaction": {
"duration": {
"us": 43103
},
"result": "HTTP 2xx",
"name": "controller#getList",
"span_count": {
"dropped": 0,
"started": 11
},
"id": "d81af3d8a4aff5f5",
"type": "request",
"sampled": true
}
}
After further investigation it seems the issue might be in Kibana that might rely on an obsolete attribute that is no longer populated with Java agent 1.27.0 and later.
I have opened [APM] Hostname broken in APM / JVM Tab · Issue #119737 · elastic/kibana · GitHub.
Thanks for the quick response/fix!
This topic was automatically closed 20 days after the last reply. New replies are no longer allowed.
© 2020. All Rights Reserved - Elasticsearch
Apache, Apache Lucene, Apache Hadoop, Hadoop, HDFS and the yellow elephant logo are trademarks of the Apache Software Foundation in the United States and/or other countries.