Java APM agent version `unknown` and Host `N/A` due to missing `host.hostname`

Elastic Observability APM
1

Kibana version: 7.15.0

Elasticsearch version: 7.15.0

APM Server version: 7.15.0

APM Agent language and version: Java 1.27.0 (Java 8)

Description of the problem including expected versus actual behavior. Please include screenshots (if relevant):
agent.version has unknown value after upgrading to Elastic APM Java Agent 1.27.0.

image

2

Hi @hendry.lim ,

  • Do you see that from Kibana only ?
  • Can you send a few documents that appear to be sent by the recently upgraded agents ?
  • Also, Can you share the agent logs please ? See procedure in documentation for details.
3

I also noticed that host.hostname is missing from the document sent by agent v1.27.0, while we had that in 1.26.0. This affects the APM UI which is displaying Host N/A on the JVMs tab.

image

image

Sample doc from another app running Java 11 (some fields were removed) - 1.27.0

{
  "container": {
    "id": "9328fd947c04ddc0e17437f42408aeaa42bcf45da557d6688c65c60c2fe4986d"
  },
  "agent": {
    "name": "java",
    "ephemeral_id": "e4ec45b1-ed97-4b6e-bc5a-5b51ec989022",
    "version": "unknown"
  },
  "process": {
    "pid": 1,
    "title": "/opt/java/openjdk/bin/java"
  },
  "processor": {
    "name": "transaction",
    "event": "transaction"
  },
  "url": {
    "path": "/index.html",
    "scheme": "https",
    "port": 443,
    "domain": "xxx",
    "full": "xxx"
  },
  "observer": {
    "hostname": "xxx",
    "id": "c4e5c03a-3714-4b33-bf8e-e7dd958d7bbe",
    "ephemeral_id": "17c9f1f3-5e4b-4898-8f14-d1bc715bc0de",
    "type": "apm-server",
    "version": "7.15.0",
    "version_major": 7
  },
  "trace": {
    "id": "b52dba2843c6af3abf08e9a3a60ffaa1"
  },
  "@timestamp": "2021-11-24T14:13:22.032Z",
  "ecs": {
    "version": "1.11.0"
  },
  "service": {
    "node": {
      "name": "xxx"
    },
    "runtime": {
      "name": "Java",
      "version": "11.0.11"
    },
    "language": {
      "name": "Java",
      "version": "11.0.11"
    }
  },
  "host": {
    "os": {
      "platform": "Linux"
    },
    "ip": "127.0.0.1",
    "name": "xxx",
    "architecture": "amd64"
  },
  "event": {
    "ingested": "2021-11-24T14:13:23.433948504Z",
    "outcome": "success"
  },
  "timestamp": {
    "us": 1637763202032015
  }
}

Comparison with 1.26.0

{
  "container": {
    "id": "9a1f43470e789214ffae06ba0a29fe3193d6fe80f5e6079e201f8882ed3a18f2"
  },
  "agent": {
    "name": "java",
    "ephemeral_id": "46e72620-6668-486f-af71-54c303ce3f12",
    "version": "1.26.0"
  },
  "observer": {
    "hostname": "xxx",
    "id": "c4e5c03a-3714-4b33-bf8e-e7dd958d7bbe",
    "ephemeral_id": "17c9f1f3-5e4b-4898-8f14-d1bc715bc0de",
    "type": "apm-server",
    "version": "7.15.0",
    "version_major": 7
  },
  "trace": {
    "id": "77d3c6093d678e43ad14bca138a9cffb"
  },
  "ecs": {
    "version": "1.11.0"
  },
  "host": {
    "hostname": "xxx",
    "os": {
      "platform": "Linux"
    },
    "ip": "127.0.0.1",
    "name": "xxx",
    "architecture": "amd64"
  },
  "event": {
    "ingested": "2021-11-24T12:35:13.845309666Z",
    "outcome": "success"
  },
  "process": {
    "pid": 1,
    "title": "/opt/java/openjdk/bin/java"
  },
  "processor": {
    "name": "transaction",
    "event": "transaction"
  },
  "url": {
    "path": "xxx",
    "scheme": "https",
    "port": 443,
    "domain": "xxx",
    "full": "xxx"
  },
  "@timestamp": "2021-11-24T12:35:09.445Z",
  "service": {
    "node": {
      "name": "xxx"
    },
    "name": "xxx",
    "runtime": {
      "name": "Java",
      "version": "11.0.11"
    },
    "language": {
      "name": "Java",
      "version": "11.0.11"
    }
  }
}

Logs

2021-11-24 14:09:40,575 [main] INFO  co.elastic.apm.agent.util.JmxUtils - Found JVM-specific OperatingSystemMXBean interface: com.sun.management.OperatingSystemMXBean
2021-11-24 14:09:40,651 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - Starting Elastic APM unknown as xxx (null) on Java 11.0.11 Runtime version: 11.0.11+9 VM version: 11.0.11+9 (AdoptOpenJDK) Linux 3.10.0-1160.42.2.el7.x86_64
2021-11-24 14:09:40,652 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - service_name: 'xxx' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,652 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - service_node_name: 'xxx' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,652 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - hostname: 'xxx' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,652 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - environment: 'production' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,652 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - transaction_max_spans: '2000' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,653 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - capture_body: 'ALL' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,654 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - trace_methods: '*ServiceImpl#*' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,654 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - config_file: '/usr/local/tomcat/conf/elastic-apm-agent.properties' (source: Java System Properties)
2021-11-24 14:09:40,654 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - server_url: 'http://127.0.0.1:8200' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,654 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - verify_server_cert: 'false' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,654 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - application_packages: 'xxx' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:40,654 [main] INFO  co.elastic.apm.agent.configuration.StartupInfo - enable_log_correlation: 'true' (source: /usr/local/tomcat/conf/elastic-apm-agent.properties)
2021-11-24 14:09:43,185 [main] INFO  co.elastic.apm.agent.impl.ElasticApmTracer - Tracer switched to RUNNING state
2021-11-24 14:09:43,258 [elastic-apm-server-healthcheck] INFO  co.elastic.apm.agent.report.ApmServerHealthChecker - Elastic APM server is available: {  "build_date": "2021-09-16T02:05:39Z",  "build_sha": "a183f675ecd03fca4a897cbe85fda3511bc3ca43",  "version": "7.15.0"}
4

Thanks !

I'm investigating this today, I'll keep you posted when we have a PR to fix that.

5

Thanks for the update.

6

There are two issues:

  • I've just opened a PR for the agent version
  • some fields are missing in the provided documents by doing a diff (details below)

host.hostname appears to be missing in 1.27.0, which might be related to this issue.
Can you provide an obfuscated version of the value you had with 1.26.0 and the provided value of hostname agent configuration ? Replacing all alphabetic characters with x should be fine.

Also, service.name appears to is missing in 1.27.0, thus I'll have to investigate this too.

7

The service.name is available in 1.27.0. No issue there, but yes the host.hostname is missing. The service.name was missing because I removed it from the sample doc.

You referred to hostname agent configuration, were you referring to service_node_name agent configuration?

Here is another sample doc for 1.27.0. I have kept most of the fields intact.

{
  "container": {
    "id": "9328fd947c04ddc0e17437f42408aeaa42bcf45da557d6688c65c60c2fe4986d"
  },
  "parent": {
    "id": "676d853ebfccf1e1"
  },
  "agent": {
    "name": "java",
    "ephemeral_id": "e4ec45b1-ed97-4b6e-bc5a-5b51ec989022",
    "version": "unknown"
  },
  "source": {
    "ip": "1.1.1.1"
  },
  "observer": {
    "hostname": "xxx",
    "id": "c4e5c03a-3714-4b33-bf8e-e7dd958d7bbe",
    "ephemeral_id": "17c9f1f3-5e4b-4898-8f14-d1bc715bc0de",
    "type": "apm-server",
    "version": "7.15.0",
    "version_major": 7
  },
  "trace": {
    "id": "e516efe8b31e9395f839f152a56e9fe0"
  },
  "ecs": {
    "version": "1.11.0"
  },
  "host": {
    "os": {
      "platform": "Linux"
    },
    "ip": "127.0.0.1",
    "name": "offsite.domain.com",
    "architecture": "amd64"
  },
  "@version": "1",
  "client": {
    "geo": {
      ...
    },
    "ip": "1.1.1.1"
  },
  "event": {
    "ingested": "2021-11-25T10:59:35.860366371Z",
    "outcome": "success"
  },
  "user_agent": {
    "original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36",
    "os": {
      "name": "Windows",
      "version": "10",
      "full": "Windows 10"
    },
    "name": "Chrome",
    "device": {
      "name": "Other"
    },
    "version": "96.0.4664.45"
  },
  "timestamp": {
    "us": 1637837971346000
  },
  "process": {
    "pid": 1,
    "title": "/opt/java/openjdk/bin/java"
  },
  "processor": {
    "name": "transaction",
    "event": "transaction"
  },
  "url": {
    "path": "/api/xxx",
    "scheme": "https",
    "port": 443,
    "domain": "crm.domain.com",
    "full": "https://crm.domain.com/api/xxx"
  },
  "@timestamp": "2021-11-25T10:59:31.346Z",
  "service": {
    "node": {
      "name": "offsite.domain.com"
    },
    "environment": "production",
    "framework": {
      "name": "Spring Web MVC",
      "version": "xxx"
    },
    "name": "somename",
    "runtime": {
      "name": "Java",
      "version": "11.0.11"
    },
    "language": {
      "name": "Java",
      "version": "11.0.11"
    }
  },
  "http": {
    "request": {
      "headers": {
        ...
      },
      "method": "GET",
      "socket": {
        "remote_address": "1.1.1.1"
      }
    },
    "response": {
      "headers": {
        ...
      },
      "status_code": 200,
      "finished": true,
      "headers_sent": true
    },
    "version": "1.0"
  },
  "user": {
    "name": "xxx"
  },
  "transaction": {
    "duration": {
      "us": 71422
    },
    "result": "HTTP 2xx",
    "name": "controller#getList",
    "span_count": {
      "dropped": 0,
      "started": 1
    },
    "id": "3c3eaf0ea6c3cd5c",
    "type": "request",
    "sampled": true
  }
}

1.26.0

{
  "container": {
    "id": "9a1f43470e789214ffae06ba0a29fe3193d6fe80f5e6079e201f8882ed3a18f2"
  },
  "parent": {
    "id": "5a6a1b27673b681e"
  },
  "agent": {
    "name": "java",
    "ephemeral_id": "46e72620-6668-486f-af71-54c303ce3f12",
    "version": "1.26.0"
  },
  "source": {
    "ip": "1.1.1.1"
  },
  "observer": {
    "hostname": "xxx",
    "id": "c4e5c03a-3714-4b33-bf8e-e7dd958d7bbe",
    "ephemeral_id": "17c9f1f3-5e4b-4898-8f14-d1bc715bc0de",
    "type": "apm-server",
    "version": "7.15.0",
    "version_major": 7
  },
  "trace": {
    "id": "72e2c582d5f4a7a29951d8631b618065"
  },
  "ecs": {
    "version": "1.11.0"
  },
  "host": {
    "hostname": "offsite.domain.com",
    "os": {
      "platform": "Linux"
    },
    "ip": "127.0.0.1",
    "name": "offsite.domain.com",
    "architecture": "amd64"
  },
  "@version": "1",
  "client": {
    "geo": {
      ...
    },
    "ip": "1.1.1.1"
  },
  "event": {
    "ingested": "2021-11-24T11:04:50.303760566Z",
    "outcome": "success"
  },
  "user_agent": {
    "original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36",
    "os": {
      "name": "Windows",
      "version": "10",
      "full": "Windows 10"
    },
    "name": "Chrome",
    "device": {
      "name": "Other"
    },
    "version": "96.0.4664.45"
  },
  "timestamp": {
    "us": 1637751879654000
  },
  "process": {
    "pid": 1,
    "title": "/opt/java/openjdk/bin/java"
  },
  "processor": {
    "name": "transaction",
    "event": "transaction"
  },
  "url": {
    "path": "/api/xxx",
    "scheme": "https",
    "port": 443,
    "domain": "crm.domain.com",
    "full": "https://crm.domain.com/api/xxx"
  },
  "@timestamp": "2021-11-24T11:04:39.654Z",
  "service": {
    "node": {
      "name": "offsite.domain.com"
    },
    "environment": "production",
    "framework": {
      "name": "Spring Web MVC",
      "version": "xxx"
    },
    "name": "somename",
    "runtime": {
      "name": "Java",
      "version": "11.0.11"
    },
    "language": {
      "name": "Java",
      "version": "11.0.11"
    }
  },
  "http": {
    "request": {
      "headers": {
        ...
      },
      "method": "GET",
      "socket": {
        "encrypted": true,
        "remote_address": "1.1.1.1"
      }
    },
    "response": {
      "headers": {
        ...
      },
      "status_code": 200,
      "finished": true,
      "headers_sent": true
    },
    "version": "1.0"
  },
  "user": {
    "name": "xxx"
  },
  "transaction": {
    "duration": {
      "us": 43103
    },
    "result": "HTTP 2xx",
    "name": "controller#getList",
    "span_count": {
      "dropped": 0,
      "started": 11
    },
    "id": "d81af3d8a4aff5f5",
    "type": "request",
    "sampled": true
  }
}
8

After further investigation it seems the issue might be in Kibana that might rely on an obsolete attribute that is no longer populated with Java agent 1.27.0 and later.

I have opened [APM] Hostname broken in APM / JVM Tab · Issue #119737 · elastic/kibana · GitHub.

9

Thanks for the quick response/fix!

10

This topic was automatically closed 20 days after the last reply. New replies are no longer allowed.