"No JVMs were found" in JVM tab after APM upgrade from 1.38 to 1.43

Elastic Observability APM
,
21

JSON document with APM 1.43:

{
  "_index": "apm-7.16.3-transaction-000144",
  "_type": "_doc",
  "_id": "37",
  "_version": 1,
  "_score": 1,
  "_source": {
    "agent": {
      "name": "java",
      "ephemeral_id": "5",
      "version": "1.43.0"
    },
    "process": {
      "pid": 1758526,
      "title": "/usr/lib/jvm/temurin-17-jdk-amd64/bin/java",
      "ppid": 1
    },
    "processor": {
      "name": "transaction",
      "event": "transaction"
    },
    "cloud": {
      "availability_zone": "eu-west-3b",
      "instance": {
        "id": "i-instance"
      },
      "provider": "aws",
      "machine": {
        "type": "r5.large"
      },
      "service": {
        "name": "ec2"
      },
      "region": "eu-west-3",
      "account": {
        "id": "1"
      }
    },
    "observer": {
      "hostname": "host.com",
      "id": "b",
      "type": "apm-server",
      "ephemeral_id": "7",
      "version": "7.16.3",
      "version_major": 7
    },
    "trace": {
      "id": "b"
    },
    "@timestamp": "2023-11-20T15:37:50.819Z",
    "ecs": {
      "version": "1.12.0"
    },
    "service": {
      "node": {
        "name": "host.com"
      },
      "environment": "staging",
      "framework": {
        "name": "Servlet API"
      },
      "name": "load-test2",
      "runtime": {
        "name": "Java",
        "version": "17.0.9"
      },
      "language": {
        "name": "Java",
        "version": "17.0.9"
      },
      "version": "no-version"
    },
    "host": {
      "hostname": "hostname.com",
      "os": {
        "platform": "Linux"
      },
      "ip": "10.100.228.246",
      "name": "hostname.com",
      "architecture": "amd64"
    },
    "event": {
      "outcome": "success"
    },
    "transaction": {
      "result": "HTTP 3xx",
      "duration": {
        "us": 18625
      },
      "name": "GET unknown route",
      "span_count": {
        "dropped": 0,
        "started": 0
      },
      "id": "c",
      "type": "request"
    },
    "timestamp": {
      "us": 1700494670819007
    }
  },
  "fields": {
    "transaction.name.text": [
      "GET unknown route"
    ],
    "service.node.name": [
      "hostname.com"
    ],
    "host.hostname": [
      "hostname.com"
    ],
    "process.pid": [
      1758526
    ],
    "service.language.name": [
      "Java"
    ],
    "transaction.result": [
      "HTTP 3xx"
    ],
    "cloud.availability_zone": [
      "eu-west-3b"
    ],
    "transaction.id": [
      "c"
    ],
    "host.ip": [
      "127.0.0.1"
    ],
    "cloud.instance.id": [
      "i-instance"
    ],
    "trace.id": [
      "b"
    ],
    "transaction.span_count.dropped": [
      0
    ],
    "processor.event": [
      "transaction"
    ],
    "agent.name": [
      "java"
    ],
    "host.name": [
      "hostname.com"
    ],
    "event.outcome": [
      "success"
    ],
    "service.environment": [
      "staging"
    ],
    "service.name": [
      "load-test2"
    ],
    "cloud.region": [
      "eu-west-3"
    ],
    "service.framework.name": [
      "Servlet API"
    ],
    "process.ppid": [
      1
    ],
    "service.runtime.name": [
      "Java"
    ],
    "processor.name": [
      "transaction"
    ],
    "transaction.duration.us": [
      18625
    ],
    "service.runtime.version": [
      "17.0.9"
    ],
    "observer.version_major": [
      7
    ],
    "observer.hostname": [
      "hostname.net"
    ],
    "transaction.type": [
      "request"
    ],
    "host.architecture": [
      "amd64"
    ],
    "transaction.span_count.started": [
      0
    ],
    "observer.id": [
      "b"
    ],
    "timestamp.us": [
      1700494670819007
    ],
    "cloud.provider": [
      "aws"
    ],
    "cloud.machine.type": [
      "r5.large"
    ],
    "@timestamp": [
      "2023-11-20T15:37:50.819Z"
    ],
    "service.version": [
      "no-version"
    ],
    "observer.ephemeral_id": [
      "7"
    ],
    "cloud.service.name": [
      "ec2"
    ],
    "observer.version": [
      "7.16.3"
    ],
    "host.os.platform": [
      "Linux"
    ],
    "cloud.account.id": [
      "1"
    ],
    "ecs.version": [
      "1.12.0"
    ],
    "observer.type": [
      "apm-server"
    ],
    "service.language.version": [
      "17.0.9"
    ],
    "agent.ephemeral_id": [
      "5"
    ],
    "transaction.name": [
      "GET unknown route"
    ],
    "agent.version": [
      "1.43.0"
    ],
    "process.title": [
      "/usr/lib/jvm/temurin-17-jdk-amd64/bin/java"
    ]
  }
}

And with APM 1.38 :

{
  "_index": "apm-7.16.3-metric-000165",
  "_type": "_doc",
  "_id": "Z",
  "_version": 1,
  "_score": 1,
  "_source": {
    "system.process.cpu.total.norm.pct": 0.9602671118530884,
    "jvm.memory.heap.used": 1323138048,
    "agent": {
      "name": "java",
      "ephemeral_id": "1",
      "version": "1.38.0"
    },
    "jvm.memory.non_heap.committed": 240320512,
    "system.cpu.total.norm.pct": 0.9901234567901235,
    "cloud": {
      "availability_zone": "eu-west-3b",
      "instance": {
        "id": "i-instance"
      },
      "provider": "aws",
      "machine": {
        "type": "r5.large"
      },
      "service": {
        "name": "ec2"
      },
      "region": "eu-west-3",
      "account": {
        "id": "1"
      }
    },
    "system.process.cgroup.memory.mem.usage.bytes": 2620325888,
    "observer": {
      "hostname": "hostname.net",
      "id": "b",
      "ephemeral_id": "7",
      "type": "apm-server",
      "version": "7.16.3",
      "version_major": 7
    },
    "system.process.memory.size": 14456860672,
    "ecs": {
      "version": "1.12.0"
    },
    "system.memory.actual.free": 12416540672,
    "jvm.memory.heap.committed": 2147483648,
    "host": {
      "hostname": "hostname.com",
      "os": {
        "platform": "Linux"
      },
      "ip": "10.100.228.246",
      "name": "hostname.com",
      "architecture": "amd64"
    },
    "system.memory.total": 16479371264,
    "jvm.thread.count": 70,
    "jvm.memory.non_heap.used": 237526208,
    "process": {
      "pid": 1758950,
      "title": "/usr/lib/jvm/temurin-17-jdk-amd64/bin/java",
      "ppid": 1
    },
    "jvm.memory.heap.max": 11114905600,
    "jvm.gc.alloc": 8940628232,
    "processor": {
      "name": "metric",
      "event": "metric"
    },
    "metricset.name": "app",
    "@timestamp": "2023-11-20T15:44:24.998Z",
    "service": {
      "node": {
        "name": "hostname.com"
      },
      "environment": "staging",
      "name": "load-test2",
      "runtime": {
        "name": "Java",
        "version": "17.0.9"
      },
      "language": {
        "name": "Java",
        "version": "17.0.9"
      },
      "version": "no-version"
    },
    "jvm.memory.non_heap.max": -1
  },
  "fields": {
    "system.process.cpu.total.norm.pct": [
      0.96
    ],
    "host.hostname": [
      "hostname.com"
    ],
    "process.pid": [
      1758950
    ],
    "service.language.name": [
      "Java"
    ],
    "cloud.availability_zone": [
      "eu-west-3b"
    ],
    "system.process.memory.size": [
      14456860672
    ],
    "processor.event": [
      "metric"
    ],
    "system.memory.actual.free": [
      12416540672
    ],
    "agent.name": [
      "java"
    ],
    "system.memory.total": [
      16479371264
    ],
    "host.name": [
      "hostname.com"
    ],
    "cloud.region": [
      "eu-west-3"
    ],
    "process.ppid": [
      1
    ],
    "processor.name": [
      "metric"
    ],
    "service.runtime.version": [
      "17.0.9"
    ],
    "host.architecture": [
      "amd64"
    ],
    "cloud.provider": [
      "aws"
    ],
    "observer.id": [
      "b"
    ],
    "cloud.machine.type": [
      "r5.large"
    ],
    "cloud.service.name": [
      "ec2"
    ],
    "observer.version": [
      "7.16.3"
    ],
    "observer.type": [
      "apm-server"
    ],
    "ecs.version": [
      "1.12.0"
    ],
    "jvm.memory.non_heap.max": [
      -1
    ],
    "agent.version": [
      "1.38.0"
    ],
    "process.title": [
      "/usr/lib/jvm/temurin-17-jdk-amd64/bin/java"
    ],
    "jvm.memory.heap.used": [
      1323138050
    ],
    "jvm.memory.non_heap.committed": [
      240320512
    ],
    "service.node.name": [
      "hostname.com"
    ],
    "system.cpu.total.norm.pct": [
      0.99
    ],
    "host.ip": [
      "127.0.0.1"
    ],
    "cloud.instance.id": [
      "i-instance"
    ],
    "system.process.cgroup.memory.mem.usage.bytes": [
      2620325888
    ],
    "jvm.memory.heap.committed": [
      2147483650
    ],
    "jvm.thread.count": [
      70
    ],
    "service.environment": [
      "staging"
    ],
    "jvm.memory.non_heap.used": [
      237526208
    ],
    "jvm.memory.heap.max": [
      11114905600
    ],
    "service.name": [
      "load-test2"
    ],
    "jvm.gc.alloc": [
      8940628000
    ],
    "service.runtime.name": [
      "Java"
    ],
    "observer.version_major": [
      7
    ],
    "observer.hostname": [
      "hostname.net"
    ],
    "metricset.name": [
      "app"
    ],
    "@timestamp": [
      "2023-11-20T15:44:24.998Z"
    ],
    "service.version": [
      "no-version"
    ],
    "observer.ephemeral_id": [
      "7"
    ],
    "cloud.account.id": [
      "669976527417"
    ],
    "host.os.platform": [
      "Linux"
    ],
    "service.language.version": [
      "17.0.9"
    ],
    "agent.ephemeral_id": [
      "1"
    ]
  }
}
22

The 1.38 is from the metrics index but the 1.43 is from transaction, seems odd. Are there no metric docs at all?

23

To confirm what Sylvain saw, I created a 7.16.3 server, using a simple hello world app and ran the 2 different agent versions with everything else the same (except service name to distinguish) and I see JVM metrics for both versions. So this is not a simple change in the agent, it's some interaction that's not at all obvious yet

24

It's worth doing the same test in your environment, a simple Hello world app with a long sleep, leave it running 5 mins under each version - it will generate metrics and we have a bit more evidence to consider

25

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.