Kibana version: 9.0.4
APM Server version: 9.0.4
APM Agent language and version: elastic otel dotnet - 1.1.0
Fresh install or upgraded from other version?
migrating from apm-agent to elastic otel dotnet (EDOT)
Is there anything special in your setup? For example, are you using the Logstash or Kafka outputs? Are you using a load balancer in front of the APM Servers? Have you changed index pattern, generated custom templates, changed agent configuration etc.
Description of the problem including expected versus actual behavior. Please include screenshots (if relevant):
We are using chef configuration tool to setup, following is the powershell script, that gets executed by the tool
Import-Module WebAdministration
$appPoolsTemp = Get-WebConfigurationProperty -Filter 'system.applicationHost/applicationPools/add' -Name 'name'
$appPools = @()
foreach ($appPool in $appPoolsTemp) {
$appPoolName = $appPool.Value
$applications = Get-WebConfigurationProperty -Filter "system.applicationHost/sites/site/application[@applicationPool='$appPoolName']" -Name 'path'
if ($applications) {
$appPools += $appPoolName
}
}
$edotServerUrl = '<%= @edot_server_url %>'
$edotSecretToken = '<%= @edot_secret_token %>'
$edotEnvironment = '<%= @edot_environment %>'
$edotLogLevel = '<%= @edot_log_level %>'
$edotLogFolder = '<%= @edot_log_folder %>'
$nodeHostName = '<%= @node_host_name %>'
$dotnetStartupHooks = '<%= @dotnet_startup_hooks %>'
$dotnetSharedStore = '<%= @dotnet_shared_store %>'
$dotnetAdditionalDeps = '<%= @dotnet_additional_deps %>'
$otelDotnetAutoHome = '<%= @otel_dotnet_auto_home %>'
$otelDotnetAutoPlugins = '<%= @otel_dotnet_auto_plugins %>'
$profiler32Path = '<%= @profiler_path_32 %>'
$profiler64Path = '<%= @profiler_path_64 %>'
$profilerGuid = '<%= @profiler_guid %>'
$serviceName = 'govaccess-vision-'
if ($edotEnvironment -eq "prod") {
$serviceName = $serviceName + $edotEnvironment + "-"
}
else {
$serviceName = $serviceName + "preprod-" + $edotEnvironment + "-"
}
$tempValue = $nodeHostName -split "-"
switch -Regex ($tempValue[1]) {
"cms" {
$tempValue = $tempValue[1] -split "cms"
$serviceName = $serviceName + "cms-" + $tempValue[1].Substring(0, $tempValue[1].Length - 1)
}
"mis" {
$tempValue = $tempValue[1] -split "mis"
$cluster = $tempValue[1].Substring(0, $tempValue[1].Length - 1)
if ([string]::IsNullOrEmpty($cluster)) {
$serviceName = $serviceName + "mis"
}
else {
$serviceName = $serviceName + "mis-" + $cluster
}
}
default {
exit 1
}
}
foreach ($appPool in $appPools) {
$appPoolConfig = Get-WebConfigurationProperty -Filter "system.applicationHost/applicationPools/add[@name='$appPool']" -Name 'managedRuntimeVersion'
$dotnetVersion = $appPoolConfig.Value
$environment = @{}
if ($dotnetVersion -eq "v4.0") {
$environment = @{
"COR_ENABLE_PROFILING" = "1";
"COR_PROFILER" = $profilerGuid;
"COR_PROFILER_PATH_32" = $profiler32Path;
"COR_PROFILER_PATH_64" = $profiler64Path;
}
}
else {
$environment = @{
"CORECLR_ENABLE_PROFILING" = "1";
"CORECLR_PROFILER" = $profilerGuid;
"CORECLR_PROFILER_PATH_32" = $profiler32Path;
"CORECLR_PROFILER_PATH_64" = $profiler64Path;
}
}
$environment["DOTNET_STARTUP_HOOKS"] = $dotnetStartupHooks
$environment["DOTNET_SHARED_STORE"] = $dotnetSharedStore
$environment["DOTNET_ADDITIONAL_DEPS"] = $dotnetAdditionalDeps
$environment["OTEL_DOTNET_AUTO_HOME"] = $otelDotnetAutoHome
$environment["OTEL_DOTNET_AUTO_PLUGINS"] = $otelDotnetAutoPlugins
$environment["OTEL_EXPORTER_OTLP_ENDPOINT"] = $edotServerUrl
$environment["OTEL_METRICS_EXPORTER"] = 'otlp'
$environment["OTEL_LOGS_EXPORTER"] = 'otlp'
$environment["OTEL_EXPORTER_OTLP_HEADERS"] = "Authorization=Bearer $edotSecretToken"
$environment["OTEL_RESOURCE_ATTRIBUTES"] = "deployment.environment=$edotEnvironment,service.name=$serviceName,service.version=1.0.0,host.name=$nodeHostName,service.node.name=$nodeHostName"
$environment["OTEL_LOG_LEVEL"] = $edotLogLevel
$environment["OTEL_DOTNET_AUTO_LOG_DIRECTORY"] = $edotLogFolder
& "$env:systemroot\system32\inetsrv\AppCmd.exe" set config -section:system.applicationHost/applicationPools "/-[name='$appPool'].environmentVariables"
<% if @is_edot_enabled %>
foreach ($key in $environment.Keys) {
& "$env:systemroot\system32\inetsrv\AppCmd.exe" set config -section:system.applicationHost/applicationPools "/+[name='$appPool'].environmentVariables.[name='$key',value='$($environment[$key])']"
}
<% end %>
}
net stop was /y
net start w3svc
iisreset
Errors in browser console:
We are getting random id’s in the dashboard with EDOT, while using apm-agent we are able to see the hostname.
Provide logs and/or server output (if relevant):
Log received by the elastic dashboard:
{
"@timestamp": [
"2025-08-26T13:34:16.256Z"
],
"agent.name": [
"opentelemetry/dotnet/elastic"
],
"agent.name.text": [
"opentelemetry/dotnet/elastic"
],
"agent.version": [
"1.1.0"
],
"data_stream.dataset": [
"apm.app.govaccess_vision_preprod_dev_cms_a"
],
"data_stream.namespace": [
"default"
],
"data_stream.type": [
"metrics"
],
"db.client.operation.duration": [
{
"values": [
0.003
],
"counts": [
12
]
}
],
"host.hostname": [
"mse2d-viscmsa1"
],
"host.id": [
"690033b3-2aff-42e2-b3da-fcde427d3cfa"
],
"host.name": [
"mse2d-viscmsa1"
],
"host.name.text": [
"mse2d-viscmsa1"
],
"host.os.full": [
"Microsoft Windows NT 10.0.14393.0"
],
"host.os.full.text": [
"Microsoft Windows NT 10.0.14393.0"
],
"host.os.name": [
"Windows Server 2016 Datacenter"
],
"host.os.name.text": [
"Windows Server 2016 Datacenter"
],
"host.os.platform": [
"windows"
],
"host.os.type": [
"windows"
],
"host.os.version": [
"6.3"
],
"labels.db_system": [
"mssql"
],
"labels.os_build_id": [
"14393"
],
"labels.process_runtime_description": [
".NET Framework 4.7.4135.0"
],
"labels.server_address": [
"mse2d-smi1.16356df7f545.database.windows.net"
],
"labels.service_node_name": [
"mse2d-viscmsa1"
],
"metricset.name": [
"app"
],
"metricset.name.text": [
"app"
],
"observer.hostname": [
"mse2m-logstash1"
],
"observer.type": [
"apm-server"
],
"observer.version": [
"9.0.4"
],
"process.pid": [
3776
],
"processor.event": [
"metric"
],
"service.environment": [
"dev"
],
"service.framework.name": [
"OpenTelemetry.Instrumentation.SqlClient"
],
"service.framework.name.text": [
"OpenTelemetry.Instrumentation.SqlClient"
],
"service.framework.version": [
"1.12.0-beta.2"
],
"service.language.name": [
"dotnet"
],
"service.language.name.text": [
"dotnet"
],
"service.name": [
"govaccess-vision-preprod-dev-cms-a"
],
"service.name.text": [
"govaccess-vision-preprod-dev-cms-a"
],
"service.node.name": [
"d5c1e78b-6c0b-424f-a41b-9b39212bd4d8"
],
"service.node.name.text": [
"d5c1e78b-6c0b-424f-a41b-9b39212bd4d8"
],
"service.runtime.name": [
".NET Framework"
],
"service.runtime.name.text": [
".NET Framework"
],
"service.runtime.version": [
"4.7.2"
],
"service.version": [
"1.0.0"
],
"user.name": [
"ga-d-vision-aa02"
],
"user.name.text": [
"ga-d-vision-aa02"
],
"_id": "hfeW5pgBb1dn8eJkKLzS",
"_index": ".ds-metrics-apm.app.govaccess_vision_preprod_dev_cms_a-default-2025.08.04-000001",
"_score": null
}
EDOT Debug logs
[2025-08-26T13:10:08.857264600Z] [2452|11340] [info] OpenTelemetry CLR Profiler 1.12.0 on Windows (amd64)
[2025-08-26T13:10:08.857396500Z] [2452|11340] [debug] ClassFactory::CreateInstance
[2025-08-26T13:10:08.857436000Z] [2452|11340] [debug] Initialize
[2025-08-26T13:10:08.857480700Z] [2452|11340] [debug] Environment variables:
[2025-08-26T13:10:08.857529000Z] [2452|11340] [debug] APP_POOL_ID=DefaultAppPool
[2025-08-26T13:10:08.857535000Z] [2452|11340] [debug] OTEL_METRICS_EXPORTER=otlp
[2025-08-26T13:10:08.857540300Z] [2452|11340] [debug] COR_PROFILER={918728DD-259F-4A6A-AC2B-B85E1B658318}
[2025-08-26T13:10:08.857549600Z] [2452|11340] [debug] OTEL_RESOURCE_ATTRIBUTES=deployment.environment=dev,service.name=govaccess-vision-preprod-dev-cms-a,service.version=1.0.0,host.name=mse2d-viscmsa1,service.node.name=mse2d-viscmsa1,service.instance.id=mse2d-viscmsa1
[2025-08-26T13:10:08.857558100Z] [2452|11340] [debug] COR_PROFILER_PATH_32=C:\Program Files\Elastic\EDOT\win-x86\OpenTelemetry.AutoInstrumentation.Native.dll
[2025-08-26T13:10:08.857565700Z] [2452|11340] [debug] OTEL_EXPORTER_OTLP_ENDPOINT=https://<URL>:8443
[2025-08-26T13:10:08.857572400Z] [2452|11340] [debug] OTEL_DOTNET_AUTO_HOME=C:\Program Files\Elastic\EDOT
[2025-08-26T13:10:08.857580100Z] [2452|11340] [debug] OTEL_DOTNET_AUTO_LOG_DIRECTORY=D:\ProgramData\elastic\edot\logs
[2025-08-26T13:10:08.857586900Z] [2452|11340] [debug] DOTNET_STARTUP_HOOKS=C:\Program Files\Elastic\EDOT\net\OpenTelemetry.AutoInstrumentation.StartupHook.dll
[2025-08-26T13:10:08.857592100Z] [2452|11340] [debug] OTEL_LOGS_EXPORTER=otlp
[2025-08-26T13:10:08.857599900Z] [2452|11340] [debug] OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer <KEY>
[2025-08-26T13:10:08.857605100Z] [2452|11340] [debug] OTEL_LOG_LEVEL=debug
[2025-08-26T13:10:08.857611200Z] [2452|11340] [debug] DOTNET_ADDITIONAL_DEPS=C:\Program Files\Elastic\EDOT\AdditionalDeps
[2025-08-26T13:10:08.857619000Z] [2452|11340] [debug] OTEL_DOTNET_AUTO_PLUGINS=Elastic.OpenTelemetry.AutoInstrumentationPlugin, Elastic.OpenTelemetry.AutoInstrumentation
[2025-08-26T13:10:08.857624900Z] [2452|11340] [debug] DOTNET_SHARED_STORE=C:\Program Files\Elastic\EDOT\store
[2025-08-26T13:10:08.857630100Z] [2452|11340] [debug] COR_ENABLE_PROFILING=1
[2025-08-26T13:10:08.857637800Z] [2452|11340] [debug] COR_PROFILER_PATH_64=C:\Program Files\Elastic\EDOT\win-x64\OpenTelemetry.AutoInstrumentation.Native.dll
[2025-08-26T13:10:08.857650600Z] [2452|11340] [debug] .NET Runtime: .NET Framework
[2025-08-26T13:10:08.857929400Z] [2452|11340] [info] JIT Inlining is enabled.
[2025-08-26T13:10:08.860857800Z] [2452|11340] [info] NGEN is disabled.
[2025-08-26T13:10:08.861033800Z] [2452|11340] [info] Profiler filepath: C:\Program Files\Elastic\EDOT\win-x64\OpenTelemetry.AutoInstrumentation.Native.dll
[2025-08-26T13:10:08.861060400Z] [2452|11340] [info] Profiler attached.
[2025-08-26T13:10:08.910714800Z] [2452|11340] [debug] AssemblyLoadStarted: 1598072754144
[2025-08-26T13:10:08.910750200Z] [2452|11340] [debug] ModuleLoadStarted: 140733591470112
[2025-08-26T13:10:08.914833600Z] [2452|11340] [debug] ModuleLoadFinished: 140733591470112 mscorlib AppDomain 140735202508704 [EE Shared Assembly Repository] | IsNGEN = false | IsDynamic = false | IsResource = false
[2025-08-26T13:10:08.919491200Z] [2452|1116] [info] Initializing ReJIT request thread.
[2025-08-26T13:10:08.921992200Z] [2452|11340] [info] COR library: mscorlib 4.0.0
[2025-08-26T13:10:08.929741200Z] [2452|11340] [debug] GenerateLoaderType: Setting the PInvoke native profiler library path to C:\Program Files\Elastic\EDOT\win-x64\OpenTelemetry.AutoInstrumentation.Native.dll
[2025-08-26T13:10:08.930930800Z] [2452|11340] [debug] AssemblyLoadFinished: 1598072754144 0
[2025-08-26T13:10:08.940826200Z] [2452|11340] [debug] AppDomainCreationStarted: 140735202510192
[2025-08-26T13:10:08.940835000Z] [2452|11340] [debug] AppDomainCreationFinished: 140735202510192 hrStatus=0x00000000
[2025-08-26T13:10:08.940837200Z] [2452|11340] [debug] AppDomainCreationStarted: 1598057814272
[2025-08-26T13:10:08.940840100Z] [2452|11340] [debug] AppDomainCreationFinished: 1598057814272 hrStatus=0x00000000
[2025-08-26T13:10:08.940841800Z] [2452|11340] [debug] AppDomainCreationStarted: 140735202508704
[2025-08-26T13:10:08.940844900Z] [2452|11340] [debug] AppDomainCreationFinished: 140735202508704 hrStatus=0x00000000
Server ApplicationHost config file
<add name="vision_live_two" autoStart="true" managedRuntimeVersion="v4.0"
managedPipelineMode="Integrated" startMode="AlwaysRunning">
<processModel identityType="SpecificUser" userName="obt.dc.gdi\ga-d-kansascity"
password="[enc:IISWASOnlyCngProvider:yikijODuwQiP1HurzBi5DzG1DDUDs5Woc7EeErb9+3jORnNJgELmK93Xr6kibwJ9W35QqIJ6FvZRED5GsFuUzrJF4uJ6SxsbzVLaQ2ERAuMpkmwTCi8S4EE5EJdz0Gzj:enc]"
loadUserProfile="true" idleTimeout="00:00:00" idleTimeoutAction="Suspend"
pingingEnabled="false" />
<recycling>
<periodicRestart time="00:00:00" />
</recycling>
<cpu limit="40000" action="Throttle" />
<environmentVariables>
<add name="OTEL_METRICS_EXPORTER" value="otlp" />
<add name="COR_PROFILER" value="{918728DD-259F-4A6A-AC2B-B85E1B658318}" />
<add name="OTEL_RESOURCE_ATTRIBUTES"
value="deployment.environment=dev,service.name=govaccess-vision-preprod-dev-cms-a,service.version=1.0.0,host.name=mse2d-viscmsa1,service.node.name=mse2d-viscmsa1,service.instance.id=mse2d-viscmsa1" />
<add name="COR_PROFILER_PATH_32"
value="C:\Program Files\Elastic\EDOT\win-x86\OpenTelemetry.AutoInstrumentation.Native.dll" />
<add name="OTEL_EXPORTER_OTLP_ENDPOINT"
value="https://<URL>:8443" />
<add name="OTEL_DOTNET_AUTO_HOME" value="C:\Program Files\Elastic\EDOT" />
<add name="OTEL_DOTNET_AUTO_LOG_DIRECTORY"
value="D:\ProgramData\elastic\edot\logs" />
<add name="DOTNET_STARTUP_HOOKS"
value="C:\Program Files\Elastic\EDOT\net\OpenTelemetry.AutoInstrumentation.StartupHook.dll" />
<add name="OTEL_LOGS_EXPORTER" value="otlp" />
<add name="OTEL_EXPORTER_OTLP_HEADERS"
value="Authorization=Bearer <KEY>" />
<add name="OTEL_LOG_LEVEL" value="debug" />
<add name="DOTNET_ADDITIONAL_DEPS"
value="C:\Program Files\Elastic\EDOT\AdditionalDeps" />
<add name="OTEL_DOTNET_AUTO_PLUGINS"
value="Elastic.OpenTelemetry.AutoInstrumentationPlugin, Elastic.OpenTelemetry.AutoInstrumentation" />
<add name="DOTNET_SHARED_STORE" value="C:\Program Files\Elastic\EDOT\store" />
<add name="COR_ENABLE_PROFILING" value="1" />
<add name="COR_PROFILER_PATH_64"
value="C:\Program Files\Elastic\EDOT\win-x64\OpenTelemetry.AutoInstrumentation.Native.dll" />
</environmentVariables>
</add>
Screenshots:
APM-Agent:
Our Findings:
When we observe the logs coming to dashboard we see service.node.name is being set by the script, but it is added in labels rather than actual place it needs to be present where id is present.
