Kibana i snot showing data when I go to Observability->APM->Services
It doesn't show data neither in services, nor traces, nor dependencies
Kibana version: 8.15.3
Elasticsearch version: 8.15.3
APM Server version: 8.15.3 APM
Agent language and version: go go.elastic.co/apm/v2 v2.6.2
As soon as I press the APM option, it shows is if it was querying for a second or less and then the initial image as if APM was not integrated.
APM integration was added and I can see data in Discover. See the image.
I have debug logs to share (attached)
You will see this log:
[2025-03-12T00:19:06.298+00:00][INFO ][plugins.apm] create static data view apm_static_data_view_id_default
This is the name of the APM view
Pasting part of the logs around this line:
[2025-03-12T00:19:06.261+00:00][DEBUG][elasticsearch.query.data] 200 - 227.0B
POST /.kibana_task_manager/_update_by_query?ignore_unavailable=true&refresh=true
{"query":{"bool":{"must":[{"term":{"type":"task"}},{"bool":{"must":[{"bool":{"must":[{"term":{"task.enabled":true}}]}},{"bool":{"should":[{"bool":{"must":[{"term":{"task.status":"idle"}},{"range":{"task.runAt":{"lte":"now"}}}]}},{"bool":{"must":[{"bool":{"should":[{"term":{"task.status":"running"}},{"term":{"task.status":"claiming"}}]}},{"range":{"task.retryAt":{"lte":"now"}}}]}}]}}],"filter":[{"bool":{"must_not":[{"bool":{"should":[{"term":{"task.status":"running"}},{"term":{"task.status":"claiming"}}],"minimum_should_match":1,"must":{"range":{"task.retryAt":{"gt":"now"}}}}}]}}]}}]}},"script":{"source":"\n if (params.claimableTaskTypes.contains(ctx._source.task.taskType)) {\n if(ctx._source.task.retryAt != null && ZonedDateTime.parse(ctx._source.task.retryAt).toInstant().toEpochMilli() < params.now) {\n ctx._source.task.scheduledAt=ctx._source.task.retryAt;\n } else {\n ctx._source.task.scheduledAt=ctx._source.task.runAt;\n }\n ctx._source.task.status = \"claiming\"; ctx._source.task.ownerId=params.fieldUpdates.ownerId; ctx._source.task.retryAt=params.fieldUpdates.retryAt;\n } else if (params.unusedTaskTypes.contains(ctx._source.task.taskType)) {\n ctx._source.task.status = \"unrecognized\";\n } else {\n ctx.op = \"noop\";\n }","lang":"painless","params":{"now":1741738746246,"fieldUpdates":{"ownerId":"kibana:72180710-bd9c-4f02-9f5c-b928fa3781c6","retryAt":"2025-03-12T00:19:36.207Z"},"claimableTaskTypes":["session_cleanup","actions_telemetry","observabilityAIAssistant:indexQueuedDocumentsTaskType","ad_hoc_run-backfill","alerting_telemetry","alerts_invalidate_api_keys","alerting_health_check","dashboard_telemetry","alerting:transform_health","actions:.email","actions:.index","actions:.pagerduty","actions:.swimlane","actions:.server-log","actions:.slack","actions:.slack_api","actions:.webhook","actions:.cases-webhook","actions:.xmatters","actions:.servicenow","actions:.servicenow-sir","actions:.servicenow-itom","actions:.jira","actions:.teams","actions:.torq","actions:.opsgenie","actions:.tines","actions:.gen-ai","actions:.bedrock","actions:.gemini","actions:.d3security","actions:.resilient","actions:.sentinelone","actions:.crowdstrike","alerting:.index-threshold","alerting:.geo-containment","alerting:.es-query","cases-telemetry-task","actions:.cases","ML:saved-objects-sync","alerting:xpack.ml.anomaly_detection_alert","alerting:xpack.ml.anomaly_detection_jobs_health","Fleet-Usage-Sender","Fleet-Usage-Logger","Fleet-Metrics-Task","fleet:reassign_action:retry","fleet:unenroll_action:retry","fleet:upgrade_action:retry","fleet:update_agent_tags:retry","fleet:request_diagnostics:retry","fleet:check-deleted-files-task","osquery:telemetry-packs","osquery:telemetry-saved-queries","osquery:telemetry-configs","cloud_security_posture-stats_task","risk_engine:risk_scoring","alerting:siem.eqlRule","alerting:siem.esqlRule","alerting:siem.savedQueryRule","alerting:siem.indicatorRule","alerting:siem.mlRule","alerting:siem.queryRule","alerting:siem.thresholdRule","alerting:siem.newTermsRule","alerting:siem.notifications","endpoint:user-artifact-packager","endpoint:complete-external-response-actions","security:endpoint-diagnostics","security:endpoint-meta-telemetry","security:telemetry-lists","security:telemetry-detection-rules","security:telemetry-prebuilt-rule-alerts","security:telemetry-timelines","security:telemetry-diagnostic-timelines","security:telemetry-configuration","security:telemetry-filterlist-artifact","endpoint:metadata-check-transforms-task","alerting:observability.rules.custom_threshold","alerting:xpack.uptime.alerts.monitorStatus","alerting:xpack.uptime.alerts.tlsCertificate","alerting:xpack.uptime.alerts.durationAnomaly","alerting:xpack.uptime.alerts.tls","UPTIME:SyntheticsService:Sync-Saved-Monitor-Objects","Synthetics:Clean-Up-Package-Policies","alerting:xpack.synthetics.alerts.monitorStatus","alerting:xpack.synthetics.alerts.tls","alerting:slo.rules.burnRate","SLO:ORPHAN_SUMMARIES-CLEANUP-TASK","actions:.observability-ai-assistant","alerting:logs.alert.document.count","alerting:metrics.alert.inventory.threshold","alerting:metrics.alert.threshold","alerting:monitoring_alert_cluster_health","alerting:monitoring_alert_license_expiration","alerting:monitoring_alert_cpu_usage","alerting:monitoring_alert_missing_monitoring_data","alerting:monitoring_alert_disk_usage","alerting:monitoring_alert_thread_pool_search_rejections","alerting:monitoring_alert_thread_pool_write_rejections","alerting:monitoring_alert_jvm_memory_usage","alerting:monitoring_alert_nodes_changed","alerting:monitoring_alert_logstash_version_mismatch","alerting:monitoring_alert_kibana_version_mismatch","alerting:monitoring_alert_elasticsearch_version_mismatch","alerting:monitoring_ccr_read_exceptions","alerting:monitoring_shard_size","apm-telemetry-task","alerting:apm.transaction_duration","alerting:apm.anomaly","alerting:apm.error_rate","alerting:apm.transaction_error_rate","apm-source-map-migration-task"],"skippedTaskTypes":["report:execute"],"unusedTaskTypes":["sampleTaskRemovedType","alerting:siem.signals","search_sessions_monitor","search_sessions_cleanup","search_sessions_expire","cleanup_failed_action_executions","reports:monitor"],"taskMaxAttempts":{"session_cleanup":3,"actions_telemetry":3,"observabilityAIAssistant:indexQueuedDocumentsTaskType":2,"ad_hoc_run-backfill":3,"alerting_telemetry":3,"alerts_invalidate_api_keys":3,"alerting_health_check":3,"dashboard_telemetry":3,"alerting:transform_health":3,"actions:.email":3,"actions:.index":3,"actions:.pagerduty":3,"actions:.swimlane":3,"actions:.server-log":3,"actions:.slack":3,"actions:.slack_api":3,"actions:.webhook":3,"actions:.cases-webhook":3,"actions:.xmatters":3,"actions:.servicenow":3,"actions:.servicenow-sir":3,"actions:.servicenow-itom":3,"actions:.jira":3,"actions:.teams":3,"actions:.torq":3,"actions:.opsgenie":3,"actions:.tines":3,"actions:.gen-ai":3,"actions:.bedrock":3,"actions:.gemini":3,"actions:.d3security":3,"actions:.resilient":3,"actions:.sentinelone":3,"actions:.crowdstrike":3,"alerting:.index-threshold":3,"alerting:.geo-containment":3,"alerting:.es-query":3,"cases-telemetry-task":3,"actions:.cases":3,"ML:saved-objects-sync":3,"alerting:xpack.ml.anomaly_detection_alert":3,"alerting:xpack.ml.anomaly_detection_jobs_health":3,"Fleet-Usage-Sender":1,"Fleet-Usage-Logger":1,"Fleet-Metrics-Task":1,"fleet:reassign_action:retry":1,"fleet:unenroll_action:retry":1,"fleet:upgrade_action:retry":1,"fleet:update_agent_tags:retry":1,"fleet:request_diagnostics:retry":1,"fleet:check-deleted-files-task":3,"osquery:telemetry-packs":3,"osquery:telemetry-saved-queries":3,"osquery:telemetry-configs":3,"cloud_security_posture-stats_task":3,"risk_engine:risk_scoring":3,"alerting:siem.eqlRule":3,"alerting:siem.esqlRule":3,"alerting:siem.savedQueryRule":3,"alerting:siem.indicatorRule":3,"alerting:siem.mlRule":3,"alerting:siem.queryRule":3,"alerting:siem.thresholdRule":3,"alerting:siem.newTermsRule":3,"alerting:siem.notifications":3,"endpoint:user-artifact-packager":3,"endpoint:complete-external-response-actions":3,"security:endpoint-diagnostics":3,"security:endpoint-meta-telemetry":3,"security:telemetry-lists":3,"security:telemetry-detection-rules":3,"security:telemetry-prebuilt-rule-alerts":3,"security:telemetry-timelines":3,"security:telemetry-diagnostic-timelines":3,"security:telemetry-configuration":3,"security:telemetry-filterlist-artifact":3,"endpoint:metadata-check-transforms-task":3,"alerting:observability.rules.custom_threshold":3,"alerting:xpack.uptime.alerts.monitorStatus":3,"alerting:xpack.uptime.alerts.tlsCertificate":3,"alerting:xpack.uptime.alerts.durationAnomaly":3,"alerting:xpack.uptime.alerts.tls":3,"UPTIME:SyntheticsService:Sync-Saved-Monitor-Objects":3,"Synthetics:Clean-Up-Package-Policies":3,"alerting:xpack.synthetics.alerts.monitorStatus":3,"alerting:xpack.synthetics.alerts.tls":3,"alerting:slo.rules.burnRate":3,"SLO:ORPHAN_SUMMARIES-CLEANUP-TASK":1,"actions:.observability-ai-assistant":3,"alerting:logs.alert.document.count":3,"alerting:metrics.alert.inventory.threshold":3,"alerting:metrics.alert.threshold":3,"alerting:monitoring_alert_cluster_health":3,"alerting:monitoring_alert_license_expiration":3,"alerting:monitoring_alert_cpu_usage":3,"alerting:monitoring_alert_missing_monitoring_data":3,"alerting:monitoring_alert_disk_usage":3,"alerting:monitoring_alert_thread_pool_search_rejections":3,"alerting:monitoring_alert_thread_pool_write_rejections":3,"alerting:monitoring_alert_jvm_memory_usage":3,"alerting:monitoring_alert_nodes_changed":3,"alerting:monitoring_alert_logstash_version_mismatch":3,"alerting:monitoring_alert_kibana_version_mismatch":3,"alerting:monitoring_alert_elasticsearch_version_mismatch":3,"alerting:monitoring_ccr_read_exceptions":3,"alerting:monitoring_shard_size":3,"apm-telemetry-task":3,"alerting:apm.transaction_duration":3,"alerting:apm.anomaly":3,"alerting:apm.error_rate":3,"alerting:apm.transaction_error_rate":3,"apm-source-map-migration-task":5}}},"sort":[{"_script":{"type":"number","order":"desc","script":{"lang":"painless","source":"\n String taskType = doc['task.taskType'].value;\n if (params.priority_map.containsKey(taskType)) {\n return params.priority_map[taskType];\n } else {\n return 50;\n }\n ","params":{"priority_map":{"ad_hoc_run-backfill":1}}}}},{"_script":{"type":"number","order":"asc","script":{"lang":"painless","source":"\nif (doc['task.retryAt'].size()!=0) {\n return doc['task.retryAt'].value.toInstant().toEpochMilli();\n}\nif (doc['task.runAt'].size()!=0) {\n return doc['task.runAt'].value.toInstant().toEpochMilli();\n}\n "}}}],"max_docs":10,"conflicts":"proceed"}
[2025-03-12T00:19:06.262+00:00][DEBUG][plugins.taskManager.metrics-subscribe-debugger] subscribed metrics {"last_update":"2025-03-12T00:19:06.262Z","metrics":{"task_claim":{"timestamp":"2025-03-12T00:19:03.215Z","value":{"success":15,"total":15,"total_errors":0,"duration":{"counts":[13,2],"values":[100,200]},"duration_values":[15,13,119,14,108,12,14,13,15,14,15,13,14,15,16]}},"task_run":{"timestamp":"2025-03-12T00:19:06.262Z","value":{"overall":{"success":3,"not_timed_out":3,"total":3,"total_errors":0,"user_errors":0,"framework_errors":0,"delay":{"counts":[3],"values":[10]},"delay_values":[3,3,3]},"by_type":{"alerting:observability__rules__custom_threshold":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"alerting":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"alerting:apm__transaction_duration":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"endpoint:complete-external-response-actions":{"success":1,"not_timed_out":1,"total":1,"total_errors":0,"user_errors":0,"framework_errors":0},"Fleet-Metrics-Task":{"success":1,"not_timed_out":1,"total":1,"total_errors":0,"user_errors":0,"framework_errors":0},"apm-source-map-migration-task":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"observabilityAIAssistant:indexQueuedDocumentsTaskType":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"dashboard_telemetry":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"endpoint:user-artifact-packager":{"success":1,"not_timed_out":1,"total":1,"total_errors":0,"user_errors":0,"framework_errors":0}}}},"task_overdue":{"timestamp":"2025-03-12T00:19:06.228Z","value":{"by_type":{},"overall":{"overdue_by":{"counts":[],"values":[]},"overdue_by_values":[]}}}}}
[2025-03-12T00:19:06.263+00:00][DEBUG][plugins.taskManager.metrics-subscribe-debugger] subscribed metrics {"last_update":"2025-03-12T00:19:06.263Z","metrics":{"task_claim":{"timestamp":"2025-03-12T00:19:03.215Z","value":{"success":15,"total":15,"total_errors":0,"duration":{"counts":[13,2],"values":[100,200]},"duration_values":[15,13,119,14,108,12,14,13,15,14,15,13,14,15,16]}},"task_run":{"timestamp":"2025-03-12T00:19:06.263Z","value":{"overall":{"success":3,"not_timed_out":3,"total":3,"total_errors":0,"user_errors":0,"framework_errors":0,"delay":{"counts":[3],"values":[10]},"delay_values":[3,3,3]},"by_type":{"alerting:observability__rules__custom_threshold":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"alerting":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"alerting:apm__transaction_duration":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"endpoint:complete-external-response-actions":{"success":1,"not_timed_out":1,"total":1,"total_errors":0,"user_errors":0,"framework_errors":0},"Fleet-Metrics-Task":{"success":1,"not_timed_out":1,"total":1,"total_errors":0,"user_errors":0,"framework_errors":0},"apm-source-map-migration-task":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"observabilityAIAssistant:indexQueuedDocumentsTaskType":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"dashboard_telemetry":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"endpoint:user-artifact-packager":{"success":1,"not_timed_out":1,"total":1,"total_errors":0,"user_errors":0,"framework_errors":0}}}},"task_overdue":{"timestamp":"2025-03-12T00:19:06.228Z","value":{"by_type":{},"overall":{"overdue_by":{"counts":[],"values":[]},"overdue_by_values":[]}}}}}
[2025-03-12T00:19:06.263+00:00][DEBUG][plugins.taskManager.metrics-subscribe-debugger] subscribed metrics {"last_update":"2025-03-12T00:19:06.263Z","metrics":{"task_claim":{"timestamp":"2025-03-12T00:19:06.263Z","value":{"success":16,"total":16,"total_errors":0,"duration":{"counts":[14,2],"values":[100,200]},"duration_values":[15,13,119,14,108,12,14,13,15,14,15,13,14,15,16,55]}},"task_run":{"timestamp":"2025-03-12T00:19:06.263Z","value":{"overall":{"success":3,"not_timed_out":3,"total":3,"total_errors":0,"user_errors":0,"framework_errors":0,"delay":{"counts":[3],"values":[10]},"delay_values":[3,3,3]},"by_type":{"alerting:observability__rules__custom_threshold":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"alerting":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"alerting:apm__transaction_duration":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"endpoint:complete-external-response-actions":{"success":1,"not_timed_out":1,"total":1,"total_errors":0,"user_errors":0,"framework_errors":0},"Fleet-Metrics-Task":{"success":1,"not_timed_out":1,"total":1,"total_errors":0,"user_errors":0,"framework_errors":0},"apm-source-map-migration-task":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"observabilityAIAssistant:indexQueuedDocumentsTaskType":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"dashboard_telemetry":{"success":0,"not_timed_out":0,"total":0,"total_errors":0,"user_errors":0,"framework_errors":0},"endpoint:user-artifact-packager":{"success":1,"not_timed_out":1,"total":1,"total_errors":0,"user_errors":0,"framework_errors":0}}}},"task_overdue":{"timestamp":"2025-03-12T00:19:06.228Z","value":{"by_type":{},"overall":{"overdue_by":{"counts":[],"values":[]},"overdue_by_values":[]}}}}}
[2025-03-12T00:19:06.264+00:00][DEBUG][elasticsearch.query.data] 200 - 172.0B
POST /_security/user/_has_privileges
[redacted]
[2025-03-12T00:19:06.270+00:00][DEBUG][http.server.response] GET /internal/apm/has_data 200 166ms - 17.0B
[2025-03-12T00:19:06.271+00:00][DEBUG][elasticsearch.query.data] 200
POST /.entities-observability.latest-*/_search
{"size":1000,"track_total_hits":false,"_source":["agent.name","entity","data_stream.type"],"query":{"bool":{"filter":[{"range":{"entity.firstSeenTimestamp":{"gte":1741737846015}}},{"range":{"entity.lastSeenTimestamp":{"lte":1741738746015}}}]}}}
[2025-03-12T00:19:06.275+00:00][DEBUG][elasticsearch.query.data] 200 - 143.0B
POST /metrics-apm*%2Capm-*%2Clogs-apm*/_terms_enum
{"case_insensitive":true,"field":"service.environment","size":100,"string":"","index_filter":{"range":{"@timestamp":{"gte":1741737846007,"lte":1741738746007,"format":"epoch_millis"}}}}
[2025-03-12T00:19:06.278+00:00][DEBUG][elasticsearch.query.data] 200 - 510.0B
GET /.kibana_8.15.3/_doc/apm-indices%3Aapm-indices
[2025-03-12T00:19:06.280+00:00][DEBUG][elasticsearch.query.data] 200 - 910.0B
GET /.kibana_8.15.3/_doc/config%3A8.15.3
[2025-03-12T00:19:06.282+00:00][DEBUG][elasticsearch.query.data] 200
POST /metrics-apm*%2Capm-*/_search?ignore_unavailable=true&preference=any&expand_wildcards=open%2Chidden
{"track_total_hits":1,"size":0,"terminate_after":1,"query":{"bool":{"filter":[{"terms":{"processor.event":["metric"]}}],"must":[{"bool":{"filter":[{"range":{"@timestamp":{"gte":1741737846006,"lte":1741738746006,"format":"epoch_millis"}}},{"bool":{"filter":[{"term":{"metricset.name":"service_destination"}}],"must_not":{"terms":{"metricset.interval":["10m","60m"]}}}}]}}]}}}
[2025-03-12T00:19:06.284+00:00][DEBUG][elasticsearch.query.data] 200
POST /_msearch
{"index":["metrics-apm*","apm-*"],"preference":"any","ignore_unavailable":true,"expand_wildcards":["open","hidden"]}
{"track_total_hits":1,"size":0,"terminate_after":1,"query":{"bool":{"filter":[{"bool":{"filter":[{"range":{"@timestamp":{"gte":1741737846006,"lte":1741738746006,"format":"epoch_millis"}}}]}},{"terms":{"processor.event":["metric"]}},{"bool":{"filter":[{"term":{"metricset.name":"service_transaction"}},{"term":{"metricset.interval":"1m"}}]}}]}}}
{"index":["metrics-apm*","apm-*"],"preference":"any","ignore_unavailable":true,"expand_wildcards":["open","hidden"]}
{"track_total_hits":1,"size":0,"terminate_after":1,"query":{"bool":{"filter":[{"bool":{"filter":[{"range":{"@timestamp":{"gte":1741737846006,"lte":1741738746006,"format":"epoch_millis"}}}]}},{"terms":{"processor.event":["metric"]}},{"bool":{"filter":[{"term":{"metricset.name":"service_transaction"}},{"term":{"metricset.interval":"10m"}}]}}]}}}
{"index":["metrics-apm*","apm-*"],"preference":"any","ignore_unavailable":true,"expand_wildcards":["open","hidden"]}
{"track_total_hits":1,"size":0,"terminate_after":1,"query":{"bool":{"filter":[{"bool":{"filter":[{"range":{"@timestamp":{"gte":1741737846006,"lte":1741738746006,"format":"epoch_millis"}}}]}},{"terms":{"processor.event":["metric"]}},{"bool":{"filter":[{"term":{"metricset.name":"service_transaction"}},{"term":{"metricset.interval":"60m"}}]}}]}}}
{"index":["metrics-apm*","apm-*"],"preference":"any","ignore_unavailable":true,"expand_wildcards":["open","hidden"]}
{"track_total_hits":1,"size":0,"terminate_after":1,"query":{"bool":{"filter":[{"bool":{"filter":[{"range":{"@timestamp":{"gte":1741737846006,"lte":1741738746006,"format":"epoch_millis"}}}]}},{"terms":{"processor.event":["metric"]}},{"bool":{"filter":[{"bool":{"filter":[{"exists":{"field":"transaction.duration.histogram"}}],"must_not":[{"terms":{"metricset.interval":["10m","60m"]}},{"term":{"metricset.name":"service_transaction"}}]}}]}}]}}}
{"index":["metrics-apm*","apm-*"],"preference":"any","ignore_unavailable":true,"expand_wildcards":["open","hidden"]}
{"track_total_hits":1,"size":0,"terminate_after":1,"query":{"bool":{"filter":[{"bool":{"filter":[{"range":{"@timestamp":{"gte":1741737846006,"lte":1741738746006,"format":"epoch_millis"}}},{"bool":{"must_not":[{"exists":{"field":"transaction.duration.summary"}}]}}]}},{"terms":{"processor.event":["metric"]}},{"bool":{"filter":[{"bool":{"filter":[{"exists":{"field":"transaction.duration.histogram"}}],"must_not":[{"terms":{"metricset.interval":["10m","60m"]}},{"term":{"metricset.name":"service_transaction"}}]}}]}}]}}}
{"index":["metrics-apm*","apm-*"],"preference":"any","ignore_unavailable":true,"expand_wildcards":["open","hidden"]}
{"track_total_hits":1,"size":0,"terminate_after":1,"query":{"bool":{"filter":[{"bool":{"filter":[{"range":{"@timestamp":{"gte":1741737846006,"lte":1741738746006,"format":"epoch_millis"}}}]}},{"terms":{"processor.event":["metric"]}},{"bool":{"filter":[{"term":{"metricset.name":"transaction"}},{"term":{"metricset.interval":"10m"}}]}}]}}}
{"index":["metrics-apm*","apm-*"],"preference":"any","ignore_unavailable":true,"expand_wildcards":["open","hidden"]}
{"track_total_hits":1,"size":0,"terminate_after":1,"query":{"bool":{"filter":[{"bool":{"filter":[{"range":{"@timestamp":{"gte":1741737846006,"lte":1741738746006,"format":"epoch_millis"}}}]}},{"terms":{"processor.event":["metric"]}},{"bool":{"filter":[{"term":{"metricset.name":"transaction"}},{"term":{"metricset.interval":"60m"}}]}}]}}}
[2025-03-12T00:19:06.286+00:00][DEBUG][elasticsearch.query.data] 200 - 172.0B
POST /_security/user/_has_privileges
[redacted]
[2025-03-12T00:19:06.289+00:00][DEBUG][http.server.response] GET /internal/apm/entities/services?environment=ENVIRONMENT_ALL&kuery=&start=2025-03-12T00%3A04%3A06.015Z&end=2025-03-12T00%3A19%3A06.015Z 200 179ms - 15.0B
[2025-03-12T00:19:06.289+00:00][DEBUG][http.server.response] GET /internal/apm/suggestions?start=2025-03-12T00%3A04%3A06.007Z&end=2025-03-12T00%3A19%3A06.007Z&fieldName=service.environment&fieldValue= 200 186ms - 77.0B
[2025-03-12T00:19:06.294+00:00][DEBUG][elasticsearch.query.data] 200 - 167.0B
POST /_security/user/_has_privileges
[redacted]
[2025-03-12T00:19:06.298+00:00][INFO ][plugins.apm] create static data view apm_static_data_view_id_default
[2025-03-12T00:19:06.302+00:00][DEBUG][http.server.response] GET /internal/apm/time_range_metadata?start=2025-03-12T00%3A04%3A06.006Z&end=2025-03-12T00%3A19%3A06.006Z&kuery=&useSpanName=false&enableServiceTransactionMetrics=true&enableContinuousRollups=true 200 194ms - 823.0B
[2025-03-12T00:19:06.302+00:00][DEBUG][http.server.response] GET /internal/apm/data_view/index_pattern 200 193ms - 82.0B
[2025-03-12T00:19:06.320+00:00][DEBUG][execution_context] {"type":"application","name":"apm","url":"/app/apm/services","page":"/services"}
[2025-03-12T00:19:06.322+00:00][DEBUG][execution_context] {"type":"application","name":"apm","url":"/app/apm/services","page":"/services"}
[2025-03-12T00:19:06.326+00:00][DEBUG][elasticsearch.query.data] 200 - 744.0B
GET /.kibana_security_session/_doc/pwPhsPtgHd8O54BqV9iMOO4lHYhRGwv%2B%2FAUl04ICLgw%3D
[2025-03-12T00:19:06.327+00:00][DEBUG][elasticsearch.query.data] 200 - 744.0B
GET /.kibana_security_session/_doc/pwPhsPtgHd8O54BqV9iMOO4lHYhRGwv%2B%2FAUl04ICLgw%3D
[2025-03-12T00:19:06.336+00:00][DEBUG][execution_context] {"type":"application","name":"apm","url":"/app/apm/services","page":"/services"}
[2025-03-12T00:19:06.339+00:00][DEBUG][execution_context] {"type":"application","name":"apm","url":"/app/apm/services","page":"/services"}
[2025-03-12T00:19:06.344+00:00][DEBUG][elasticsearch.query.data] 200 - 744.0B
GET /.kibana_security_session/_doc/pwPhsPtgHd8O54BqV9iMOO4lHYhRGwv%2B%2FAUl04ICLgw%3D
[2025-03-12T00:19:06.344+00:00][DEBUG][plugins.security.basic.basic] Trying to authenticate user request to /internal/apm/fleet/has_apm_policies.
[2025-03-12T00:19:06.345+00:00][DEBUG][plugins.security.basic.basic] Trying to authenticate via state.
[2025-03-12T00:19:06.347+00:00][DEBUG][elasticsearch.query.data] 200 - 744.0B
GET /.kibana_security_session/_doc/pwPhsPtgHd8O54BqV9iMOO4lHYhRGwv%2B%2FAUl04ICLgw%3D
[2025-03-12T00:19:06.347+00:00][DEBUG][elasticsearch.query.data] 200 - 339.0B
GET /_security/_authenticate
[redacted]
[2025-03-12T00:19:06.348+00:00][DEBUG][plugins.security.basic.basic] Request has been authenticated via state.
[2025-03-12T00:19:06.348+00:00][DEBUG][plugins.security.session.Ul04ICLgw=] Successfully extended existing session.
[2025-03-12T00:19:06.367+00:00][DEBUG][plugins.security.basic.basic] Trying to authenticate user request to /internal/apm/data_view/index_pattern.
[2025-03-12T00:19:06.367+00:00][DEBUG][plugins.security.basic.basic] Trying to authenticate via state.
[2025-03-12T00:19:06.369+00:00][DEBUG][plugins.security.basic.basic] Trying to authenticate user request to /internal/apm/entities/services?environment=ENVIRONMENT_ALL&kuery=&start=2025-03-12T00%3A04%3A06.236Z&end=2025-03-12T00%3A19%3A06.236Z.
[2025-03-12T00:19:06.369+00:00][DEBUG][plugins.security.basic.basic] Trying to authenticate via state.
[2025-03-12T00:19:06.371+00:00][DEBUG][plugins.security.basic.basic] Trying to authenticate user request to /internal/security/me.
[2025-03-12T00:19:06.371+00:00][DEBUG][plugins.security.basic.basic] Trying to authenticate via state.
[2025-03-12T00:19:06.374+00:00][DEBUG][elasticsearch.query.data] 200
POST /.kibana_ingest_8.15.3/_search?rest_total_hits_as_int=true
{"size":20,"seq_no_primary_term":true,"from":0,"query":{"bool":{"filter":[{"bool":{"should":[{"match":{"ingest-package-policies.package.name":"apm"}}],"minimum_should_match":1}},{"bool":{"should":[{"bool":{"must":[{"term":{"type":"ingest-package-policies"}}],"must_not":[{"exists":{"field":"namespace"}},{"exists":{"field":"namespaces"}}]}}],"minimum_should_match":1}}]}},"sort":[{"ingest-package-policies.updated_at":{"order":"desc","unmapped_type":"date"}}]}
[2025-03-12T00:19:06.378+00:00][DEBUG][elasticsearch.query.data] 200 - 339.0B
GET /_security/_authenticate
[redacted]
[2025-03-12T00:19:06.378+00:00][DEBUG][plugins.security.basic.basic] Request has been authenticated via state.
[2025-03-12T00:19:06.379+00:00][DEBUG][plugins.security.session.Ul04ICLgw=] Successfully extended existing session.
[2025-03-12T00:19:06.382+00:00][DEBUG][elasticsearch.query.data] 200 - 339.0B
GET /_security/_authenticate
[redacted]
[2025-03-12T00:19:06.383+00:00][DEBUG][plugins.security.basic.basic] Request has been authenticated via state.
[2025-03-12T00:19:06.384+00:00][DEBUG][plugins.security.session.Ul04ICLgw=] Successfully extended existing session.
[2025-03-12T00:19:06.387+00:00][DEBUG][elasticsearch.query.data] 200 - 151.0B
POST /_security/user/_has_privileges
[redacted]
[2025-03-12T00:19:06.392+00:00][DEBUG][plugins.security.api-authorization] User authorized for "/internal/apm/data_view/index_pattern"
[2025-03-12T00:19:06.396+00:00][DEBUG][elasticsearch.query.data] 200 - 339.0B
GET /_security/_authenticate
[redacted]
[2025-03-12T00:19:06.397+00:00][DEBUG][plugins.security.basic.basic] Request has been authenticated via state.
[2025-03-12T00:19:06.402+00:00][DEBUG][elasticsearch.query.data] 200 - 539.0B
POST /.kibana_8.15.3/_update/usage-counters%3Aapm%3A12032025%3Asuccess%3AGET%20%2Finternal%2Fapm%2Ffleet%2Fhas_apm_policies?refresh=wait_for&require_alias=true
{"script":{"source":"\n for (int i = 0; i < params.counterFieldNames.length; i++) {\n def counterFieldName = params.counterFieldNames[i];\n def count = params.counts[i];\n\n if (ctx._source[params.type][counterFieldName] == null) {\n ctx._source[params.type][counterFieldName] = count;\n }\n else {\n ctx._source[params.type][counterFieldName] += count;\n }\n }\n ctx._source.updated_at = params.time;\n ","lang":"painless","params":{"counts":[1],"counterFieldNames":["count"],"time":"2025-03-12T00:19:06.206Z","type":"usage-counters"}},"upsert":{"usage-counters":{"domainId":"apm","counterName":"GET /internal/apm/fleet/has_apm_policies","counterType":"success","count":1},"type":"usage-counters","managed":false,"coreMigrationVersion":"8.8.0","updated_at":"2025-03-12T00:19:06.206Z"},"_source":true}
[2025-03-12T00:19:06.402+00:00][DEBUG][plugins.usageCollection.usage-collection.usage-counters-service] Store counters into savedObjects
[2025-03-12T00:19:06.403+00:00][DEBUG][http.server.response] GET /internal/security/me 200 64ms - 488.0B
[2025-03-12T00:19:06.409+00:00][DEBUG][elasticsearch.query.data] 200 - 151.0B
POST /_security/user/_has_privileges
[redacted]