Hello,
We are running fleet server in ECK environment. Recently we had to restart fleet server for tolerations related changes to make sure fleet server is not running on master node of our K8s cluster. But post that changes Fleet server went to degraded state. Upon Looking at the logs we found the below logs. Can someone please help us on this. Just for minute changes we are now and then seeing issues with Fleet server. We had similar issue last time. Upon restarting operator,fleet server and after creating new policy it has fixed the issue. But this time none of them are working and we wanted to know the root cause of what causing the issue.
"log.level":"warn","@timestamp":"2024-10-18T13:00:26.177Z","message":"missing config fleet.agent.id; acceptable until Elastic Agent has enrolled","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"service.type":"fleet-server","ctx":"policy leader manager","ecs.version":"1.6.0","service.name":"fleet-server","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:26.638Z","message":"Running on policy with Fleet Server integration: eck-fleet-server-dev; missing config fleet.agent.id (expected during bootstrap process)","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"service.name":"fleet-server","service.type":"fleet-server","state":"DEGRADED","ecs.version":"1.6.0","ecs.version":"1.6.0"}
{"log.level":"warn","@timestamp":"2024-10-18T13:00:26.639Z","log.origin":{"file.name":"coordinator/coordinator.go","file.line":624},"message":"Unit state changed fleet-server-default-fleet-server (STARTING->DEGRADED): Running on policy with Fleet Server integration: eck-fleet-server-dev; missing config fleet.agent.id (expected during bootstrap process)","log":{"source":"elastic-agent"},"component":{"id":"fleet-server-default","state":"HEALTHY"},"unit":{"id":"fleet-server-default-fleet-server","type":"input","state":"DEGRADED","old_state":"STARTING"},"ecs.version":"1.6.0"}
{"log.level":"warn","@timestamp":"2024-10-18T13:00:26.639Z","log.origin":{"file.name":"coordinator/coordinator.go","file.line":624},"message":"Unit state changed fleet-server-default (STARTING->DEGRADED): Running on policy with Fleet Server integration: eck-fleet-server-dev; missing config fleet.agent.id (expected during bootstrap process)","log":{"source":"elastic-agent"},"component":{"id":"fleet-server-default","state":"HEALTHY"},"unit":{"id":"fleet-server-default","type":"output","state":"DEGRADED","old_state":"STARTING"},"ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:26.639Z","message":"applying new components data","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"http.request.id":"01JAFVE303FTHX82FSHC6BRTX5","fleet.access.apikey.id":"nEnOZpIB6CxI4R5zP6Si","service.type":"fleet-server","server.address":"","fleet.agent.id":"ff7a85d6-ff8b-4eaa-a26e-77a86ed83faf","req.Components":[{"id":"logstash/metrics-default","message":"Healthy: communicating with pid '2068519'","status":"HEALTHY","type":"logstash/metrics","units":[{"id":"logstash/metrics-default-logstash/metrics-logstash-71f4507e-ac12-4488-947f-d34a7a6fe1f5","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"logstash/metrics-default","message":"Healthy","status":"HEALTHY","type":"output"}]},{"id":"beat/metrics-monitoring","message":"Healthy: communicating with pid '3111793'","status":"HEALTHY","type":"beat/metrics","units":[{"id":"beat/metrics-monitoring-metrics-monitoring-beats","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"beat/metrics-monitoring","message":"Healthy","status":"HEALTHY","type":"output"}]},{"id":"http/metrics-monitoring","message":"Healthy: communicating with pid '3535115'","status":"HEALTHY","type":"http/metrics","units":[{"id":"http/metrics-monitoring-metrics-monitoring-agent","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"http/metrics-monitoring","message":"Healthy","status":"HEALTHY","type":"output"}]},{"id":"kibana/metrics-default","message":"Healthy: communicating with pid '3537644'","status":"HEALTHY","type":"kibana/metrics","units":[{"id":"kibana/metrics-default-kibana/metrics-kibana-84c7437c-c4c7-4d8e-a453-665dc54f99b8","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"kibana/metrics-default","message":"Healthy","status":"HEALTHY","type":"output"}]},{"id":"elasticsearch/metrics-default","message":"Healthy: communicating with pid '3542226'","status":"HEALTHY","type":"elasticsearch/metrics","units":[{"id":"elasticsearch/metrics-default-elasticsearch/metrics-elasticsearch-50acf376-56ec-4d02-8820-a76c14ff74f3","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"elasticsearch/metrics-default","message":"Healthy","status":"HEALTHY","type":"output"}]},{"id":"kubernetes/metrics-default","message":"Healthy: communicating with pid '3542826'","status":"HEALTHY","type":"kubernetes/metrics","units":[{"id":"kubernetes/metrics-default-kubernetes/metrics-kubelet-d083f6ce-4345-4fb8-a970-c8c5d2ffef52","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"kubernetes/metrics-default-kubernetes/metrics-kube-proxy-d083f6ce-4345-4fb8-a970-c8c5d2ffef52","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"kubernetes/metrics-default","message":"Healthy","status":"HEALTHY","type":"output"}]},{"id":"system/metrics-default","message":"Healthy: communicating with pid '3543203'","status":"HEALTHY","type":"system/metrics","units":[{"id":"system/metrics-default","message":"Healthy","status":"HEALTHY","type":"output"},{"id":"system/metrics-default-system/metrics-system-5362c5be-2052-4889-9551-07047dfb38aa","message":"Healthy","status":"HEALTHY","type":"input"}]},{"id":"filestream-default","message":"Healthy: communicating with pid '3543525'","status":"HEALTHY","type":"filestream","units":[{"id":"filestream-default","message":"Healthy","status":"HEALTHY","type":"output"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-b9f85973-db46-40e9-ab19-a5304d49143f.antrea-agent","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-8ec1cfac-4ec8-448e-a230-dae76602f91e.multisearch-address-api","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-f7976630-425a-4fe5-9eeb-3221a1c7e067.elastic-internal-init-keystore","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-f7976630-425a-4fe5-9eeb-3221a1c7e067.elastic-internal-init-filesystem","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-b9f85973-db46-40e9-ab19-a5304d49143f.install-cni","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-f7976630-425a-4fe5-9eeb-3221a1c7e067.elastic-internal-suspend","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-5f00fdb1-3a79-4dbb-86fb-efd4b0cb6e21.geoserver","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-f7976630-425a-4fe5-9eeb-3221a1c7e067.elasticsearch","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-81870d40-2edb-45d2-8991-9cd6ada38784.elastic-agent","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-f4ed16dc-794a-4c7a-8b2c-08f8877cf01d.vsphere-csi-node","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-b9f85973-db46-40e9-ab19-a5304d49143f.antrea-agent-tweaker","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-9143c7bd-f5e1-4521-81ab-fc309f578785.init-volume","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-75412458-9d48-4991-a3f0-eb8db96e85c7.secretgen-controller","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-9143c7bd-f5e1-4521-81ab-fc309f578785.agent","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-a62cbbd8-6c4e-4fc5-b330-c113069868ad.manager","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-b7598613-de58-4168-997d-f939ea9d08b3.istio-proxy","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-263d6d9e-6118-4cf5-87d6-cbdd88021bb5.metricbeat","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-9143c7bd-f5e1-4521-81ab-fc309f578785.init-config","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-5a3a9812-2fb3-4d7e-b707-22703b4ebbba.falcon-node-sensor","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-ec6b5c7c-6955-477e-8541-fc5bc45e639e.llminsights-api","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-ca1fa263-3f37-42c4-bf90-ef54a2594931.metricbeat","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-9143c7bd-f5e1-4521-81ab-fc309f578785.process-agent","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-b9f85973-db46-40e9-ab19-a5304d49143f.antrea-ovs","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-cfb71d7c-a935-4ea4-8045-821ee92fdca6.docker-registry","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-5a3a9812-2fb3-4d7e-b707-22703b4ebbba.init-falconstore","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-f4ed16dc-794a-4c7a-8b2c-08f8877cf01d.node-driver-registrar","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-c8705f5b-e8f5-4a55-b06e-5ecfd9857c3f.kube-proxy","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-7a34b531-65df-4241-a341-4eb70d4a773e.filebeat","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-81870d40-2edb-45d2-8991-9cd6ada38784.manage-agent-hostpath-permissions","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"filestream-default-filestream-container-logs-d083f6ce-4345-4fb8-a970-c8c5d2ffef52-kubernetes-f4ed16dc-794a-4c7a-8b2c-08f8877cf01d.liveness-probe","message":"Healthy","status":"HEALTHY","type":"input"}]}],"ecs.version":"1.6.0","service.name":"fleet-server","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:26.894Z","message":"force load on unknown policyId","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"ecs.version":"1.6.0","fleet.agent.id":"ff7a85d6-ff8b-4eaa-a26e-77a86ed83faf","service.name":"fleet-server","service.type":"fleet-server","ctx":"policy agent monitor","fleet.policy.id":"41ec1f3c-ab94-4e48-80c3-8af43f2ebbb6","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.084Z","log.origin":{"file.name":"cmd/enroll_cmd.go","file.line":846},"message":"Fleet Server - Running on policy with Fleet Server integration: eck-fleet-server-dev; missing config fleet.agent.id (expected during bootstrap process)","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.194Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"service.type":"fleet-server","ctx":"policy agent monitor","fleet.coordinator_idx":0,"service.name":"fleet-server","ecs.version":"1.6.0","fleet.policy.id":"eck-agent","fleet.revision_idx":94,"ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.194Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"fleet.coordinator_idx":0,"service.name":"fleet-server","ctx":"policy agent monitor","fleet.revision_idx":44,"ecs.version":"1.6.0","service.type":"fleet-server","fleet.policy.id":"761bccba-85b4-419d-86a2-0a10e3cce216","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.194Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"service.name":"fleet-server","fleet.coordinator_idx":0,"fleet.policy.id":"a999c197-be16-4be0-9248-b471a7b8f28a","fleet.revision_idx":32,"ecs.version":"1.6.0","service.type":"fleet-server","ctx":"policy agent monitor","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.194Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"ecs.version":"1.6.0","service.type":"fleet-server","ctx":"policy agent monitor","fleet.revision_idx":28,"fleet.coordinator_idx":0,"service.name":"fleet-server","fleet.policy.id":"f3cc9a9b-ea82-457a-a76a-bbbed3da6988","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.195Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"fleet.revision_idx":19,"service.name":"fleet-server","service.type":"fleet-server","fleet.coordinator_idx":0,"ecs.version":"1.6.0","ctx":"policy agent monitor","fleet.policy.id":"eck-fleet-server-3","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.195Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"service.name":"fleet-server","fleet.policy.id":"d25e8c09-b8d5-4862-bbba-fcbdf2d0a19d","fleet.revision_idx":28,"ecs.version":"1.6.0","service.type":"fleet-server","ctx":"policy agent monitor","fleet.coordinator_idx":0,"ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.195Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"ecs.version":"1.6.0","service.type":"fleet-server","ctx":"policy agent monitor","fleet.coordinator_idx":0,"service.name":"fleet-server","fleet.policy.id":"d5315f22-60c0-47dd-840c-92491ae175c7","fleet.revision_idx":54,"ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.196Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"fleet.policy.id":"6efc29d2-7cd6-49d8-ad8e-5418b79d80db","fleet.revision_idx":48,"fleet.coordinator_idx":0,"service.name":"fleet-server","service.type":"fleet-server","ctx":"policy agent monitor","ecs.version":"1.6.0","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.196Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"service.name":"fleet-server","service.type":"fleet-server","ctx":"policy agent monitor","fleet.policy.id":"eck-agent-1","fleet.revision_idx":36,"fleet.coordinator_idx":0,"ecs.version":"1.6.0","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.196Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"service.name":"fleet-server","service.type":"fleet-server","ctx":"policy agent monitor","fleet.revision_idx":37,"fleet.coordinator_idx":0,"ecs.version":"1.6.0","fleet.policy.id":"fleet-server-policy","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.197Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"fleet.policy.id":"091b0241-6b60-46c8-bccf-b77c7b3ee8ff","fleet.revision_idx":34,"fleet.coordinator_idx":0,"ecs.version":"1.6.0","ctx":"policy agent monitor","service.type":"fleet-server","service.name":"fleet-server","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.197Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"fleet.policy.id":"022491cc-7ef6-4811-97fd-80a495f1f5c9","ecs.version":"1.6.0","service.name":"fleet-server","fleet.coordinator_idx":0,"service.type":"fleet-server","ctx":"policy agent monitor","fleet.revision_idx":32,"ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.197Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"fleet.policy.id":"b51efe83-086d-4b32-851c-b5858b79f2e5","fleet.revision_idx":29,"fleet.coordinator_idx":0,"ecs.version":"1.6.0","service.name":"fleet-server","ctx":"policy agent monitor","service.type":"fleet-server","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.198Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"fleet.revision_idx":14,"fleet.coordinator_idx":0,"ecs.version":"1.6.0","service.name":"fleet-server","service.type":"fleet-server","ctx":"policy agent monitor","fleet.policy.id":"eck-fleet-server-dev","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.198Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"fleet.revision_idx":205,"fleet.coordinator_idx":0,"ecs.version":"1.6.0","service.type":"fleet-server","service.name":"fleet-server","ctx":"policy agent monitor","fleet.policy.id":"41ec1f3c-ab94-4e48-80c3-8af43f2ebbb6","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.198Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"ecs.version":"1.6.0","service.name":"fleet-server","service.type":"fleet-server","fleet.coordinator_idx":0,"ctx":"policy agent monitor","fleet.policy.id":"8f853ee4-78cf-4f5d-a150-04f2af5d0321","fleet.revision_idx":54,"ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.198Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"ecs.version":"1.6.0","service.name":"fleet-server","service.type":"fleet-server","ctx":"policy agent monitor","fleet.policy.id":"42dda84f-d4c1-430e-93bc-0a8158ea6b2b","fleet.revision_idx":37,"fleet.coordinator_idx":0,"ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.199Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"fleet.coordinator_idx":0,"service.type":"fleet-server","fleet.policy.id":"2dc5966d-2604-4c89-8c23-dbd0e9848a4c","fleet.revision_idx":29,"ecs.version":"1.6.0","service.name":"fleet-server","ctx":"policy agent monitor","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.199Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"service.type":"fleet-server","fleet.revision_idx":35,"fleet.coordinator_idx":0,"ecs.version":"1.6.0","service.name":"fleet-server","ctx":"policy agent monitor","fleet.policy.id":"eck-fleet-server-2","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.199Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"service.name":"fleet-server","ctx":"policy agent monitor","ecs.version":"1.6.0","service.type":"fleet-server","fleet.policy.id":"b24a801a-c505-455f-af64-132dc4c34856","fleet.revision_idx":27,"fleet.coordinator_idx":0,"ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.199Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"fleet.coordinator_idx":0,"service.name":"fleet-server","service.type":"fleet-server","ctx":"policy agent monitor","fleet.policy.id":"eck-fleet-server-4","fleet.revision_idx":19,"ecs.version":"1.6.0","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.277Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"fleet.policy.id":"eck-fleet-server","fleet.revision_idx":41,"service.type":"fleet-server","ctx":"policy agent monitor","fleet.coordinator_idx":0,"@timestamp":"2024-10-18T13:00:27.2Z","ecs.version":"1.6.0","service.name":"fleet-server","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.278Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"ecs.version":"1.6.0","service.name":"fleet-server","service.type":"fleet-server","ctx":"policy agent monitor","@timestamp":"2024-10-18T13:00:27.2Z","fleet.policy.id":"458aff81-abc3-48a8-bd58-5ed75916b4c5","fleet.revision_idx":32,"fleet.coordinator_idx":0,"ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.278Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"service.type":"fleet-server","ctx":"policy agent monitor","fleet.coordinator_idx":0,"@timestamp":"2024-10-18T13:00:27.2Z","ecs.version":"1.6.0","service.name":"fleet-server","fleet.policy.id":"0be068e6-774c-45d6-afae-b32f4db97071","fleet.revision_idx":31,"ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:27.645Z","log.origin":{"file.name":"cmd/enroll_cmd.go","file.line":517},"message":"Starting enrollment to URL: https://sosh-fleet-server-dev-agent-http.sosh-monitoring-dev.svc:8220/","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:31.779Z","message":"applying new components data","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"service.name":"fleet-server","fleet.agent.id":"105fa518-6f50-4fbe-b638-c2b6afeca50f","req.Components":[{"id":"beat/metrics-monitoring","message":"Healthy: communicating with pid '2228040'","status":"HEALTHY","type":"beat/metrics","units":[{"id":"beat/metrics-monitoring-metrics-monitoring-beats","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"beat/metrics-monitoring","message":"Healthy","status":"HEALTHY","type":"output"}]},{"id":"http/metrics-monitoring","message":"Healthy: communicating with pid '3616730'","status":"HEALTHY","type":"http/metrics","units":[{"id":"http/metrics-monitoring-metrics-monitoring-agent","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"http/metrics-monitoring","message":"Healthy","status":"HEALTHY","type":"output"}]},{"id":"logstash/metrics-default","message":"Healthy: communicating with pid '1453833'","status":"HEALTHY","type":"logstash/metrics","units":[{"id":"logstash/metrics-default-logstash/metrics-logstash-5376a97f-9699-4454-bcc9-19b2de13eb34","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"logstash/metrics-default","message":"Healthy","status":"HEALTHY","type":"output"}]},{"id":"kibana/metrics-default","message":"Healthy: communicating with pid '1461415'","status":"HEALTHY","type":"kibana/metrics","units":[{"id":"kibana/metrics-default","message":"Healthy","status":"HEALTHY","type":"output"},{"id":"kibana/metrics-default-kibana/metrics-kibana-78a8a973-7824-4ef7-be8a-879e322a7a02","message":"Healthy","status":"HEALTHY","type":"input"}]},{"id":"system/metrics-default","message":"Healthy: communicating with pid '1461902'","status":"HEALTHY","type":"system/metrics","units":[{"id":"system/metrics-default","message":"Healthy","status":"HEALTHY","type":"output"},{"id":"system/metrics-default-system/metrics-system-776a633b-01e6-41cc-8a0a-05b07a486374","message":"Healthy","status":"HEALTHY","type":"input"}]},{"id":"elasticsearch/metrics-default","message":"Healthy: communicating with pid '1461903'","status":"HEALTHY","type":"elasticsearch/metrics","units":[{"id":"elasticsearch/metrics-default-elasticsearch/metrics-elasticsearch-b3f17546-3c45-4f60-8679-0dabebedc18e","message":"Healthy","status":"HEALTHY","type":"input"},{"id":"elasticsearch/metrics-default","message":"Healthy","status":"HEALTHY","type":"output"}]},{"id":"filestream-monitoring","message":"Healthy: communicating with pid '1462224'","status":"HEALTHY","type":"filestream","units":[{"id":"filestream-monitoring-filestream-monitoring-agent","message":"Starting: spawned pid '1462224'","status":"STARTING","type":"input"},{"id":"filestream-monitoring","message":"Starting: spawned pid '1462224'","status":"STARTING","type":"output"}]},{"id":"filestream-default","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"filestream","units":[{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-b37e883a-0c79-4312-9a1c-f7f87aaa10af.elastic-internal-init-keystore","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-a6657ad2-d918-4bc8-964e-57bcef6bfdf3.node-driver-registrar","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-895646e4-2eae-427e-9167-17b4b758f9bc.falcon-node-sensor","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-b37e883a-0c79-4312-9a1c-f7f87aaa10af.elastic-internal-init-filesystem","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-8136cfea-447f-49c3-a661-71b5368027c8.manager","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-8908c3b3-db42-4182-9cc9-9782d03619b5.istio-proxy","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-5474777e-157a-4b03-b909-8334e70a268a.keyahead-api","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-dcb65d1d-046e-4afc-801e-ea23f9be7796.kube-proxy","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-75b488cb-f5e7-4577-ba2e-a9aa7d95519f.elastic-agent","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-a6657ad2-d918-4bc8-964e-57bcef6bfdf3.liveness-probe","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-895646e4-2eae-427e-9167-17b4b758f9bc.init-falconstore","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-5ad4e107-b7ae-41a0-83f1-ffd14f110d8e.docker-registry","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-d63e7005-61cf-462b-a986-a16a064a8028.llm-insights-ui-app","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-b37e883a-0c79-4312-9a1c-f7f87aaa10af.elasticsearch","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-210f8a36-14c5-4509-9e95-0190a9a5f042.antrea-agent","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-90cc5058-9755-493d-9b08-754173c830fe.init-config","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"output"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-210f8a36-14c5-4509-9e95-0190a9a5f042.antrea-agent-tweaker","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-0d2a8bb4-141c-4e39-9fcb-3a896eb4c80d.filebeat","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-b37e883a-0c79-4312-9a1c-f7f87aaa10af.elastic-internal-suspend","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-a6657ad2-d918-4bc8-964e-57bcef6bfdf3.vsphere-csi-node","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-90cc5058-9755-493d-9b08-754173c830fe.process-agent","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-90cc5058-9755-493d-9b08-754173c830fe.init-volume","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-210f8a36-14c5-4509-9e95-0190a9a5f042.install-cni","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-e206bd1a-92d0-4ec6-b880-5da50b2ba025.metricbeat","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-210f8a36-14c5-4509-9e95-0190a9a5f042.antrea-ovs","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-75b488cb-f5e7-4577-ba2e-a9aa7d95519f.manage-agent-hostpath-permissions","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"},{"id":"filestream-default-filestream-container-logs-3bd6a9c6-5292-428f-9300-18abee8bbe7b-kubernetes-90cc5058-9755-493d-9b08-754173c830fe.agent","message":"Failed: pid '1462164' exited with code '-1'","status":"FAILED","type":"input"}]}],"ecs.version":"1.6.0","service.type":"fleet-server","http.request.id":"01JAFVE7PCSJD6H5Y6Y9X71JXB","server.address":"","fleet.access.apikey.id":"7f0DdZIB6CxI4R5z1mT_","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:31.779Z","message":"force load on unknown policyId","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"ecs.version":"1.6.0","fleet.policy.id":"d5315f22-60c0-47dd-840c-92491ae175c7","service.type":"fleet-server","ctx":"policy agent monitor","fleet.agent.id":"105fa518-6f50-4fbe-b638-c2b6afeca50f","service.name":"fleet-server","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:32.074Z","message":"Running on policy with Fleet Server integration: eck-fleet-server-dev; missing config fleet.agent.id (expected during bootstrap process)","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"ecs.version":"1.6.0","service.name":"fleet-server","service.type":"fleet-server","state":"DEGRADED","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:32.079Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"service.name":"fleet-server","service.type":"fleet-server","fleet.revision_idx":37,"fleet.coordinator_idx":0,"ecs.version":"1.6.0","ctx":"policy agent monitor","fleet.policy.id":"fleet-server-policy","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-10-18T13:00:32.079Z","message":"Ignore policy that has not passed through coordinator","component":{"binary":"fleet-server","dataset":"elastic_agent.fleet_server","id":"fleet-server-default","type":"fleet-server"},"log":{"source":"fleet-server-default"},"ecs.version":"1.6.0","service.name":"fleet-server","ctx":"policy agent