Have several integrations that break trying to update them.
Kibana shows me that Integrations have Updates but when trying update the dialog shows its used by 0 agents, which is not true. Continuing it shows Kibana trying to install it but only seconds later I get a message saying it failed and it´s reverting to the prior version.
After that the Integration vanishes from Installed, if you go to Available and search for it it just shows the menu as if it is installed (Settings Menu, Upgrade/Reinstall/etc.).
The existing installation of the integration des continue to work as it was but can effectively not be updatet.
I have a custom ILM, for the datastream, including namespace, index template, component template but even as a default never ever changed installation of a new plugin/integration the update process will break as soon as an update is available and I want to install it.
The custom setup followed Tutorial: Customize data retention policies | Fleet and Elastic Agent Guide [8.8] | Elastic but as it doesnt even work with a default install it shouldnt matter.
Following is the log from kibana.log when you start an update, same thing for every plugin I just choose the latest I tried.
Funny thing is that System metrics plugin updates just fine.
The only fix is a complete removal, deinstall integration, delete Policy and reinstall the newest version of the plugin, create new policy and add it to the agent.
ES version is 8.7.0
{"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.0"},"@timestamp":"2023-06-12T07:35:13.800+00:00","message":"Failure to install package [checkpoint]: [Error: could not update index template settings for logs-checkpoint.firewall-production]","log":{"level":"WARN","logger":"plugins.fleet"},"process":{"pid":863},"trace":{"id":"bd80407cb936139992eae950eead9c36"},"transaction":{"id":"ac9d203b0917aefb"}}
{"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.0"},"@timestamp":"2023-06-12T07:35:14.422+00:00","message":"could not update index template settings for logs-checkpoint.firewall-production","error":{"message":"could not update index template settings for logs-checkpoint.firewall-production","type":"Error","stack_trace":"Error: could not update index template settings for logs-checkpoint.firewall-production\n at updateExistingDataStream (/usr/share/kibana/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.js:576:11)\n at runMicrotasks (<anonymous>)\n at processTicksAndRejections (node:internal/process/task_queues:96:5)\n at async Promise.all (index 0)\n at updateAllDataStreams (/usr/share/kibana/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.js:528:3)\n at _installPackage (/usr/share/kibana/x-pack/plugins/fleet/server/services/epm/packages/_install_package.js:178:5)\n at installPackageCommon (/usr/share/kibana/x-pack/plugins/fleet/server/services/epm/packages/install.js:387:12)\n at installPackageFromRegistry (/usr/share/kibana/x-pack/plugins/fleet/server/services/epm/packages/install.js:288:12)\n at installPackage (/usr/share/kibana/x-pack/plugins/fleet/server/services/epm/packages/install.js:553:22)\n at installPackageFromRegistryHandler (/usr/share/kibana/x-pack/plugins/fleet/server/routes/epm/handlers.js:283:15)\n at Router.handle (/usr/share/kibana/node_modules/@kbn/core-http-router-server-internal/src/router.js:141:30)\n at handler (/usr/share/kibana/node_modules/@kbn/core-http-router-server-internal/src/router.js:107:50)\n at exports.Manager.execute (/usr/share/kibana/node_modules/@hapi/hapi/lib/toolkit.js:60:28)\n at Object.internals.handler (/usr/share/kibana/node_modules/@hapi/hapi/lib/handler.js:46:20)\n at exports.execute (/usr/share/kibana/node_modules/@hapi/hapi/lib/handler.js:31:20)\n at Request._lifecycle (/usr/share/kibana/node_modules/@hapi/hapi/lib/request.js:371:32)\n at Request._execute (/usr/share/kibana/node_modules/@hapi/hapi/lib/request.js:281:9)"},"log":{"level":"ERROR","logger":"plugins.fleet"},"process":{"pid":863},"trace":{"id":"bd80407cb936139992eae950eead9c36"},"transaction":{"id":"ac9d203b0917aefb"}}
{"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.0"},"@timestamp":"2023-06-12T07:35:25.830+00:00","message":"Failure to install package [checkpoint]: [Error: could not update index template settings for logs-checkpoint.firewall-production]","log":{"level":"WARN","logger":"plugins.fleet"},"process":{"pid":863},"trace":{"id":"9b7ec748390372bd55a0ee1e2979eb8a"},"transaction":{"id":"609ca7b42d75151b"}}
{"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.0"},"@timestamp":"2023-06-12T07:35:26.448+00:00","message":"rolling back to checkpoint-1.21.0 after error installing checkpoint-1.22.0","log":{"level":"ERROR","logger":"plugins.fleet"},"process":{"pid":863},"trace":{"id":"9b7ec748390372bd55a0ee1e2979eb8a"},"transaction":{"id":"609ca7b42d75151b"}}
{"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.0"},"@timestamp":"2023-06-12T07:35:27.853+00:00","message":"Failure to install package [checkpoint]: [Error: could not update index template settings for logs-checkpoint.firewall-production]","log":{"level":"WARN","logger":"plugins.fleet"},"process":{"pid":863},"trace":{"id":"9b7ec748390372bd55a0ee1e2979eb8a"},"transaction":{"id":"609ca7b42d75151b"}}
{"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.0"},"@timestamp":"2023-06-12T07:35:28.470+00:00","message":"could not update index template settings for logs-checkpoint.firewall-production","error":{"message":"could not update index template settings for logs-checkpoint.firewall-production","type":"Error","stack_trace":"Error: could not update index template settings for logs-checkpoint.firewall-production\n at updateExistingDataStream (/usr/share/kibana/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.js:576:11)\n at runMicrotasks (<anonymous>)\n at processTicksAndRejections (node:internal/process/task_queues:96:5)\n at async Promise.all (index 0)\n at updateAllDataStreams (/usr/share/kibana/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.js:528:3)\n at _installPackage (/usr/share/kibana/x-pack/plugins/fleet/server/services/epm/packages/_install_package.js:178:5)\n at installPackageCommon (/usr/share/kibana/x-pack/plugins/fleet/server/services/epm/packages/install.js:387:12)\n at installPackageFromRegistry (/usr/share/kibana/x-pack/plugins/fleet/server/services/epm/packages/install.js:288:12)\n at installPackage (/usr/share/kibana/x-pack/plugins/fleet/server/services/epm/packages/install.js:553:22)\n at installPackageFromRegistryHandler (/usr/share/kibana/x-pack/plugins/fleet/server/routes/epm/handlers.js:283:15)\n at Router.handle (/usr/share/kibana/node_modules/@kbn/core-http-router-server-internal/src/router.js:141:30)\n at handler (/usr/share/kibana/node_modules/@kbn/core-http-router-server-internal/src/router.js:107:50)\n at exports.Manager.execute (/usr/share/kibana/node_modules/@hapi/hapi/lib/toolkit.js:60:28)\n at Object.internals.handler (/usr/share/kibana/node_modules/@hapi/hapi/lib/handler.js:46:20)\n at exports.execute (/usr/share/kibana/node_modules/@hapi/hapi/lib/handler.js:31:20)\n at Request._lifecycle (/usr/share/kibana/node_modules/@hapi/hapi/lib/request.js:371:32)\n at Request._execute (/usr/share/kibana/node_modules/@hapi/hapi/lib/request.js:281:9)"},"log":{"level":"ERROR","logger":"plugins.fleet"},"process":{"pid":863},"trace":{"id":"9b7ec748390372bd55a0ee1e2979eb8a"},"transaction":{"id":"609ca7b42d75151b"}}
{"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.0"},"@timestamp":"2023-06-12T07:38:55.241+00:00","message":"Running Fleet Usage telemetry send task","log":{"level":"INFO","logger":"plugins.fleet"},"process":{"pid":863},"trace":{"id":"05b75e16acad44f54ca4f80b8887d13f"},"transaction":{"id":"0a990e3c77e7499c"}}
{"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.0"},"@timestamp":"2023-06-12T07:40:07.742+00:00","message":"Fleet Usage: {\"agents_enabled\":true,\"agents\":{\"total_enrolled\":15,\"healthy\":15,\"unhealthy\":0,\"offline\":0,\"inactive\":0,\"unenrolled\":0,\"total_all_statuses\":15,\"updating\":0},\"fleet_server\":{\"total_enrolled\":1,\"healthy\":1,\"unhealthy\":0,\"offline\":0,\"updating\":0,\"total_all_statuses\":1,\"num_host_urls\":1}}","log":{"level":"INFO","logger":"plugins.fleet"},"process":{"pid":863},"trace":{"id":"05b75e16acad44f54ca4f80b8887d13f"},"transaction":{"id":"489a87fd7c8f502f"}}
Oh and the included dashboards from the Integration are gone aswell, they are still listed but trying to open them gets you an error.
