As part of our backup strategy, we are using source only repos and snapshots.
As soon as such a snapshot is restored, the prometheus queries fail.
Is there any chance to filter / skip these indices for the metrics?
[2021-12-06T09:19:57,816][DEBUG][o.e.a.a.i.s.TransportIndicesStatsAction] default-0] [indices:monitor/stats] failed to execute operation for shard [[xyz_reindex][4], node[ac], [P], s[STARTED], a[id=RPVl4_oHTx-Q01cEKobKdw]]
java.lang.UnsupportedOperationException: _source only indices can't be searched or filtered
at org.elasticsearch.snapshots.SeqIdGeneratingFilterReader$SeqIdGeneratingSubReaderWrapper$1.terms(SeqIdGeneratingFilterReader.java:159) ~[?:?]
at org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:366) ~[lucene-core-8.8.0.jar:8.8.0 b10659f0fc18b58b90929cfdadde94544d202c4a - noble - 2021-01-25 19:07:45]
at org.elasticsearch.index.engine.CompletionStatsCache.lambda$get$0(CompletionStatsCache.java:79) ~[elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:329) ~[elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.index.engine.CompletionStatsCache.get(CompletionStatsCache.java:71) ~[elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.index.engine.ReadOnlyEngine.completionStats(ReadOnlyEngine.java:574) ~[elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.index.shard.IndexShard.completionStats(IndexShard.java:1135) ~[elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:196) ~[elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:95) ~[elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:38) ~[elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:434) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:396) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:61) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.transport.TransportService$8.doRun(TransportService.java:914) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.12.1.jar:7.12.1]
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:?]
at java.lang.Thread.run(Unknown Source) [?:?]
[10:43] Siegwart, Christian
[2021-12-06T09:19:58,439][WARN ][r.suppressed ] [perf-elasticsearch-default-0] path: /_prometheus/metrics, params: {}
org.elasticsearch.ElasticsearchException: Cluster state request failed
at org.elasticsearch.action.TransportNodePrometheusMetricsAction$AsyncAction$1.onFailure(TransportNodePrometheusMetricsAction.java:141) [prometheus-exporter-7.12.1.0.jar:7.12.1.0]
at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.lambda$doStart$2(TransportMasterNodeAction.java:151) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.ActionListener$2.onFailure(ActionListener.java:148) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:77) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:28) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.doStart(TransportMasterNodeAction.java:155) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.support.master.TransportMasterNodeAction.doExecute(TransportMasterNodeAction.java:104) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.support.master.TransportMasterNodeAction.doExecute(TransportMasterNodeAction.java:46) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:173) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:149) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:77) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:86) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:66) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:402) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.client.support.AbstractClient$ClusterAdmin.execute(AbstractClient.java:695) [elasticsearch-7.12.1.jar:7.12.1]
at org.elasticsearch.client.support.AbstractClient$ClusterAdmin.state(AbstractClient.java:725) [elasticsearch-7.12.1.jar:7.12.1]