Elastic GCS bucket snapshot failed and we cannot able to retrive the old inremental backup in that bucket,
Is there any possibility to recover the back
FYI, we can able to view the data storage inside that bucket in the compute engine
[2023-08-24T21:39:41,840][WARN ][o.e.s.SnapshotsService ] [xxxx] [xxxx] failed to finalize snapshot
com.google.cloud.storage.StorageException: Error writing request body to server
at com.google.cloud.storage.spi.v1.HttpStorageRpc.translate(HttpStorageRpc.java:227) ~[?:?]
at com.google.cloud.storage.spi.v1.HttpStorageRpc.write(HttpStorageRpc.java:762) ~[?:?]
at com.google.cloud.storage.BlobWriteChannel$1.run(BlobWriteChannel.java:60) ~[?:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_131]
at com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:105) ~[?:?]
at com.google.cloud.RetryHelper.run(RetryHelper.java:76) ~[?:?]
at com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:50) ~[?:?]
at com.google.cloud.storage.BlobWriteChannel.flushBuffer(BlobWriteChannel.java:53) ~[?:?]
at com.google.cloud.BaseWriteChannel.flush(BaseWriteChannel.java:112) ~[?:?]
at com.google.cloud.BaseWriteChannel.write(BaseWriteChannel.java:139) ~[?:?]
at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStore$2.lambda$write$0(GoogleCloudStorageBlobStore.java:264) ~[?:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_131]
at org.elasticsearch.repositories.gcs.SocketAccess.doPrivilegedIOException(SocketAccess.java:44) ~[?:?]
at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStore$2.write(GoogleCloudStorageBlobStore.java:264) ~[?:?]
at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) ~[?:1.8.0_131]
at java.nio.channels.Channels.writeFully(Channels.java:101) ~[?:1.8.0_131]
at java.nio.channels.Channels.access$000(Channels.java:61) ~[?:1.8.0_131]
at java.nio.channels.Channels$1.write(Channels.java:174) ~[?:1.8.0_131]
at org.elasticsearch.core.internal.io.Streams.copy(Streams.java:55) ~[elasticsearch-core-7.4.0.jar:7.4.0]
at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStore.writeBlobResumable(GoogleCloudStorageBlobStore.java:250) ~[?:?]
at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStore.writeBlob(GoogleCloudStorageBlobStore.java:219) ~[?:?]
at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobContainer.writeBlob(GoogleCloudStorageBlobContainer.java:67) ~[?:?]
at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobContainer.writeBlobAtomic(GoogleCloudStorageBlobContainer.java:72) ~[?:?]
at org.elasticsearch.repositories.blobstore.BlobStoreRepository.writeAtomic(BlobStoreRepository.java:906) ~[elasticsearch-7.4.0.jar:7.4.0]
at org.elasticsearch.repositories.blobstore.BlobStoreRepository.writeIndexGen(BlobStoreRepository.java:824) ~[elasticsearch-7.4.0.jar:7.4.0]
at org.elasticsearch.repositories.blobstore.BlobStoreRepository.finalizeSnapshot(BlobStoreRepository.java:645) ~[elasticsearch-7.4.0.jar:7.4.0]
at org.elasticsearch.snapshots.SnapshotsService$5.doRun(SnapshotsService.java:982) [elasticsearch-7.4.0.jar:7.4.0]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773) [elasticsearch-7.4.0.jar:7.4.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.4.0.jar:7.4.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_131]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_131]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
Caused by: java.io.IOException: Error writing request body to server
at sun.net.www.protocol.http.HttpURLConnection$StreamingOutputStream.checkError(HttpURLConnection.java:3518) ~[?:?]
at sun.net.www.protocol.http.HttpURLConnection$StreamingOutputStream.write(HttpURLConnection.java:3501) ~[?:?]
at com.google.api.client.util.ByteStreams.copy(ByteStreams.java:53) ~[?:?]
at com.google.api.client.util.IOUtils.copy(IOUtils.java:87) ~[?:?]
at com.google.api.client.http.AbstractInputStreamContent.writeTo(AbstractInputStreamContent.java:69) ~[?:?]
at com.google.api.client.http.javanet.NetHttpRequest$DefaultOutputWriter.write(NetHttpRequest.java:71) ~[?:?]