hi guys,
i have ES cluster of 3 nodes, and when i am trying to register repository for my snapshots via curl command - i get empty server reply from server, and elasticsearch service ends up failed and disconnected from the cluster.
ES version : 5.6.13
Curl command :
curl -X PUT http://$MASTERIP:9200/_snapshot/s3_snapshots -H 'Content-Type: application/json' -H 'cache-control: no-cache' -d '{"type": "s3", "settings": {"max_retries": "3", "access_key": "KEY", "secret_key": "KEY", "bucket": "bucketname", "base_path": "bucketname"}}'
i ve turned on debug logs for plugin, but can't find any reason why it keeps failing and disconnecting my node from cluster.
Logs:
20181129/us-east-1/s3/aws4_request
cc2e03269fd1dc72b3d0664780f455957b98667b125593b804ee57bf353148b6"
[2018-11-29T15:35:11,126][DEBUG][c.a.request ] Received successful response: 200, AWS Request ID: E753D8E8AACEE878
[2018-11-29T15:35:11,126][DEBUG][c.a.requestId ] x-amzn-RequestId: not available
[2018-11-29T15:35:11,126][DEBUG][c.a.requestId ] AWS Request ID: E753D8E8AACEE878
[2018-11-29T15:35:11,132][ERROR][o.e.b.ElasticsearchUncaughtExceptionHandler] [ip-10-0-105-4] fatal error in thread [elasticsearch[ip-10-0-105-4][clusterService#updateTask][T#1]], exiting
java.lang.ExceptionInInitializerError: null
at com.amazonaws.services.s3.AmazonS3Client.putObject(AmazonS3Client.java:1748) ~[?:?]
at org.elasticsearch.repositories.s3.S3BlobContainer.executeSingleUpload(S3BlobContainer.java:231) ~[?:?]
at org.elasticsearch.repositories.s3.S3BlobContainer.writeBlob(S3BlobContainer.java:113) ~[?:?]
at org.elasticsearch.repositories.blobstore.BlobStoreRepository.startVerification(BlobStoreRepository.java:724) ~[elasticsearch-5.6.13.jar:5.6.13]
at org.elasticsearch.repositories.RepositoriesService.verifyRepository(RepositoriesService.java:210) ~[elasticsearch-5.6.13.jar:5.6.13]
at org.elasticsearch.repositories.RepositoriesService$VerifyingRegisterRepositoryListener.onResponse(RepositoriesService.java:413) ~[elasticsearch-5.6.13.jar:5.6.13]
at org.elasticsearch.repositories.RepositoriesService$VerifyingRegisterRepositoryListener.onResponse(RepositoriesService.java:398) ~[elasticsearch-5.6.13.jar:5.6.13]
at org.elasticsearch.cluster.AckedClusterStateUpdateTask.onAllNodesAcked(AckedClusterStateUpdateTask.java:64) ~[elasticsearch-5.6.13.jar:5.6.13]
at org.elasticsearch.cluster.service.ClusterService$SafeAckedClusterStateTaskListener.onAllNodesAcked(ClusterService.java:1005) ~[elasticsearch-5.6.13.jar:5.6.13]
at org.elasticsearch.cluster.service.ClusterService$AckCountDownListener.onNodeAck(ClusterService.java:1204) ~[elasticsearch-5.6.13.jar:5.6.13]
at org.elasticsearch.cluster.service.ClusterService$DelegetingAckListener.onNodeAck(ClusterService.java:1144) ~[elasticsearch-5.6.13.jar:5.6.13]
at org.elasticsearch.cluster.service.ClusterService.publishAndApplyChanges(ClusterService.java:786) ~[elasticsearch-5.6.13.jar:5.6.13]
at org.elasticsearch.cluster.service.ClusterService.runTasks(ClusterService.java:587) ~[elasticsearch-5.6.13.jar:5.6.13]
at org.elasticsearch.cluster.service.ClusterService$ClusterServiceTaskBatcher.run(ClusterService.java:263) ~[elasticsearch-5.6.13.jar:5.6.13]
at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) ~[elasticsearch-5.6.13.jar:5.6.13]
at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) ~[elasticsearch-5.6.13.jar:5.6.13]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:576) ~[elasticsearch-5.6.13.jar:5.6.13]
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:247) ~[elasticsearch-5.6.13.jar:5.6.13]
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:210) ~[elasticsearch-5.6.13.jar:5.6.13]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
at java.lang.Thread.run(Thread.java:844) [?:?]
Caused by: java.security.AccessControlException: access denied ("java.lang.RuntimePermission" "getClassLoader")
at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472) ~[?:?]
at java.security.AccessController.checkPermission(AccessController.java:895) ~[?:?]
at java.lang.SecurityManager.checkPermission(SecurityManager.java:335) ~[?:?]
at java.lang.ClassLoader.checkClassLoaderPermission(ClassLoader.java:2042) ~[?:?]
at java.lang.Thread.getContextClassLoader(Thread.java:1539) ~[?:?]
at javax.xml.bind.ContextFinder.find(ContextFinder.java:390) ~[?:?]
at javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:618) ~[?:?]
at javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:565) ~[?:?]
at com.amazonaws.util.Base64.<clinit>(Base64.java:44) ~[?:?]
... 22 more
elasticsearch.yaml:
cluster.name: terraform-elasticsearch
node.name: ${var.hostname}
network.host: _ec2_
cloud.node.auto_attributes: true
discovery.ec2.tag.es_cluster: terraform-elasticsearch
discovery.zen.hosts_provider: ec2
discovery.zen.minimum_master_nodes: 2
cluster.routing.allocation.awareness.attributes: aws_availability_zone
cluster.routing.allocation.total_shards_per_node: 2
path.data: [ "/mnt/data/shard_0/elasticsearch", "/mnt/data/shard_1/elasticsearch", "/mnt/data/shard_2/elasticsearch", "/mnt/data/shard_3/elasticsearch" ]
path.repo: ["/var/lib/elasticsearch/snapshots", "/var/lib/elasticsearch/longterm_snapshots"]
http.port: 9200
reindex.remote.whitelist : 10.0.105.*:9200,10.0.106.*:9200
http.cors.enabled : true
http.cors.allow-origin : "*"
http.cors.allow-methods : OPTIONS, HEAD, GET, POST, PUT, DELETE
http.cors.allow-headers : X-Requested-With, X-Auth-Token, Content-Type, Content-Length
script.engine.groovy.inline: true
script.engine.groovy.inline.aggs: true
script.engine.groovy.inline.update: true
script.engine.groovy.stored.aggs: true
script.engine.groovy.stored.search: true
indices.query.bool.max_clause_count: 10240
cloud.aws.access_key: KEY
cloud.aws.secret_key: KEY
cloud.aws.protocol: https
cloud.aws.region: us-east-1
cloud.aws.read_timeout: 60s
s3.client.my_config.access_key: "KEY"
s3.client.my_config.secret_key: "KEY"
s3.client.my_config.endpoint: s3.us-east-1.amazonaws.com
Could someone help me with that? Because i am playing with it almost for a week