Hi
Can You help me with fix it?
During install elasticsearch over docker swarm/docker compose I've met a common issue with
java.lang.IllegalStateException: failed to obtain node locks
I'm pretty sure
that only one node was hit to this directory
this installation was executed from below command
docker stack deploy -c elk_cluster.yml elk
docker --version
Docker version 20.10.12, build e91ed57
java --version
openjdk 11.0.14.1 2022-02-08 LTS
OpenJDK Runtime Environment 18.9 (build 11.0.14.1+1-LTS)
OpenJDK 64-Bit Server VM 18.9 (build 11.0.14.1+1-LTS, mixed mode, sharing)
and volumes are mounted:
es_data_hdd_1_3:
image: private_repo/elk-docker/elasticsearch/elasticsearch:8.0.1
container_name: es_data_hdd_1_3
user: elasticsearch
environment:
- xpack.security.enabled=true
- xpack.security.http.ssl.enabled=true
- xpack.security.http.ssl.key=/usr/share/elasticsearch/config/certificates/es_data_hdd_1_3/es_data_hdd_1_3.key
- xpack.security.http.ssl.certificate_authorities=/usr/share/elasticsearch/config/certificates/ca/ca.crt
- xpack.security.http.ssl.certificate=/usr/share/elasticsearch/config/certificates/es_data_hdd_1_3/es_data_hdd_1_3.crt
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.security.transport.ssl.certificate_authorities=/usr/share/elasticsearch/config/certificates/ca/ca.crt
- xpack.security.transport.ssl.certificate=/usr/share/elasticsearch/config/certificates/es_data_hdd_1_3/es_data_hdd_1_3.crt
- xpack.security.transport.ssl.key=/usr/share/elasticsearch/config/certificates/es_data_hdd_1_3/es_data_hdd_1_3.key
- ELASTIC_USERNAME=elastic
- ELASTIC_PASSWORD=changeme
- cluster.name=new_elk_cluster
- discovery.seed_hosts=es_coordination_1,es_master_1_1,es_master_2_1,es_data_ssd_3_1_ingest,es_data_ssd_1_1,es_data_ssd_2_1,es_data_ssd_3_1,es_data_ssd_4_1,es_data_ssd_5_1,es_data_hdd_1_1,es_data_hdd_2_1,es_data_hdd_3_1,es_data_hdd_4_1,es_data_hdd_5_1,es_data_hdd_6_1,es_data_hdd_7_1,es_data_hdd_8_1,es_data_hdd_9_1,es_coordination_2,es_master_1_2,es_master_2_2,es_data_ssd_3_2_ingest,es_data_ssd_1_2,es_data_ssd_2_2,es_data_ssd_3_2,es_data_ssd_4_2,es_data_ssd_5_2,es_data_hdd_1_2,es_data_hdd_2_2,es_data_hdd_3_2,es_data_hdd_4_2,es_data_hdd_5_2,es_data_hdd_6_2,es_data_hdd_7_2,es_data_hdd_8_2,es_data_hdd_9_2,es_coordination_3,es_master_1_3,es_master_2_3,es_data_ssd_3_3_ingest,es_data_ssd_1_3,es_data_ssd_2_3,es_data_ssd_3_3,es_data_ssd_4_3,es_data_ssd_5_3,es_data_hdd_2_3,es_data_hdd_3_3,es_data_hdd_4_3,es_data_hdd_5_3,es_data_hdd_6_3,es_data_hdd_7_3,es_data_hdd_8_3,es_data_hdd_9_3
- cluster.initial_master_nodes=es_master_1_1,es_master_2_1,es_master_1_2,es_master_2_2,es_master_1_3,es_master_2_3
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms8g -Xmx8g"
- node.roles= data_warm,data_content
networks:
- mynet
ports:
- "9245:9200"
- "9345:9300"
volumes:
- /home/elasticsearch/cer/:/usr/share/elasticsearch/config/certificates
- /opt/hdd/disk1/data_warm/:/usr/share/elasticsearch/data
ulimits:
memlock:
soft: -1
hard: -1
deploy:
placement:
constraints:
- node.hostname == host2
mode: "replicated"
replicas: 1
resources:
limits:
memory: 8G
drwxr-xr-x. 3 elasticsearch elasticsearch 23 Mar 4 11:32 disk1
drwxr-xr-x. 3 elasticsearch elasticsearch 23 Mar 4 11:33 disk2
drwxr-x---. 3 elasticsearch elasticsearch 23 Mar 4 11:33 disk3
drwxr-xr-x. 3 elasticsearch elasticsearch 23 Mar 4 11:33 disk4
drwxr-xr-x. 3 elasticsearch elasticsearch 23 Mar 4 11:33 disk5
drwxr-xr-x. 3 elasticsearch elasticsearch 23 Mar 4 11:33 disk6
drwxr-xr-x. 3 elasticsearch elasticsearch 23 Mar 4 11:34 disk7
drwxr-xr-x. 3 elasticsearch elasticsearch 23 Mar 4 11:34 disk8
drwxr-xr-x. 3 elasticsearch elasticsearch 23 Mar 4 11:34 disk9
[elasticsearch@host2]$ cd disk1
[elasticsearch@host2]$ ls -ltr
total 0
drwxr-xr-x. 2 elasticsearch elasticsearch 6 Mar 3 13:32 data_warm
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | {"@timestamp":"2022-03-07T15:47:19.806Z", "log.level":"ERROR", "message":"uncaught exception in thread [main]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.bootstrap.ElasticsearchUncaughtExceptionHandler","elasticsearch.node.name":"71bfee888c95","elasticsearch.cluster.name":"new_elk_cluster","error.type":"org.elasticsearch.bootstrap.StartupException","error.message":"java.lang.IllegalStateException: failed to obtain node locks, tried [/usr/share/elasticsearch/data]; maybe these locations are not writable or multiple nodes were started on the same data path?","error.stack_trace":"java.lang.IllegalStateException: failed to obtain node locks, tried [/usr/share/elasticsearch/data]; maybe these locations are not writable or multiple nodes were started on the same data path?\nLikely root cause: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/node.lock\n\tat java.base/sun.nio.fs.UnixException.translateToIOException(UnixException.java:92)\n\tat java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106)\n\tat java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111)\n\tat java.base/sun.nio.fs.UnixPath.toRealPath(UnixPath.java:825)\n\tat org.apache.lucene.store.NativeFSLockFactory.obtainFSLock(NativeFSLockFactory.java:94)\n\tat org.apache.lucene.store.FSLockFactory.obtainLock(FSLockFactory.java:43)\n\tat org.apache.lucene.store.BaseDirectory.obtainLock(BaseDirectory.java:44)\n\tat org.elasticsearch.env.NodeEnvironment$NodeLock.<init>(NodeEnvironment.java:218)\n\tat org.elasticsearch.env.NodeEnvironment$NodeLock.<init>(NodeEnvironment.java:193)\n\tat org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:266)\n\tat org.elasticsearch.node.Node.<init>(Node.java:388)\n\tat org.elasticsearch.node.Node.<init>(Node.java:277)\n\tat org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:234)\n\tat org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:234)\n\tat org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:358)\n\tat org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:166)\n\tat org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:157)\n\tat org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:77)\n\tat org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:112)\n\tat org.elasticsearch.cli.Command.main(Command.java:77)\n\tat org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:122)\n\tat org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:80)\nFor complete error details, refer to the log at /usr/share/elasticsearch/logs/new_elk_cluster.log\n"}
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | uncaught exception in thread [main]
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | java.lang.IllegalStateException: failed to obtain node locks, tried [/usr/share/elasticsearch/data]; maybe these locations are not writable or multiple nodes were started on the same data path?
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | Likely root cause: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/node.lock
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at java.base/sun.nio.fs.UnixException.translateToIOException(UnixException.java:92)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at java.base/sun.nio.fs.UnixPath.toRealPath(UnixPath.java:825)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.apache.lucene.store.NativeFSLockFactory.obtainFSLock(NativeFSLockFactory.java:94)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.apache.lucene.store.FSLockFactory.obtainLock(FSLockFactory.java:43)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.apache.lucene.store.BaseDirectory.obtainLock(BaseDirectory.java:44)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.elasticsearch.env.NodeEnvironment$NodeLock.<init>(NodeEnvironment.java:218)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.elasticsearch.env.NodeEnvironment$NodeLock.<init>(NodeEnvironment.java:193)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:266)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.elasticsearch.node.Node.<init>(Node.java:388)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.elasticsearch.node.Node.<init>(Node.java:277)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:234)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:234)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:358)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:166)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:157)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:77)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:112)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.elasticsearch.cli.Command.main(Command.java:77)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:122)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:80)
elk_es_data_hdd_1_3.1.fgfnwjsdcwz1@host3 | For complete error details, refer to the log at /usr/share/elasticsearch/logs/new_elk_cluster.log