I have a three node Elasticsearch cluster with a shared NFS volume. I have set a path.repo value in elasticsearch.yml and restarted elasticsearch (one node at a time, my cluster needs to stay up). I have confirmed all three nodes can read from and write to this path like so:
Node1:
root@facility-es01:~# su -s /bin/bash - elasticsearch
su: warning: cannot change directory to /nonexistent: No such file or directory
elasticsearch@facility-es01:/root$ grep repo /etc/elasticsearch/elasticsearch.yml
path.repo: "/tmp/elastic"
elasticsearch@facility-es01:/root$ echo 'test1' >> /tmp/elastic/test
elasticsearch@facility-es01:/root$ ls -lha /tmp/elastic/
total 5.0K
drwxrwxrwx 2 elasticsearch elasticsearch 3 Jul 26 10:12 .
drwxrwxrwt 11 root root 4.0K Jul 26 10:13 ..
-rw-rw-rw- 1 elasticsearch elasticsearch 6 Jul 26 10:12 test
Node2:
root@facility-es02:~# su -s /bin/bash - elasticsearch
su: warning: cannot change directory to /nonexistent: No such file or directory
elasticsearch@facility-es02:/root$ grep repo /etc/elasticsearch/elasticsearch.yml
path.repo: "/tmp/elastic"
elasticsearch@facility-es02:/root$ ls -lha /tmp/elastic/
total 5.0K
drwxrwxrwx 2 elasticsearch elasticsearch 3 Jul 26 10:12 .
drwxrwxrwt 11 root root 4.0K Jul 26 10:15 ..
-rw-rw-rw- 1 elasticsearch elasticsearch 6 Jul 26 10:12 test
elasticsearch@facility-es02:/root$ cat /tmp/elastic/test
test1
elasticsearch@facility-es02:/root$ echo 'test2' >> /tmp/elastic/test
elasticsearch@facility-es02:/root$
Node3:
root@facility-es03:~# su -s /bin/bash - elasticsearch
su: warning: cannot change directory to /nonexistent: No such file or directory
elasticsearch@facility-es03:/root$ grep repo /etc/elasticsearch/elasticsearch.yml
path.repo: "/tmp/elastic"
elasticsearch@facility-es03:/root$ ls -lha /tmp/elastic/
total 5.0K
drwxrwxrwx 2 elasticsearch elasticsearch 3 Jul 26 10:12 .
drwxrwxrwt 11 root root 4.0K Jul 26 10:16 ..
-rw-rw-rw- 1 elasticsearch elasticsearch 12 Jul 26 10:15 test
elasticsearch@facility-es03:/root$ echo 'test3' >> /tmp/elastic/test
elasticsearch@facility-es03:/root$ cat /tmp/elastic/test
test1
test2
test3
However when I attempt to verify the repository I still get the error message like they cannot write to that path like this:
root@facility-es03:~# curl -s -XPOST 'localhost:9200/_snapshot/backup1/_verify'| jq
{
"error": {
"root_cause": [
{
"type": "repository_verification_exception",
"reason": "[backup1] [[_AAzUpbdSLmH8Eu2fATC8w, 'RemoteTransportException[[facility-es03.ftc][10.40.8.53:9300][internal:admin/repository/verify]]; nested: RepositoryVerificationException[[backup1] a file written by master to the store [/tmp/elastic] cannot be accessed on the node [{facility-es03.ftc}{_AAzUpbdSLmH8Eu2fATC8w}{ZHNN9EcmTjWTzmcyDt-h_g}{10.40.8.53}{10.40.8.53:9300}{cdhilmrstw}{ml.machine_memory=12256530432, xpack.installed=true, transform.node=true, ml.max_open_jobs=20, ml.max_jvm_size=8589934592}]. This might indicate that the store [/tmp/elastic] is not shared between this node and the master node or that permissions on the store don't allow reading files written by the master node]; nested: NoSuchFileException[/tmp/elastic/tests-NDjRPrGHQD6AHCURtQV5Hg/master.dat];'], [PFSVaur8STWZap533rLEXg, 'RemoteTransportException[[facility-es01.ftc][10.40.8.51:9300][internal:admin/repository/verify]]; nested: RepositoryVerificationException[[backup1] a file written by master to the store [/tmp/elastic] cannot be accessed on the node [{facility-es01.ftc}{PFSVaur8STWZap533rLEXg}{_WH1JcQzQySVGxICnW8O7w}{10.40.8.51}{10.40.8.51:9300}{cdhilmrstw}{ml.machine_memory=12594253824, xpack.installed=true, transform.node=true, ml.max_open_jobs=20, ml.max_jvm_size=8589934592}]. This might indicate that the store [/tmp/elastic] is not shared between this node and the master node or that permissions on the store don't allow reading files written by the master node]; nested: NoSuchFileException[/tmp/elastic/tests-NDjRPrGHQD6AHCURtQV5Hg/master.dat];']]"
}
],
"type": "repository_verification_exception",
"reason": "[backup1] [[_AAzUpbdSLmH8Eu2fATC8w, 'RemoteTransportException[[facility-es03.ftc][10.40.8.53:9300][internal:admin/repository/verify]]; nested: RepositoryVerificationException[[backup1] a file written by master to the store [/tmp/elastic] cannot be accessed on the node [{facility-es03.ftc}{_AAzUpbdSLmH8Eu2fATC8w}{ZHNN9EcmTjWTzmcyDt-h_g}{10.40.8.53}{10.40.8.53:9300}{cdhilmrstw}{ml.machine_memory=12256530432, xpack.installed=true, transform.node=true, ml.max_open_jobs=20, ml.max_jvm_size=8589934592}]. This might indicate that the store [/tmp/elastic] is not shared between this node and the master node or that permissions on the store don't allow reading files written by the master node]; nested: NoSuchFileException[/tmp/elastic/tests-NDjRPrGHQD6AHCURtQV5Hg/master.dat];'], [PFSVaur8STWZap533rLEXg, 'RemoteTransportException[[facility-es01.ftc][10.40.8.51:9300][internal:admin/repository/verify]]; nested: RepositoryVerificationException[[backup1] a file written by master to the store [/tmp/elastic] cannot be accessed on the node [{facility-es01.ftc}{PFSVaur8STWZap533rLEXg}{_WH1JcQzQySVGxICnW8O7w}{10.40.8.51}{10.40.8.51:9300}{cdhilmrstw}{ml.machine_memory=12594253824, xpack.installed=true, transform.node=true, ml.max_open_jobs=20, ml.max_jvm_size=8589934592}]. This might indicate that the store [/tmp/elastic] is not shared between this node and the master node or that permissions on the store don't allow reading files written by the master node]; nested: NoSuchFileException[/tmp/elastic/tests-NDjRPrGHQD6AHCURtQV5Hg/master.dat];']]"
},
"status": 500
}
What am I missing?