Elasticsearch - major design flaw with node ID

Elastic Stack Elasticsearch
7

The deployment configuration is below.
Some notes:-
The data path is persisted to a local volume. The cluster works fine (read/write) so I assume the Node ID is appropriately stored.
The issue can easily be recreated by scaling the Elasticsearch node replica down and up (I’ll confirm tomorrow and get logs)

version: "3.3"

services:

Elasticsearch:
image: lcl02164.test.softwaregrp.net/efk/elasticsearch-7-16-1
logging:
driver: "json-file"
options:
max-size: 30m
environment:
- discovery.type=zen
- node.name=node-01
- network.host=0.0.0.0
- cluster.name=k-test-cluster
- discovery.seed_hosts=es_2,es_3
- cluster.initial_master_nodes=node-01,node-02,node-03
- bootstrap.memory_lock=false
- "ES_JAVA_OPTS=-Xms16384m -Xmx16384m"
- xpack.security.enabled=true
- ELASTIC_PASSWORD=changeMe
ports:
- "9200:9200"
- 9300
networks:
- private
volumes:
- /opt/mount1/Elasticsearch/data:/usr/share/Elasticsearch/data
- /opt/mount1/Elasticsearch/logs:/usr/share/Elasticsearch/logs
deploy:
placement:
constraints: [node.hostname == lcl02165.test.softwaregrp.net]
replicas: 1

es_2:
image: lcl02164.test.softwaregrp.net/efk/elasticsearch-7-16-1
logging:
driver: "json-file"
options:
max-size: 30m

environment:
  - discovery.type=zen
  - node.name=node-02
  - network.host=0.0.0.0
  - cluster.name=k-test-cluster
  - discovery.seed_hosts=elasticsearch,es_3
  - cluster.initial_master_nodes=node-01,node-02,node-03
  - bootstrap.memory_lock=false
  - "ES_JAVA_OPTS=-Xms16384m -Xmx16384m"
  - xpack.security.enabled=true
  - ELASTIC_PASSWORD=changeMe
ports:
  - "9201:9200"
  - 9300
networks:
  - private
volumes:
  - /opt/mount1/elasticsearch/data:/usr/share/elasticsearch/data
  - /opt/mount1/elasticsearch/logs:/usr/share/elasticsearch/logs
deploy:
  placement:
    constraints: [node.hostname == lcl02446.test.softwaregrp.net]

es_3:
image: lcl02164.test.softwaregrp.net/efk/elasticsearch-7-16-1
logging:
driver: "json-file"
options:
max-size: 30m

environment:
  - discovery.type=zen
  - node.name=node-03
  - network.host=0.0.0.0
  - cluster.name=k-test-cluster
  - discovery.seed_hosts=elasticsearch,es_2
  - cluster.initial_master_nodes=node-01,node-02,node-03
  - bootstrap.memory_lock=false
  - "ES_JAVA_OPTS=-Xms16384m -Xmx16384m"
  - xpack.security.enabled=true
  - ELASTIC_PASSWORD=changeMe
ports:
  - "9202:9200"
  - 9300
networks:
  - private
volumes:
  - /opt/mount1/elasticsearch/data:/usr/share/elasticsearch/data
  - /opt/mount1/elasticsearch/logs:/usr/share/elasticsearch/logs
deploy:
  placement:
    constraints: [node.hostname == lcl02586.test.softwaregrp.net]

kibana:
image: lcl02164.test.softwaregrp.net/efk/kibana-7-12-1
logging:
driver: "json-file"
options:
max-size: 30m
environment:
ELASTICSEARCH_URL: http://elasticsearch:9200
SERVER_SSL_ENABLED: "true"
ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: /usr/share/kibana/config/server-chain.crt
SERVER_SSL_KEY: /usr/share/kibana/config/localhost.key
SERVER_SSL_CERTIFICATE: /usr/share/kibana/config/localhost.crt
volumes:
- ./kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
- ./kibana/server-chain.crt:/usr/share/kibana/config/server-chain.crt
- ./kibana/localhost.crt:/usr/share/kibana/config/localhost.crt
- ./kibana/localhost.key:/usr/share/kibana/config/localhost.key
ports:
- 5601:5601
networks:
- private
depends_on:
- Elasticsearch
- es_2
- es_3
deploy:
placement:
constraints: [node.role == manager]

networks:
private:
external: true

.......................