Hi,
I'm working on a test environment elastic stack on docker from scratch to get a good grip on things. I started with "sebp/elk" docker image, then i made a docker-compose stack with 2 logstash containers one for docker logs and the other for application logs. Stack is working fine and the compose.yaml is like this:
version: "3"
services:
elasticsearch:
image: elasticsearch:7.7.0
container_name: elasticsearch
hostname: elasticsearch
environment:
- "discovery.type=single-node"
- "ES_JAVA_OPTS=-Xms2g -Xmx2g"
- "ES_CONNECT_RETRY=300"
- "ES_HEAP_SIZE=2g"
ports:
- 9200:9200
- 9300:9300
networks:
- elknetwork
kibana:
image: kibana:7.7.0
container_name: kibana
hostname: kibana
ports:
- 5601:5601
links:
- elasticsearch:elasticsearch
depends_on:
- elasticsearch
networks:
- elknetwork
logstash:
image: logstasttag:latest
container_name: logstash
hostname: logstash
environment:
- "xpack.monitoring.enabled=true"
- "path.config=/usr/share/logstash/pipeline"
ports:
- 9600:9600
- 8089:8089
- 12201:12201/udp
links:
- elasticsearch:elasticsearch
depends_on:
- elasticsearch
networks:
- elknetwork
logstash2:
image: logstashprogram:latest
container_name: logstash2
hostname: logstash
environment:
- "xpack.monitoring.enabled=true"
- "path.config=/usr/share/logstash/pipeline"
ports:
- 9601:9601
- 8088:8088
- 5001:5001/udp
links:
- elasticsearch:elasticsearch
depends_on:
- elasticsearch
networks:
- elknetwork
networks:
elknetwork:
driver: bridge
Then i decided to test out a 3 node cluster with compose and got a reference from https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html. First i tried to test out the example compose and as expected it worked. Then i started to modify my compose and the final result is this:
version: "3"
services:
esnode1:
image: elasticsearch:7.7.0
container_name: esnode1
hostname: elasticsearch
environment:
- "node.name=esnode1"
- "cluster.name=es-docker-cluster"
- "discovery.seed_hosts=esnode2,esnode3"
- "cluster.initial_master_nodes=esnode1,esnode2,esnode3"
- "bootstrap.memory_lock=true"
- "ES_JAVA_OPTS=-Xms768m -Xmx768m"
- "ES_CONNECT_RETRY=300"
- "ES_HEAP_SIZE=768m"
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- data01:/usr/share/elasticsearch/data
ports:
- 9200:9200
- 9300:9300
networks:
- elknetwork
esnode2:
image: elasticsearch:7.7.0
container_name: esnode2
hostname: elasticsearch
environment:
- "node.name=esnode2"
- "cluster.name=es-docker-cluster"
- "discovery.seed_hosts=esnode1,esnode3"
- "cluster.initial_master_nodes=esnode1,esnode2,esnode3"
- "bootstrap.memory_lock=true"
- "ES_JAVA_OPTS=-Xms768m -Xmx768m"
- "ES_CONNECT_RETRY=300"
- "ES_HEAP_SIZE=768m"
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- data02:/usr/share/elasticsearch/data
ports:
- 9201:9201
- 9301:9301
networks:
- elknetwork
esnode3:
image: elasticsearch:7.7.0
container_name: esnode3
hostname: elasticsearch
environment:
- "node.name=esnode3"
- "cluster.name=es-docker-cluster"
- "discovery.seed_hosts=esnode1,esnode2"
- "cluster.initial_master_nodes=esnode1,esnode2,esnode3"
- "bootstrap.memory_lock=true"
- "ES_JAVA_OPTS=-Xms768m -Xmx768m"
- "ES_CONNECT_RETRY=300"
- "ES_HEAP_SIZE=768m"
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- data03:/usr/share/elasticsearch/data
ports:
- 9202:9202
- 9302:9302
networks:
- elknetwork
kibana:
image: kibana:7.7.0
container_name: kibana
hostname: kibana
ports:
- 5601:5601
environment:
- "ELASTICSEARCH_URL=http://esnode1:9200"
- "ELASTICSEARCH_HOSTS=http://esnode1:9200"
links:
- esnode1:esnode1
depends_on:
- esnode1
networks:
- elknetwork
logstash:
image: logstasttag:latest
container_name: logstash
hostname: logstash
environment:
- "xpack.monitoring.enabled=true"
- "path.config=/usr/share/logstash/pipeline"
ports:
- 9600:9600
- 8089:8089
- 12201:12201/udp
links:
- esnode1:esnode1
depends_on:
- esnode1
networks:
- elknetwork
logstash2:
image: logstashprogram:latest
container_name: logstash2
hostname: logstash
environment:
- "xpack.monitoring.enabled=true"
- "path.config=/usr/share/logstash/pipeline"
ports:
- 9601:9601
- 8088:8088
- 5001:5001/udp
links:
- esnode1:esnode1
depends_on:
- esnode1
networks:
- elknetwork
volumes:
data01:
driver: local
data02:
driver: local
data03:
driver: local
networks:
elknetwork:
driver: bridge
When i run this compose i get these logs from all 3 esnodes:
uncaught exception in thread [main]
ElasticsearchException[failed to bind service]; nested: IndexFormatTooNewException[Format version is not supported (resource BufferedChecksumIndexInput(SimpleFSIndexInput(path="/usr/share/elasticsearch/data/nodes/0/_state/segments_4"))): 10 (needs to be between 7 and 9)];
Likely root cause: org.apache.lucene.index.IndexFormatTooNewException: Format version is not supported (resource BufferedChecksumIndexInput(SimpleFSIndexInput(path="/usr/share/elasticsearch/data/nodes/0/_state/segments_4"))): 10 (needs to be between 7 and 9)
at org.apache.lucene.codecs.CodecUtil.checkHeaderNoMagic(CodecUtil.java:216)
{"type": "server", "timestamp": "2020-11-02T08:16:45,562Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "cluster.name": "es-docker-cluster", "node.name": "esnode1", "message": "uncaught exception in thread [main]",
at org.apache.lucene.index.SegmentInfos.readCommit(SegmentInfos.java:305)
at org.apache.lucene.index.SegmentInfos.readCommit(SegmentInfos.java:289)
at org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:64)
at org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:61)
at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:680)
at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:84)
at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:76)
at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:64)
at org.elasticsearch.gateway.PersistedClusterStateService.nodeMetaData(PersistedClusterStateService.java:262)
at org.elasticsearch.env.NodeEnvironment.loadNodeMetaData(NodeEnvironment.java:408)
at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:329)
at org.elasticsearch.node.Node.<init>(Node.java:284)
at org.elasticsearch.node.Node.<init>(Node.java:264)
at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:227)
at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:227)
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:393)
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:170)
at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:161)
at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:86)
at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:127)
at org.elasticsearch.cli.Command.main(Command.java:90)
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:126)
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:92)
For complete error details, refer to the log at /usr/share/elasticsearch/logs/es-docker-cluster.log
Then after some repeating process with the same errors the elasticsearch containers shut down automatically. what did i mess up with in my compose to cause this, i can't figure it out.