I am trying to deploy an Elasticsearch 8.0.0 4 node cluster with 2 masters and 3 datas (1 host will have both a master and a data). I am deploying it in portainer 2.9.0 and docker 20.10.11, so my docker compose is the following:
version: "3.8"
x-master-opts: &master
ES_JAVA_OPTS: "-Xmx2g -Xms2g"
discovery.seed_resolver.timeout: 60s
discovery.seed_hosts: master1, master2
cluster.initial_master_nodes: master1,master2
network.host: 0.0.0.0
xpack.security.enabled: "false"
x-data-opts: &data
ES_JAVA_OPTS: "-Xmx2g -Xms2g"
discovery.seed_resolver.timeout: 60s
discovery.seed_hosts: master1, master2
cluster.initial_master_nodes: master1, master2
network.host: 0.0.0.0
xpack.security.enabled: "false"
networks:
net:
external: true
services:
master1:
image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0
environment:
<<: *master
node.name: "master1"
node.roles: "master"
ulimits:
memlock:
soft: -1
hard: -1
ports:
- "9200:9200"
- "9300:9300"
networks:
net:
aliases:
- master1
volumes:
- /var/volumes/esmaster1:/usr/share/elasticsearch/data_m1
deploy:
placement:
constraints:
- node.hostname == <node_dir>
mode: "replicated"
replicas: 1
resources:
limits:
memory: 2G
master2:
depends_on:
- master1
image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0
environment:
<<: *master
node.name: "master2"
node.roles: "master, data"
ulimits:
memlock:
soft: -1
hard: -1
networks:
net:
aliases:
- master2
volumes:
- /var/volumes/esmaster2:/usr/share/elasticsearch/data_m2
deploy:
placement:
constraints:
- node.hostname == <node_dir>
mode: "replicated"
replicas: 1
resources:
limits:
memory: 2G
data1:
image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0
environment:
<<: *data
node.name: "data1"
node.roles: "data"
ulimits:
memlock:
soft: -1
hard: -1
healthcheck:
test: curl -fs http://localhost:9200/_cat/health || exit 1
interval: 30s
timeout: 5s
retries: 3
start_period: 45s
networks:
net:
aliases:
- data1
volumes:
- /var/volumes/esdata1:/usr/share/elasticsearch/data1
deploy:
placement:
constraints:
- node.hostname == <node_dir>
mode: "replicated"
replicas: 1
resources:
limits:
memory: 2G
data2:
image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0
environment:
<<: *data
node.name: "data2"
node.roles: "data"
ulimits:
memlock:
soft: -1
hard: -1
healthcheck:
test: curl -fs http://localhost:9200/_cat/health || exit 1
interval: 30s
timeout: 5s
retries: 3
start_period: 45s
networks:
net:
aliases:
- data2
volumes:
- /var/volumes/esdata2:/usr/share/elasticsearch/data2
deploy:
placement:
constraints:
- node.hostname == <node_dir>
mode: "replicated"
replicas: 1
resources:
limits:
memory: 2G
data3:
image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0
environment:
<<: *data
node.name: "data3"
node.roles: "data"
ulimits:
memlock:
soft: -1
hard: -1
healthcheck:
test: curl -fs http://localhost:9200/_cat/health || exit 1
interval: 30s
timeout: 5s
retries: 3
start_period: 45s
networks:
net:
aliases:
- data3
volumes:
- /var/volumes/esdata3:/usr/share/elasticsearch/data3
deploy:
placement:
constraints:
- node.hostname == <node_dir>
mode: "replicated"
replicas: 1
resources:
limits:
memory: 2G
The problem is that the nodes suppossed to be masters are not elected as such and the datas can't discover them.
master1 log:
{"@timestamp":"2022-03-21T10:30:09.114Z", "log.level": "INFO", "message":"loaded module [repository-url]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.114Z", "log.level": "INFO", "message":"loaded module [runtime-fields-common]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.115Z", "log.level": "INFO", "message":"loaded module [search-business-rules]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.115Z", "log.level": "INFO", "message":"loaded module [searchable-snapshots]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.115Z", "log.level": "INFO", "message":"loaded module [snapshot-based-recoveries]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.115Z", "log.level": "INFO", "message":"loaded module [snapshot-repo-test-kit]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.116Z", "log.level": "INFO", "message":"loaded module [spatial]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.116Z", "log.level": "INFO", "message":"loaded module [transform]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.116Z", "log.level": "INFO", "message":"loaded module [transport-netty4]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.116Z", "log.level": "INFO", "message":"loaded module [unsigned-long]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.116Z", "log.level": "INFO", "message":"loaded module [vector-tile]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.117Z", "log.level": "INFO", "message":"loaded module [vectors]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.117Z", "log.level": "INFO", "message":"loaded module [wildcard]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.117Z", "log.level": "INFO", "message":"loaded module [x-pack-aggregate-metric]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.117Z", "log.level": "INFO", "message":"loaded module [x-pack-analytics]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.117Z", "log.level": "INFO", "message":"loaded module [x-pack-async]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.118Z", "log.level": "INFO", "message":"loaded module [x-pack-async-search]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.118Z", "log.level": "INFO", "message":"loaded module [x-pack-autoscaling]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.118Z", "log.level": "INFO", "message":"loaded module [x-pack-ccr]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.118Z", "log.level": "INFO", "message":"loaded module [x-pack-core]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.118Z", "log.level": "INFO", "message":"loaded module [x-pack-data-streams]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.118Z", "log.level": "INFO", "message":"loaded module [x-pack-deprecation]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.119Z", "log.level": "INFO", "message":"loaded module [x-pack-enrich]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.119Z", "log.level": "INFO", "message":"loaded module [x-pack-eql]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.119Z", "log.level": "INFO", "message":"loaded module [x-pack-fleet]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.119Z", "log.level": "INFO", "message":"loaded module [x-pack-graph]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.119Z", "log.level": "INFO", "message":"loaded module [x-pack-identity-provider]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.120Z", "log.level": "INFO", "message":"loaded module [x-pack-ilm]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.120Z", "log.level": "INFO", "message":"loaded module [x-pack-logstash]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.120Z", "log.level": "INFO", "message":"loaded module [x-pack-ml]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.120Z", "log.level": "INFO", "message":"loaded module [x-pack-monitoring]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.120Z", "log.level": "INFO", "message":"loaded module [x-pack-ql]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.120Z", "log.level": "INFO", "message":"loaded module [x-pack-rollup]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.121Z", "log.level": "INFO", "message":"loaded module [x-pack-security]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.121Z", "log.level": "INFO", "message":"loaded module [x-pack-shutdown]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.121Z", "log.level": "INFO", "message":"loaded module [x-pack-sql]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.121Z", "log.level": "INFO", "message":"loaded module [x-pack-stack]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.121Z", "log.level": "INFO", "message":"loaded module [x-pack-text-structure]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.122Z", "log.level": "INFO", "message":"loaded module [x-pack-voting-only-node]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.122Z", "log.level": "INFO", "message":"loaded module [x-pack-watcher]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.123Z", "log.level": "INFO", "message":"no plugins loaded", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.167Z", "log.level": "INFO", "message":"using [1] data paths, mounts [[/ (overlay)]], net usable_space [39.5gb], net total_space [97.9gb], types [overlay]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.env.NodeEnvironment","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.167Z", "log.level": "INFO", "message":"heap size [2gb], compressed ordinary object pointers [true]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.env.NodeEnvironment","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:09.378Z", "log.level": "INFO", "message":"node name [master1], node ID [TAtZ_19eQQa0EthJtr3Gpg], cluster name [docker-cluster], roles [master]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:17.719Z", "log.level": "INFO", "message":"[controller/306] [Main.cc@123] controller (64 bit): Version 8.0.0 (Build 5e85495ea85316) Copyright (c) 2022 Elasticsearch BV", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"ml-cpp-log-tail-thread","log.logger":"org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:18.050Z", "log.level": "INFO", "message":"Security is disabled", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.xpack.security.Security","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:19.829Z", "log.level": "INFO", "message":"creating NettyAllocator with the following configs: [name=elasticsearch_configured, chunk_size=1mb, suggested_max_allocation_size=1mb, factors={es.unsafe.use_netty_default_chunk_and_page_size=false, g1gc_enabled=true, g1gc_region_size=4mb}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.transport.netty4.NettyAllocator","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:19.923Z", "log.level": "INFO", "message":"using discovery type [zen] and seed hosts providers [settings]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.discovery.DiscoveryModule","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:22.026Z", "log.level": "INFO", "message":"initialized", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:22.027Z", "log.level": "INFO", "message":"starting ...", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:22.037Z", "log.level": "INFO", "message":"deprecation component started", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.xpack.deprecation.logging.DeprecationIndexingComponent","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:22.213Z", "log.level": "INFO", "message":"publish_address {10.0.0.67:9300}, bound_addresses {0.0.0.0:9300}", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.transport.TransportService","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:23.040Z", "log.level": "INFO", "message":"bound or publishing to a non-loopback address, enforcing bootstrap checks", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.bootstrap.BootstrapChecks","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:33.058Z", "log.level": "WARN", "message":"master not discovered yet, this node has not previously joined a bootstrapped cluster, and this node must discover master-eligible nodes [master1, master2] to bootstrap a cluster: have discovered [{master1}{TAtZ_19eQQa0EthJtr3Gpg}{apO2KvRlSEexXZv9angn2A}{10.0.0.67}{10.0.0.67:9300}{m}]; discovery will continue using [10.0.3.154:9300] from hosts providers and [{master1}{TAtZ_19eQQa0EthJtr3Gpg}{apO2KvRlSEexXZv9angn2A}{10.0.0.67}{10.0.0.67:9300}{m}] from last-known cluster state; node term 0, last-accepted version 0 in term 0", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[master1][generic][T#2]","log.logger":"org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:43.060Z", "log.level": "WARN", "message":"master not discovered yet, this node has not previously joined a bootstrapped cluster, and this node must discover master-eligible nodes [master1, master2] to bootstrap a cluster: have discovered [{master1}{TAtZ_19eQQa0EthJtr3Gpg}{apO2KvRlSEexXZv9angn2A}{10.0.0.67}{10.0.0.67:9300}{m}]; discovery will continue using [10.0.3.154:9300] from hosts providers and [{master1}{TAtZ_19eQQa0EthJtr3Gpg}{apO2KvRlSEexXZv9angn2A}{10.0.0.67}{10.0.0.67:9300}{m}] from last-known cluster state; node term 0, last-accepted version 0 in term 0", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[master1][generic][T#4]","log.logger":"org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:53.062Z", "log.level": "WARN", "message":"master not discovered yet, this node has not previously joined a bootstrapped cluster, and this node must discover master-eligible nodes [master1, master2] to bootstrap a cluster: have discovered [{master1}{TAtZ_19eQQa0EthJtr3Gpg}{apO2KvRlSEexXZv9angn2A}{10.0.0.67}{10.0.0.67:9300}{m}]; discovery will continue using [10.0.3.154:9300] from hosts providers and [{master1}{TAtZ_19eQQa0EthJtr3Gpg}{apO2KvRlSEexXZv9angn2A}{10.0.0.67}{10.0.0.67:9300}{m}] from last-known cluster state; node term 0, last-accepted version 0 in term 0", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[master1][generic][T#4]","log.logger":"org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:53.062Z", "log.level": "WARN", "message":"timed out while waiting for initial discovery state - timeout: 30s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[master1][generic][T#2]","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:53.076Z", "log.level": "INFO", "message":"publish_address {10.0.0.67:9200}, bound_addresses {0.0.0.0:9200}", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.http.AbstractHttpServerTransport","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:30:53.077Z", "log.level": "INFO", "message":"started", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:31:03.063Z", "log.level": "WARN", "message":"master not discovered yet, this node has not previously joined a bootstrapped cluster, and this node must discover master-eligible nodes [master1, master2] to bootstrap a cluster: have discovered [{master1}{TAtZ_19eQQa0EthJtr3Gpg}{apO2KvRlSEexXZv9angn2A}{10.0.0.67}{10.0.0.67:9300}{m}]; discovery will continue using [10.0.3.154:9300] from hosts providers and [{master1}{TAtZ_19eQQa0EthJtr3Gpg}{apO2KvRlSEexXZv9angn2A}{10.0.0.67}{10.0.0.67:9300}{m}] from last-known cluster state; node term 0, last-accepted version 0 in term 0", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[master1][generic][T#3]","log.logger":"org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:31:13.066Z", "log.level": "WARN", "message":"master not discovered yet, this node has not previously joined a bootstrapped cluster, and this node must discover master-eligible nodes [master1, master2] to bootstrap a cluster: have discovered [{master1}{TAtZ_19eQQa0EthJtr3Gpg}{apO2KvRlSEexXZv9angn2A}{10.0.0.67}{10.0.0.67:9300}{m}]; discovery will continue using [10.0.3.154:9300] from hosts providers and [{master1}{TAtZ_19eQQa0EthJtr3Gpg}{apO2KvRlSEexXZv9angn2A}{10.0.0.67}{10.0.0.67:9300}{m}] from last-known cluster state; node term 0, last-accepted version 0 in term 0", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[master1][generic][T#2]","log.logger":"org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
{"@timestamp":"2022-03-21T10:31:16.886Z", "log.level": "INFO", "message":"setting initial configuration to VotingConfiguration{kFr8RqDITsGzB8xNcm_q7Q,TAtZ_19eQQa0EthJtr3Gpg}", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[master1][generic][T#3]","log.logger":"org.elasticsearch.cluster.coordination.Coordinator","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster"},
F{"@timestamp":"2022-03-21T10:31:23.075Z", "log.level": "WARN", "message":"last failed join attempt was 532ms ago, failed to join {master1}{TAtZ_19eQQa0EthJtr3Gpg}{apO2KvRlSEexXZv9angn2A}{10.0.0.67}{10.0.0.67:9300}{m}{xpack.installed=true} with JoinRequest{sourceNode={master1}{TAtZ_19eQQa0EthJtr3Gpg}{apO2KvRlSEexXZv9angn2A}{10.0.0.67}{10.0.0.67:9300}{m}{xpack.installed=true}, minimumTerm=6, optionalJoin=Optional[Join{term=7, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={master1}{TAtZ_19eQQa0EthJtr3Gpg}{apO2KvRlSEexXZv9angn2A}{10.0.0.67}{10.0.0.67:9300}{m}{xpack.installed=true}, targetNode={master1}{TAtZ_19eQQa0EthJtr3Gpg}{apO2KvRlSEexXZv9angn2A}{10.0.0.67}{10.0.0.67:9300}{m}{xpack.installed=true}}]}", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[master1][generic][T#4]","log.logger":"org.elasticsearch.cluster.coordination.JoinHelper","elasticsearch.node.name":"master1","elasticsearch.cluster.name":"docker-cluster","error.type":"org.elasticsearch.transport.RemoteTransportException","error.message":"[master1][10.0.0.67:9300][internal:cluster/coordination/join]","error.stack_trace":"org.elasticsearch.transport.RemoteTransportException: [master1][10.0.0.67:9300][internal:cluster/coordination/join]\nCaused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {master1}{TAtZ_19eQQa0EthJtr3Gpg}{apO2KvRlSEexXZv9angn2A}{10.0.0.67}{10.0.0.67:9300}{m}{xpack.installed=true}\n\tat org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:448)\n\tat org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:661)\n\tat org.elasticsearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$8(Coordinator.java:559)\n\tat org.elasticsearch.action.ActionListener$DelegatingFailureActionListener.onResponse(ActionListener.java:219)\n\tat
Due to char limit I was forced to put the data1 log in another message and to cut the master1 log