@Tomo_M @Christian_Dahlqvist
i change docker-compose like this:
version: "3.0"
services:
elasticsearch:
hostname: log-01.domain.com
container_name: elasticsearch
image: harbor.domain.com/tsi/elasticsearch:8.2.0
restart: always
extra_hosts:
- "log-01.domain.com:192.168.57.66"
- "log-02.domain.com:192.168.57.67"
- "log-03.domain.com:192.168.57.68"
volumes:
# - elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
- /app/elasticsearch/certs:/usr/share/elasticsearch/config/certs
- /app/elasticsearch/data:/usr/share/elasticsearch/data
- /app/elasticsearch/log:/usr/share/elasticsearch/log
environment:
- node.name=log-01
- "ES_JAVA_OPTS=-Xms32g -Xmx32g"
- cluster.name=log-cluster
- cluster.initial_master_nodes=log-01,log-02,log-03
- discovery.seed_hosts=log-01,log-02,log-03
- ELASTIC_PASSWORD=Aa123456
- network.host=0.0.0.0
- bootstrap.memory_lock=true
- xpack.security.enabled=true
- xpack.security.http.ssl.enabled=true
- xpack.security.http.ssl.key=certs/privkey.pem
- xpack.security.http.ssl.certificate=certs/fullchain.pem
- xpack.security.http.ssl.certificate_authorities=certs/fullchain.pem
- xpack.security.http.ssl.verification_mode=certificate
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.key=certs/privkey.pem
- xpack.security.transport.ssl.certificate=certs/fullchain.pem
- xpack.security.transport.ssl.certificate_authorities=certs/fullchain.pem
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.license.self_generated.type=basic
# mem_limit: 1073741824
ulimits:
memlock:
soft: -1
hard: -1
ports:
- 9200:9200
- 9300:9300
- 9400:9400
networks:
- elastic
networks:
elastic:
driver: bridge
https://log-01.doamin.com:9200:
name "log-01"
cluster_name "log-cluster"
cluster_uuid "G-cYJCEfSGuOWy-liT1Ing"
version
number "8.2.0"
build_flavor "default"
build_type "docker"
build_hash "b174af62e8dd9f4ac4d25875e9381ffe2b9282c5"
build_date "2022-04-20T10:35:10.180408517Z"
build_snapshot false
lucene_version "9.1.0"
minimum_wire_compatibility_version "7.17.0"
minimum_index_compatibility_version "7.0.0"
tagline "You Know, for Search"
https://log-02.doamin.com:9200:
name "log-02"
cluster_name "log-cluster"
cluster_uuid "_na_"
version
number "8.2.0"
build_flavor "default"
build_type "docker"
build_hash "b174af62e8dd9f4ac4d25875e9381ffe2b9282c5"
build_date "2022-04-20T10:35:10.180408517Z"
build_snapshot false
lucene_version "9.1.0"
minimum_wire_compatibility_version "7.17.0"
minimum_index_compatibility_version "7.0.0"
tagline "You Know, for Search"
https://log-03.doamin.com:9200:
name "log-03"
cluster_name "log-cluster"
cluster_uuid "_na_"
version
number "8.2.0"
build_flavor "default"
build_type "docker"
build_hash "b174af62e8dd9f4ac4d25875e9381ffe2b9282c5"
build_date "2022-04-20T10:35:10.180408517Z"
build_snapshot false
lucene_version "9.1.0"
minimum_wire_compatibility_version "7.17.0"
minimum_index_compatibility_version "7.0.0"
tagline "You Know, for Search"
node log-03 full log:
{"@timestamp":"2022-05-30T07:33:37.424Z", "log.level": "INFO", "message":"version[8.2.0], pid[7], build[default/docker/b174af62e8dd9f4ac4d25875e9381ffe2b9282c5/2022-04-20T10:35:10.180408517Z], OS[Linux/5.4.0-109-generic/amd64], JVM[Eclipse Adoptium/OpenJDK 64-Bit Server VM/18/18+36]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:37.428Z", "log.level": "INFO", "message":"JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:37.428Z", "log.level": "INFO", "message":"JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -Djava.security.manager=allow, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Dlog4j2.formatMsgNoLookups=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-6004893836504698192, -XX:+HeapDumpOnOutOfMemoryError, -XX:+ExitOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xms32g, -Xmx32g, -XX:MaxDirectMemorySize=17179869184, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=25, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:41.138Z", "log.level": "INFO", "message":"no plugins loaded", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:41.177Z", "log.level": "INFO", "message":"using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/sda2)]], net usable_space [76.8gb], net total_space [97.9gb], types [ext4]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.env.NodeEnvironment","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:41.177Z", "log.level": "INFO", "message":"heap size [32gb], compressed ordinary object pointers [false]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.env.NodeEnvironment","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:41.234Z", "log.level": "INFO", "message":"node name [log-03], node ID [ofOy5rfHRBGVx9_GKHfu1Q], cluster name [log-cluster], roles [data_frozen, ml, data_hot, transform, data_content, data_warm, master, remote_cluster_client, data, data_cold, ingest]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:47.988Z", "log.level": "INFO", "message":"[controller/606] [Main.cc@123] controller (64 bit): Version 8.2.0 (Build a8c0a88ede0ff2) Copyright (c) 2022 Elasticsearch BV", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"ml-cpp-log-tail-thread","log.logger":"org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:48.121Z", "log.level": "INFO", "message":"Security is enabled", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.xpack.security.Security","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:48.432Z", "log.level": "INFO", "message":"license mode is [trial], currently licensed security realms are [reserved/reserved,file/default_file,native/default_native]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.xpack.security.authc.Realms","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:48.440Z", "log.level": "INFO", "message":"parsed [0] roles from file [/usr/share/elasticsearch/config/roles.yml]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.xpack.security.authz.store.FileRolesStore","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:50.399Z", "log.level": "INFO", "message":"creating NettyAllocator with the following configs: [name=elasticsearch_configured, chunk_size=1mb, suggested_max_allocation_size=1mb, factors={es.unsafe.use_netty_default_chunk_and_page_size=false, g1gc_enabled=true, g1gc_region_size=16mb}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.transport.netty4.NettyAllocator","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:50.427Z", "log.level": "INFO", "message":"using rate limit [40mb] with [default=40mb, read=0b, write=0b, max=0b]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.indices.recovery.RecoverySettings","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:50.458Z", "log.level": "INFO", "message":"using discovery type [multi-node] and seed hosts providers [settings]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.discovery.DiscoveryModule","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:51.984Z", "log.level": "INFO", "message":"initialized", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:51.985Z", "log.level": "INFO", "message":"starting ...", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:52.002Z", "log.level": "INFO", "message":"persistent cache index loaded", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.xpack.searchablesnapshots.cache.full.PersistentCache","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:52.003Z", "log.level": "INFO", "message":"deprecation component started", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.xpack.deprecation.logging.DeprecationIndexingComponent","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}{"@timestamp":"2022-05-30T07:33:52.099Z", "log.level": "INFO", "message":"publish_address {172.28.0.2:9300}, bound_addresses {0.0.0.0:9300}", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.transport.TransportService","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:33:52.213Z", "log.level": "INFO", "message":"bound or publishing to a non-loopback address, enforcing bootstrap checks", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.bootstrap.BootstrapChecks","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:34:02.225Z", "log.level": "WARN", "message":"master not discovered yet, this node has not previously joined a bootstrapped cluster, and this node must discover master-eligible nodes [log-01, log-02, log-03] to bootstrap a cluster: have discovered [{log-03}{ofOy5rfHRBGVx9_GKHfu1Q}{go_31sH5TLKrJ3Y4gP6ZbQ}{172.28.0.2}{172.28.0.2:9300}{cdfhilmrstw}]; discovery will continue using [192.168.57.66:9300, 192.168.57.67:9300] from hosts providers and [{log-03}{ofOy5rfHRBGVx9_GKHfu1Q}{go_31sH5TLKrJ3Y4gP6ZbQ}{172.28.0.2}{172.28.0.2:9300}{cdfhilmrstw}] from last-known cluster state; node term 0, last-accepted version 0 in term 0", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[log-03][cluster_coordination][T#1]","log.logger":"org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:34:12.226Z", "log.level": "WARN", "message":"master not discovered yet, this node has not previously joined a bootstrapped cluster, and this node must discover master-eligible nodes [log-01, log-02, log-03] to bootstrap a cluster: have discovered [{log-03}{ofOy5rfHRBGVx9_GKHfu1Q}{go_31sH5TLKrJ3Y4gP6ZbQ}{172.28.0.2}{172.28.0.2:9300}{cdfhilmrstw}]; discovery will continue using [192.168.57.66:9300, 192.168.57.67:9300] from hosts providers and [{log-03}{ofOy5rfHRBGVx9_GKHfu1Q}{go_31sH5TLKrJ3Y4gP6ZbQ}{172.28.0.2}{172.28.0.2:9300}{cdfhilmrstw}] from last-known cluster state; node term 0, last-accepted version 0 in term 0", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[log-03][cluster_coordination][T#1]","log.logger":"org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:34:22.228Z", "log.level": "WARN", "message":"master not discovered yet, this node has not previously joined a bootstrapped cluster, and this node must discover master-eligible nodes [log-01, log-02, log-03] to bootstrap a cluster: have discovered [{log-03}{ofOy5rfHRBGVx9_GKHfu1Q}{go_31sH5TLKrJ3Y4gP6ZbQ}{172.28.0.2}{172.28.0.2:9300}{cdfhilmrstw}]; discovery will continue using [192.168.57.66:9300, 192.168.57.67:9300] from hosts providers and [{log-03}{ofOy5rfHRBGVx9_GKHfu1Q}{go_31sH5TLKrJ3Y4gP6ZbQ}{172.28.0.2}{172.28.0.2:9300}{cdfhilmrstw}] from last-known cluster state; node term 0, last-accepted version 0 in term 0", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[log-03][cluster_coordination][T#1]","log.logger":"org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:34:22.236Z", "log.level": "WARN", "message":"timed out while waiting for initial discovery state - timeout: 30s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[log-03][generic][T#1]","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:34:22.240Z", "log.level": "INFO", "message":"publish_address {172.28.0.2:9200}, bound_addresses {0.0.0.0:9200}", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.http.AbstractHttpServerTransport","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:34:22.241Z", "log.level": "INFO", "message":"started", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
{"@timestamp":"2022-05-30T07:34:22.873Z", "log.level": "WARN", "message":"completed handshake with [{log-02}{TomH6eGvRf2olxDud932ug}{ZV_BAj8oRo-14cyjQQUQ9g}{172.29.0.2}{172.29.0.2:9300}{cdfhilmrstw}] at [192.168.57.67:9300] but followup connection to [172.29.0.2:9300] failed", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[log-03][generic][T#3]","log.logger":"org.elasticsearch.discovery.HandshakingTransportAddressConnector","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster","error.type":"org.elasticsearch.transport.ConnectTransportException","error.message":"[log-02][172.29.0.2:9300] connect_timeout[30s]","error.stack_trace":"org.elasticsearch.transport.ConnectTransportException: [log-02][172.29.0.2:9300] connect_timeout[30s]\n\tat org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onTimeout(TcpTransport.java:1112)\n\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:714)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n"}
{"@timestamp":"2022-05-30T07:34:22.874Z", "log.level": "WARN", "message":"completed handshake with [{log-01}{FsGAh9A1SI6N_R50q_dniw}{2y6qX8QIShiCpgH32yI8Gg}{192.168.192.2}{192.168.192.2:9300}{cdfhilmrstw}] at [192.168.57.66:9300] but followup connection to [192.168.192.2:9300] failed", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[log-03][generic][T#1]","log.logger":"org.elasticsearch.discovery.HandshakingTransportAddressConnector","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster","error.type":"org.elasticsearch.transport.ConnectTransportException","error.message":"[log-01][192.168.192.2:9300] connect_timeout[30s]","error.stack_trace":"org.elasticsearch.transport.ConnectTransportException: [log-01][192.168.192.2:9300] connect_timeout[30s]\n\tat org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onTimeout(TcpTransport.java:1112)\n\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:714)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n"}
{"@timestamp":"2022-05-30T07:34:32.230Z", "log.level": "WARN", "message":"master not discovered yet, this node has not previously joined a bootstrapped cluster, and this node must discover master-eligible nodes [log-01, log-02, log-03] to bootstrap a cluster: have discovered [{log-03}{ofOy5rfHRBGVx9_GKHfu1Q}{go_31sH5TLKrJ3Y4gP6ZbQ}{172.28.0.2}{172.28.0.2:9300}{cdfhilmrstw}]; discovery will continue using [192.168.57.66:9300, 192.168.57.67:9300] from hosts providers and [{log-03}{ofOy5rfHRBGVx9_GKHfu1Q}{go_31sH5TLKrJ3Y4gP6ZbQ}{172.28.0.2}{172.28.0.2:9300}{cdfhilmrstw}] from last-known cluster state; node term 0, last-accepted version 0 in term 0", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[log-03][cluster_coordination][T#1]","log.logger":"org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper","elasticsearch.node.name":"log-03","elasticsearch.cluster.name":"log-cluster"}
