Issues with clustering on Docker

Elastic Stack Elasticsearch
1

Hi everyone, having an issue with setting up an elasticsearch cluster over talking over 2 centos machines.

Here are my compse files:

VM 1 -

version: '2.2'
services:
  es-8.2.2:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.2.2
    restart: always
    volumes:
      - esdata-8.2.2:/usr/share/elasticsearch/data
      - esconfig-8.2.2:/usr/share/elasticsearch/config

    environment:
      - cluster.initial_master_nodes=prod-01
      - cluster.name=production-cluster
      - discovery.seed_hosts=["10.0.0.105:9301","10.0.0.106:901"]
      - node.name=prod-01
      - network.host=0.0.0.0
      - network.publish_host=10.0.0.105
      - http.publish_port=9301
      - transport.publish_host=10.0.0.105
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTIONS=-Xms4000m -Xmx4000m"
      - xpack.security.enabled=false
      - xpack.security.transport.ssl.enabled=false
      - transport.host=0.0.0.0
      - node.roles=["master","data"]
    extra_hosts:
      - "prod-01:10.0.0.105"
      - "prod-02:10.0.0.106"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    ports:
      - 9202:9200
      - 9301:9300

volumes:
  esdata-8.2.2:
    driver: local
  esconfig-8.2.2:
    driver: local

VM 2 -

version: '2.2'
services:
  es-8.2:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.2.2
    restart: always
    volumes:
      - esdata-8.2:/usr/share/elasticsearch/data
      - esconfig-8.2:/usr/share/elasticsearch/config

    environment:
      - cluster.name=production-cluster
      - cluster.initial_master_nodes=10.0.0.105
      - node.name=prod-02
      - network.host=0.0.0.0
      - network.publish_host=10.0.0.106
      - transport.publish_host=10.0.0.106
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms4000m -Xmx4000m"
      - xpack.security.enabled=false
      - discovery.seed_hosts=["10.0.0.105:9301","10.0.0.106:9301"]
      - transport.host=0.0.0.0
      - xpack.security.transport.ssl.enabled=false
      - node.roles=data
    extra_hosts:
      - "prod-01:10.0.0.105"
      - "prod-02:10.0.0.106"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    ports:
      - 9202:9200
      - 9301:9300

volumes:
  esdata-8.2:
    driver: local
  esconfig-8.2:
    driver: local

Error wise, im getting this:


{"@timestamp":"2022-06-24T10:56:49.896Z", "log.level": "WARN", "message":"completed handshake with [{prod-01}{zV7jyfEHRyGTswqzwLtP6g}{7fUpZd4bRhKuEf2fWQcViw}{10.0.0.105}{10.0.0.105:9300}{dm}] at [10.0.0.105:9301] but followup connection to [10.0.0.105:9300] failed", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[prod-02][generic][T#2]","log.logger":"org.elasticsearch.discovery.HandshakingTransportAddressConnector","elasticsearch.node.name":"prod-02","elasticsearch.cluster.name":"production-cluster","error.type":"org.elasticsearch.transport.ConnectTransportException","error.message":"[prod-01][10.0.0.105:9300] general node connection failure","error.stack_trace":"org.elasticsearch.transport.ConnectTransportException: [prod-01][10.0.0.105:9300] general node connection failure\n\tat org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.lambda$onResponse$2(TcpTransport.java:1094)\n\tat org.elasticsearch.action.ActionListener$2.onFailure(ActionListener.java:170)\n\tat org.elasticsearch.transport.TransportHandshaker$HandshakeResponseHandler.handleLocalException(TransportHandshaker.java:170)\n\tat org.elasticsearch.transport.TransportHandshaker.lambda$sendHandshake$0(TransportHandshaker.java:60)\n\tat org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:267)\n\tat org.elasticsearch.action.ActionListener.lambda$toBiConsumer$0(ActionListener.java:303)\n\tat org.elasticsearch.core.CompletableContext.lambda$addListener$0(CompletableContext.java:30)\n\tat java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)\n\tat java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)\n\tat java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)\n\tat org.elasticsearch.core.CompletableContext.complete(CompletableContext.java:52)\n\tat org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:56)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:578)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:571)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:550)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:491)\n\tat io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:616)\n\tat io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:605)\n\tat io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104)\n\tat io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)\n\tat io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:1164)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:755)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:731)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:620)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.closeOnRead(AbstractNioByteChannel.java:105)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:174)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\nCaused by: org.elasticsearch.transport.TransportException: handshake failed because connection reset\n\t... 31 more\n"}
{"@timestamp":"2022-06-24T10:56:50.895Z", "log.level": "WARN", "message":"completed handshake with [{prod-01}{zV7jyfEHRyGTswqzwLtP6g}{7fUpZd4bRhKuEf2fWQcViw}{10.0.0.105}{10.0.0.105:9300}{dm}] at [10.0.0.105:9301] but followup connection to [10.0.0.105:9300] failed", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[prod-02][generic][T#4]","log.logger":"org.elasticsearch.discovery.HandshakingTransportAddressConnector","elasticsearch.node.name":"prod-02","elasticsearch.cluster.name":"production-cluster","error.type":"org.elasticsearch.transport.ConnectTransportException","error.message":"[prod-01][10.0.0.105:9300] general node connection failure","error.stack_trace":"org.elasticsearch.transport.ConnectTransportException: [prod-01][10.0.0.105:9300] general node connection failure\n\tat org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.lambda$onResponse$2(TcpTransport.java:1094)\n\tat org.elasticsearch.action.ActionListener$2.onFailure(ActionListener.java:170)\n\tat org.elasticsearch.transport.TransportHandshaker$HandshakeResponseHandler.handleLocalException(TransportHandshaker.java:170)\n\tat org.elasticsearch.transport.TransportHandshaker.lambda$sendHandshake$0(TransportHandshaker.java:60)\n\tat org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:267)\n\tat org.elasticsearch.action.ActionListener.lambda$toBiConsumer$0(ActionListener.java:303)\n\tat org.elasticsearch.core.CompletableContext.lambda$addListener$0(CompletableContext.java:30)\n\tat java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)\n\tat java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)\n\tat java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)\n\tat org.elasticsearch.core.CompletableContext.complete(CompletableContext.java:52)\n\tat org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:56)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:578)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:571)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:550)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:491)\n\tat io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:616)\n\tat io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:605)\n\tat io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104)\n\tat io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)\n\tat io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:1164)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:755)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:731)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:620)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.closeOnRead(AbstractNioByteChannel.java:105)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:174)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\nCaused by: org.elasticsearch.transport.TransportException: handshake failed because connection reset\n\t... 31 more\n"}
{"@timestamp":"2022-06-24T10:56:51.897Z", "log.level": "WARN", "message":"completed handshake with [{prod-01}{zV7jyfEHRyGTswqzwLtP6g}{7fUpZd4bRhKuEf2fWQcViw}{10.0.0.105}{10.0.0.105:9300}{dm}] at [10.0.0.105:9301] but followup connection to [10.0.0.105:9300] failed", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[prod-02][generic][T#3]","log.logger":"org.elasticsearch.discovery.HandshakingTransportAddressConnector","elasticsearch.node.name":"prod-02","elasticsearch.cluster.name":"production-cluster","error.type":"org.elasticsearch.transport.ConnectTransportException","error.message":"[prod-01][10.0.0.105:9300] general node connection failure","error.stack_trace":"org.elasticsearch.transport.ConnectTransportException: [prod-01][10.0.0.105:9300] general node connection failure\n\tat org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.lambda$onResponse$2(TcpTransport.java:1094)\n\tat org.elasticsearch.action.ActionListener$2.onFailure(ActionListener.java:170)\n\tat org.elasticsearch.transport.TransportHandshaker$HandshakeResponseHandler.handleLocalException(TransportHandshaker.java:170)\n\tat org.elasticsearch.transport.TransportHandshaker.lambda$sendHandshake$0(TransportHandshaker.java:60)\n\tat org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:267)\n\tat org.elasticsearch.action.ActionListener.lambda$toBiConsumer$0(ActionListener.java:303)\n\tat org.elasticsearch.core.CompletableContext.lambda$addListener$0(CompletableContext.java:30)\n\tat java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)\n\tat java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)\n\tat java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)\n\tat org.elasticsearch.core.CompletableContext.complete(CompletableContext.java:52)\n\tat org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:56)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:578)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:571)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:550)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:491)\n\tat io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:616)\n\tat io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:605)\n\tat io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104)\n\tat io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)\n\tat io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:1164)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:755)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:731)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:620)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.closeOnRead(AbstractNioByteChannel.java:105)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:174)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\nCaused by: org.elasticsearch.transport.TransportException: handshake failed because connection reset\n\t... 31 more\n"}
{"@timestamp":"2022-06-24T10:56:52.878Z", "log.level": "WARN", "message":"master not discovered yet: have discovered [{prod-02}{5Qbe2K43RRKuVe7VajyRlQ}{LVhor2WqS7eluXGjlIH8WA}{10.0.0.106}{10.0.0.106:9300}{d}]; discovery will continue using [10.0.0.105:9301, 10.0.0.106:9301] from hosts providers and [] from last-known cluster state; node term 0, last-accepted version 0 in term 0", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[prod-02][cluster_coordination][T#1]","log.logger":"org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper","elasticsearch.node.name":"prod-02","elasticsearch.cluster.name":"production-cluster"}
{"@timestamp":"2022-06-24T10:56:52.894Z", "log.level": "WARN", "message":"completed handshake with [{prod-01}{zV7jyfEHRyGTswqzwLtP6g}{7fUpZd4bRhKuEf2fWQcViw}{10.0.0.105}{10.0.0.105:9300}{dm}] at [10.0.0.105:9301] but followup connection to [10.0.0.105:9300] failed", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[prod-02][generic][T#3]","log.logger":"org.elasticsearch.discovery.HandshakingTransportAddressConnector","elasticsearch.node.name":"prod-02","elasticsearch.cluster.name":"production-cluster","error.type":"org.elasticsearch.transport.ConnectTransportException","error.message":"[prod-01][10.0.0.105:9300] general node connection failure","error.stack_trace":"org.elasticsearch.transport.ConnectTransportException: [prod-01][10.0.0.105:9300] general node connection failure\n\tat org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.lambda$onResponse$2(TcpTransport.java:1094)\n\tat org.elasticsearch.action.ActionListener$2.onFailure(ActionListener.java:170)\n\tat org.elasticsearch.transport.TransportHandshaker$HandshakeResponseHandler.handleLocalException(TransportHandshaker.java:170)\n\tat org.elasticsearch.transport.TransportHandshaker.lambda$sendHandshake$0(TransportHandshaker.java:60)\n\tat org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:267)\n\tat org.elasticsearch.action.ActionListener.lambda$toBiConsumer$0(ActionListener.java:303)\n\tat org.elasticsearch.core.CompletableContext.lambda$addListener$0(CompletableContext.java:30)\n\tat java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)\n\tat java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)\n\tat java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)\n\tat org.elasticsearch.core.CompletableContext.complete(CompletableContext.java:52)\n\tat org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:56)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:578)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:571)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:550)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:491)\n\tat io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:616)\n\tat io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:605)\n\tat io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104)\n\tat io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)\n\tat io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:1164)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:755)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:731)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:620)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.closeOnRead(AbstractNioByteChannel.java:105)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:174)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\nCaused by: org.elasticsearch.transport.TransportException: handshake failed because connection reset\n\t... 31 more\n"}
{"@timestamp":"2022-06-24T10:56:53.894Z", "log.level": "WARN", "message":"completed handshake with [{prod-01}{zV7jyfEHRyGTswqzwLtP6g}{7fUpZd4bRhKuEf2fWQcViw}{10.0.0.105}{10.0.0.105:9300}{dm}] at [10.0.0.105:9301] but followup connection to [10.0.0.105:9300] failed", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[prod-02][generic][T#4]","log.logger":"org.elasticsearch.discovery.HandshakingTransportAddressConnector","elasticsearch.node.name":"prod-02","elasticsearch.cluster.name":"production-cluster","error.type":"org.elasticsearch.transport.ConnectTransportException","error.message":"[prod-01][10.0.0.105:9300] general node connection failure","error.stack_trace":"org.elasticsearch.transport.ConnectTransportException: [prod-01][10.0.0.105:9300] general node connection failure\n\tat org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.lambda$onResponse$2(TcpTransport.java:1094)\n\tat org.elasticsearch.action.ActionListener$2.onFailure(ActionListener.java:170)\n\tat org.elasticsearch.transport.TransportHandshaker$HandshakeResponseHandler.handleLocalException(TransportHandshaker.java:170)\n\tat org.elasticsearch.transport.TransportHandshaker.lambda$sendHandshake$0(TransportHandshaker.java:60)\n\tat org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:267)\n\tat org.elasticsearch.action.ActionListener.lambda$toBiConsumer$0(ActionListener.java:303)\n\tat org.elasticsearch.core.CompletableContext.lambda$addListener$0(CompletableContext.java:30)\n\tat java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)\n\tat java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)\n\tat java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)\n\tat org.elasticsearch.core.CompletableContext.complete(CompletableContext.java:52)\n\tat org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:56)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:578)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:571)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:550)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:491)\n\tat io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:616)\n\tat io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:605)\n\tat io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104)\n\tat io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)\n\tat io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:1164)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:755)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:731)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:620)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.closeOnRead(AbstractNioByteChannel.java:105)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:174)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\nCaused by: org.elasticsearch.transport.TransportException: handshake failed because connection reset\n\t... 31 more\n"}
{"@timestamp":"2022-06-24T10:56:54.894Z", "log.level": "WARN", "message":"completed handshake with [{prod-01}{zV7jyfEHRyGTswqzwLtP6g}{7fUpZd4bRhKuEf2fWQcViw}{10.0.0.105}{10.0.0.105:9300}{dm}] at [10.0.0.105:9301] but followup connection to [10.0.0.105:9300] failed", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[prod-02][generic][T#1]","log.logger":"org.elasticsearch.discovery.HandshakingTransportAddressConnector","elasticsearch.node.name":"prod-02","elasticsearch.cluster.name":"production-cluster","error.type":"org.elasticsearch.transport.ConnectTransportException","error.message":"[prod-01][10.0.0.105:9300] general node connection failure","error.stack_trace":"org.elasticsearch.transport.ConnectTransportException: [prod-01][10.0.0.105:9300] general node connection failure\n\tat org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.lambda$onResponse$2(TcpTransport.java:1094)\n\tat org.elasticsearch.action.ActionListener$2.onFailure(ActionListener.java:170)\n\tat org.elasticsearch.transport.TransportHandshaker$HandshakeResponseHandler.handleLocalException(TransportHandshaker.java:170)\n\tat org.elasticsearch.transport.TransportHandshaker.lambda$sendHandshake$0(TransportHandshaker.java:60)\n\tat org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:267)\n\tat org.elasticsearch.action.ActionListener.lambda$toBiConsumer$0(ActionListener.java:303)\n\tat org.elasticsearch.core.CompletableContext.lambda$addListener$0(CompletableContext.java:30)\n\tat java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)\n\tat java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)\n\tat java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)\n\tat org.elasticsearch.core.CompletableContext.complete(CompletableContext.java:52)\n\tat org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:56)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:578)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:571)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:550)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:491)\n\tat io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:616)\n\tat io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:605)\n\tat io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104)\n\tat io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)\n\tat io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:1164)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:755)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:731)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:620)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.closeOnRead(AbstractNioByteChannel.java:105)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:174)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\nCaused by: org.elasticsearch.transport.TransportException: handshake failed because connection reset\n\t... 31 more\n"}
{"@timestamp":"2022-06-24T10:56:55.894Z", "log.level": "WARN", "message":"completed handshake with [{prod-01}{zV7jyfEHRyGTswqzwLtP6g}{7fUpZd4bRhKuEf2fWQcViw}{10.0.0.105}{10.0.0.105:9300}{dm}] at [10.0.0.105:9301] but followup connection to [10.0.0.105:9300] failed", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[prod-02][generic][T#4]","log.logger":"org.elasticsearch.discovery.HandshakingTransportAddressConnector","elasticsearch.node.name":"prod-02","elasticsearch.cluster.name":"production-cluster","error.type":"org.elasticsearch.transport.ConnectTransportException","error.message":"[prod-01][10.0.0.105:9300] general node connection failure","error.stack_trace":"org.elasticsearch.transport.ConnectTransportException: [prod-01][10.0.0.105:9300] general node connection failure\n\tat org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.lambda$onResponse$2(TcpTransport.java:1094)\n\tat org.elasticsearch.action.ActionListener$2.onFailure(ActionListener.java:170)\n\tat org.elasticsearch.transport.TransportHandshaker$HandshakeResponseHandler.handleLocalException(TransportHandshaker.java:170)\n\tat org.elasticsearch.transport.TransportHandshaker.lambda$sendHandshake$0(TransportHandshaker.java:60)\n\tat org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:267)\n\tat org.elasticsearch.action.ActionListener.lambda$toBiConsumer$0(ActionListener.java:303)\n\tat org.elasticsearch.core.CompletableContext.lambda$addListener$0(CompletableContext.java:30)\n\tat java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)\n\tat java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)\n\tat java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)\n\tat org.elasticsearch.core.CompletableContext.complete(CompletableContext.java:52)\n\tat org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:56)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:578)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:571)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:550)\n\tat io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:491)\n\tat io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:616)\n\tat io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:605)\n\tat io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104)\n\tat io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)\n\tat io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:1164)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:755)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:731)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:620)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.closeOnRead(AbstractNioByteChannel.java:105)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:174)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\nCaused by: org.elasticsearch.transport.TransportException: handshake failed because connection reset\n\t... 31 more\n"}

From my understanding, prod-02 can see and talk to prod-01 but its just not completting. I have to run this on port 9202/9301 due to production running on the other steps (this is an inhertied env). Any help would be amazing because im banging my head against a wall.

Thanks!

2

Remapping the ports like this is going to cause problems. Try setting transport.port: 9301 and http.port: 9202 and then just expose the right ports.

3

Absolute hero, thats cracked it !

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.