Hello,
I'm trying to deploy a dockerized 3 node Elasticsearch (8.9.1) cluster in AWS ECS. The cluster's bootstraping seens to be okay, but when I try to join a node into it, I get a connection refused error.
These are the node's logs I'm trying to join, where 172.31.15.210 it's the private IP address for my bootstraped master node.
{"@timestamp":"2023-08-30T14:27:25.406Z", "log.level": "WARN", "message":"address [127.0.0.1:9300], node [null], requesting [false] discovery result: [9f4af867bc48][172.31.12.77:9300] successfully discovered local node {9f4af867bc48}{OAfYP4mtQ4CowSjlKTKBaA}{Ca_RcbmyTK2RIEXg5CBS8g}{9f4af867bc48}{172.31.12.77}{172.31.12.77:9300}{dm}{8.9.1} at [127.0.0.1:9300]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[9f4af867bc48][generic][T#3]","log.logger":"org.elasticsearch.discovery.PeerFinder","elasticsearch.node.name":"9f4af867bc48","elasticsearch.cluster.name":"collection-es-cluster"}
{"@timestamp":"2023-08-30T14:27:26.345Z", "log.level": "WARN", "message":"address [127.0.0.1:9302], node [null], requesting [false] discovery result: [][127.0.0.1:9302] connect_exception: Failed execution: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.0.0.1:9302: Connection refused: /127.0.0.1:9302: Connection refused", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[9f4af867bc48][generic][T#1]","log.logger":"org.elasticsearch.discovery.PeerFinder","elasticsearch.node.name":"9f4af867bc48","elasticsearch.cluster.name":"collection-es-cluster"}
{"@timestamp":"2023-08-30T14:27:26.346Z", "log.level": "WARN", "message":"address [127.0.0.1:9301], node [null], requesting [false] discovery result: [][127.0.0.1:9301] connect_exception: Failed execution: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.0.0.1:9301: Connection refused: /127.0.0.1:9301: Connection refused", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[9f4af867bc48][generic][T#3]","log.logger":"org.elasticsearch.discovery.PeerFinder","elasticsearch.node.name":"9f4af867bc48","elasticsearch.cluster.name":"collection-es-cluster"}
{"@timestamp":"2023-08-30T14:27:26.346Z", "log.level": "WARN", "message":"address [127.0.0.1:9303], node [null], requesting [false] discovery result: [][127.0.0.1:9303] connect_exception: Failed execution: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.0.0.1:9303: Connection refused: /127.0.0.1:9303: Connection refused", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[9f4af867bc48][generic][T#4]","log.logger":"org.elasticsearch.discovery.PeerFinder","elasticsearch.node.name":"9f4af867bc48","elasticsearch.cluster.name":"collection-es-cluster"}
{"@timestamp":"2023-08-30T14:27:26.346Z", "log.level": "WARN", "message":"address [127.0.0.1:9305], node [null], requesting [false] discovery result: [][127.0.0.1:9305] connect_exception: Failed execution: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.0.0.1:9305: Connection refused: /127.0.0.1:9305: Connection refused", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[9f4af867bc48][generic][T#6]","log.logger":"org.elasticsearch.discovery.PeerFinder","elasticsearch.node.name":"9f4af867bc48","elasticsearch.cluster.name":"collection-es-cluster"}
{"@timestamp":"2023-08-30T14:27:26.348Z", "log.level": "WARN", "message":"address [127.0.0.1:9304], node [null], requesting [false] discovery result: [][127.0.0.1:9304] connect_exception: Failed execution: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.0.0.1:9304: Connection refused: /127.0.0.1:9304: Connection refused", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[9f4af867bc48][generic][T#5]","log.logger":"org.elasticsearch.discovery.PeerFinder","elasticsearch.node.name":"9f4af867bc48","elasticsearch.cluster.name":"collection-es-cluster"}
{"@timestamp":"2023-08-30T14:27:26.348Z", "log.level": "WARN", "message":"address [172.31.15.210:9300], node [null], requesting [false] discovery result: [][172.31.15.210:9300] connect_exception: Failed execution: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /172.31.15.210:9300: Connection refused: /172.31.15.210:9300: Connection refused", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[9f4af867bc48][generic][T#7]","log.logger":"org.elasticsearch.discovery.PeerFinder","elasticsearch.node.name":"9f4af867bc48","elasticsearch.cluster.name":"collection-es-cluster"}
The elasticsearch.yml file for the node I'm trying to join:
cluster:
name: ${CLUSTER_NAME}
routing.allocation.awareness.attributes: aws_availability_zone
routing.allocation.disk.threshold_enabled: false
max_shards_per_node: 5100
node:
roles: [master, data]
network:
host: 0.0.0.0
publish_host: _ec2:privateIpv4_
transport:
publish_host: _ec2:privateIpv4_
discovery:
seed_providers: ec2
ec2:
tag.ElasticSearch: es
endpoint: ec2.${REGION}.amazonaws.com
host_type: private_ip
any_group: true
cloud.node.auto_attributes: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.client_authentication: required
xpack.security.transport.ssl.keystore.path: elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: elastic-certificates.p12
Dockerfile for the node I'm trying to join:
# Dockerfile for elasticsearch deployment
FROM docker.elastic.co/elasticsearch/elasticsearch:8.9.1
ENV CLUSTER_NAME collection-es-cluster
ENV REGION us-east-1
COPY --chown=elasticsearch:elasticsearch elasticsearch.yml /usr/share/elasticsearch/config/
COPY --chown=elasticsearch:elasticsearch elastic-certificates.p12 /usr/share/elasticsearch/config/
COPY --chown=elasticsearch:elasticsearch initial_entrypoint.sh /usr/share/elasticsearch/
USER root
RUN chown elasticsearch:elasticsearch /usr/share/elasticsearch/config/users
RUN chown elasticsearch:elasticsearch /usr/share/elasticsearch/config/users_roles
USER elasticsearch
RUN bin/elasticsearch-plugin install -b discovery-ec2
ENTRYPOINT [ "/usr/share/elasticsearch/initial_entrypoint.sh" ]
Docker entrypoint for the node I'm trying to join:
#!/bin/bash
echo Configuring passwords for keystore and truststore
{ echo $KEYSTORE_PASSWORD ; echo $KEYSTORE_PASSWORD ; echo "y" ; } | bin/elasticsearch-keystore create -p
{ echo $KEYSTORE_PASSWORD | echo $ES_KEYSTORE_SECURE_PASSWORD ; } | bin/elasticsearch-keystore add -xf xpack.security.transport.ssl.keystore.secure_password
{ echo $KEYSTORE_PASSWORD | echo $ES_TRUSTSTORE_SECURE_PASSWORD ; } | bin/elasticsearch-keystore add -xf xpack.security.transport.ssl.truststore.secure_password
# Resume to base image entrypoint
/bin/tini -s -- /usr/local/bin/docker-entrypoint.sh "$@"
The bootstraped master elasticsearch.yml file:
cluster:
name: ${CLUSTER_NAME}
routing.allocation.awareness.attributes: aws_availability_zone
routing.allocation.disk.threshold_enabled: false
max_shards_per_node: 5100
initial_master_nodes:
- election_node
node:
name: election_node
network:
host: 0.0.0.0
publish_host: _ec2:privateIpv4_
http:
port: 9200
transport:
publish_host: _ec2:privateIpv4_
port: 9300
discovery:
seed_providers: ec2
ec2:
tag.ElasticSearch: es
endpoint: ec2.${REGION}.amazonaws.com
host_type: private_ip
any_group: true
cloud.node.auto_attributes: true
Dockerfile for bootstraped master node
# Dockerfile for elasticsearch deployment
FROM docker.elastic.co/elasticsearch/elasticsearch:8.9.1
ENV CLUSTER_NAME collection-es-cluster
ENV REGION us-east-1
COPY --chown=elasticsearch:elasticsearch elasticsearch.yml /usr/share/elasticsearch/config/
COPY --chown=elasticsearch:elasticsearch initial_entrypoint.sh /usr/share/elasticsearch/
USER root
RUN chown elasticsearch:elasticsearch /usr/share/elasticsearch/config/users
RUN chown elasticsearch:elasticsearch /usr/share/elasticsearch/config/users_roles
USER elasticsearch
RUN bin/elasticsearch-plugin install -b discovery-ec2
ENTRYPOINT [ "/usr/share/elasticsearch/initial_entrypoint.sh" ]
Docker entrypoint for bootstraped master node
#!/bin/bash
# Create a encrypted keystore
echo Creating a encrypted keystore
{ echo $KEYSTORE_PASSWORD ; echo $KEYSTORE_PASSWORD ; echo "y" ; } | bin/elasticsearch-keystore create -p
# Bootstrap elastic user password
echo Bootstrap elastic user password
{ echo $KEYSTORE_PASSWORD ; echo $ELASTIC_PASSWORD ; } | bin/elasticsearch-keystore add -f bootstrap.password
echo Generating CA
echo $ES_KEYSTORE_SECURE_PASSWORD | bin/elasticsearch-certutil ca --silent --out elastic-stack-ca.p12
echo Generating certificates
bin/elasticsearch-certutil cert --silent --ca elastic-stack-ca.p12 --ca-pass $ES_KEYSTORE_SECURE_PASSWORD --out elastic-certificates.p12 --pass $ES_TRUSTSTORE_SECURE_PASSWORD
# Resume to base image entrypoint
echo starting elasticsearch
/bin/tini -s -- /usr/local/bin/docker-entrypoint.sh "$@"
I'm allowing just for tests purposes all TCP from income and outcome in the machines security groups. I've tried to do the same with the Elasticsearch security disabled (using xpack.security.enabled: false), but the error persists.
Thanks in advance.