Hello everyone,
I want to use Elastic Stack behind nginx reverse proxies.
So Kibana and Elasticsearch have a reverse proxy and they work very well together.
But Fleet Server can't connect to Elasticsearch through its reverse proxy.
When I register it, I can see it in Kibana but no log is sent:
In Fleet Server logs, I got this:
{"log.level":"info","@timestamp":"2023-09-25T12:20:12.477Z","message":"Non-zero metrics in the last 30s","component":{"binary":"filebeat","dataset":"elastic_agent.filebeat","id":"filestream-monitoring","type":"filestream"},"log":{"source":"filestream-monitoring"},"log.logger":"monitoring","log.origin":{"file.line":187,"file.name":"log/log.go"},"service.name":"filebeat","monitoring":{"ecs.version":"1.6.0","metrics":{"beat":{"cgroup":{"memory":{"mem":{"usage":{"bytes":202788864}}}},"cpu":{"system":{"ticks":1470},"total":{"ticks":4160,"value":4160},"user":{"ticks":2690}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":12},"info":{"ephemeral_id":"e378e296-9be0-4edb-9d7e-b3be7d2cfeff","uptime":{"ms":8130268},"version":"8.9.1"},"memstats":{"gc_next":39452920,"memory_alloc":19920856,"memory_total":215809456,"rss":106037248},"runtime":{"goroutines":46}},"filebeat":{"events":{"active":2116,"added":6},"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":1}},"output":{"events":{"active":0}},"pipeline":{"clients":1,"events":{"active":167,"filtered":6,"total":6}}},"registrar":{"states":{"current":0}},"system":{"load":{"1":1.14,"15":0.9,"5":0.93,"norm":{"1":0.57,"15":0.45,"5":0.465}}}}},"ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2023-09-25T12:20:12.560Z","message":"Non-zero metrics in the last 30s","component":{"binary":"metricbeat","dataset":"elastic_agent.metricbeat","id":"beat/metrics-monitoring","type":"beat/metrics"},"log":{"source":"beat/metrics-monitoring"},"monitoring":{"ecs.version":"1.6.0","metrics":{"beat":{"cgroup":{"memory":{"mem":{"usage":{"bytes":202788864}}}},"cpu":{"system":{"ticks":1410,"time":{"ms":10}},"total":{"ticks":4940,"time":{"ms":50},"value":4940},"user":{"ticks":3530,"time":{"ms":40}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":11},"info":{"ephemeral_id":"7495e6f5-a49f-4df8-979b-8fd43d807b01","uptime":{"ms":8130361},"version":"8.9.1"},"memstats":{"gc_next":58492152,"memory_alloc":29353008,"memory_total":202801408,"rss":129208320},"runtime":{"goroutines":39}},"libbeat":{"config":{"module":{"running":1}},"output":{"events":{"active":0}},"pipeline":{"clients":2,"events":{"active":1626,"published":6,"total":6}}},"metricbeat":{"beat":{"state":{"events":3,"success":3},"stats":{"events":3,"success":3}}},"system":{"load":{"1":1.14,"15":0.9,"5":0.93,"norm":{"1":0.57,"15":0.45,"5":0.465}}}}},"log.logger":"monitoring","log.origin":{"file.line":187,"file.name":"log/log.go"},"service.name":"metricbeat","ecs.version":"1.6.0"}
{"log.level":"error","@timestamp":"2023-09-25T12:20:38.135Z","message":"Failed to connect to backoff(elasticsearch(https://192.168.1.102:9201)): Get \"https://192.168.1.102:9201\": context deadline exceeded","component":{"binary":"metricbeat","dataset":"elastic_agent.metricbeat","id":"beat/metrics-monitoring","type":"beat/metrics"},"log":{"source":"beat/metrics-monitoring"},"ecs.version":"1.6.0","log.logger":"publisher_pipeline_output","log.origin":{"file.line":148,"file.name":"pipeline/client_worker.go"},"service.name":"metricbeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2023-09-25T12:20:38.135Z","message":"Attempting to reconnect to backoff(elasticsearch(https://192.168.1.102:9201)) with 62 reconnect attempt(s)","component":{"binary":"metricbeat","dataset":"elastic_agent.metricbeat","id":"beat/metrics-monitoring","type":"beat/metrics"},"log":{"source":"beat/metrics-monitoring"},"log.logger":"publisher_pipeline_output","log.origin":{"file.line":139,"file.name":"pipeline/client_worker.go"},"service.name":"metricbeat","ecs.version":"1.6.0","ecs.version":"1.6.0"}
{"log.level":"error","@timestamp":"2023-09-25T12:20:39.372Z","message":"Error dialing dial tcp 192.168.1.102:9201: i/o timeout","component":{"binary":"filebeat","dataset":"elastic_agent.filebeat","id":"filestream-monitoring","type":"filestream"},"log":{"source":"filestream-monitoring"},"address":"192.168.1.102:9201","network":"tcp","service.name":"filebeat","ecs.version":"1.6.0","log.logger":"esclientleg","log.origin":{"file.line":38,"file.name":"transport/logging.go"},"ecs.version":"1.6.0"}
I'm using docker-compose to setup the Stack.
This is my Fleet setup:
fleet-setup:
depends_on:
kibana-reverse-proxy:
condition: service_healthy
image: alpine:3.18.0
volumes:
- cacert:/tmp/certs/ca:ro
- fleetsetupcheck:/tmp/checks
group_add:
- 0
command: >
sh -c '
apk update;
apk add curl;
apk add jq;
echo "Check if fleet is setup...";
if [ -f /tmp/checks/fleet_setup.check ];then
echo "Fleet is already setup!";
else
echo "Fleet setup";
curl -k -XPOST --cacert /tmp/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" https://kibana-reverse-proxy:443/api/fleet/setup --header "kbn-xsrf: true" && touch /tmp/checks/fleet_setup.check;
fi;
echo "Check if Fleet Server Policy already exists...";
if [ -f /tmp/checks/fleet_policy.check ];then
echo "Fleet Server Policy already exists!";
else
echo && echo "Generate Server Policy";
curl -k -XPOST --cacert /tmp/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" https://kibana-reverse-proxy:443/api/fleet/agent_policies?sys_monitoring=true --header "content-type: application/json" --header "kbn-xsrf: true" --data "{\"id\":\"fleet-server-policy\",\"name\":\"Fleet Server Policy\",\"description\":\"Fleet Server Policy\",\"namespace\":\"default\",\"monitoring_enabled\":[\"logs\",\"metrics\"],\"has_fleet_server\":\"true\"}" && touch /tmp/checks/fleet_policy.check;
fi;
echo "Check if Fleet Server URL is updated...";
if [ -f /tmp/checks/fleet_url.check ];then
echo "Fleet Server URL is already updated";
else
echo && echo "Update Fleet Server URL";
curl -k -XPUT --cacert /tmp/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" https://kibana-reverse-proxy:443/api/fleet/settings --header "kbn-xsrf: true" --header "Content-Type: application/json" --data "{\"fleet_server_hosts\":[\"https://192.168.1.102:8220\"]}" && touch /tmp/checks/fleet_url.check;
fi;
echo && echo "All done!";
'
This is my configuration for Fleet Server:
fleet:
depends_on:
kibana-reverse-proxy:
condition: service_healthy
fleet-setup:
condition: service_completed_successfully
image: docker.elastic.co/beats/elastic-agent:${STACK_VERSION}
user: elastic-agent
volumes:
- cacert:/opt/Elastic/certs/ca:ro
- fleetcert:/opt/Elastic/certs/fleet:ro
- fleetdata:/usr/share/elastic-agent/data
group_add:
- 0
ports:
- 8220:8220
environment:
- FLEET_SERVER_ENABLE=true
- FLEET_SERVER_ELASTICSEARCH_HOST=https://elasticsearch-reverse-proxy:9201
- FLEET_SERVER_ELASTICSEARCH_CA=/opt/Elastic/certs/ca/ca.crt
- FLEET_SERVER_POLICY=fleet-server-policy
- FLEET_SERVER_ELASTICSEARCH_USERNAME=elastic
- FLEET_SERVER_ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
- FLEET_SERVER_CERT=/opt/Elastic/certs/fleet/fleet-server.crt
- FLEET_SERVER_CERT_KEY=/opt/Elastic/certs/fleet/fleet-server.key
- FLEET_ENROLL=1
- FLEET_URL=https://localhost:8220
- FLEET_CA=/opt/Elastic/certs/ca/ca.crt
- KIBANA_FLEET_SETUP=1
- KIBANA_HOST=https://kibana-reverse-proxy:443
- KIBANA_USERNAME=elastic
- KIBANA_PASSWORD=${ELASTIC_PASSWORD}
- KIBANA_CA=/opt/Elastic/certs/ca/ca.crt
- ELASTICSEARCH_HOST=https://elasticsearch-reverse-proxy:9201
- ELASTICSEARCH_USERNAME=elastic
- ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
- ELATICSEARCH_CA=/opt/Elastic/certs/ca/ca.crt
- CERTIFICATE_AUTHORITIES=/opt/Elastic/certs/ca/ca.crt
mem_limit: ${MEM_LIMIT}
restart: on-failure
healthcheck:
test:
[
"CMD-SHELL",
"curl -s --cacert /opt/Elastic/certs/ca/ca.crt -I https://localhost:8220 | grep -q 'HTTP/2 404'",
]
interval: 10s
timeout: 10s
retries: 120
This is my configuration for Elasticsearch reverse proxy:
elasticsearch-reverse-proxy:
depends_on:
es03:
condition: service_healthy
image: nginx:latest
ports:
- 9201:9201
volumes:
- ./nginx_conf/elasticsearch:/etc/nginx/conf.d
- elasticsearchreversecert:/etc/nginx/certs/nginx:ro
- cacert:/etc/nginx/certs/ca:ro
healthcheck:
test:
[
"CMD-SHELL",
"curl -s -I --cacert /etc/nginx/certs/ca/ca.crt https://localhost:9201 | grep -q 'HTTP/1.1 401 Unauthorized'",
]
interval: 10s
timeout: 10s
retries: 120
With this nginx configuration:
# Load balancing
upstream elasticsearch {
server es01:9200;
server es02:9200;
server es03:9200;
}
server {
# Between browser and nginx
listen 9201 ssl;
ssl_certificate /etc/nginx/certs/nginx/nginx-elasticsearch.crt;
ssl_certificate_key /etc/nginx/certs/nginx/nginx-elasticsearch.key;
ssl_client_certificate /etc/nginx/certs/ca/ca.crt;
ssl_verify_client optional;
location / {
# Between nginx and elasticsearch
proxy_set_header X-Real-IP $remote_addr;
proxy_pass https://elasticsearch;
}
}