Trying to deploy elk as a docker service with certificate

[Spyros_Agriopoulos]

Hello all.

Here is the problem: I have deployed an elk stack using docker compose, which worked fine up until recently. Now after a while one or more of the containers go down for some reason, and I am guessing it may be because of the workload. Anyways, I have decided to deploy it as a docker service, so the containers can automatically go up again if they drop.

When I deployed it, the containers started to crash all the time, saying that the certificate folder doesn't exist. I connected to one of the containers with bash, and the permissions for the certificate folder was changed to root, instead of Elasticsearch or kibana. When I use docker compose this doesn't happen.

Any ideas what could be causing this?

version: '3'
services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    environment:
      - node.name=es01
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es02,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - ELASTIC_PASSWORD=pass
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=/usr/share/elasticsearch/config/certificates/es01/es01.key
   -xpack.security.http.ssl.certificate_authorities=/usr/share/elasticsearch/config/certificates/ca/ca.crt
      - xpack.security.http.ssl.certificate=/usr/share/elasticsearch/config/certificates/es01/es01.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.certificate_authorities=/usr/share/elasticsearch/config/certificates/ca/ca.crt
      - xpack.security.transport.ssl.certificate=/usr/share/elasticsearch/config/certificates/es01/es01.crt
      - xpack.security.transport.ssl.key=/usr/share/elasticsearch/config/certificates/es01/es01.key
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data01:/usr/share/elasticsearch/data', 'certs:/usr/share/elasticsearch/config/certificates/']
    ports:
      - 9200:9200
    healthcheck:
      test: curl --cacert /usr/share/elasticsearch/config/certificates/ca/ca.crt -s https://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - elastic
  es02:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    environment:
      - node.name=es02
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - ELASTIC_PASSWORD=pass
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=/usr/share/elasticsearch/config/certificates/es02/es02.key
      - xpack.security.http.ssl.certificate_authorities=/usr/share/elasticsearch/config/certificates/ca/ca.crt
      - xpack.security.http.ssl.certificate=/usr/share/elasticsearch/config/certificates/es02/es02.crt
- xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.certificate_authorities=/usr/share/elasticsearch/config/certificates/ca/ca.crt
      - xpack.security.transport.ssl.certificate=/usr/share/elasticsearch/config/certificates/es02/es02.crt
      - xpack.security.transport.ssl.key=/usr/share/elasticsearch/config/certificates/es02/es02.key
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data02:/usr/share/elasticsearch/data', 'certs:/usr/share/elasticsearch/config/certificates']
    networks:
      - elastic

  es03:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    environment:
      - node.name=es03
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es02
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - ELASTIC_PASSWORD=pass
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=/usr/share/elasticsearch/config/certificates/es03/es03.key
      - xpack.security.http.ssl.certificate_authorities=/usr/share/elasticsearch/config/certificates/ca/ca.crt
      - xpack.security.http.ssl.certificate=/usr/share/elasticsearch/config/certificates/es03/es03.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.certificate_authorities=/usr/share/elasticsearch/config/certificates/ca/ca.crt
      - xpack.security.transport.ssl.certificate=/usr/share/elasticsearch/config/certificates/es03/es03.crt
      - xpack.security.transport.ssl.key=/usr/share/elasticsearch/config/certificates/es03/es03.key
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data03:/usr/share/elasticsearch/data', 'certs:/usr/share/elasticsearch/config/certificates']
    networks:
      - elastic

  kibana:
    image: docker.elastic.co/kibana/kibana:7.15.1
    environment:
            #- server.publicBaseUrl=http://ip:5601
            #SERVER_NAME: kibana.example.org
      - ELASTICSEARCH_HOSTS=["https://es01:9200","https://es02:9200","https://es03:9200"]
      - XPACK_SECURITY_ENABLED=true
                   - ELASTICSEARCH_USERNAME=user
      - ELASTICSEARCH_PASSWORD=pass
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=/usr/share/kibana/config/certificates/ca/ca.crt
      - ELASTICSEARCH_SSL_VERIFICATIONMODE=certificate
      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=d3e5bd37d75a348b5e0736cbccf515b6
    ports:
      - 5601:5601
    volumes: ['certs:/usr/share/kibana/config/certificates']
    networks:
      - elastic


volumes:
  data01:
    driver: local
  data02:
    driver: local
  data03:
    driver: local
  certs:
    driver: local

networks:
  elastic:
          driver: overlay