Logstash don't trust Elasticsearch's certificate

Elastic Stack Logstash
1

I am stuck for 1 week already.. :')

Elasticsearch's cluster and Kibana is working fine except for logstash only.

I kept getting these logs from elasticsearch:
'''
es01 | {"type": "server", "timestamp": "2020-04-15T08:42:19,757Z", "level": "WARN", "component": "o.e.x.s.t.n.SecurityNetty4HttpServerTransport", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "http client did not trust this server's certificate, closing connection Netty4HttpChannel{localAddress=/172.19.0.2:9200, remoteAddress=/10.0.2.2:51810}", "cluster.uuid": "ijOAusq0ToCH6JrwKxvnGw", "node.id": "8kSU_txBQ5qqCsLQyBPuSw" }
es01 | {"type": "server", "timestamp": "2020-04-15T08:42:19,855Z", "level": "WARN", "component": "o.e.x.s.t.n.SecurityNetty4HttpServerTransport", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "http client did not trust this server's certificate, closing connection Netty4HttpChannel{localAddress=/172.19.0.2:9200, remoteAddress=/10.0.2.2:51811}", "cluster.uuid": "ijOAusq0ToCH6JrwKxvnGw", "node.id": "8kSU_txBQ5qqCsLQyBPuSw" }
'''

I generate the self-signed certs following this guide https://www.elastic.co/guide/en/elasticsearch/reference/current/configuring-tls-docker.html

docker-compose.yml

version: "3.2"

services:
  es01:
    container_name: es01
    build:
      context: elasticsearch/
      args:
        ELK_VERSION: $ELK_VERSION
    volumes:
      - es-volume01:/usr/share/elasticsearch/data
      - ./elasticsearch/config/synonyms.txt:/usr/share/elasticsearch/config/synonyms.txt
      - ./certs:$CERTS_DIR
    environment:
      - node.name=es01
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es02,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=${ES_JAVA_OPTS}"
      - xpack.license.self_generated.type=basic # <1>
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true # <2>
      - xpack.security.http.ssl.key=$CERTS_DIR/es01/es01.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.transport.ssl.enabled=true # <3>
      - xpack.security.transport.ssl.verification_mode=certificate # <4>
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es01/es01.key
      - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
    ulimits:
      memlock:
        soft: -1
        hard: -1
    ports:
      - 9200:9200
    networks:
      - es-network

  es02:
    container_name: es02
    build:
      context: elasticsearch/
      args:
        ELK_VERSION: $ELK_VERSION
    volumes:
      - es-volume02:/usr/share/elasticsearch/data
      - ./elasticsearch/config/synonyms.txt:/usr/share/elasticsearch/config/synonyms.txt
      - ./certs:$CERTS_DIR
    environment:
      - node.name=es02
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=${ES_JAVA_OPTS}"
      - xpack.license.self_generated.type=basic
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=$CERTS_DIR/es02/es02.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es02/es02.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es02/es02.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es02/es02.key
      - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
    ulimits:
      memlock:
        soft: -1
        hard: -1
    ports:
      - 9201:9201
    networks:
      - es-network

  es03:
    container_name: es03
    build:
      context: elasticsearch/
      args:
        ELK_VERSION: $ELK_VERSION
    volumes:
      - es-volume03:/usr/share/elasticsearch/data
      - ./elasticsearch/config/synonyms.txt:/usr/share/elasticsearch/config/synonyms.txt
      - ./certs:$CERTS_DIR
    environment:
      - node.name=es03
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es02
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=${ES_JAVA_OPTS}"
      - xpack.license.self_generated.type=basic
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=$CERTS_DIR/es03/es03.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es03/es03.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es03/es03.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es03/es03.key
      - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
      - CERTS_DIR=$CERTS_DIR
    ulimits:
      memlock:
        soft: -1
        hard: -1
    ports:
      - 9202:9202
    networks:
      - es-network

  logstash:
    build:
      context: logstash/
      args:
        ELK_VERSION: $ELK_VERSION
    volumes:
      - ./certs:/certs
      - type: bind
        source: ./logstash/pipeline
        target: /usr/share/logstash/pipeline
        read_only: true
      - type: bind
        source: ./lib/mysql-connector-java-8.0.19.jar
        target: /usr/share/logstash/logstash-core/lib/jars/mysql-connector-java.jar
        read_only: true
      - type: bind
        source: ./logstash/log
        target: /etc/logstash
    ports:
      - "5000:5000/tcp"
      - "5000:5000/udp"
      - "9600:9600"
    environment:
      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
      http.host: 0.0.0.0
      xpack.monitoring.enabled: "true"
      xpack.monitoring.elasticsearch.username: $ELASTIC_USERNAME
      xpack.monitoring.elasticsearch.password: $ELASTIC_PASSWORD
      xpack.monitoring.elasticsearch.hosts: https://es01:9200
      xpack.monitoring.elasticsearch.url: https://es01:9200
      xpack.ssl.key: /certs/logstash/logstash.key
      xpack.ssl.certificate: /certs/logstash/logstash.crt
      xpack.ssl.certificate_authorities: "/certs/ca/ca.crt"
      xpack.security.http.ssl.enabled: "true"
      xpack.monitoring.elasticsearch.ssl.certificate_authority: "/certs/ca/ca.crt"
      # for logstash.conf
      SQL_URL: $SQL_URL
      SQL_USER: $SQL_USER
      SQL_PASSWORD: $SQL_PASSWORD
      ELASTIC_USERNAME: $ELASTIC_USERNAME
      ELASTIC_PASSWORD: $ELASTIC_PASSWORD
    networks:
      - es-network
    depends_on:
      - es01
      - es02
      - es03

  kib01:
    container_name: kib01
    build:
      context: kibana/
      args:
        ELK_VERSION: $ELK_VERSION
    volumes:
      - ./certs:$CERTS_DIR
    environment:
      SERVERNAME: localhost
      ELASTICSEARCH_URL: https://es01:9200
      ELASTICSEARCH_HOSTS: https://es01:9200
      ELASTICSEARCH_USERNAME: $ELASTIC_USERNAME
      ELASTICSEARCH_PASSWORD: $ELASTIC_PASSWORD
      ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: $CERTS_DIR/ca/ca.crt
      SERVER_SSL_ENABLED: "true"
      SERVER_SSL_KEY: $CERTS_DIR/kib01/kib01.key
      SERVER_SSL_CERTIFICATE: $CERTS_DIR/kib01/kib01.crt
    ports:
      - 5601:5601
    depends_on:
      - es01
      - es02
      - es03
    networks:
      - es-network

volumes:
  es-volume01:
    driver: local
  es-volume02:
    driver: local
  es-volume03:
    driver: local

networks:
  es-network:
    driver: bridge

logstash.conf

input {
  jdbc {
    jdbc_driver_library => "/usr/share/logstash/logstash-core/lib/jars/mysql-connector-java.jar"
    jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
    jdbc_connection_string => "jdbc:mysql://${SQL_URL}/celestine?useTimezone=true&useLegacyDatetimeCode=false&serverTimezone=UTC"
    jdbc_user => "${SQL_USER}"
    jdbc_password => "${SQL_PASSWORD}"
    jdbc_paging_enabled => true
    tracking_column => "unix_ts_in_secs"
    last_run_metadata_path => "/etc/logstash/user.logstash_jdbc_last_run"
    use_column_value => true
    tracking_column_type => "numeric"
    schedule => "*/10 * * * * *"
    statement => "SELECT *, UNIX_TIMESTAMP(updated_at) AS unix_ts_in_secs FROM users WHERE (UNIX_TIMESTAMP(updated_at) > :sql_last_value AND updated_at < NOW()) ORDER BY updated_at ASC"
    type => "user"
  }
}

filter {
  mutate {
    copy => { "id" => "[@metadata][_id]"}
    remove_field => ["id", "@version", "unix_ts_in_secs"]
    rename => { "type" => "[@metadata][type]" }
  }
}

output {
  if [@metadata][type] == "user" {
    elasticsearch {
      index => "user"
      document_type => "_doc"
      document_id => "%{[@metadata][_id]}"
      hosts => ["https://es01:9200"]
      user => "${ELASTIC_USERNAME}"
      password => "${ELASTIC_PASSWORD}"
      doc_as_upsert => true
      action => "update"
      ssl => true
      ssl_certificate_verification => false
      cacert => "/certs/ca/ca.crt"
    }
  }
}

Any advice is highly appreciated.

2

Have you tried setting ssl_certificate_verification to false?

3

Yes, i have used that property, it is written in the logstash.conf above.

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.