Logstash output adding ES cluster exception

Elastic Stack Logstash
,
1

Logstash
docker

I plan to use Docker Compose to deploy ELK in Docker, where ES is a cluster
The version is 8.11.3

  • .env:
ELASTIC_PASSWORD=123456

KIBANA_PASSWORD=123456

STACK_VERSION=8.11.3

CLUSTER_NAME=docker-cluster

LICENSE=basic
#LICENSE=trial

ES_PORT=9200

KIBANA_PORT=5601

LOGSTASH_PORT=9600

MEM_LIMIT=1073741824

COMPOSE_PROJECT_NAME=myproject
  • docker-compose.yml：
version: "1.0"

services:
  setup:
    container_name: setup
    image: elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
    user: "0"
    networks:
      - elastic
    command: >
      bash -c '
        if [ x${ELASTIC_PASSWORD} == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x${KIBANA_PASSWORD} == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
          exit 1;
        fi;
        if [ ! -f config/certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
        fi;
        if [ ! -f config/certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \
          "instances:\n"\
          "  - name: es01\n"\
          "    dns:\n"\
          "      - es01\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          "  - name: es02\n"\
          "    dns:\n"\
          "      - es02\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          "  - name: es03\n"\
          "    dns:\n"\
          "      - es03\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          > config/certs/instances.yml;
          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
          unzip config/certs/certs.zip -d config/certs;
        fi;
        echo "Setting file permissions"
        chown -R root:root config/certs;
        find . -type d -exec chmod 750 \{\} \;;
        find . -type f -exec chmod 640 \{\} \;;
        echo "Waiting for Elasticsearch availability";
        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
        echo "All done!";
      '
    healthcheck:
      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
      interval: 1s
      timeout: 5s
      retries: 120

  es01:
    container_name: es01
    depends_on:
      setup:
        condition: service_healthy
    image: elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - ./esdata01/data:/usr/share/elasticsearch/data
    ports:
      - ${ES_PORT}:9200
    environment:
      - node.name=es01
      - cluster.name=${CLUSTER_NAME}
      - cluster.initial_master_nodes=es01,es02,es03
      - discovery.seed_hosts=es01,es02,es03
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es01/es01.key
      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es01/es01.key
      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${LICENSE}
    networks:
      - elastic
    restart: always
    mem_limit: ${MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  es02:
    container_name: es02
    depends_on:
      - es01
    image: elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - ./esdata02/data:/usr/share/elasticsearch/data
    environment:
      - node.name=es02
      - cluster.name=${CLUSTER_NAME}
      - cluster.initial_master_nodes=es01,es02,es03
      - discovery.seed_hosts=es01,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es02/es02.key
      - xpack.security.http.ssl.certificate=certs/es02/es02.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es02/es02.key
      - xpack.security.transport.ssl.certificate=certs/es02/es02.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${LICENSE}
    networks:
      - elastic
    restart: always
    mem_limit: ${MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  es03:
    container_name: es03
    depends_on:
      - es02
    image: elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - ./esdata03/data:/usr/share/elasticsearch/data
    environment:
      - node.name=es03
      - cluster.name=${CLUSTER_NAME}
      - cluster.initial_master_nodes=es01,es02,es03
      - discovery.seed_hosts=es01,es02
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es03/es03.key
      - xpack.security.http.ssl.certificate=certs/es03/es03.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es03/es03.key
      - xpack.security.transport.ssl.certificate=certs/es03/es03.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${LICENSE}
    networks:
      - elastic
    restart: always
    mem_limit: ${MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  kibana:
    container_name: kibana
    depends_on:
      es01:
        condition: service_healthy
      es02:
        condition: service_healthy
      es03:
        condition: service_healthy
    image: kibana:${STACK_VERSION}
    volumes:
      - certs:/usr/share/kibana/config/certs
      - ./kibanadata/data:/usr/share/kibana/data
    ports:
      - ${KIBANA_PORT}:5601
    environment:
      - SERVERNAME=kibana
      - ELASTICSEARCH_HOSTS=https://es01:9200
      - ELASTICSEARCH_USERNAME=kibana_system
      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
      - I18N_LOCALE=zh-CN
    networks:
      - elastic
    restart: always
    mem_limit: ${MEM_LIMIT}
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  logstash:
    container_name: logstash
    image: logstash:${STACK_VERSION}
    user: "0"
    volumes:
      - certs:/usr/share/logstash/config/certs
      - ./logstash/data:/usr/share/logstash/data
    ports:
      - ${LOGSTASH_PORT}:9600
    environment:
      - "LS_JAVA_OPTS=-Xmx256m -Xms256m"
    networks:
      - elastic
    mem_limit: ${MEM_LIMIT}
    depends_on:
      es01:
        condition: service_healthy
      es02:
        condition: service_healthy
      es03:
        condition: service_healthy
    restart: always
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s -I http://localhost:9600 | grep -q 'HTTP/1.1 302 Found'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

volumes:
  certs:
    driver: local
#  esdata01:
#    driver: local
#  esdata02:
#    driver: local
#  esdata03:
#    driver: local
#  kibanadata:
#    driver: local

networks:
  elastic:
    driver: bridge
    ipam:
      driver: default
      config:
        - subnet: 172.24.0.0/16

http.host: "0.0.0.0"
pipeline.ordered: auto
  • logstash.conf
input {
  beats {
    port => 5044
  }
}

output {
  stdout {
    codec => rubydebug
  }

  elasticsearch {
    hosts => ["https://localhost:9200"]
    index => "stdin-%{+YYYY.MM.dd}"
    user => "elastic"
    password => "123456"
    ssl_enabled => true
    ssl_certificate_authorities => "/usr/share/logstash/config/certs/ca/ca.crt"
    ssl_certificate => "/usr/share/logstash/config/certs/es01/es01.crt"
    ssl_key => "/usr/share/logstash/config/certs/es01/es01.key"
  }
}

ES cluster and kibana are both running normally
But logstash did report an error, the error message is as follows

2024-01-04 09:05:59 [2024-01-04T01:05:59,555][ERROR][logstash.javapipeline    ][main] Pipeline error {:pipeline_id=>"main", :exception=>#<Java::JavaSecuritySpec::InvalidKeySpecException: java.security.InvalidKeyException: IOException : algid parse error, not a sequence>, :backtrace=>["sun.security.rsa.RSAKeyFactory.engineGeneratePrivate(sun/security/rsa/RSAKeyFactory.java:253)", "java.security.KeyFactory.generatePrivate(java/security/KeyFactory.java:389)", "jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)", "jdk.internal.reflect.NativeMethodAccessorImpl.invoke(jdk/internal/reflect/NativeMethodAccessorImpl.java:77)", "jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(jdk/internal/reflect/DelegatingMethodAccessorImpl.java:43)", "java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:568)", "org.jruby.javasupport.JavaMethod.invokeDirectWithExceptionHandling(org/jruby/javasupport/JavaMethod.java:315)", "org.jruby.javasupport.JavaMethod.invokeDirect(org/jruby/javasupport/JavaMethod.java:176)", "RUBY.setup_key_store(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/manticore-0.9.1-java/lib/manticore/client.rb:751)", "org.jruby.RubyArray.each(org/jruby/RubyArray.java:1989)", "org.jruby.RubyArray$INVOKER$i$0$0$each.call(org/jruby/RubyArray$INVOKER$i$0$0$each.gen)", "RUBY.setup_key_store(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/manticore-0.9.1-java/lib/manticore/client.rb:746)", "RUBY.ssl_socket_factory_from_options(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/manticore-0.9.1-java/lib/manticore/client.rb:693)", "RUBY.pool_builder(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/manticore-0.9.1-java/lib/manticore/client.rb:454)", "RUBY.pool(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/manticore-0.9.1-java/lib/manticore/client.rb:462)", "RUBY.initialize(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/manticore-0.9.1-java/lib/manticore/client.rb:227)", "org.jruby.RubyClass.new(org/jruby/RubyClass.java:904)", "org.jruby.RubyClass$INVOKER$i$newInstance.call(org/jruby/RubyClass$INVOKER$i$newInstance.gen)", "RUBY.initialize(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/logstash-output-elasticsearch-11.19.0-java/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb:26)", "org.jruby.RubyClass.new(org/jruby/RubyClass.java:917)", "org.jruby.RubyClass$INVOKER$i$newInstance.call(org/jruby/RubyClass$INVOKER$i$newInstance.gen)", "RUBY.build_adapter(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/logstash-output-elasticsearch-11.19.0-java/lib/logstash/outputs/elasticsearch/http_client.rb:336)", "RUBY.build_pool(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/logstash-output-elasticsearch-11.19.0-java/lib/logstash/outputs/elasticsearch/http_client.rb:352)", "RUBY.initialize(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/logstash-output-elasticsearch-11.19.0-java/lib/logstash/outputs/elasticsearch/http_client.rb:63)", "org.jruby.RubyClass.new(org/jruby/RubyClass.java:904)", "org.jruby.RubyClass$INVOKER$i$newInstance.call(org/jruby/RubyClass$INVOKER$i$newInstance.gen)", "RUBY.create_http_client(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/logstash-output-elasticsearch-11.19.0-java/lib/logstash/outputs/elasticsearch/http_client_builder.rb:106)", "RUBY.build(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/logstash-output-elasticsearch-11.19.0-java/lib/logstash/outputs/elasticsearch/http_client_builder.rb:102)", "RUBY.build_client(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/logstash-output-elasticsearch-11.19.0-java/lib/logstash/plugin_mixins/elasticsearch/common.rb:42)", "RUBY.register(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/logstash-output-elasticsearch-11.19.0-java/lib/logstash/outputs/elasticsearch.rb:301)", "org.jruby.RubyClass.finvoke(org/jruby/RubyClass.java:580)", "org.jruby.RubyBasicObject.callMethod(org/jruby/RubyBasicObject.java:349)", "org.logstash.config.ir.compiler.OutputStrategyExt$SimpleAbstractOutputStrategyExt.reg(org/logstash/config/ir/compiler/OutputStrategyExt.java:275)", "org.logstash.config.ir.compiler.OutputStrategyExt$AbstractOutputStrategyExt.register(org/logstash/config/ir/compiler/OutputStrategyExt.java:131)", "org.logstash.config.ir.compiler.OutputDelegatorExt.doRegister(org/logstash/config/ir/compiler/OutputDelegatorExt.java:126)", "org.logstash.config.ir.compiler.AbstractOutputDelegatorExt.register(org/logstash/config/ir/compiler/AbstractOutputDelegatorExt.java:69)", "org.logstash.config.ir.compiler.AbstractOutputDelegatorExt$INVOKER$i$0$0$register.call(org/logstash/config/ir/compiler/AbstractOutputDelegatorExt$INVOKER$i$0$0$register.gen)", "RUBY.register_plugins(/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:237)", "org.jruby.RubyArray.each(org/jruby/RubyArray.java:1989)", "org.jruby.RubyArray$INVOKER$i$0$0$each.call(org/jruby/RubyArray$INVOKER$i$0$0$each.gen)", "RUBY.register_plugins(/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:236)", "RUBY.maybe_setup_out_plugins(/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:610)", "RUBY.start_workers(/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:249)", "RUBY.run(/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:194)", "RUBY.start(/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:146)", "org.jruby.RubyProc.call(org/jruby/RubyProc.java:352)", "java.lang.Thread.run(java/lang/Thread.java:840)"], "pipeline.sources"=>["/usr/share/logstash/pipeline/logstash.conf"], :thread=>"#<Thread:0x58c2c140 /usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:134 run>"}
2024-01-04 09:05:59 [2024-01-04T01:05:59,557][INFO ][logstash.javapipeline    ][main] Pipeline terminated {"pipeline.id"=>"main"}
2024-01-04 09:05:59 [2024-01-04T01:05:59,565][ERROR][logstash.agent           ] Failed to execute action {:id=>:main, :action_type=>LogStash::ConvergeResult::FailedAction, :message=>"Could not execute action: PipelineAction::Create<main>, action_result: false", :backtrace=>nil}
2024-01-04 09:05:59 [2024-01-04T01:05:59,589][INFO ][logstash.runner          ] Logstash shut down.
2024-01-04 09:05:59 [2024-01-04T01:05:59,610][FATAL][org.logstash.Logstash    ] Logstash stopped processing because of an error: (SystemExit) exit
2024-01-04 09:05:59 org.jruby.exceptions.SystemExit: (SystemExit) exit
2024-01-04 09:05:59 at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:808) ~[jruby.jar:?]
2024-01-04 09:05:59 at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:767) ~[jruby.jar:?]
2024-01-04 09:05:59 at usr.share.logstash.lib.bootstrap.environment.<main>(/usr/share/logstash/lib/bootstrap/environment.rb:90) ~[?:?]

May I ask how to solve it？

2

It is telling you that one of your keys or certs is in the wrong format.

  • ssl_certificate expects an OpenSSL-style X.509 certificate file
  • ssl_certificate_authorities should be a .cer or .pem file (this may be your problem)
  • ssl_key should be an OpenSSL-style RSA private key
3

Thank you very much for your guidance

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.