I've got a logstash.conf file defining the following:
input { jdbc { jdbc_validate_connection => true jdbc_driver_library => "" jdbc_driver_class => "" jdbc_connection_string => "jdbc:mysql://x.mysql.mysql.database.azure.com:3306/x" jdbc_user => "xx" jdbc_password => "xx" schedule => "* * * * *" clean_run => false statement => "select * from x" } } output { elasticsearch { hosts => ["https://xxxx.westeurope.azure.elastic-cloud.com:9243"] user => "elastic" password => "xxx" data_stream => "auto" index => "xxx_%{[@metadata][index]}" } elastic_app_search { host => "https://xxxx.ent.westeurope.azure.elastic-cloud.com" api_key => "private-xxxxx" engine => "myengine" } stdout { codec => rubydebug } }
The push to Elasticsearch is working, but I can't manage to push the data to elastic_app_search, I get the following stack trace:
{"level":"ERROR","loggerName":"logstash.javapipeline","timeMillis":1622016095616,"thread":"[main]-pipeline-manager","logEvent":{"message":"Pipeline error","pipeline_id":"main","exception":{"metaClass":{"metaClass":{"metaClass":{"exception":"Failed to connect to App Search. Failed to open TCP connection to https:443 (initialize: name or service not known)","backtrace":["/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elastic_app_search-1.1.1/lib/logstash/outputs/elastic_app_search.rb:36:in `register'","org/logstash/config/ir/compiler/OutputStrategyExt.java:131:in `register'","org/logstash/config/ir/compiler/AbstractOutputDelegatorExt.java:68:in `register'","/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:228:in `block in register_plugins'","org/jruby/RubyArray.java:1809:in `each'","/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:227:in `register_plugins'","/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:585:in `maybe_setup_out_plugins'","/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:240:in `start_workers'","/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:185:in `run'","/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:137:in `block in start'"],"pipeline.sources":["/usr/share/logstash/pipeline/logstash.conf"],"thread":"#<Thread:0x6956ec45 run>"}}}}}}
{"level":"INFO","loggerName":"logstash.javapipeline","timeMillis":1622016095627,"thread":"[main]-pipeline-manager","logEvent":{"message":"Pipeline terminated","pipeline.id":"main"}}
{"level":"ERROR","loggerName":"logstash.agent","timeMillis":1622016095654,"thread":"Converge PipelineAction::Create<main>","logEvent":{"message":"Failed to execute action","id":{"metaClass":{"metaClass":{"metaClass":{"id":"main","action_type":{"metaClass":{"metaClass":{"action_type":"LogStash::ConvergeResult::FailedAction","message":"Could not execute action: PipelineAction::Create<main>, action_result: false","backtrace":null}}}}}}}}}
{"level":"TRACE","loggerName":"logstash.agent","timeMillis":1622016095664,"thread":"Agent thread","logEvent":{"message":"Converge results","success":false,"failed_actions":["id: main, action_type: LogStash::PipelineAction::Create, message: Could not execute action: PipelineAction::Create<main>, action_result: false"],"successful_actions":[]}}
{"level":"DEBUG","loggerName":"logstash.instrument.periodicpoller.os","timeMillis":1622016095713,"thread":"LogStash::Runner","logEvent":{"message":"Stopping"}}
{"level":"DEBUG","loggerName":"logstash.instrument.periodicpoller.jvm","timeMillis":1622016095727,"thread":"LogStash::Runner","logEvent":{"message":"Stopping"}}
For reference; this is my dockerfile:
FROM docker.elastic.co/logstash/logstash:7.13.0
# Install necessary logstash filters
# By default logstash-filter-jdbc and streaming is enabled
RUN /usr/share/logstash/bin/logstash-plugin install logstash-filter-aggregate
RUN /usr/share/logstash/bin/logstash-plugin install logstash-filter-mutate
RUN /usr/share/logstash/bin/logstash-plugin install logstash-output-elastic_app_search
USER logstash
# Copy mysql connector for java to logstash core dir
COPY --chown=logstash:root ./libs/mysql-connector-java-8.0.25.jar /usr/share/logstash/logstash-core/lib/jars/mysql-connector-java.jar
# Copy logstash config files into conf directory
COPY ./logstash.conf /usr/share/logstash/pipeline/logstash.conf
COPY ./logstash.yml /usr/share/logstash/config/logstash.yml