Getting error while connecting logstash to elastic

Elastic Stack Logstash
1

Hi all,

elastic -> sudo docker run -d --name es01 --net elastic -p 9200:9200 -e "discovery.type=single-node" -it -m 2GB docker.elastic.co/elasticsearch/elasticsearch:8.14.3

kibana-> sudo docker run -d --name kib01 --net elastic -p 5601:5601 docker.elastic.co/kibana/kibana:8.14.3

logstash ->
logstash: image: docker.elastic.co/logstash/logstash:8.14.3 container_name: logstash networks: - elastic volumes: - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf - ./logs/temp.log:/usr/share/logstash/logs/logfile.log ports: - "5044:5044" - "9600:9600"

logstash.conf ->
`input {
file {
hosts => ["localhost:9200"]
path => "/path to file/filename.log"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}

filter {
json {
source => "message"
}
}

output {
elasticsearch {
index => "nodejs-logs-%{+YYYY.MM.dd}",
ssl_certificate_verification => false # add this to test
user => "user",
password => "password"
# ssl => true
}
stdout {
codec => rubydebug
}
}`

elastic and kibana working fine. logstash I am getting below error:-
[2024-07-26T07:10:24,721][INFO ][logstash.licensechecker.licensereader] Failed to perform request {:message=>"elasticsearch: Temporary failure in name resolution", :exception=>Manticore::ResolutionFailure, :cause=>#<Java::JavaNet::UnknownHostException: elasticsearch: Temporary failure in name resolution>} [2024-07-26T07:10:24,722][WARN ][logstash.licensechecker.licensereader] Attempted to resurrect connection to dead ES instance, but got an error {:url=>"http://elasticsearch:9200/", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :message=>"Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch: Temporary failure in name resolution"} [2024-07-26T07:10:24,729][INFO ][logstash.licensechecker.licensereader] Failed to perform request {:message=>"elasticsearch", :exception=>Manticore::ResolutionFailure, :cause=>#<Java::JavaNet::UnknownHostException: elasticsearch>} [2024-07-26T07:10:24,729][WARN ][logstash.licensechecker.licensereader] Marking url as dead. Last error: [LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError] Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch {:url=>http://elasticsearch:9200/, :error_message=>"Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch", :error_class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError"} [2024-07-26T07:10:24,731][WARN ][logstash.licensechecker.licensereader] Attempt to fetch Elasticsearch cluster info failed. Sleeping for 0.02 {:fail_count=>1, :exception=>"Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch"} [2024-07-26T07:10:24,752][ERROR][logstash.licensechecker.licensereader] Unable to retrieve Elasticsearch cluster info. {:message=>"No Available connections", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError} [2024-07-26T07:10:24,754][ERROR][logstash.licensechecker.licensereader] Unable to retrieve license information from license server {:message=>"No Available connections"} [2024-07-26T07:10:24,766][ERROR][logstash.monitoring.internalpipelinesource] Failed to fetch X-Pack information from Elasticsearch. This is likely due to failure to reach a live Elasticsearch cluster. [2024-07-26T07:10:24,837][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600, :ssl_enabled=>false} [2024-07-26T07:10:24,961][ERROR][logstash.agent ] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"LogStash::ConfigurationError", :message=>"Expected one of [ \\t\\r\\n], \"#\", \"{\", \"}\" at line 18, column 44 (byte 323) after output {\n elasticsearch {\n hosts => [\"https://localhost:9200\"]\n index => \"shortify-logs-%{+YYYY.MM.dd}\"", :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:32:in compile_imperative'", "org/logstash/execution/AbstractPipelineExt.java:239:in initialize'", "org/logstash/execution/AbstractPipelineExt.java:173:in initialize'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:48:in initialize'", "org/jruby/RubyClass.java:949:in new'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:49:in execute'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:386:in block in converge_state'"]}`

please help.. tried all your topics and google also .. but getting correct solution

2

logstash does not like whatever comes after the index option. The example configuration you show has a comma there, which would cause an exception. Remove the commas there and after user => "user".

1 Like
3

thank you for reply sir.

now my logstash.conf looks like this

`input {
file {
path => "/path to file/filename.log"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}

filter {
json {
source => "message"
}
}

output {
elasticsearch {
hosts => ["localhost:9200"]
index => "shortify-logs-%{+YYYY.MM.dd}"
ssl_certificate_verification => false
user => "user"
password => "password"
ssl => true
}
stdout {
codec => rubydebug
}
}`

and I am getting error->
[2024-07-27T11:54:08,041][INFO ][logstash.licensechecker.licensereader] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://elasticsearch:9200/]}} [2024-07-27T11:54:08,095][INFO ][logstash.licensechecker.licensereader] Failed to perform request {:message=>"elasticsearch: Temporary failure in name resolution", :exception=>Manticore::ResolutionFailure, :cause=>#<Java::JavaNet::UnknownHostException: elasticsearch: Temporary failure in name resolution>} [2024-07-27T11:54:08,097][WARN ][logstash.licensechecker.licensereader] Attempted to resurrect connection to dead ES instance, but got an error {:url=>"http://elasticsearch:9200/", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :message=>"Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch: Temporary failure in name resolution"} [2024-07-27T11:54:08,111][INFO ][logstash.licensechecker.licensereader] Failed to perform request {:message=>"elasticsearch", :exception=>Manticore::ResolutionFailure, :cause=>#<Java::JavaNet::UnknownHostException: elasticsearch>} [2024-07-27T11:54:08,113][WARN ][logstash.licensechecker.licensereader] Marking url as dead. Last error: [LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError] Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch {:url=>http://elasticsearch:9200/, :error_message=>"Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch", :error_class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError"} [2024-07-27T11:54:08,115][WARN ][logstash.licensechecker.licensereader] Attempt to fetch Elasticsearch cluster info failed. Sleeping for 0.02 {:fail_count=>1, :exception=>"Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch"} [2024-07-27T11:54:08,137][ERROR][logstash.licensechecker.licensereader] Unable to retrieve Elasticsearch cluster info. {:message=>"No Available connections", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError} [2024-07-27T11:54:08,139][ERROR][logstash.licensechecker.licensereader] Unable to retrieve license information from license server {:message=>"No Available connections"} [2024-07-27T11:54:08,156][ERROR][logstash.monitoring.internalpipelinesource] Failed to fetch X-Pack information from Elasticsearch. This is likely due to failure to reach a live Elasticsearch cluster. [2024-07-27T11:54:08,166][INFO ][logstash.config.source.local.configpathloader] No config files found in path {:path=>"/logstash/logstash.conf"} [2024-07-27T11:54:08,167][ERROR][logstash.config.sourceloader] No configuration found in the configured sources. [2024-07-27T11:54:08,250][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600, :ssl_enabled=>false} [2024-07-27T11:54:08,260][INFO ][logstash.runner ] Logstash shut down. [2024-07-27T11:54:08,270][FATAL][org.logstash.Logstash ] Logstash stopped processing because of an error: (SystemExit) exit org.jruby.exceptions.SystemExit: (SystemExit) exit at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:921) ~[jruby.jar:?] at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:880) ~[jruby.jar:?] at usr.share.logstash.lib.bootstrap.environment.<main>(/usr/share/logstash/lib/bootstrap/environment.rb:90) ~[?:?]

4

You do not seem to be running the configuration you shared, please double check which configuration you are running and share it.

In the configuration you shared your output is configured to localhost:9200, but in your error log it is showing elasticsearch:9200, also your container name is es01 not elasticsearch, so you should use es01.

You have another error regarding configuration:

[2024-07-27T11:54:08,166][INFO ][logstash.config.source.local.configpathloader] No config files found in path {:path=>"/logstash/logstash.conf"} [2024-07-27T11:54:08,167][ERROR][logstash.config.sourceloader] No configuration found in the configured sources.

It is looking for a configuration in /logstash/logstash.conf, which also does not match the command line you shared.

Please double check which command you are running and your configurations.

1 Like
5

Thank you for quick reply sir,
still getting same error

[2024-07-27T12:44:12,722][INFO ][logstash.licensechecker.licensereader] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://elasticsearch:9200/]}} [2024-07-27T12:44:12,797][INFO ][logstash.licensechecker.licensereader] Failed to perform request {:message=>"elasticsearch: Temporary failure in name resolution", :exception=>Manticore::ResolutionFailure, :cause=>#<Java::JavaNet::UnknownHostException: elasticsearch: Temporary failure in name resolution>} [2024-07-27T12:44:12,799][WARN ][logstash.licensechecker.licensereader] Attempted to resurrect connection to dead ES instance, but got an error {:url=>"http://elasticsearch:9200/", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :message=>"Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch: Temporary failure in name resolution"} [2024-07-27T12:44:12,812][INFO ][logstash.licensechecker.licensereader] Failed to perform request {:message=>"elasticsearch", :exception=>Manticore::ResolutionFailure, :cause=>#<Java::JavaNet::UnknownHostException: elasticsearch>} [2024-07-27T12:44:12,814][WARN ][logstash.licensechecker.licensereader] Marking url as dead. Last error: [LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError] Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch {:url=>http://elasticsearch:9200/, :error_message=>"Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch", :error_class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError"} [2024-07-27T12:44:12,816][WARN ][logstash.licensechecker.licensereader] Attempt to fetch Elasticsearch cluster info failed. Sleeping for 0.02 {:fail_count=>1, :exception=>"Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch"} [2024-07-27T12:44:12,838][ERROR][logstash.licensechecker.licensereader] Unable to retrieve Elasticsearch cluster info. {:message=>"No Available connections", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError} [2024-07-27T12:44:12,841][ERROR][logstash.licensechecker.licensereader] Unable to retrieve license information from license server {:message=>"No Available connections"} [2024-07-27T12:44:12,858][ERROR][logstash.monitoring.internalpipelinesource] Failed to fetch X-Pack information from Elasticsearch. This is likely due to failure to reach a live Elasticsearch cluster. [2024-07-27T12:44:12,925][INFO ][logstash.config.source.local.configpathloader] No config files found in path {:path=>"/logstash/logstash.conf"} [2024-07-27T12:44:12,927][ERROR][logstash.config.sourceloader] No configuration found in the configured sources. [2024-07-27T12:44:13,049][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600, :ssl_enabled=>false} [2024-07-27T12:44:13,062][INFO ][logstash.runner ] Logstash shut down. [2024-07-27T12:44:13,072][FATAL][org.logstash.Logstash ] Logstash stopped processing because of an error: (SystemExit) exit org.jruby.exceptions.SystemExit: (SystemExit) exit at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:921) ~[jruby.jar:?] at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:880) ~[jruby.jar:?] at usr.share.logstash.lib.bootstrap.environment.<main>(/usr/share/logstash/lib/bootstrap/environment.rb:90) ~[?:?]

i have kept folder (logstash and logs) in root level of my node express-js repo.. but still dont know why it is not picking.
docker-compose file ->
logstash: image: docker.elastic.co/logstash/logstash:8.14.3 container_name: logstash networks: - elastic volumes: - ./logstash/logstash.conf:/usr/share/logstash/pipeline/logstash.conf - ./logs/filename.log:/usr/share/logstash/logs/logfile.log command: logstash -f /logstash/logstash.conf ports: - "9600:9600"

i have kept localhost only in my .conf file.. but still dont know from where it is taking elasticsearch.

`input {
file {
path => "/path to file/filename.log"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}

filter {
json {
source => "message"
}
}

output {
es01 {
hosts => ["localhost:9200"]
index => "nodejs-logs-%{+YYYY.MM.dd}"
ssl_certificate_verification => false # add this to test
user => "user"
password => "password"
# ssl => true
}
stdout {
codec => rubydebug
}
}
`

is it possible for you to join meet call? here is my email id amol.nagotkar@vivaconnect.co ... we can connect on google chat.

6

You are not running the configuration you think you are, the one you shared is not the configuration your docker container is running.

Your error is the same because you didn't change anything in the file it is using, also, the changes you made are invalid.

output {
es01 {
hosts => ["localhost:9200"]
index => "nodejs-logs-%{+YYYY.MM.dd}"
ssl_certificate_verification => false # add this to test
user => "user"
password => "password"
# ssl => true
}

This is invalid, the es01 should be in the hosts, not as a plugin output.

It should look like this:

output {
elasticsearch {
hosts => ["es01:9200"]
index => "nodejs-logs-%{+YYYY.MM.dd}"
ssl_certificate_verification => false # add this to test
user => "user"
password => "password"
# ssl => true
}

But this needs to be done in the configuration file being used, which is not the one you shared.

Please provide the command line you are using to run the logstash docker container.

I'm sorry, but it is not possible, keep in mind that this forum has no SLA and everyone here answer when they have time to do it.

7

Thanks for the reply sir
es01 is my container name by the way.

elastic and kibana->
`sudo docker run -d --name es01 --net elastic -p 9200:9200 -e "discovery.type=single-node" -it -m 2GB docker.elastic.co/elasticsearch/elasticsearch:8.14.3

sudo docker run -d --name kib01 --net elastic -p 5601:5601 docker.elastic.co/kibana/kibana:8.14.3 `

.conf file ->
`input {
file {
path => "path to file/filename.log"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}

filter {
json {
source => "message"
}
}

output {
elasticsearch {
hosts => ["localhost:9200"]
index => "nodejs-logs-%{+YYYY.MM.dd}"
ssl_certificate_verification => false # add this to test
user => "user"
password => "password"
# ssl => true
}
stdout {
codec => rubydebug
}
}`

docker-compose file ->
`
version: '3'
services:
logstash:
image: docker.elastic.co/logstash/logstash:8.14.3
container_name: logstash
networks:
- elastic
volumes:
- ./logstash/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
- ./logs/filename.log:/usr/share/logstash/logs/logfile.log
command: logstash -f /logstash/logstash.conf
ports:
- "9600:9600"
networks:
elastic:
driver: bridge

`

command to up logstash
sudo docker-compose -f docker-compose.yml up -d

8

Remove this, this is not required, per default logstash will run the configuration files inside /usr/share/logstash/pipeline and you are already mounting a configuration there.

Please look at this file ./logstash/logstash.conf and validate the configuration, as already mentioned the configuration your container is running is not the one you shared, you are running some configuration where the outuput host is set to elasticsearch.

The output hosts needs to be the name of your elasticsearch container, not localhost.

output {
    elasticsearch {
        hosts => ["es01:9200"]
        index => "nodejs-logs-%{+YYYY.MM.dd}"
        ssl_certificate_verification => false # add this to test
        user => "user"
        password => "password"
       # ssl => true
}
9

Thank you for the reply sir.

no luck

.conf file ->

`input {
file {
path => "path to file/filename.log"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}

filter {
json {
source => "message"
}
}

output {
elasticsearch {
hosts => ["es01:9200"]
index => "nodejs-logs-%{+YYYY.MM.dd}"
ssl_certificate_verification => false # add this to test
user => "user"
password => "password"
# ssl => true
}
stdout {
codec => rubydebug
}
}
`

docker-compose file ->

version: '3'
services:
logstash:
image: docker.elastic.co/logstash/logstash:8.14.3
container_name: logstash
networks:
- elastic
volumes:
- ./logstash/logstash.conf:/logstash.conf
- ./logs/filename.log:/logfile.log
command: logstash -f /logstash/logstash.conf
ports:
- "9600:9600"
networks:
elastic:
driver: bridge

still same error ->
[2024-07-27T13:32:49,388][INFO ][logstash.licensechecker.licensereader] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://elasticsearch:9200/]}} [2024-07-27T13:32:49,446][INFO ][logstash.licensechecker.licensereader] Failed to perform request {:message=>"elasticsearch: Temporary failure in name resolution", :exception=>Manticore::ResolutionFailure, :cause=>#<Java::JavaNet::UnknownHostException: elasticsearch: Temporary failure in name resolution>} [2024-07-27T13:32:49,449][WARN ][logstash.licensechecker.licensereader] Attempted to resurrect connection to dead ES instance, but got an error {:url=>"http://elasticsearch:9200/", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :message=>"Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch: Temporary failure in name resolution"} [2024-07-27T13:32:49,468][INFO ][logstash.licensechecker.licensereader] Failed to perform request {:message=>"elasticsearch", :exception=>Manticore::ResolutionFailure, :cause=>#<Java::JavaNet::UnknownHostException: elasticsearch>} [2024-07-27T13:32:49,470][WARN ][logstash.licensechecker.licensereader] Marking url as dead. Last error: [LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError] Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch {:url=>http://elasticsearch:9200/, :error_message=>"Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch", :error_class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError"} [2024-07-27T13:32:49,472][WARN ][logstash.licensechecker.licensereader] Attempt to fetch Elasticsearch cluster info failed. Sleeping for 0.02 {:fail_count=>1, :exception=>"Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::ResolutionFailure] elasticsearch"} [2024-07-27T13:32:49,496][ERROR][logstash.licensechecker.licensereader] Unable to retrieve Elasticsearch cluster info. {:message=>"No Available connections", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError} [2024-07-27T13:32:49,499][ERROR][logstash.licensechecker.licensereader] Unable to retrieve license information from license server {:message=>"No Available connections"} [2024-07-27T13:32:49,517][ERROR][logstash.monitoring.internalpipelinesource] Failed to fetch X-Pack information from Elasticsearch. This is likely due to failure to reach a live Elasticsearch cluster. [2024-07-27T13:32:49,531][INFO ][logstash.config.source.local.configpathloader] No config files found in path {:path=>"/logstash/logstash.conf"} [2024-07-27T13:32:49,533][ERROR][logstash.config.sourceloader] No configuration found in the configured sources. [2024-07-27T13:32:49,638][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600, :ssl_enabled=>false} [2024-07-27T13:32:49,648][INFO ][logstash.runner ] Logstash shut down. [2024-07-27T13:32:49,658][FATAL][org.logstash.Logstash ] Logstash stopped processing because of an error: (SystemExit) exit org.jruby.exceptions.SystemExit: (SystemExit) exit at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:921) ~[jruby.jar:?] at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:880) ~[jruby.jar:?] at usr.share.logstash.lib.bootstrap.environment.<main>(/usr/share/logstash/lib/bootstrap/environment.rb:90) ~[?:?]

Please help.

10

You need to remove this as mentioned in the previous answer.

Remove the command parameter you have in your docker compose, this is not needed.

You also need to validate your config file as mentioned before, the logs shows that you are not running this configuration file you are sharing.

Please look at this file in your system: ./logstash/logstash.conf.

Your logstash is still using a configuration file where you have an elasticsearch output pointing to http://elasticsearch:9200, the configuration you shared does not have this, so it is running another different file.

I would recommend that you stop this docker compose, check the logstash.conf file and try to start it again.

11

Try to create a new local configuration file named new-config.conf

With this content:

input {
    file {
        path => "/usr/share/logstash/logs/filename.log"
        start_position => "beginning"
        sincedb_path => "/dev/null"
    }
}

filter {
    json {
        source => "message"
    }
}

output {
    elasticsearch {
        hosts => ["es01:9200"]
        index => "nodejs-logs-%{+YYYY.MM.dd}"
        ssl_certificate_verification => false
        user => "user"
        password => "password"
        #ssl => true
    }
    stdout {
        codec => rubydebug
    }
}

Then change your compose to use it:

- ./logstash/new-config.conf:/usr/share/logstash/pipeline/logstash.conf

And start it to see what changes.

Your error has nothing to do with Logstash, it is something wrong in your docker configuration.

12

thanks for reply sir

new docker-compose file ->
version: '3'
services:
logstash:
image: docker.elastic.co/logstash/logstash:8.14.3
container_name: logstash
networks:
- elastic
volumes:
# - ./logstash/logstash.conf:/logstash.conf
- ./logstash/new-config.conf:/usr/share/logstash/pipeline/logstash.conf
- ./logs/shortify-logs-2024-07-27.log:/logfile.log
ports:
- "9600:9600"
networks:
elastic:
driver: bridge

new .conf file->
`input {
file {
path => "/usr/share/logstash/logs/shortify-logs-2024-07-27.log"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}

filter {
json {
source => "message"
}
}

output {
elasticsearch {
hosts => ["es01:9200"]
index => "shortify-logs-%{+YYYY.MM.dd}"
ssl_certificate_verification => false # add this to test
user => "user"
password => "password"
# ssl => true
}
stdout {
codec => rubydebug
}
}
`

now error is ->

[2024-07-27T14:38:32,578][WARN ][logstash.outputs.elasticsearch][main] Attempted to resurrect connection to dead ES instance, but got an error {:url=>"http://elastic:xxxxxx@es01:9200/", :exception=>LogStash::Outputs::Elasticsearch::HttpClient::Pool::HostUnreachableError, :message=>"Elasticsearch Unreachable: [http://es01:9200/][Manticore::ResolutionFailure] es01"}
[2024-07-27T14:38:37,585][INFO ][logstash.outputs.elasticsearch][main] Failed to perform request {:message=>"es01: Temporary failure in name resolution", :exception=>Manticore::ResolutionFailure, :cause=>#<Java::JavaNet::UnknownHostException: es01: Temporary failure in name resolution>}
[2024-07-27T14:38:37,587][WARN ][logstash.outputs.elasticsearch][main] Attempted to resurrect connection to dead ES instance, but got an error {:url=>"http://elastic:xxxxxx@es01:9200/", :exception=>LogStash::Outputs::Elasticsearch::HttpClient::Pool::HostUnreachableError, :message=>"Elasticsearch Unreachable: [http://es01:9200/][Manticore::ResolutionFailure] es01: Temporary failure in name resolution"}

my es01 container running perfectly. user and password is also correct. still above error.

thanks in advance.

13

Now the error has changed and you are running the configuration in your docker compose.

Since you are running Elasticsearch 8.X, it uses https per default, you need to add https to your host in the elasticsearch output, change your hosts to this:

hosts => ["https://es01:9200"]
14

Thank you for reply

now .conf file ->

input {
file {
path => "/usr/share/logstash/logs/shortify-logs-2024-07-27.log"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}

filter {
json {
source => "message"
}
}

output {
elasticsearch {
hosts => ["https://es01:9200"]
index => "shortify-logs-%{+YYYY.MM.dd}"
ssl_certificate_verification => false # add this to test
user => "user"
password => "password"
# ssl => true
}
stdout {
codec => rubydebug
}
}

still error is ->

[2024-07-27T14:55:36,679][WARN ][logstash.outputs.elasticsearch][main] Attempted to resurrect connection to dead ES instance, but got an error {:url=>"https://elastic:xxxxxx@es01:9200/", :exception=>LogStash::Outputs::Elasticsearch::HttpClient::Pool::HostUnreachableError, :message=>"Elasticsearch Unreachable: [https://es01:9200/][Manticore::ResolutionFailure] es01"}
[2024-07-27T14:55:41,698][INFO ][logstash.outputs.elasticsearch][main] Failed to perform request {:message=>"es01: Temporary failure in name resolution", :exception=>Manticore::ResolutionFailure, :cause=>#<Java::JavaNet::UnknownHostException: es01: Temporary failure in name resolution>}

plus one more thing... how to handle dynamic file name..which will get changed everyday as per date. will my logstash will show last X date datas??

Thanks in advance.

15

This means that logstash is not able to connect to your elasticsearch container, is it running?

You need to confirm that your elasticsearch container is running and using the same docker network as Logstash.

In the way you are running no, you need to change it, you would need to mount the path with your logs, not the file.

Something like this in your volumes configuration:

- ./logs/:/log/path/inside/container

Then your file output would need to look at this log path

path => "/log/path/inside/container/*.log

Keep in mind that those configurations and issues you are having are unrelated to logstash, they are docker issues.

16

hello sir,

actually i search little bit and came to know that to see logs ....written in nodejs application with winston package .., not necessary i should use logstash. so by just adding Elasticsearch URL, username and password and it should be able to see my logs in Observability ->Logs- > Stream.
and my elastic & kibana is already working.

logger file in js
`
const winston = require('winston');
const { ElasticsearchTransport } = require('winston-elasticsearch');

const esTransportOpts = {
level: 'info',
clientOpts: {
node: 'https://localhost:9200', // Elasticsearch URL
auth: {
username: 'username', // Elasticsearch username
password: 'password' // Elasticsearch password
},
log: 'info' // Enable detailed logging
}
};

const esTransport = new ElasticsearchTransport(esTransportOpts);

const logger = winston.createLogger({
level: 'info',
format: winston.format.json(),
transports: [
new winston.transports.Console(),
esTransport
]
});

module.exports = logger;`

and i have added logs in controller like this->
logger.info("Test 1 Log ");
logger.info("Test 2 Log ");

but my logs are not coming here ->Observability ->Logs- > Stream.
In discover also my logs are not coming.
what else i am missing ?

thanks in advance.

17

anyone available ?

18

I am not sure about the exact issue you are facing, but there is a bug I have logged via the support case as our code pipeline fails for recent containers and work for 8.12.x [1] 8.14.0 xpack.management.pipeline.id * no longer working · Issue #16200 · elastic/logstash · GitHub