Log stash not able comm with Elastic search pls find the below logs for Elasticsearch and Logstash
Elastic search:
dress=/10.244.0.53:9200, remoteAddress=/10.224.0.5:48848}", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[quickstart-es-default-0][transport_worker][T#1]","log.logger":"org.elasticsearch.xpack.security.transport.netty4.SecurityNetty4HttpServerTransport","elasticsearch.cluster.uuid":"cagIA0bBQS6HgnlAuBIX3g","elasticsearch.node.id":"egnJAMwKR_Od0Xebr6S7oA","elasticsearch.node.name":"quickstart-es-default-0","elasticsearch.cluster.name":"quickstart"}
{"@timestamp":"2023-02-28T02:37:25.240Z", "log.level": "WARN", "message":"http client did not trust this server's certificate, closing connection Netty4HttpChannel{localAddress=/10.244.0.53:9200, remoteAddress=/10.224.0.5:38053}", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[quickstart-es-default-0][transport_worker][T#2]","log.logger":"org.elasticsearch.xpack.security.transport.netty4.SecurityNetty4HttpServerTransport","elasticsearch.cluster.uuid":"cagIA0bBQS6HgnlAuBIX3g","elasticsearch.node.id":"egnJAMwKR_Od0Xebr6S7oA","elasticsearch.node.name":"quickstart-es-default-0","elasticsearch.cluster.name":"quickstart"}
{"@timestamp":"2023-02-28T02:37:55.256Z", "log.level": "WARN", "message":"http client did not trust this server's certificate, closing connection Netty4HttpChannel{localAddress=/10.244.0.53:9200, remoteAddress=/10.224.0.5:61467}", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[quickstart-es-default-0][transport_worker][T#1]","log.logger":"org.elasticsearch.xpack.security.transport.netty4.SecurityNetty4HttpServerTransport","elasticsearch.cluster.uuid":"cagIA0bBQS6HgnlAuBIX3g","elasticsearch.node.id":"egnJAMwKR_Od0Xebr6S7oA","elasticsearch.node.name":"quickstart-es-default-0","elasticsearch.cluster.name":"quickstart"}
{"@timestamp":"2023-02-28T02:38:25.272Z", "log.level": "WARN", "message":"http client did not trust this server's certificate, closing connection Netty4HttpChannel{localAddress=/10.244.0.53:9200, remoteAddress=/10.224.0.5:48463}", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[quickstart-es-default-0][transport_worker][T#2]","log.logger":"org.elasticsearch.xpack.security.transport.netty4.SecurityNetty4HttpServerTransport","elasticsearch.cluster.uuid":"cagIA0bBQS6HgnlAuBIX3g","elasticsearch.node.id":"egnJAMwKR_Od0Xebr6S7oA","elasticsearch.node.name":"quickstart-es-default-0","elasticsearch.cluster.name":"quickstart"}
{"@timestamp":"2023-02-28T02:38:55.287Z", "log.level": "WARN", "message":"http client did not trust this server's certificate, closing connection Netty4HttpChannel{localAddress=/10.244.0.53:9200, remoteAddress=/10.224.0.5:41699}", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[quickstart-es-default-0][transport_worker][T#2]","log.logger":"org.elasticsearch.xpack.security.transport.netty4.SecurityNetty4HttpServerTransport","elasticsearch.cluster.uuid":"cagIA0bBQS6HgnlAuBIX3g","elasticsearch.node.id":"egnJAMwKR_Od0Xebr6S7oA","elasticsearch.node.name":"quickstart-es-default-0","elasticsearch.cluster.name":"quickstart"}
{"@timestamp":"2023-02-28T02:39:25.310Z", "log.level": "WARN", "message":"http client did not trust this server's certificate, closing connection Netty4HttpChannel{localAddress=/10.244.0.53:9200, remoteAddress=/10.224.0.5:33520}", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[quickstart-es-default-0][transport_worker][T#1]","log.logger":"org.elasticsearch.xpack.security.transport.netty4.SecurityNetty4HttpServerTransport","elasticsearch.cluster.uuid":"cagIA0bBQS6HgnlAuBIX3g","elasticsearch.node.id":"egnJAMwKR_Od0Xebr6S7oA","elasticsearch.node.name":"quickstart-es-default-0","elasticsearch.cluster.name":"quickstart"}
Logstash:
2023/02/27 12:45:06 Setting 'xpack.monitoring.elasticsearch.hosts' from environment.
2023/02/27 12:45:06 Setting 'xpack.monitoring.enabled' from environment.
Using bundled JDK: /usr/share/logstash/jdk
Sending Logstash logs to /usr/share/logstash/logs which is now configured via log4j2.properties
[2023-02-27T12:45:19,972][INFO ][logstash.runner ] Log4j configuration path used is: /usr/share/logstash/config/log4j2.properties
[2023-02-27T12:45:19,976][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"8.6.2", "jruby.version"=>"jruby 9.3.10.0 (2.6.8) 2023-02-01 107b2e6697 OpenJDK 64-Bit Server VM 17.0.6+10 on 17.0.6+10 +indy +jit [x86_64-linux]"}
[2023-02-27T12:45:19,979][INFO ][logstash.runner ] JVM bootstrap flags: [-Xms1g, -Xmx1g, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djruby.compile.invokedynamic=true, -XX:+HeapDumpOnOutOfMemoryError, -Djava.security.egd=file:/dev/urandom, -Dlog4j2.isThreadContextMapInheritable=true, -Dls.cgroup.cpuacct.path.override=/, -Dls.cgroup.cpu.path.override=/, -Djruby.regexp.interruptible=true, -Djdk.io.File.enableADS=true, --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED, --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED, --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED, --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED, --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED, --add-opens=java.base/java.security=ALL-UNNAMED, --add-opens=java.base/java.io=ALL-UNNAMED, --add-opens=java.base/java.nio.channels=ALL-UNNAMED, --add-opens=java.base/sun.nio.ch=ALL-UNNAMED, --add-opens=java.management/sun.management=ALL-UNNAMED]
[2023-02-27T12:45:19,988][INFO ][logstash.settings ] Creating directory {:setting=>"path.queue", :path=>"/usr/share/logstash/data/queue"}
[2023-02-27T12:45:19,989][INFO ][logstash.settings ] Creating directory {:setting=>"path.dead_letter_queue", :path=>"/usr/share/logstash/data/dead_letter_queue"}
[2023-02-27T12:45:20,207][INFO ][logstash.agent ] No persistent UUID file found. Generating new UUID {:uuid=>"01febd9f-20b9-4964-9b34-de031fd07b36", :path=>"/usr/share/logstash/data/uuid"}
[2023-02-27T12:45:20,608][WARN ][deprecation.logstash.monitoringextension.pipelineregisterhook] Internal collectors option for Logstash monitoring is deprecated and targeted for removal in the next major version.
Please configure Metricbeat to monitor Logstash. Documentation can be found at:
[2023-02-27T12:45:20,901][INFO ][logstash.licensechecker.licensereader] Elasticsearch pool URLs updated {:changes=>{:removed=>, :added=>[https://20.120.49.206:9200/]}}
[2023-02-27T12:45:21,160][INFO ][logstash.licensechecker.licensereader] Failed to perform request {:message=>"PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target", :exception=>Manticore::ClientProtocolException, :cause=>#<Java::JavaxNetSsl::SSLHandshakeException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target>}
[2023-02-27T12:45:21,161][WARN ][logstash.licensechecker.licensereader] Attempted to resurrect connection to dead ES instance, but got an error {:url=>"https://20.120.49.206:9200/", :exception=>LogStash::Outputs::Elasticsearch::HttpClient::Pool::HostUnreachableError, :message=>"Elasticsearch Unreachable: [https://20.120.49.206:9200/][Manticore::ClientProtocolException] PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target"}
[2023-02-27T12:45:21,202][INFO ][logstash.licensechecker.licensereader] Failed to perform request {:message=>"PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target", :exception=>Manticore::ClientProtocolException, :cause=>#<Java::JavaxNetSsl::SSLHandshakeException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target>}
[2023-02-27T12:45:21,203][WARN ][logstash.licensechecker.licensereader] Marking url as dead. Last error: [LogStash::Outputs::Elasticsearch::HttpClient::Pool::HostUnreachableError] Elasticsearch Unreachable: [https://20.120.49.206:9200/_xpack][Manticore::ClientProtocolException] PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target {:url=>https://20.120.49.206:9200/, :error_message=>"Elasticsearch Unreachable: [https://20.120.49.206:9200/_xpack][Manticore::ClientProtocolException] PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target", :error_class=>"LogStash::Outputs::Elasticsearch::HttpClient::Pool::HostUnreachableError"}
[2023-02-27T12:45:21,205][WARN ][logstash.licensechecker.licensereader] Attempt to validate Elasticsearch license failed. Sleeping for 0.02 {:fail_count=>1, :exception=>"Elasticsearch Unreachable: [https://20.120.49.206:9200/_xpack][Manticore::ClientProtocolException] PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target"}
[2023-02-27T12:45:21,227][ERROR][logstash.licensechecker.licensereader] Unable to retrieve license information from license server {:message=>"No Available connections"}
[2023-02-27T12:45:21,235][ERROR][logstash.monitoring.internalpipelinesource] Failed to fetch X-Pack information from Elasticsearch. This is likely due to failure to reach a live Elasticsearch cluster.
[2023-02-27T12:45:21,369][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600, :ssl_enabled=>false}
[2023-02-27T12:45:21,501][INFO ][org.reflections.Reflections] Reflections took 85 ms to scan 1 urls, producing 127 keys and 444 values
[2023-02-27T12:45:21,752][INFO ][logstash.javapipeline ] Pipeline main is configured with pipeline.ecs_compatibility: v8 setting. All plugins in this pipeline will default to ecs_compatibility => v8 unless explicitly configured otherwise.
[2023-02-27T12:45:21,804][INFO ][logstash.javapipeline ][main] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>250, "pipeline.sources"=>["/usr/share/logstash/pipeline/logstash.conf"], :thread=>"#<Thread:0x343a5633@/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:131 run>"}
[2023-02-27T12:45:22,231][INFO ][logstash.javapipeline ][main] Pipeline Java execution initialization time {"seconds"=>0.43}
[2023-02-27T12:45:22,237][INFO ][logstash.inputs.beats ][main] Starting input listener {:address=>"0.0.0.0:5044"}
[2023-02-27T12:45:22,243][INFO ][logstash.javapipeline ][main] Pipeline started {"pipeline.id"=>"main"}
[2023-02-27T12:45:22,253][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>}
[2023-02-27T12:45:22,306][INFO ][org.logstash.beats.Server][main][0710cad67e8f47667bc7612580d5b91f691dd8262a4187d9eca8cf87229d04aa] Starting server on port: 5044
[2023-02-27T12:45:51,234][ERROR][logstash.licensechecker.licensereader] Unable to retrieve license information from license server {:message=>"No Available connections"}
[2023-02-27T12:45:51,238][INFO ][logstash.licensechecker.licensereader] Failed to perform request {:message=>"PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target", :exception=>Manticore::ClientProtocolException, :cause=>#<Java::JavaxNetSsl::SSLHandshakeException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target>}
[2023-02-27T12:45:51,241][WARN ][logstash.licensechecker.licensereader] Attempted to resurrect connection to dead ES instance, but got an error {:url=>"https://20.120.49.206:9200/", :exception=>LogStash::Outputs::Elasticsearch::HttpClient::Pool::HostUnreachableError, :message=>"Elasticsearch Unreachable: [https://20.120.49.206:9200/][Manticore::ClientProtocolException] PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target"}
[2023-02-27T12:46:21,234][ERROR][logstash.licensechecker.licensereader] Unable to retrieve license information from license server {:message=>"No Available connections"}
[2023-02-27T12:46:21,291][INFO ][logstash.licensechecker.licensereader] Failed to perform request {:message=>"PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target", :exception
Elasticsearch yaml:
cat <<EOF | kubectl apply -f -
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
name: quickstart
spec:
version: 8.6.2
nodeSets:
- name: default
count: 1
config:
node.store.allow_mmap: false
EOF
Logstash yaml:
apiVersion: apps/v1
kind: Deployment
metadata:
name: logstash
spec:
replicas: 1
selector:
matchLabels:
app: logstash
template:
metadata:
labels:
app: logstash
spec:
volumes:
- name: azure
persistentVolumeClaim:
claimName: mountfileshare
- name: config-volume
configMap:
name: logstash-config
containers:
- name: logstash
image: docker.elastic.co/logstash/logstash:8.6.2
ports:
- containerPort: 5044
name: tcp
- containerPort: 9600
name: http
volumeMounts:
- name: config-volume
mountPath: /usr/share/logstash/pipeline/logstash.conf
subPath: logstash.conf
- name: azure
mountPath: /mnt/azure/
Logstash-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash-config
data:
logstash.conf: |
input {
azureblob {
storage_account_name => "elkstoragefreetrail"
storage_access_key => "3Rg57GPSFEo7JRyo+wHM8zwzY7i0cKjUC7aYMYIEMH2zilxyrQsKgqMmTmybpb15bsASDIB6whDA+AStndxo1g=="
container => "elktest"
path => "bwappnode4.log"
}
}
filter {
json {
source => "message"
}
}
output {
elasticsearch {
hosts => "${ELASTICSEARCH_HOSTS}"
ssl => true
cacert => "/mnt/azure/certs/ca.crt"
cert => "/mnt/azure/certs/tls.crt"
key => "/mnt/azure/certs/tls.key"
verify_mode => "peer"
username => "elastic"
password => "65jeyzS7N9T9OvZu4I2wS548"
index => "my_index"
document_id => "%{[@metadata][id]}"
manage_template => false
}
}
kindly check the above logs,yaml files and let me to fix the issue