Hello everybody,
First of all, happy new year to everybody 
I need your help with the ELK stack.
I have installed Elastic , Kibana and Fluentd with HelmChart.
I have healthy cluster of Elasticsearch ,I can confirm it via CURL and version is 8.5.1.
{
"name" : "elasticsearch-master-0",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "uXctgao6QEqbfKhzw5TLuA",
"version" : {
"number" : "8.5.1",
"build_flavor" : "default",
"build_type" : "docker",
"build_hash" : "c1310c45fc534583afe2c1c03046491efba2bba2",
"build_date" : "2022-11-09T21:02:20.169855900Z",
"build_snapshot" : false,
"lucene_version" : "9.4.1",
"minimum_wire_compatibility_version" : "7.17.0",
"minimum_index_compatibility_version" : "7.0.0"
},
"tagline" : "You Know, for Search"
}
{
"cluster_name" : "elasticsearch",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 3,
"number_of_data_nodes" : 3,
"active_primary_shards" : 11,
"active_shards" : 22,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
I have modified the following details on the config map of fluentd-forwarder-cm and restarted the daemon.
apiVersion: v1
kind: ConfigMap
metadata:
name: fluentd-forwarder-cm
namespace: monitoring
uid: 9ee610ce-eb53-4faa-b47a-fe53da264892
resourceVersion: '30569'
creationTimestamp: '2023-01-01T15:06:22Z'
labels:
app.kubernetes.io/component: forwarder
app.kubernetes.io/instance: fluentd
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: fluentd
helm.sh/chart: fluentd-5.5.11
annotations:
meta.helm.sh/release-name: fluentd
meta.helm.sh/release-namespace: monitoring
managedFields:
- manager: helm
operation: Update
apiVersion: v1
time: '2023-01-01T15:06:22Z'
fieldsType: FieldsV1
fieldsV1:
f:data:
.: {}
f:fluentd.conf: {}
f:metrics.conf: {}
f:metadata:
f:annotations:
.: {}
f:meta.helm.sh/release-name: {}
f:meta.helm.sh/release-namespace: {}
f:labels:
.: {}
f:app.kubernetes.io/component: {}
f:app.kubernetes.io/instance: {}
f:app.kubernetes.io/managed-by: {}
f:app.kubernetes.io/name: {}
f:helm.sh/chart: {}
- manager: node-fetch
operation: Update
apiVersion: v1
time: '2023-01-01T15:39:44Z'
fieldsType: FieldsV1
fieldsV1:
f:data:
f:fluentd-inputs.conf: {}
f:fluentd-output.conf: {}
selfLink: /api/v1/namespaces/monitoring/configmaps/fluentd-forwarder-cm
data:
fluentd-inputs.conf: |
# HTTP input for the liveness and readiness probes
<source>
@type http
port 9880
</source>
# Get the logs from the containers running in the node
<source>
@type tail
path /var/log/containers/*-app*.log
pos_file /opt/bitnami/fluentd/logs/buffers/fluentd-docker.pos
tag kubernetes.*
read_from_head true
format json
</source>
# enrich with kubernetes metadata
<filter kubernetes.**>
@type kubernetes_metadata
</filter>
fluentd-output.conf: |
# Throw the healthcheck to the standard output instead of forwarding it
<match fluentd.healthcheck>
@type null
</match>
# Forward all logs to the aggregators
<match kubernetes.var.log.containers.**java-app**.log>
@type elasticsearch
include_tag_key true
host "https://elasticsearch-master.monitoring.svc.cluster.local:443"
port "9200"
index_name "java-app-logs"
scheme https
ssl_verify false
<buffer>
@type file
path /opt/bitnami/fluentd/logs/buffers/java-logs.buffer
flush_thread_count 2
flush_interval 5s
</buffer>
</match>
# <match **>
# @type forward
# <server>
# host fluentd-0.fluentd-headless.monitoring.svc.cluster.local
# port 24224
# </server>
# <buffer>
# @type file
# path /opt/bitnami/fluentd/logs/buffers/logs.buffer
# flush_thread_count 2
# flush_interval 5s
# </buffer>
# </match>
fluentd.conf: |
# Ignore fluentd own events
<match fluent.**>
@type null
</match>
@include fluentd-inputs.conf
@include fluentd-output.conf
metrics.conf: |
# Prometheus Exporter Plugin
# input plugin that exports metrics
<source>
@type prometheus
port 24231
</source>
# input plugin that collects metrics from MonitorAgent
<source>
@type prometheus_monitor
<labels>
host ${hostname}
</labels>
</source>
# input plugin that collects metrics for output plugin
<source>
@type prometheus_output_monitor
<labels>
host ${hostname}
</labels>
</source>
# input plugin that collects metrics for in_tail plugin
<source>
@type prometheus_tail_monitor
<labels>
host ${hostname}
</labels>
</source>
binaryData: {}
After restart ,I see that my fluentd pods are failing with following logs.
2023-01-01 16:59:38 +0000 [warn]: #0 Could not communicate to Elasticsearch, resetting connection and trying again. no address for https (Resolv::ResolvError)
2023-01-01 16:59:38 +0000 [warn]: #0 Remaining retry: 10. Retry to communicate after 32 second(s).
The client is unable to verify that the server is Elasticsearch. Some functionality may not be compatible if the server is running an unsupported product.
2023-01-01 17:00:19 +0000 [info]: Received graceful stop
2023-01-01 17:00:42 +0000 [warn]: #0 Could not communicate to Elasticsearch, resetting connection and trying again. no address for https (Resolv::ResolvError)
2023-01-01 17:00:42 +0000 [warn]: #0 Remaining retry: 9. Retry to communicate after 64 second(s).
The client is unable to verify that the server is Elasticsearch. Some functionality may not be compatible if the server is running an unsupported product.
I changed this line host "https://elasticsearch-master.monitoring.svc.cluster.local:443" to host "elasticsearch-master.monitoring.svc.cluster.local" and now I see the following logs
2023-01-01 17:05:52 +0000 [warn]: #0 Could not communicate to Elasticsearch, resetting connection and trying again. [401] {"error":{"root_cause":[{"type":"security_exception","reason":"missing authentication credentials for REST request [/]","header":{"WWW-Authenticate":["Basic realm=\"security\" charset=\"UTF-8\"","Bearer realm=\"security\"","ApiKey"]}}],"type":"security_exception","reason":"missing authentication credentials for REST request [/]","header":{"WWW-Authenticate":["Basic realm=\"security\" charset=\"UTF-8\"","Bearer realm=\"security\"","ApiKey"]}},"status":401}
2023-01-01 17:05:52 +0000 [warn]: #0 Remaining retry: 12. Retry to communicate after 8 second(s).
The client is unable to verify that the server is Elasticsearch due to security privileges on the server side. Some functionality may not be compatible if the server is running an unsupported product.
Can you please help me?