After upgrade ELK Stack (ES, Kibana, Metricbeat, Logstash) from 7.17 to 8.9.2 all node in cluster status offline without elasticsearch in same kibana node
In index management
Before upgrade to 8.9.2
Data insert to .monitoring-es-7-mb-2023.09.12
After upgrade to 8.9.2
Data insert to data stream ".monitoring-es-8-mb" and ".ds-.monitoring-es-8-mb-2023.09.11-000004" index in data stream
It looks like your metrics aren't being collected for those other nodes, the rest of the table displays no metrics so it's not only the availability status.
Can you share your Metricbeat config?
Do you have any errors in your Metricbeat logs about failing to collect certain metric sets?
Since you're on a new version of Kibana, you'll have access to the Stack Monitoring UI health API https://github.com/elastic/kibana/blob/main/x-pack/plugins/monitoring/server/routes/api/v1/_health/README.md
It's a Kibana API, if you can share the result of that we might see why the data is missing.
Please remember to clear any sensitive information (such as IPs etc.) before sharing.
Can you share your Metricbeat config?
setup.kibana:
****no config
output.elasticsearch:
hosts: ["https://esnwarm03:9200"]
username: "xxxxxxxxxx"
password: "xxxxxxxx"
ssl.certificate_authorities: ["/etc/elasticsearch/ca.crt"]
Do you have any errors in your Metricbeat logs about failing to collect certain metric sets?
{"log.level":"debug","@timestamp":"2023-09-12T15:08:08.321+0700","log.logger":"processes","log.origin":{"file.name":"process/process.go","file.line":173},"message":"Error fetching PID info for 3912004, skipping: FillPidMetrics: error getting metadata for pid 3912004: error fetching exe from pid 3912004: readlink /proc/3912004/exe: no such file or directory","service.name":"metricbeat","ecs.version":"1.6.0"}
Stack Monitoring UI health API Result:
"monitoredClusters":{
"clusters":{
"NhoD8ETfTgeRz0bS-ri0Sg":{
"logstash":{
"20593bf1-f76e-4e0a-be59-e219c63846bb":{
"node_stats":{
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T08:02:42.107Z"
}
}
},
"f88280ee-6660-4945-a201-30da0b255712":{
"node_stats":{
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T08:02:42.077Z"
}
}
},
"beats":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.121Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"beats_palo":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.122Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"bluecoat":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.123Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"forescout":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.121Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"fortinet":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.121Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"heartbeat":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.122Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"metricbeat":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.123Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"oracle_xml":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.124Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"oslinux":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.123Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"proofpoint":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.124Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"sonicwall":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.123Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"ias":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.124Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"apmf5":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.122Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"appoe":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.123Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"appop":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.124Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"appsap":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.123Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"appsapuser":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.123Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"aruba":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.123Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"cyberark":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.122Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"nutanix":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.121Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"useradmail":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:07.124Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"apigw":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:00.056Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"deep":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:00.055Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"sepm":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:00.055Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"appinfra":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:00.055Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"f5":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.12",
"lastSeen":"2023-09-12T00:00:00.055Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-logstash-8-mb-2023.09.12-000001",
"lastSeen":"2023-09-12T04:03:32.045Z"
}
}
},
"pulsesecure":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.06",
"lastSeen":"2023-09-06T13:03:27.245Z"
}
}
},
"app_netcore":{
"node":{
"metricbeat-7":{
"index":".monitoring-logstash-7-mb-2023.09.06",
"lastSeen":"2023-09-06T10:51:46.194Z"
}
}
}
},
"cluster":{
"NhoD8ETfTgeRz0bS-ri0Sg":{
"index_summary":{
"metricbeat-8":{
"index":".ds-.monitoring-es-8-mb-2023.09.11-000004",
"lastSeen":"2023-09-12T08:02:42.462Z"
}
},
"index_recovery":{
"metricbeat-8":{
"index":".ds-.monitoring-es-8-mb-2023.09.11-000004",
"lastSeen":"2023-09-12T08:02:41.303Z"
}
},
"index":{
"metricbeat-8":{
"index":".ds-.monitoring-es-8-mb-2023.09.11-000004",
"lastSeen":"2023-09-12T08:02:41.273Z"
}
},
"cluster_stats":{
"metricbeat-8":{
"index":".ds-.monitoring-es-8-mb-2023.09.11-000004",
"lastSeen":"2023-09-12T08:02:42.739Z"
}
}
}
},
"elasticsearch":{
"1LAH6Zl5Sye_V-ltx-iwEA":{
"shard":{
"metricbeat-7":{
"index":".monitoring-es-7-mb-2023.09.10",
"lastSeen":"2023-09-10T16:13:47.269Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-es-8-mb-2023.09.09-000002",
"lastSeen":"2023-09-10T10:47:41.859Z"
}
}
},
"Ep7F4HMuRxmDPj7aVVXa3Q":{
"shard":{
"metricbeat-7":{
"index":".monitoring-es-7-mb-2023.09.10",
"lastSeen":"2023-09-10T16:13:47.269Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-es-8-mb-2023.09.09-000002",
"lastSeen":"2023-09-10T10:47:41.859Z"
}
}
},
"Hw9v2asTQRa3NApF6gN8BQ":{
"shard":{
"metricbeat-7":{
"index":".monitoring-es-7-mb-2023.09.10",
"lastSeen":"2023-09-10T16:13:47.269Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-es-8-mb-2023.09.09-000002",
"lastSeen":"2023-09-10T10:47:41.859Z"
}
}
},
"gHgnVJGJSKKY_YlcN53n2g":{
"shard":{
"metricbeat-7":{
"index":".monitoring-es-7-mb-2023.09.10",
"lastSeen":"2023-09-10T16:13:47.269Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-es-8-mb-2023.09.09-000002",
"lastSeen":"2023-09-10T10:47:41.859Z"
}
}
},
"zLxCEmAVRVKDt6bzUAOBhg":{
"shard":{
"metricbeat-7":{
"index":".monitoring-es-7-mb-2023.09.10",
"lastSeen":"2023-09-10T16:13:47.269Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-es-8-mb-2023.09.09-000002",
"lastSeen":"2023-09-10T10:47:41.859Z"
}
}
},
"vIkhlLagSP6f8-brPx1bow":{
"shard":{
"metricbeat-7":{
"index":".monitoring-es-7-mb-2023.09.10",
"lastSeen":"2023-09-10T16:13:47.267Z"
},
"metricbeat-8":{
"index":".ds-.monitoring-es-8-mb-2023.09.09-000002",
"lastSeen":"2023-09-10T10:47:41.859Z"
}
}
},
"RvhjnE3vTqS3xzIffJUzMg":{
"node_stats":{
"metricbeat-8":{
"index":".ds-.monitoring-es-8-mb-2023.09.11-000004",
"lastSeen":"2023-09-12T08:02:42.034Z"
}
}
}
},
"kibana":{
"c3b0a3a8-3ed1-4c26-a7e7-8b307bdb174b":{
"stats":{
"metricbeat-8":{
"index":".ds-.monitoring-kibana-8-mb-2023.09.11-000001",
"lastSeen":"2023-09-12T08:02:42.127Z"
}
}
}
}
}
},
"execution":{
"timedOut":false,
"errors":[
]
}
},
"metricbeatErrors":{
"products":{
"elasticsearch":{
"ccr":[
{
"message":"error determining if connected Elasticsearch node is master: error making http request: Get \"https://eswarm02.cpf.co.th:9200/_nodes/_local/nodes\": dial tcp <eswarm02_ip>:9200: i/o timeout",
"lastSeen":"2023-09-12T08:02:33.255Z"
},
{
"message":"error determining if connected Elasticsearch node is master: HTTP error 401 in : 401 Unauthorized",
"lastSeen":"2023-09-12T08:02:33.243Z"
}
],
"cluster_stats":[
{
"message":"error determining if connected Elasticsearch node is master: HTTP error 401 in : 401 Unauthorized",
"lastSeen":"2023-09-12T08:02:33.253Z"
},
{
"message":"error determining if connected Elasticsearch node is master: error making http request: Get \"https://eswarm02.cpf.co.th:9200/_nodes/_local/nodes\": dial tcp <eswarm02_ip>:9200: i/o timeout",
"lastSeen":"2023-09-12T08:02:33.241Z"
}
],
"enrich":[
{
"message":"error determining if connected Elasticsearch node is master: error making http request: Get \"https://eswarm02.cpf.co.th:9200/_nodes/_local/nodes\": dial tcp <eswarm02_ip>:9200: i/o timeout",
"lastSeen":"2023-09-12T08:02:33.255Z"
},
{
"message":"error determining if connected Elasticsearch node is master: HTTP error 401 in : 401 Unauthorized",
"lastSeen":"2023-09-12T08:02:33.249Z"
}
],
"index":[
{
"message":"error determining if connected Elasticsearch node is master: HTTP error 401 in : 401 Unauthorized",
"lastSeen":"2023-09-12T08:02:33.253Z"
},
{
"message":"error determining if connected Elasticsearch node is master: error making http request: Get \"https://eswarm02.cpf.co.th:9200/_nodes/_local/nodes\": dial tcp <eswarm02_ip>:9200: i/o timeout",
"lastSeen":"2023-09-12T08:02:33.244Z"
}
],
"index_recovery":[
{
"message":"error determining if connected Elasticsearch node is master: error making http request: Get \"https://eswarm02.cpf.co.th:9200/_nodes/_local/nodes\": dial tcp <eswarm02_ip>:9200: i/o timeout",
"lastSeen":"2023-09-12T08:02:33.250Z"
},
{
"message":"error determining if connected Elasticsearch node is master: HTTP error 401 in : 401 Unauthorized",
"lastSeen":"2023-09-12T08:02:33.243Z"
}
],
"index_summary":[
{
"message":"error determining if connected Elasticsearch node is master: error making http request: Get \"https://eswarm02.cpf.co.th:9200/_nodes/_local/nodes\": dial tcp <eswarm02_ip>:9200: i/o timeout",
"lastSeen":"2023-09-12T08:02:33.259Z"
},
{
"message":"error determining if connected Elasticsearch node is master: HTTP error 401 in : 401 Unauthorized",
"lastSeen":"2023-09-12T08:02:33.253Z"
}
],
"ml_job":[
{
"message":"error determining if connected Elasticsearch node is master: error making http request: Get \"https://eswarm02.cpf.co.th:9200/_nodes/_local/nodes\": dial tcp <eswarm02_ip>:9200: i/o timeout",
"lastSeen":"2023-09-12T08:02:33.244Z"
},
{
"message":"error determining if connected Elasticsearch node is master: HTTP error 401 in : 401 Unauthorized",
"lastSeen":"2023-09-12T08:02:33.243Z"
}
],
"node_stats":[
{
"message":"HTTP error 401 in : 401 Unauthorized",
"lastSeen":"2023-09-12T08:02:33.250Z"
},
{
"message":"error making http request: Get \"https://eswarm02.cpf.co.th:9200/_nodes/_local/stats\": dial tcp <eswarm02_ip>:9200: i/o timeout",
"lastSeen":"2023-09-12T08:02:33.244Z"
}
],
"shard":[
{
"message":"error determining if connected Elasticsearch node is master: HTTP error 401 in : 401 Unauthorized",
"lastSeen":"2023-09-12T08:02:33.261Z"
},
{
"message":"error determining if connected Elasticsearch node is master: error making http request: Get \"https://eswarm02.cpf.co.th:9200/_nodes/_local/nodes\": dial tcp <eswarm02_ip>:9200: i/o timeout",
"lastSeen":"2023-09-12T08:02:33.250Z"
}
]
},
"kibana":{
"cluster_actions":[
{
"message":"error making http request: Get \"http://localhost:5601/api/status\": EOF",
"lastSeen":"2023-09-12T08:02:42.007Z"
}
],
"cluster_rules":[
{
"message":"error making http request: Get \"http://localhost:5601/api/status\": EOF",
"lastSeen":"2023-09-12T08:02:42.008Z"
}
],
"node_actions":[
{
"message":"error making http request: Get \"http://localhost:5601/api/status\": EOF",
"lastSeen":"2023-09-12T08:02:42.008Z"
}
],
"node_rules":[
{
"message":"error making http request: Get \"http://localhost:5601/api/status\": EOF",
"lastSeen":"2023-09-12T08:02:42.023Z"
}
],
"stats":[
{
"message":"error making http request: Get \"http://localhost:5601/api/status\": EOF",
"lastSeen":"2023-09-12T08:02:42.007Z"
}
]
}
},
"execution":{
"timedOut":false,
"errors":[
]
}
},
"packageErrors":{
"products":{
},
"execution":{
"timedOut":false,
"errors":[
]
}
},
"settings":{
"ccs":false,
"logsIndex":"filebeat-*",
"metricbeatIndex":"metricbeat-*",
"hasRemoteClusterConfigured":false
}
}"```
P.S. eswarm02 is terminated and use esnwarm02 insteadIf you look at the API response, under metricbeatErrors -> products -> elasticsearch you'll see there are errors for all Metricsets about TCP timeouts and authorization issues.
Can you also share what your module config looks like for the Elasticsearch module (since you're using Stack Monitoring UI, it will be the x-pack version)?
Most likely something is wrong there with regards to the credentials used.
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.
© 2020. All Rights Reserved - Elasticsearch
Apache, Apache Lucene, Apache Hadoop, Hadoop, HDFS and the yellow elephant logo are trademarks of the Apache Software Foundation in the United States and/or other countries.