Hi ,
version: 6.3.0
We have installed trial version of ELK and it was working fine but recently we observed below error in ELK nodes and no data is showing in kibana dashboard.
In elk-master-node we are getting below errors along with the licesne expiry message since last couple of days
[2018-08-31T18:51:59,859][WARN ][o.e.l.LicenseService ] [elk-master-node] License [will expire] on [Friday, August 31, 2018].
If you have a new license, please update it. Otherwise, please reach out to
your support contact.
Commercial plugins operate with reduced functionality on license expiration:
security
Cluster health, cluster stats and indices stats operations are blocked
All data operations (read and write) continue to work
watcher
PUT / GET watch APIs are disabled, DELETE watch API continues to work
Watches execute and write to the history
The actions of the watches don't execute
monitoring
The agent will stop collecting cluster and indices metrics
The agent will stop automatically cleaning indices older than [xpack.monitoring.history.duration]
graph
Graph explore APIs are disabled
ml
Machine learning APIs are disabled
logstash
Logstash will continue to poll centrally-managed pipelines
deprecation
Deprecation APIs are disabled
upgrade
Upgrade API is disabled
sql
SQL support is disabled
rollup
Creating and Starting rollup jobs will no longer be allowed.
Stopping/Deleting existing jobs, RollupCaps API and RollupSearch continue to function.
[2018-08-31T18:52:26,761][ERROR][o.e.x.m.c.c.ClusterStatsCollector] [elk-master-node] collector [cluster_stats] timed out when collecting data
[2018-08-31T18:53:23,022][ERROR][o.e.x.m.c.i.IndexStatsCollector] [elk-master-node] collector [index-stats] timed out when collecting data
In data nodes getting the below message:
[2018-08-31T19:00:37,918][WARN ][o.e.m.j.JvmGcMonitorService] [elk-data-node01] [gc][8066] overhead, spent [1.8s] collecting in the last [1.9s]
[2018-08-31T19:00:39,410][ERROR][o.e.x.m.c.n.NodeStatsCollector] [elk-data-node01] collector [node_stats] timed out when collecting data
[2018-08-31T19:00:39,512][WARN ][o.e.m.j.JvmGcMonitorService] [elk-data-node01] [gc][8067] overhead, spent [1.4s] collecting in the last [1.5s]
Cluster Status from master node:
{
"_nodes" : {
"total" : 3,
"successful" : 3,
"failed" : 0
},
"cluster_name" : "elk-logging-prod",
"timestamp" : 1535741657925,
"status" : "green",
"indices" : {
"count" : 172,
"shards" : {
"total" : 1432,
"primaries" : 716,
"replication" : 1.0,
"index" : {
"shards" : {
"min" : 2,
"max" : 10,
"avg" : 8.325581395348838
},
"primaries" : {
"min" : 1,
"max" : 5,
"avg" : 4.162790697674419
},
"replication" : {
"min" : 1.0,
"max" : 1.0,
"avg" : 1.0
}
}
},
"docs" : {
"count" : 146584946,
"deleted" : 96934
},
"store" : {
"size" : "105.6gb",
"size_in_bytes" : 113435405019
},
"fielddata" : {
"memory_size" : "30.4kb",
"memory_size_in_bytes" : 31216,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "163.2kb",
"memory_size_in_bytes" : 167168,
"total_count" : 66284,
"hit_count" : 5951,
"miss_count" : 60333,
"cache_size" : 59,
"cache_count" : 381,
"evictions" : 322
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 12843,
"memory" : "280.9mb",
"memory_in_bytes" : 294616635,
"terms_memory" : "217.8mb",
"terms_memory_in_bytes" : 228436330,
"stored_fields_memory" : "39.3mb",
"stored_fields_memory_in_bytes" : 41305856,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "8.3mb",
"norms_memory_in_bytes" : 8741824,
"points_memory" : "6.7mb",
"points_memory_in_bytes" : 7077349,
"doc_values_memory" : "8.6mb",
"doc_values_memory_in_bytes" : 9055276,
"index_writer_memory" : "24.3mb",
"index_writer_memory_in_bytes" : 25550466,
"version_map_memory" : "696.2kb",
"version_map_memory_in_bytes" : 712946,
"fixed_bit_set" : "1.5mb",
"fixed_bit_set_memory_in_bytes" : 1610120,
"max_unsafe_auto_id_timestamp" : 1535729108045,
"file_sizes" : { }
}
},
"nodes" : {
"count" : {
"total" : 3,
"data" : 2,
"coordinating_only" : 0,
"master" : 1,
"ingest" : 3
},
"versions" : [
"6.3.0"
],
"os" : {
"available_processors" : 24,
"allocated_processors" : 24,
"names" : [
{
"name" : "Linux",
"count" : 3
}
],
"mem" : {
"total" : "70.4gb",
"total_in_bytes" : 75689287680,
"free" : "18.7gb",
"free_in_bytes" : 20169138176,
"used" : "51.7gb",
"used_in_bytes" : 55520149504,
"free_percent" : 27,
"used_percent" : 73
}
},
"process" : {
"cpu" : {
"percent" : 33
},
"open_file_descriptors" : {
"min" : 384,
"max" : 5880,
"avg" : 4012
}
},
"jvm" : {
"max_uptime" : "3.7h",
"max_uptime_in_millis" : 13350435,
"versions" : [
{
"version" : "1.8.0_161",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "25.161-b14",
"vm_vendor" : "Oracle Corporation",
"count" : 3
}
],
"mem" : {
"heap_used" : "2.5gb",
"heap_used_in_bytes" : 2694126712,
"heap_max" : "3.8gb",
"heap_max_in_bytes" : 4153671680
},
"threads" : 493
},
"fs" : {
"total" : "1.1tb",
"total_in_bytes" : 1304314785792,
"free" : "1tb",
"free_in_bytes" : 1183926980608,
"available" : "1tb",
"available_in_bytes" : 1183926980608
},
"plugins" : [ ],
"network_types" : {
"transport_types" : {
"security4" : 3
},
"http_types" : {
"security4" : 3
}
}
}
}
[root@wielknews01 nodes]# curl -u elastic:Elastic@123 'localhost:9200'
{
"name" : "elk-master-node",
"cluster_name" : "elk-logging-prod",
"cluster_uuid" : "arew575AS2mNXNz-uWcMvQ",
"version" : {
"number" : "6.3.0",
"build_flavor" : "default",
"build_type" : "rpm",
"build_hash" : "424e937",
"build_date" : "2018-06-11T23:38:03.357887Z",
"build_snapshot" : false,
"lucene_version" : "7.3.1",
"minimum_wire_compatibility_version" : "5.6.0",
"minimum_index_compatibility_version" : "5.0.0"
},
"tagline" : "You Know, for Search"
}
Thanks
Sina