Hello, we have experiencing strange claster behavior.
If you look at the charts, the main indicators are falling.
Such as query time, total operation stats, network usage. As if the load on the cluster is reduced.
But from the client side, we observe high latency at that time. And the load did not change.
After some time the values on the graphs increase again and the delay disappears.
Attached grafana screenshots.
Can anyone help?
What is the output from the _cluster/stats?pretty&human API?
Mostly in this time we have errors like this in the log file
[2021-12-06T08:19:57,995][ERROR][c.a.o.s.s.h.n.OpenDistroSecuritySSLNettyHttpServerTransport] [p-prr-as02_node] Exception during establishing a SSL connection: java.util.NoSuchElementException: No value present
java.util.NoSuchElementException: No value present
[2021-12-06T08:20:06,922][ERROR][c.a.o.s.s.h.n.OpenDistroSecuritySSLNettyHttpServerTransport] [p-prr-as02_node] Exception during establishing a SSL connection: javax.net.ssl.SSLException: Session has no PSK
javax.net.ssl.SSLException: Session has no PSK
:
What can these errors indicate that some sessions cannot connect to the database or nodes cannot communicate with each other?
This is opendistro, which is not Elasticsearch. You will need to ask aws about this sorry, as it's their product.
Here is output of _cluster/stats?pretty&human
{
"_nodes" : {
"total" : 3,
"successful" : 3,
"failed" : 0
},
"cluster_name" : "elk-payment-receipt-repository",
"cluster_uuid" : "h37FwSqRSheiSVt677YOOA",
"timestamp" : 1638779247950,
"status" : "green",
"indices" : {
"count" : 230,
"shards" : {
"total" : 709,
"primaries" : 354,
"replication" : 1.002824858757062,
"index" : {
"shards" : {
"min" : 2,
"max" : 6,
"avg" : 3.082608695652174
},
"primaries" : {
"min" : 1,
"max" : 3,
"avg" : 1.5391304347826087
},
"replication" : {
"min" : 1.0,
"max" : 2.0,
"avg" : 1.0043478260869565
}
}
},
"docs" : {
"count" : 717647815,
"deleted" : 5570125
},
"store" : {
"size" : "2tb",
"size_in_bytes" : 2308536072597
},
"fielddata" : {
"memory_size" : "0b",
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "978.9mb",
"memory_size_in_bytes" : 1026458206,
"total_count" : 18392786761,
"hit_count" : 12156287936,
"miss_count" : 6236498825,
"cache_size" : 2791200,
"cache_count" : 50547385,
"evictions" : 47756185
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 5015,
"memory" : "1.3gb",
"memory_in_bytes" : 1494850286,
"terms_memory" : "375.8mb",
"terms_memory_in_bytes" : 394083863,
"stored_fields_memory" : "1010mb",
"stored_fields_memory_in_bytes" : 1059067224,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "5.1mb",
"norms_memory_in_bytes" : 5359296,
"points_memory" : "30.5mb",
"points_memory_in_bytes" : 32075011,
"doc_values_memory" : "4mb",
"doc_values_memory_in_bytes" : 4264892,
"index_writer_memory" : "6.4mb",
"index_writer_memory_in_bytes" : 6806724,
"version_map_memory" : "1.5kb",
"version_map_memory_in_bytes" : 1547,
"fixed_bit_set" : "480b",
"fixed_bit_set_memory_in_bytes" : 480,
"max_unsafe_auto_id_timestamp" : 1638765082369,
"file_sizes" : { }
}
},
"nodes" : {
"count" : {
"total" : 3,
"coordinating_only" : 0,
"data" : 3,
"ingest" : 3,
"master" : 3
},
"versions" : [
"7.3.2"
],
"os" : {
"available_processors" : 24,
"allocated_processors" : 24,
"names" : [
{
"name" : "Linux",
"count" : 3
}
],
"pretty_names" : [
{
"pretty_name" : "CentOS Linux 7 (Core)",
"count" : 3
}
],
"mem" : {
"total" : "46.5gb",
"total_in_bytes" : 49969078272,
"free" : "523.5mb",
"free_in_bytes" : 548995072,
"used" : "46gb",
"used_in_bytes" : 49420083200,
"free_percent" : 1,
"used_percent" : 99
}
},
"process" : {
"cpu" : {
"percent" : 62
},
"open_file_descriptors" : {
"min" : 2442,
"max" : 2956,
"avg" : 2659
}
},
"jvm" : {
"max_uptime" : "256.3d",
"max_uptime_in_millis" : 22152581680,
"versions" : [
{
"version" : "12.0.2",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "12.0.2+10",
"vm_vendor" : "Oracle Corporation",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 3
}
],
"mem" : {
"heap_used" : "11gb",
"heap_used_in_bytes" : 11816447256,
"heap_max" : "23.8gb",
"heap_max_in_bytes" : 25560612864
},
"threads" : 519
},
"fs" : {
"total" : "3.4tb",
"total_in_bytes" : 3756813361152,
"free" : "1.3tb",
"free_in_bytes" : 1445329240064,
"available" : "1.1tb",
"available_in_bytes" : 1270530682880
},
"plugins" : [
{
"name" : "opendistro_alerting",
"version" : "1.3.0.1",
"elasticsearch_version" : "7.3.2",
"java_version" : "1.8",
"description" : "Amazon OpenDistro alerting plugin",
"classname" : "com.amazon.opendistroforelasticsearch.alerting.AlertingPlugin",
"extended_plugins" : [
"lang-painless"
],
"has_native_controller" : false
},
{
"name" : "opendistro_performance_analyzer",
"version" : "1.3.0.0",
"elasticsearch_version" : "7.3.2",
"java_version" : "1.8",
"description" : "Performance Analyzer Plugin",
"classname" : "com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false
},
{
"name" : "opendistro_security",
"version" : "1.3.0.0",
"elasticsearch_version" : "7.3.2",
"java_version" : "1.8",
"description" : "Provide access control related features for Elasticsearch 7",
"classname" : "com.amazon.opendistroforelasticsearch.security.OpenDistroSecurityPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false
},
{
"name" : "opendistro-job-scheduler",
"version" : "1.3.0.0",
"elasticsearch_version" : "7.3.2",
"java_version" : "1.8",
"description" : "Open Distro for Elasticsearch job schduler plugin",
"classname" : "com.amazon.opendistroforelasticsearch.jobscheduler.JobSchedulerPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false
},
{
"name" : "opendistro_sql",
"version" : "1.3.0.0",
"elasticsearch_version" : "7.3.2",
"java_version" : "1.8",
"description" : "Open Distro for Elasticsearch SQL",
"classname" : "com.amazon.opendistroforelasticsearch.sql.plugin.SqlPlug",
"extended_plugins" : [ ],
"has_native_controller" : false
},
{
"name" : "repository-s3",
"version" : "7.3.2",
"elasticsearch_version" : "7.3.2",
"java_version" : "1.8",
"description" : "The S3 repository plugin adds S3 repositories",
"classname" : "org.elasticsearch.repositories.s3.S3RepositoryPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false
},
{
"name" : "opendistro_index_management",
"version" : "1.3.0.1",
"elasticsearch_version" : "7.3.2",
"java_version" : "1.8",
"description" : "Open Distro Index State Management Plugin",
"classname" : "com.amazon.opendistroforelasticsearch.indexstatemanagement.IndexStateManagementPlugin",
"extended_plugins" : [
"opendistro-job-scheduler"
],
"has_native_controller" : false
}
],
"network_types" : {
"transport_types" : {
"com.amazon.opendistroforelasticsearch.security.ssl.http.netty.OpenDistroSecuritySSLNettyTransport" : 3
},
"http_types" : {
"com.amazon.opendistroforelasticsearch.security.http.OpenDistroSecurityHttpServerTransport" : 3
}
},
"discovery_types" : {
"zen" : 3
},
"packaging_types" : [
{
"flavor" : "oss",
"type" : "rpm",
"count" : 3
}
]
}
}
Are there any problems here?
Again, you are using opendistro which is a bunch of custom plugins that we do not develop and support and these may be causing the issue.
You'd need to run the same workload on Elasticsearch to confirm it's not one of the plugins.
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.
© 2020. All Rights Reserved - Elasticsearch
Apache, Apache Lucene, Apache Hadoop, Hadoop, HDFS and the yellow elephant logo are trademarks of the Apache Software Foundation in the United States and/or other countries.