Fatal Error Elasticsearch

Elastic Stack Elasticsearch
1

HI Team,

We are extending the topic Fatal Error in thread Elasticsearch.

We have faced the again same issue when executing queries even we reduce heap size to around 30 GB to make use of compressed pointers.One thing to observe is we are receiving 1.8 TB per day per index.Do we have to increase number of shards as we are exceeding 50 GB per shard in case heapsize is not the issue?Currently we are using 5 shards,1 replica per index.

Regards,
Vivek

2

Do you have an indication, where this memory is used? You can take a look at the node stats API. Also use the human parameter to add fields more readable to humans by not only posting every value in bytes.

3 
 "search" : {
          "open_contexts" : 4,
          "query_total" : 278291,
          "query_time_in_millis" : 11132631,
          "query_current" : 4,
          "fetch_total" : 12174,
          "fetch_time_in_millis" : 16046,
          "fetch_current" : 0,
          "scroll_total" : 0,
          "scroll_time_in_millis" : 0,
          "scroll_current" : 0,
          "suggest_total" : 0,
          "suggest_time_in_millis" : 0,
          "suggest_current" : 0
        },
        "merges" : {
          "current" : 2,
          "current_docs" : 554704,
          "current_size_in_bytes" : 967340031,
          "total" : 1132,
          "total_time_in_millis" : 37000710,
          "total_docs" : 83823949,
          "total_size_in_bytes" : 143179119844,
          "total_stopped_time_in_millis" : 0,
          "total_throttled_time_in_millis" : 30727987,
          "total_auto_throttle_in_bytes" : 1159813837
        },
        "refresh" : {
          "total" : 5234,
          "total_time_in_millis" : 779612,
          "listeners" : 0
        },
        "flush" : {
          "total" : 233,
          "periodic" : 79,
          "total_time_in_millis" : 216288
        },
        "warmer" : {
          "current" : 0,
          "total" : 4828,
          "total_time_in_millis" : 1510
        },
        "query_cache" : {
          "memory_size_in_bytes" : 39333553,
          "total_count" : 114854744,
          "hit_count" : 58263,
          "miss_count" : 114796481,
          "cache_size" : 79034,
          "cache_count" : 87324,
          "evictions" : 8290
        },
        "fielddata" : {
          "memory_size_in_bytes" : 55984,
          "evictions" : 0
        },
        "completion" : {
          "size_in_bytes" : 0
        },
        "segments" : {
          "count" : 959,
          "memory_in_bytes" : 2364941545,
          "terms_memory_in_bytes" : 1892784157,
          "stored_fields_memory_in_bytes" : 384585872,
          "term_vectors_memory_in_bytes" : 0,
          "norms_memory_in_bytes" : 9754176,
          "points_memory_in_bytes" : 69888240,
          "doc_values_memory_in_bytes" : 7929100,
          "index_writer_memory_in_bytes" : 135035676,
          "version_map_memory_in_bytes" : 0,
          "fixed_bit_set_memory_in_bytes" : 57280,
          "max_unsafe_auto_id_timestamp" : 1581652029190,
          "file_sizes" : { }
        },
        "translog" : {
          "operations" : 1382071,
          "size_in_bytes" : 2024558792,
          "uncommitted_operations" : 274990,
          "uncommitted_size_in_bytes" : 412573920,
          "earliest_last_modified_age" : 0
        },
        "request_cache" : {
          "memory_size_in_bytes" : 17561739,
          "evictions" : 0,
          "hit_count" : 208374,
          "miss_count" : 30790
        },
        "recovery" : {
          "current_as_source" : 1,
          "current_as_target" : 0,
          "throttle_time_in_millis" : 5033632
        }
      },
      "os" : {
        "timestamp" : 1581671943825,
        "cpu" : {
          "percent" : 39,
          "load_average" : {
            "1m" : 3.73,
            "5m" : 2.74,
            "15m" : 2.41
          }
        },
        "mem" : {
          "total_in_bytes" : 67560833024,
          "free_in_bytes" : 351784960,
          "used_in_bytes" : 67209048064,
          "free_percent" : 1,
          "used_percent" : 99
        },
        "swap" : {
          "total_in_bytes" : 68716326912,
          "free_in_bytes" : 67656630272,
          "used_in_bytes" : 1059696640
        },
        "cgroup" : {
          "cpuacct" : {
            "control_group" : "/system.slice/elasticsearch.service",
            "usage_nanos" : 4722832933856926
          },
          "cpu" : {
            "control_group" : "/system.slice/elasticsearch.service",
            "cfs_period_micros" : 100000,
            "cfs_quota_micros" : -1,
            "stat" : {
              "number_of_elapsed_periods" : 0,
              "number_of_times_throttled" : 0,
              "time_throttled_nanos" : 0
            }
          },
          "memory" : {
            "control_group" : "/system.slice/elasticsearch.service",
            "limit_in_bytes" : "9223372036854771712",
            "usage_in_bytes" : "65082191872"
          }
        }
      },
      "process" : {
        "timestamp" : 1581671943825,
        "open_file_descriptors" : 871,
        "max_file_descriptors" : 65535,
        "cpu" : {
          "percent" : 38,
          "total_in_millis" : 28035050
        },
        "mem" : {
          "total_virtual_in_bytes" : 2047677554688
        }
      },
      "jvm" : {
        "timestamp" : 1581671943826,
        "uptime_in_millis" : 29473755,
        "mem" : {
          "heap_used_in_bytes" : 23209736600,
          "heap_used_percent" : 72,
          "heap_committed_in_bytes" : 32142524416,
          "heap_max_in_bytes" : 32142524416,
          "non_heap_used_in_bytes" : 197825224,
          "non_heap_committed_in_bytes" : 211595264,
          "pools" : {
            "young" : {
              "used_in_bytes" : 551656952,
              "max_in_bytes" : 558432256,
              "peak_used_in_bytes" : 558432256,
              "peak_max_in_bytes" : 558432256
            },
            "survivor" : {
              "used_in_bytes" : 39182944,
              "max_in_bytes" : 69730304,
              "peak_used_in_bytes" : 69730304,
              "peak_max_in_bytes" : 69730304
            },
            "old" : {
              "used_in_bytes" : 22618908088,
              "max_in_bytes" : 31514361856,
              "peak_used_in_bytes" : 31502313688,
              "peak_max_in_bytes" : 31514361856
            }
          }
        },
        "threads" : {
          "count" : 126,
          "peak_count" : 132
        },
        "gc" : {
          "collectors" : {
            "young" : {
              "collection_count" : 18075,
              "collection_time_in_millis" : 955820
            },
            "old" : {
              "collection_count" : 20,
              "collection_time_in_millis" : 43710
            }
          }
        },
4

Pleae let me know if you need any specific information thoseare statitics of one master node.

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.