Kibana 5.4 timing out while searching the elastic search data

Elastic Stack Elasticsearch
1

Hello we are using kibana 5.4 as a viewer to see data, but the viewer keeps getting timing out and unable to do any operations.

Here is the clusterstats,

{
  "_nodes" : {
    "total" : 7,
    "successful" : 7,
    "failed" : 0
  },
  "cluster_name" : "XXXXXXX",
  "timestamp" : 1534170777546,
  "status" : "green",
  "indices" : {
    "count" : 633,
    "shards" : {
      "total" : 5961,
      "primaries" : 2983,
      "replication" : 0.9983238350653705,
      "index" : {
        "shards" : {
          "min" : 2,
          "max" : 10,
          "avg" : 9.417061611374407
        },
        "primaries" : {
          "min" : 1,
          "max" : 5,
          "avg" : 4.712480252764613
        },
        "replication" : {
          "min" : 0.0,
          "max" : 1.0,
          "avg" : 0.9984202211690363
        }
      }
    },
    "docs" : {
      "count" : 1128173584,
      "deleted" : 56248
    },
    "store" : {
      "size" : "854gb",
      "size_in_bytes" : 917053814688,
      "throttle_time" : "0s",
      "throttle_time_in_millis" : 0
    },
    "fielddata" : {
      "memory_size" : "2.5gb",
      "memory_size_in_bytes" : 2788034680,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "257mb",
      "memory_size_in_bytes" : 269516328,
      "total_count" : 350844005,
      "hit_count" : 515808,
      "miss_count" : 350328197,
      "cache_size" : 95180,
      "cache_count" : 96423,
      "evictions" : 1243
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 45438,
      "memory" : "2.6gb",
      "memory_in_bytes" : 2803075688,
      "terms_memory" : "2gb",
      "terms_memory_in_bytes" : 2229385890,
      "stored_fields_memory" : "273.4mb",
      "stored_fields_memory_in_bytes" : 286732992,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "74.2mb",
      "norms_memory_in_bytes" : 77847552,
      "points_memory" : "35.6mb",
      "points_memory_in_bytes" : 37367938,
      "doc_values_memory" : "163.7mb",
      "doc_values_memory_in_bytes" : 171741316,
      "index_writer_memory" : "206.4mb",
      "index_writer_memory_in_bytes" : 216526244,
      "version_map_memory" : "33.1kb",
      "version_map_memory_in_bytes" : 33951,
      "fixed_bit_set" : "0b",
      "fixed_bit_set_memory_in_bytes" : 0,
      "max_unsafe_auto_id_timestamp" : 1534146193847,
      "file_sizes" : { }
    }
  },
  "nodes" : {
    "count" : {
      "total" : 7,
      "data" : 4,
      "coordinating_only" : 0,
      "master" : 3,
      "ingest" : 7
    },
    "versions" : [
      "5.4.0"
    ],
    "os" : {
      "available_processors" : 40,
      "allocated_processors" : 40,
      "names" : [
        {
          "name" : "Linux",
          "count" : 7
        }
      ],
      "mem" : {
        "total" : "266gb",
        "total_in_bytes" : 285706514432,
        "free" : "51.1gb",
        "free_in_bytes" : 54952017920,
        "used" : "214.9gb",
        "used_in_bytes" : 230754496512,
        "free_percent" : 19,
        "used_percent" : 81
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 82
      },
      "open_file_descriptors" : {
        "min" : 407,
        "max" : 3923,
        "avg" : 2338
      }
    },
    "jvm" : {
      "max_uptime" : "8.2d",
      "max_uptime_in_millis" : 715661400,
      "versions" : [
        {
          "version" : "1.8.0_72",
          "vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
          "vm_version" : "25.72-b15",
          "vm_vendor" : "Oracle Corporation",
          "count" : 7
        }
      ],
      "mem" : {
        "heap_used" : "59.1gb",
        "heap_used_in_bytes" : 63549576328,
        "heap_max" : "135.6gb",
        "heap_max_in_bytes" : 145680236544
      },
      "threads" : 526
    },
    "fs" : {
      "total" : "4tb",
      "total_in_bytes" : 4507396268032,
      "free" : "3.2tb",
      "free_in_bytes" : 3566636900352,
      "available" : "3.2tb",
      "available_in_bytes" : 3566636900352,
      "spins" : "true"
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "netty4" : 7
      },
      "http_types" : {
        "netty4" : 7
      }
    }
  }
}

Please let me know how can I manage my cluster so as to speed up my queries.

2

Hi there, how is this query being made? E.g. is this a query you're making in Dev Tools Console or are you just exploring your data using Discover? Can you share the request that's timing out? You can see what's timing out by looking in the network activity of your browser.

Thanks,
CJ

3

I work on Kibana typically, but I just spoke with an Elasticsearch engineer and his analysis is that you have an unusually large number of shards (2983) for a data set of your size (854gb). Your queries will be slowed down by the resources required to keep track of that number of shards. And some queries might require combining data from many shards, further increasing query time. You might want to take a look at https://www.elastic.co/elasticon/conf/2016/sf/quantitative-cluster-sizing to learn more about strategies for cluster management.

I hope this helps,
CJ

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.