Slow search process in kibana Discover

Aniket_Pant · May 5, 2021, 1:04pm

I am using elk 7.11..i am searching an index size of 100gb but it taking long time more than 5 minutes or i can say it is not showing me the data it can show me only 7 days data although it contain last 30 days data but it is not showing

Following is the index detail which i wanted to search/read through kibana discover

health status index                            uuid                   pri rep docs.count docs.deleted store.size pri.store.size
green  open   log-wlb-sysmon-2021.04.26-000002 p5ROhVjwT0Gpl3pgwu2bRA   1   1   69852843       103103       82gb           41gb
green  open   log-wlb-sysmon-2021.04.06-000001 IAHprh30T6-rh2hxB67TLA   1   1   84049138        52926      100gb           50gb

I am pasting some pics
pic 1 is showing last 1 hour of data

*pic2 which i wanted to search last 30 days data but i couldn't

in log entries i can see only todays data

warkolm · May 6, 2021, 12:21am

What is the output from the _cluster/stats?pretty&human API in Elasticsearch?

Aniket_Pant · May 6, 2021, 5:49am

{
  "_nodes" : {
    "total" : 11,
    "successful" : 11,
    "failed" : 0
  },
  "cluster_name" : "HELK",
  "cluster_uuid" : "vYioPHO_SzqW5yWfcZClew",
  "timestamp" : 1620280101261,
  "status" : "green",
  "indices" : {
    "count" : 56,
    "shards" : {
      "total" : 112,
      "primaries" : 56,
      "replication" : 1.0,
      "index" : {
        "shards" : {
          "min" : 2,
          "max" : 2,
          "avg" : 2.0
        },
        "primaries" : {
          "min" : 1,
          "max" : 1,
          "avg" : 1.0
        },
        "replication" : {
          "min" : 1.0,
          "max" : 1.0,
          "avg" : 1.0
        }
      }
    },
    "docs" : {
      "count" : 629360151,
      "deleted" : 464473
    },
    "store" : {
      "size" : "810.4gb",
      "size_in_bytes" : 870205016512,
      "reserved" : "0b",
      "reserved_in_bytes" : 0
    },
    "fielddata" : {
      "memory_size" : "119.7kb",
      "memory_size_in_bytes" : 122592,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "747mb",
      "memory_size_in_bytes" : 783295509,
      "total_count" : 105482268,
      "hit_count" : 10460942,
      "miss_count" : 95021326,
      "cache_size" : 3634,
      "cache_count" : 1005866,
      "evictions" : 1002232
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 1567,
      "memory" : "56.1mb",
      "memory_in_bytes" : 58868444,
      "terms_memory" : "31.4mb",
      "terms_memory_in_bytes" : 32969880,
      "stored_fields_memory" : "1.8mb",
      "stored_fields_memory_in_bytes" : 1983328,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "2.4mb",
      "norms_memory_in_bytes" : 2603776,
      "points_memory" : "0b",
      "points_memory_in_bytes" : 0,
      "doc_values_memory" : "20.3mb",
      "doc_values_memory_in_bytes" : 21311460,
      "index_writer_memory" : "310.3mb",
      "index_writer_memory_in_bytes" : 325453612,
      "version_map_memory" : "14.7mb",
      "version_map_memory_in_bytes" : 15461781,
      "fixed_bit_set" : "97.9mb",
      "fixed_bit_set_memory_in_bytes" : 102662184,
      "max_unsafe_auto_id_timestamp" : 1620279129106,
      "file_sizes" : { }
    },
    "mappings" : {
      "field_types" : [
        {
          "name" : "alias",
          "count" : 9,
          "index_count" : 3
        },
        {
          "name" : "binary",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "boolean",
          "count" : 264,
          "index_count" : 39
        },
        {
          "name" : "byte",
          "count" : 4,
          "index_count" : 4
        },
        {
          "name" : "date",
          "count" : 376,
          "index_count" : 49
        },
        {
          "name" : "date_nanos",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "date_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "double",
          "count" : 444,
          "index_count" : 12
        },
        {
          "name" : "double_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "float",
          "count" : 641,
          "index_count" : 12
        },
        {
          "name" : "float_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "geo_point",
          "count" : 40,
          "index_count" : 22
        },
        {
          "name" : "geo_shape",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "half_float",
          "count" : 117,
          "index_count" : 41
        },
        {
          "name" : "integer",
          "count" : 171,
          "index_count" : 16
        },
        {
          "name" : "integer_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "ip",
          "count" : 98,
          "index_count" : 22
        },
        {
          "name" : "ip_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "keyword",
          "count" : 5265,
          "index_count" : 49
        },
        {
          "name" : "long",
          "count" : 8569,
          "index_count" : 48
        },
        {
          "name" : "long_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "nested",
          "count" : 55,
          "index_count" : 17
        },
        {
          "name" : "object",
          "count" : 8553,
          "index_count" : 49
        },
        {
          "name" : "scaled_float",
          "count" : 389,
          "index_count" : 3
        },
        {
          "name" : "shape",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "short",
          "count" : 25,
          "index_count" : 9
        },
        {
          "name" : "text",
          "count" : 1922,
          "index_count" : 33
        }
      ]
    },
    "analysis" : {
      "char_filter_types" : [ ],
      "tokenizer_types" : [ ],
      "filter_types" : [ ],
      "analyzer_types" : [ ],
      "built_in_char_filters" : [ ],
      "built_in_tokenizers" : [ ],
      "built_in_filters" : [ ],
      "built_in_analyzers" : [ ]
    },
    "versions" : [
      {
        "version" : "7.11.2",
        "index_count" : 56,
        "primary_shard_count" : 56,
        "total_primary_size" : "403.6gb",
        "total_primary_bytes" : 433420637219
      }
    ]
  },
  "nodes" : {
    "count" : {
      "total" : 11,
      "coordinating_only" : 2,
      "data" : 5,
      "data_cold" : 0,
      "data_content" : 1,
      "data_hot" : 1,
      "data_warm" : 0,
      "ingest" : 0,
      "master" : 3,
      "ml" : 0,
      "remote_cluster_client" : 0,
      "transform" : 0,
      "voting_only" : 0
    },
    "versions" : [
      "7.11.2"
    ],
    "os" : {
      "available_processors" : 68,
      "allocated_processors" : 68,
      "names" : [
        {
          "name" : "Linux",
          "count" : 11
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "CentOS Linux 8 (Core)",
          "count" : 5
        },
        {
          "pretty_name" : "Oracle Linux Server 8.3",
          "count" : 6
        }
      ],
      "mem" : {
        "total" : "453.8gb",
        "total_in_bytes" : 487313825792,
        "free" : "53gb",
        "free_in_bytes" : 57011601408,
        "used" : "400.7gb",
        "used_in_bytes" : 430302224384,
        "free_percent" : 12,
        "used_percent" : 88
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 117
      },
      "open_file_descriptors" : {
        "min" : 610,
        "max" : 867,
        "avg" : 758
      }
    },
    "jvm" : {
      "max_uptime" : "50d",
      "max_uptime_in_millis" : 4321640635,
      "versions" : [
        {
          "version" : "15.0.1",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "15.0.1+9",
          "vm_vendor" : "AdoptOpenJDK",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 11
        }
      ],
      "mem" : {
        "heap_used" : "76.5gb",
        "heap_used_in_bytes" : 82166592784,
        "heap_max" : "224gb",
        "heap_max_in_bytes" : 240518168576
      },
      "threads" : 742
    },
    "fs" : {
      "total" : "5.4tb",
      "total_in_bytes" : 6015413645312,
      "free" : "4.6tb",
      "free_in_bytes" : 5077104328704,
      "available" : "4.6tb",
      "available_in_bytes" : 5077104328704
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "security4" : 11
      },
      "http_types" : {
        "security4" : 11
      }
    },
    "discovery_types" : {
      "zen" : 11
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "rpm",
        "count" : 11
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 0,
      "processor_stats" : { }
    }
  }
}

Aniket_Pant · May 6, 2021, 6:23am

Hey @warkolm i have solved this problem by correcting node.roles: ["data"] to node.roles: ["data_hot","data_content"] in all elasticsearch data nodes.
The reason of slow process of kibana discover was that in one of the data node i write node.roles: ["data_hot","data_content"] and in remaining 5 data nodes i had node.roles: ["data"]
So i am right here?

Aniket_Pant · May 6, 2021, 6:23am

{
  "_nodes" : {
    "total" : 11,
    "successful" : 11,
    "failed" : 0
  },
  "cluster_name" : "HELK",
  "cluster_uuid" : "vYioPHO_SzqW5yWfcZClew",
  "timestamp" : 1620282225385,
  "status" : "green",
  "indices" : {
    "count" : 61,
    "shards" : {
      "total" : 121,
      "primaries" : 61,
      "replication" : 0.9836065573770492,
      "index" : {
        "shards" : {
          "min" : 1,
          "max" : 2,
          "avg" : 1.9836065573770492
        },
        "primaries" : {
          "min" : 1,
          "max" : 1,
          "avg" : 1.0
        },
        "replication" : {
          "min" : 0.0,
          "max" : 1.0,
          "avg" : 0.9836065573770492
        }
      }
    },
    "docs" : {
      "count" : 631707796,
      "deleted" : 141743
    },
    "store" : {
      "size" : "813.5gb",
      "size_in_bytes" : 873578979010,
      "reserved" : "0b",
      "reserved_in_bytes" : 0
    },
    "fielddata" : {
      "memory_size" : "719.8kb",
      "memory_size_in_bytes" : 737088,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "2.5mb",
      "memory_size_in_bytes" : 2630888,
      "total_count" : 6093,
      "hit_count" : 493,
      "miss_count" : 5600,
      "cache_size" : 62,
      "cache_count" : 62,
      "evictions" : 0
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 1583,
      "memory" : "56.5mb",
      "memory_in_bytes" : 59329068,
      "terms_memory" : "31.6mb",
      "terms_memory_in_bytes" : 33185944,
      "stored_fields_memory" : "1.9mb",
      "stored_fields_memory_in_bytes" : 1999072,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "2.4mb",
      "norms_memory_in_bytes" : 2617408,
      "points_memory" : "0b",
      "points_memory_in_bytes" : 0,
      "doc_values_memory" : "20.5mb",
      "doc_values_memory_in_bytes" : 21526644,
      "index_writer_memory" : "191.8mb",
      "index_writer_memory_in_bytes" : 201199218,
      "version_map_memory" : "2.5mb",
      "version_map_memory_in_bytes" : 2690997,
      "fixed_bit_set" : "97.6mb",
      "fixed_bit_set_memory_in_bytes" : 102377488,
      "max_unsafe_auto_id_timestamp" : 1620300882904,
      "file_sizes" : { }
    },
    "mappings" : {
      "field_types" : [
        {
          "name" : "alias",
          "count" : 9,
          "index_count" : 3
        },
        {
          "name" : "binary",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "boolean",
          "count" : 265,
          "index_count" : 40
        },
        {
          "name" : "byte",
          "count" : 4,
          "index_count" : 4
        },
        {
          "name" : "date",
          "count" : 394,
          "index_count" : 53
        },
        {
          "name" : "date_nanos",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "date_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "double",
          "count" : 444,
          "index_count" : 12
        },
        {
          "name" : "double_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "float",
          "count" : 641,
          "index_count" : 12
        },
        {
          "name" : "float_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "geo_point",
          "count" : 45,
          "index_count" : 27
        },
        {
          "name" : "geo_shape",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "half_float",
          "count" : 127,
          "index_count" : 46
        },
        {
          "name" : "integer",
          "count" : 171,
          "index_count" : 16
        },
        {
          "name" : "integer_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "ip",
          "count" : 108,
          "index_count" : 27
        },
        {
          "name" : "ip_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "keyword",
          "count" : 5454,
          "index_count" : 53
        },
        {
          "name" : "long",
          "count" : 8606,
          "index_count" : 52
        },
        {
          "name" : "long_range",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "nested",
          "count" : 55,
          "index_count" : 17
        },
        {
          "name" : "object",
          "count" : 8594,
          "index_count" : 54
        },
        {
          "name" : "scaled_float",
          "count" : 389,
          "index_count" : 3
        },
        {
          "name" : "shape",
          "count" : 1,
          "index_count" : 1
        },
        {
          "name" : "short",
          "count" : 25,
          "index_count" : 9
        },
        {
          "name" : "text",
          "count" : 2111,
          "index_count" : 37
        }
      ]
    },
    "analysis" : {
      "char_filter_types" : [ ],
      "tokenizer_types" : [ ],
      "filter_types" : [ ],
      "analyzer_types" : [ ],
      "built_in_char_filters" : [ ],
      "built_in_tokenizers" : [ ],
      "built_in_filters" : [ ],
      "built_in_analyzers" : [ ]
    },
    "versions" : [
      {
        "version" : "7.11.2",
        "index_count" : 61,
        "primary_shard_count" : 61,
        "total_primary_size" : "406.5gb",
        "total_primary_bytes" : 436570799823
      }
    ]
  },
  "nodes" : {
    "count" : {
      "total" : 11,
      "coordinating_only" : 2,
      "data" : 0,
      "data_cold" : 0,
      "data_content" : 6,
      "data_hot" : 6,
      "data_warm" : 0,
      "ingest" : 0,
      "master" : 3,
      "ml" : 0,
      "remote_cluster_client" : 0,
      "transform" : 0,
      "voting_only" : 0
    },
    "versions" : [
      "7.11.2"
    ],
    "os" : {
      "available_processors" : 68,
      "allocated_processors" : 68,
      "names" : [
        {
          "name" : "Linux",
          "count" : 11
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "Oracle Linux Server 8.3",
          "count" : 6
        },
        {
          "pretty_name" : "CentOS Linux 8 (Core)",
          "count" : 5
        }
      ],
      "mem" : {
        "total" : "453.8gb",
        "total_in_bytes" : 487313825792,
        "free" : "51.1gb",
        "free_in_bytes" : 54930169856,
        "used" : "402.6gb",
        "used_in_bytes" : 432383655936,
        "free_percent" : 11,
        "used_percent" : 89
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 90
      },
      "open_file_descriptors" : {
        "min" : 613,
        "max" : 879,
        "avg" : 761
      }
    },
    "jvm" : {
      "max_uptime" : "50d",
      "max_uptime_in_millis" : 4323764500,
      "versions" : [
        {
          "version" : "15.0.1",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "15.0.1+9",
          "vm_vendor" : "AdoptOpenJDK",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 11
        }
      ],
      "mem" : {
        "heap_used" : "52.1gb",
        "heap_used_in_bytes" : 56019414760,
        "heap_max" : "224gb",
        "heap_max_in_bytes" : 240518168576
      },
      "threads" : 737
    },
    "fs" : {
      "total" : "5.4tb",
      "total_in_bytes" : 6015413645312,
      "free" : "4.6tb",
      "free_in_bytes" : 5077246230528,
      "available" : "4.6tb",
      "available_in_bytes" : 5077246230528
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "security4" : 11
      },
      "http_types" : {
        "security4" : 11
      }
    },
    "discovery_types" : {
      "zen" : 11
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "rpm",
        "count" : 11
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 0,
      "processor_stats" : { }
    }
  }
}

system · June 3, 2021, 6:24am

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.