Kibana 9.3.1 dashboard loads slow

[mittal_rawal]

Hi Team,

i have recently migrated elk from 8.16 to 9.3.1 i notice slowness in dashboard load.
time take to loads 1-2 minutes for all panels in dashboard i want to load that within 10sec
below i am sharing my cluster details for 9.3.1
GET _cluster/health:
{

"cluster_name": "MY custer",

"status": "green",

"timed_out": false,

"number_of_nodes": 1,

"number_of_data_nodes": 1,

"active_primary_shards": 607,

"active_shards": 607,

"relocating_shards": 0,

"initializing_shards": 0,

"unassigned_shards": 0,

"unassigned_primary_shards": 0,

"delayed_unassigned_shards": 0,

"number_of_pending_tasks": 0,

"number_of_in_flight_fetch": 0,

"task_max_waiting_in_queue_millis": 0,

"active_shards_percent_as_number": 100

}

GET _cluster/setting:
{

"persistent": {

"cluster": {

  "max_shards_per_node": "2000"

}

},

"transient": {}

}

other details
node =1
no. of fields= 120k

total data size 44gb

memory 64gb from that 28gb is for heap
no. of dashboard= 158
panel per dashboards = 20-25 charts
"attributes": {

    "ml.allocated_processors": "16",

    "ml.allocated_processors_double": "16.0",

    "ml.max_jvm_size": "30064771072",

    "ml.config_version": "12.0.0",

    "xpack.installed": "true",

    "transform.config_version": "10.0.0",

    "ml.machine_memory": "66806956032"

  },

  "indices": {

    "docs": {

      "count": 68182373,

      "deleted": 311,

      "total_size_in_bytes": 47764386963

    },

    "shard_stats": {

      "total_count": 607

    },

    "store": {

      "size_in_bytes": 47764839586,

      "total_data_set_size_in_bytes": 47764839586,

      "reserved_in_bytes": 0

    },

    "indexing": {

      "index_total": 3787,

      "index_time_in_millis": 2126,

      "index_current": 0,

      "index_failed": 2,

      "index_failed_due_to_version_conflict": 2,

      "delete_total": 538,

      "delete_time_in_millis": 70,

      "delete_current": 0,

      "noop_update_total": 0,

      "is_throttled": false,

      "throttle_time_in_millis": 0,

      "write_load": 5.424965812470846e-7,

      "recent_write_load": 4.166029632772772e-7,

      "peak_write_load": 5.497038864942717e-7

    },

"query_cache": {

      "memory_size_in_bytes": 539224,

      "total_count": 1223,

      "hit_count": 163,

      "miss_count": 1060,

      "cache_size": 24,

      "cache_count": 24,

      "evictions": 0

    },

    "fielddata": {

      "memory_size_in_bytes": 11168,

      "evictions": 7,

      "global_ordinals": {

        "build_time_in_millis": 15

      }

    },

"segments": {

      "count": 656,

      "memory_in_bytes": 0,

      "terms_memory_in_bytes": 0,

      "stored_fields_memory_in_bytes": 0,

      "term_vectors_memory_in_bytes": 0,

      "norms_memory_in_bytes": 0,

      "points_memory_in_bytes": 0,

      "doc_values_memory_in_bytes": 0,

      "index_writer_memory_in_bytes": 0,

      "version_map_memory_in_bytes": 0,

      "fixed_bit_set_memory_in_bytes": 1336,

      "max_unsafe_auto_id_timestamp": 1781841433616,

      "file_sizes": {}

    },

"os": {

    "timestamp": 1782205742811,

    "cpu": {

      "percent": 37,

      "load_average": {

        "1m": 6.37,

        "5m": 6.22,

        "15m": 6.34

      },

      "available_processors": 16

    },

    "mem": {

      "total_in_bytes": 66806956032,

      "adjusted_total_in_bytes": 66806956032,

      "free_in_bytes": 7604166656,

      "used_in_bytes": 59202789376,

      "free_percent": 11,

      "used_percent": 89

    },

    "swap": {

      "total_in_bytes": 8464101376,

      "free_in_bytes": 5831798784,

      "used_in_bytes": 2632302592

    }

  },

  "jvm": {

    "timestamp": 1782205742811,

    "uptime_in_millis": 6503357,

    "mem": {

      "heap_used_in_bytes": 1816506504,

      "heap_used_percent": 6,

      "heap_committed_in_bytes": 30064771072,

      "heap_max_in_bytes": 30064771072,

      "non_heap_used_in_bytes": 303294760,

      "non_heap_committed_in_bytes": 314572800,

      "pools": {

        "CodeHeap 'non-nmethods'": {

          "used_in_bytes": 3390464,

          "max_in_bytes": 7602176,

          "peak_used_in_bytes": 5267968,

          "peak_max_in_bytes": 7602176

        },

        "Metaspace": {

          "used_in_bytes": 190176008,

          "max_in_bytes": 0,

          "peak_used_in_bytes": 190176008,

          "peak_max_in_bytes": 0

        },

        "CodeHeap 'profiled nmethods'": {

          "used_in_bytes": 51768320,

          "max_in_bytes": 122023936,

          "peak_used_in_bytes": 53336832,

          "peak_max_in_bytes": 122023936

        },

        "Compressed Class Space": {

          "used_in_bytes": 26195872,

          "max_in_bytes": 1073741824,

          "peak_used_in_bytes": 26195872,

          "peak_max_in_bytes": 1073741824

        },

        "young": {

          "used_in_bytes": 1442840576,

          "max_in_bytes": 0,

          "peak_used_in_bytes": 11945377792,

          "peak_max_in_bytes": 0

        },

        "old": {

          "used_in_bytes": 0,

          "max_in_bytes": 30064771072,

          "peak_used_in_bytes": 0,

          "peak_max_in_bytes": 30064771072

        },

        "survivor": {

          "used_in_bytes": 373665928,

          "max_in_bytes": 0,

          "peak_used_in_bytes": 373665928,

          "peak_max_in_bytes": 0

        },

        "CodeHeap 'non-profiled nmethods'": {

          "used_in_bytes": 31764096,

          "max_in_bytes": 122032128,

          "peak_used_in_bytes": 31764096,

          "peak_max_in_bytes": 122032128

        }

      }

please tell me whats wrong i am doing also suggest me solution to load my dashboard fast within 10sec now it's taking 1-2minutes to load all panel in dashboards.

Thanks & Regards
mittal

[carly.richmond]

Welcome back @mittal_rawal!

Is this a custom dashboard that you have built? Or a dashboard from an integration?

It could be loading of the dashboard controls, or a particular query that is slow that could be causing it. I would recommend checking out the guidance in this debugging blog and see if you can narrow the slowness down to a particular process or control.

Hope that helps!

[mittal_rawal]

Hi @carly.richmond We have built a custom dashboard using Kibana visualizations. In the newer version, it seems slower compared to the older version. Previously, in version 8.16, the same queries and charts were performing faster.
also, could you also suggest any way to preload Kibana inside an iframe? Right now, whenever we open the dashboard in an iframe, it reloads the entire Kibana application, which increases the load time.

[stephenb (Stephen Brown)]

@mittal_rawal

i have recently migrated elk from 8.16 to 9.3.1

Migrated or upgraded in place?

[mittal_rawal]

we have installed new version and restore data from old version to new one with same config at same place after successful data restore, we removed old version.

[carly.richmond]

could you also suggest any way to preload Kibana inside an iframe?

It's not something I've done personally. I've only used an iframe to load the dashboard as covered in this article.
Could you try combining the article instructions with the preload attribute and see if that helps?

On the slowness, it would be useful to know if you have migrated or upgraded in place as @stephenb asks, and if you have any output from the query profile or DevTools console that shows what is causing the slowness.

Hope that helps!

[stephenb (Stephen Brown)]

I asked if Migrated / Upgraded as many times during migration the templates are not properly applied and sub--par mapping then get used which slows queries.

I think the next step, is load the dashboard, find a slow running viz, do Inspect, get the Exact Query and then run in the profiler to see what is taking so long

Other mitigations include using collapsable panels which then only load when open speed up the entire experience.

The other thing is make sure your Viz's are properly filter by index pattern / data stream this can make a huge difference (maybe your data set is larger)

Finally if still slow perhaps re-write the Viz with ESQL

But at this point it is roll-up-the-sleeves time

Oh I no nothing about framing Viz.

[mittal_rawal]

Hi @stephenb @carly.richmond ,

i have inspect the dashboard load in and attached screenshort.
i see there are multiple ese url request. right now, i have single node if i switch to 2 nodes will this optimize?

[carly.richmond]

Thanks for sharing @mittal_rawal. I wouldn't say that immediately switching from 1 to 2 nodes is going to optimize your dashboard performance.

I see you have some long running queries from the output (13.62 and 13.61s respectively) so it looks to likely be a long-running query. As @stephenb said above you need to identify which panel is taking so long and debug the query using the profiler.

I think the next step, is load the dashboard, find a slow running viz, do Inspect, get the Exact Query and then run in the profiler to see what is taking so long

Hope that helps!

[stephenb (Stephen Brown)]

Hi @mittal_rawal
The screen shot you are showing is not is not Inspect (Thats ok) but not what you want to use

First Look at the statistics you will notice 2 numbers
The Top is the Round Trip Time 128ms (Network etc)
The other is actually the Elastic Query tim 22ms

Then you can get to the actual query

They you can test that query in the query profuiler (you might need to fix the index patterns to non url encoded