Kibana is extremely slow

Elastic Stack Kibana
1

Hi everyone,
I am trying to set up elk in my system but I am noticing that kibana is very very slow, from researches that I did, it could be that I have too many shards, but I really don't know how to deal with them.

if I type
GET /_cluster/health/?pretty

this is what I receive

{
  "cluster_name": "docker-cluster",
  "status": "green",
  "timed_out": false,
  "number_of_nodes": 3,
  "number_of_data_nodes": 3,
  "active_primary_shards": 111,
  "active_shards": 151,
  "relocating_shards": 0,
  "initializing_shards": 0,
  "unassigned_shards": 0,
  "delayed_unassigned_shards": 0,
  "number_of_pending_tasks": 1,
  "number_of_in_flight_fetch": 0,
  "task_max_waiting_in_queue_millis": 0,
  "active_shards_percent_as_number": 100
}

it just turned green for no reason, but it was yellow all along. please let me know how to resolve that situation by deleting shards or other.
thanks

2

Generally if Kibana is slow it's due to Elasticsearch being slow.

What is the output from the _cluster/stats?pretty&human API?
What is your docker config?
What is your Elasticsearch and Kibana config?
What do the logs from them show?

3 
{
  "_nodes": {
    "total": 3,
    "successful": 3,
    "failed": 0
  },
  "cluster_name": "docker-cluster",
  "cluster_uuid": "mbfmRyDnRMCXZxifDx8McQ",
  "timestamp": 1674117060262,
  "status": "green",
  "indices": {
    "count": 111,
    "shards": {
      "total": 151,
      "primaries": 111,
      "replication": 0.36036036036036034,
      "index": {
        "shards": {
          "min": 1,
          "max": 2,
          "avg": 1.3603603603603605
        },
        "primaries": {
          "min": 1,
          "max": 1,
          "avg": 1
        },
        "replication": {
          "min": 0,
          "max": 1,
          "avg": 0.36036036036036034
        }
      }
    },
    "docs": {
      "count": 77456090,
      "deleted": 835757
    },
    "store": {
      "size": "12.7gb",
      "size_in_bytes": 13675648296,
      "total_data_set_size": "12.7gb",
      "total_data_set_size_in_bytes": 13675648296,
      "reserved": "0b",
      "reserved_in_bytes": 0
    },
    "fielddata": {
      "memory_size": "1.3kb",
      "memory_size_in_bytes": 1360,
      "evictions": 0
    },
    "query_cache": {
      "memory_size": "110.4kb",
      "memory_size_in_bytes": 113129,
      "total_count": 8174,
      "hit_count": 1868,
      "miss_count": 6306,
      "cache_size": 97,
      "cache_count": 99,
      "evictions": 2
    },
    "completion": {
      "size": "0b",
      "size_in_bytes": 0
    },
    "segments": {
      "count": 857,
      "memory": "0b",
      "memory_in_bytes": 0,
      "terms_memory": "0b",
      "terms_memory_in_bytes": 0,
      "stored_fields_memory": "0b",
      "stored_fields_memory_in_bytes": 0,
      "term_vectors_memory": "0b",
      "term_vectors_memory_in_bytes": 0,
      "norms_memory": "0b",
      "norms_memory_in_bytes": 0,
      "points_memory": "0b",
      "points_memory_in_bytes": 0,
      "doc_values_memory": "0b",
      "doc_values_memory_in_bytes": 0,
      "index_writer_memory": "20.5mb",
      "index_writer_memory_in_bytes": 21570896,
      "version_map_memory": "13.5kb",
      "version_map_memory_in_bytes": 13912,
      "fixed_bit_set": "246.5kb",
      "fixed_bit_set_memory_in_bytes": 252416,
      "max_unsafe_auto_id_timestamp": 1674116051750,
      "file_sizes": {}
    },
    "mappings": {
      "total_field_count": 28317,
      "total_deduplicated_field_count": 17126,
      "total_deduplicated_mapping_size": "123.7kb",
      "total_deduplicated_mapping_size_in_bytes": 126671,
      "field_types": [
        {
          "name": "alias",
          "count": 11,
          "index_count": 11,
          "script_count": 0
        },
        {
          "name": "binary",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "boolean",
          "count": 407,
          "index_count": 75,
          "script_count": 0
        },
        {
          "name": "byte",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "constant_keyword",
          "count": 310,
          "index_count": 73,
          "script_count": 0
        },
        {
          "name": "date",
          "count": 854,
          "index_count": 85,
          "script_count": 0
        },
        {
          "name": "date_nanos",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "date_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "double",
          "count": 13,
          "index_count": 3,
          "script_count": 0
        },
        {
          "name": "double_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "flattened",
          "count": 120,
          "index_count": 10,
          "script_count": 0
        },
        {
          "name": "float",
          "count": 125,
          "index_count": 22,
          "script_count": 0
        },
        {
          "name": "float_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "geo_point",
          "count": 111,
          "index_count": 24,
          "script_count": 0
        },
        {
          "name": "geo_shape",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "half_float",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "integer",
          "count": 4,
          "index_count": 2,
          "script_count": 0
        },
        {
          "name": "integer_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "ip",
          "count": 255,
          "index_count": 74,
          "script_count": 0
        },
        {
          "name": "ip_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "keyword",
          "count": 17062,
          "index_count": 85,
          "script_count": 0
        },
        {
          "name": "long",
          "count": 2500,
          "index_count": 69,
          "script_count": 0
        },
        {
          "name": "long_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "match_only_text",
          "count": 765,
          "index_count": 45,
          "script_count": 0
        },
        {
          "name": "nested",
          "count": 145,
          "index_count": 15,
          "script_count": 0
        },
        {
          "name": "object",
          "count": 5145,
          "index_count": 84,
          "script_count": 0
        },
        {
          "name": "scaled_float",
          "count": 136,
          "index_count": 22,
          "script_count": 0
        },
        {
          "name": "shape",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "short",
          "count": 7,
          "index_count": 3,
          "script_count": 0
        },
        {
          "name": "text",
          "count": 147,
          "index_count": 82,
          "script_count": 0
        },
        {
          "name": "version",
          "count": 5,
          "index_count": 5,
          "script_count": 0
        },
        {
          "name": "wildcard",
          "count": 183,
          "index_count": 19,
          "script_count": 0
        }
      ],
      "runtime_field_types": []
    },
    "analysis": {
      "char_filter_types": [],
      "tokenizer_types": [],
      "filter_types": [],
      "analyzer_types": [
        {
          "name": "pattern",
          "count": 5,
          "index_count": 5
        }
      ],
      "built_in_char_filters": [],
      "built_in_tokenizers": [],
      "built_in_filters": [],
      "built_in_analyzers": [
        {
          "name": "powershell_script_analyzer",
          "count": 2,
          "index_count": 2
        },
        {
          "name": "winlogbeat_powershell_script_analyzer",
          "count": 3,
          "index_count": 3
        }
      ]
    },
    "versions": [
      {
        "version": "8.3.2",
        "index_count": 9,
        "primary_shard_count": 9,
        "total_primary_size": "129.9mb",
        "total_primary_bytes": 136229162
      },
      {
        "version": "8.4.0",
        "index_count": 3,
        "primary_shard_count": 3,
        "total_primary_size": "3.1mb",
        "total_primary_bytes": 3279686
      },
      {
        "version": "8.4.1",
        "index_count": 12,
        "primary_shard_count": 12,
        "total_primary_size": "117.5mb",
        "total_primary_bytes": 123253457
      },
      {
        "version": "8.5.0",
        "index_count": 87,
        "primary_shard_count": 87,
        "total_primary_size": "12.2gb",
        "total_primary_bytes": 13159375353
      }
    ]
  },
  "nodes": {
    "count": {
      "total": 3,
      "coordinating_only": 0,
      "data": 3,
      "data_cold": 3,
      "data_content": 3,
      "data_frozen": 3,
      "data_hot": 3,
      "data_warm": 3,
      "ingest": 3,
      "master": 3,
      "ml": 3,
      "remote_cluster_client": 3,
      "transform": 3,
      "voting_only": 0
    },
    "versions": [
      "8.5.0"
    ],
    "os": {
      "available_processors": 24,
      "allocated_processors": 24,
      "names": [
        {
          "name": "Linux",
          "count": 3
        }
      ],
      "pretty_names": [
        {
          "pretty_name": "Ubuntu 20.04.5 LTS",
          "count": 3
        }
      ],
      "architectures": [
        {
          "arch": "amd64",
          "count": 3
        }
      ],
      "mem": {
        "total": "3gb",
        "total_in_bytes": 3221225472,
        "adjusted_total": "3gb",
        "adjusted_total_in_bytes": 3221225472,
        "free": "14.1mb",
        "free_in_bytes": 14819328,
        "used": "2.9gb",
        "used_in_bytes": 3206406144,
        "free_percent": 0,
        "used_percent": 100
      }
    },
    "process": {
      "cpu": {
        "percent": 16
      },
      "open_file_descriptors": {
        "min": 882,
        "max": 1144,
        "avg": 991
      }
    },
    "jvm": {
      "max_uptime": "54.4m",
      "max_uptime_in_millis": 3266707,
      "versions": [
        {
          "version": "19",
          "vm_name": "OpenJDK 64-Bit Server VM",
          "vm_version": "19+36-2238",
          "vm_vendor": "Oracle Corporation",
          "bundled_jdk": true,
          "using_bundled_jdk": true,
          "count": 3
        }
      ],
      "mem": {
        "heap_used": "755.4mb",
        "heap_used_in_bytes": 792164952,
        "heap_max": "1.5gb",
        "heap_max_in_bytes": 1610612736
      },
      "threads": 307
    },
    "fs": {
      "total": "4tb",
      "total_in_bytes": 4423540555776,
      "free": "3.9tb",
      "free_in_bytes": 4331099443200,
      "available": "3.7tb",
      "available_in_bytes": 4106169520128
    },
    "plugins": [],
    "network_types": {
      "transport_types": {
        "security4": 3
      },
      "http_types": {
        "security4": 3
      }
    },
    "discovery_types": {
      "multi-node": 3
    },
    "packaging_types": [
      {
        "flavor": "default",
        "type": "docker",
        "count": 3
      }
    ],
    "ingest": {
      "number_of_pipelines": 97,
      "processor_stats": {
        "append": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "community_id": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "conditional": {
          "count": 1525,
          "failed": 0,
          "current": 0,
          "time": "97ms",
          "time_in_millis": 97
        },
        "convert": {
          "count": 160,
          "failed": 0,
          "current": 0,
          "time": "1ms",
          "time_in_millis": 1
        },
        "date": {
          "count": 15016,
          "failed": 0,
          "current": 0,
          "time": "745ms",
          "time_in_millis": 745
        },
        "fingerprint": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "foreach": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "geoip": {
          "count": 990,
          "failed": 0,
          "current": 0,
          "time": "4.2s",
          "time_in_millis": 4296
        },
        "grok": {
          "count": 1158,
          "failed": 164,
          "current": 0,
          "time": "141ms",
          "time_in_millis": 141
        },
        "gsub": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "json": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "lowercase": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "pipeline": {
          "count": 14674,
          "failed": 0,
          "current": 0,
          "time": "16ms",
          "time_in_millis": 16
        },
        "registered_domain": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "remove": {
          "count": 30559,
          "failed": 0,
          "current": 0,
          "time": "93ms",
          "time_in_millis": 93
        },
        "rename": {
          "count": 4416,
          "failed": 2576,
          "current": 0,
          "time": "68ms",
          "time_in_millis": 68
        },
        "script": {
          "count": 15170,
          "failed": 0,
          "current": 0,
          "time": "228ms",
          "time_in_millis": 228
        },
        "set": {
          "count": 1986,
          "failed": 0,
          "current": 0,
          "time": "53ms",
          "time_in_millis": 53
        },
        "set_security_user": {
          "count": 14681,
          "failed": 0,
          "current": 0,
          "time": "264ms",
          "time_in_millis": 264
        },
        "split": {
          "count": 335,
          "failed": 0,
          "current": 0,
          "time": "2ms",
          "time_in_millis": 2
        },
        "uri_parts": {
          "count": 335,
          "failed": 0,
          "current": 0,
          "time": "7ms",
          "time_in_millis": 7
        },
        "urldecode": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "user_agent": {
          "count": 335,
          "failed": 0,
          "current": 0,
          "time": "87ms",
          "time_in_millis": 87
        }
      }
    },
    "indexing_pressure": {
      "memory": {
        "current": {
          "combined_coordinating_and_primary": "0b",
          "combined_coordinating_and_primary_in_bytes": 0,
          "coordinating": "0b",
          "coordinating_in_bytes": 0,
          "primary": "0b",
          "primary_in_bytes": 0,
          "replica": "0b",
          "replica_in_bytes": 0,
          "all": "0b",
          "all_in_bytes": 0
        },
        "total": {
          "combined_coordinating_and_primary": "0b",
          "combined_coordinating_and_primary_in_bytes": 0,
          "coordinating": "0b",
          "coordinating_in_bytes": 0,
          "primary": "0b",
          "primary_in_bytes": 0,
          "replica": "0b",
          "replica_in_bytes": 0,
          "all": "0b",
          "all_in_bytes": 0,
          "coordinating_rejections": 0,
          "primary_rejections": 0,
          "replica_rejections": 0
        },
        "limit": "0b",
        "limit_in_bytes": 0
      }
    }
  }
}

docker-compose.yml :

version: "2.2"

services:
  setup:
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
    user: "0"
    command: >
      bash -c '
        if [ x${ELASTIC_PASSWORD} == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x${KIBANA_PASSWORD} == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
          exit 1;
        fi;
        if [ ! -f config/certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
        fi;
        if [ ! -f config/certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \
          "instances:\n"\
          "  - name: es01\n"\
          "    dns:\n"\
          "      - es01\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          "  - name: es02\n"\
          "    dns:\n"\
          "      - es02\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          "  - name: es03\n"\
          "    dns:\n"\
          "      - es03\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          "  - name: kibana\n"\
          "    dns:\n"\
          "      - kibana\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          > config/certs/instances.yml;
          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
          unzip config/certs/certs.zip -d config/certs;
        fi;
        echo "Setting file permissions"
        chown -R root:root config/certs;
        find . -type d -exec chmod 750 \{\} \;;
        find . -type f -exec chmod 640 \{\} \;;
        echo "Waiting for Elasticsearch availability";
        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
        echo "All done!";
      '
    healthcheck:
      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
      interval: 1s
      timeout: 5s
      retries: 120

  es01:
    depends_on:
      setup:
        condition: service_healthy
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - esdata01:/usr/share/elasticsearch/data
    ports:
      - ${ES_PORT}:9200
    environment:
      - node.name=es01
      - cluster.name=${CLUSTER_NAME}
      - cluster.initial_master_nodes=es01,es02,es03
      - discovery.seed_hosts=es02,es03
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es01/es01.key
      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es01/es01.key
      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${LICENSE}
    mem_limit: ${MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  es02:
    depends_on:
      - es01
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - esdata02:/usr/share/elasticsearch/data
    environment:
      - node.name=es02
      - cluster.name=${CLUSTER_NAME}
      - cluster.initial_master_nodes=es01,es02,es03
      - discovery.seed_hosts=es01,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es02/es02.key
      - xpack.security.http.ssl.certificate=certs/es02/es02.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es02/es02.key
      - xpack.security.transport.ssl.certificate=certs/es02/es02.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${LICENSE}
    mem_limit: ${MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  es03:
    depends_on:
      - es02
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - esdata03:/usr/share/elasticsearch/data
    environment:
      - node.name=es03
      - cluster.name=${CLUSTER_NAME}
      - cluster.initial_master_nodes=es01,es02,es03
      - discovery.seed_hosts=es01,es02
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es03/es03.key
      - xpack.security.http.ssl.certificate=certs/es03/es03.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es03/es03.key
      - xpack.security.transport.ssl.certificate=certs/es03/es03.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${LICENSE}
    mem_limit: ${MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  logstash:
    depends_on:
      es01:
        condition: service_healthy
    image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
    volumes:
      - /home/admin/elk/logstash/conf.d/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro
      - /home/admin/elk/logstash/:/usr/share/logstash/config
    ports:
      - "5044:5044/udp"
    environment:
      - SERVERNAME=logstash
      - XPACK_SECURITY_ENABLED=true
      - ELASTICSEARCH_USERNAME=elastic
      - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
      - ELASTICSEARCH_HOSTS=https://es01:9200
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/ca.crt

  kibana:
    depends_on:
      es01:
        condition: service_healthy
      es02:
        condition: service_healthy
      es03:
        condition: service_healthy
    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
    volumes:
      - certs:/usr/share/kibana/config/certs
      - kibanadata:/usr/share/kibana/data
    ports:
      - ${KIBANA_PORT}:5601
    environment:
      - SERVERNAME=kibana
      - ELASTICSEARCH_HOSTS=https://es01:9200
      - ELASTICSEARCH_USERNAME=kibana_system
      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=--/ca.crt
      - ELASTICSEARCH_SSL_VERIFICATIONMODE=certificate
      - SERVER_SSL_ENABLED=true
      - SERVER_SSL_KEY=--/es01.key
      - SERVER_SSL_CERTIFICATE=--/es01.crt
      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=xxx
      - XPACK.REPORTING.ENCRYPTIONKEY=xxx
      - XPACK.SECURITY.ENCRYPTIONKEY=xxx
    mem_limit: ${MEM_LIMIT}
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

volumes:
  certs:
    driver: local
  esdata01:
    driver: local
  esdata02:
    driver: local
  esdata03:
    driver: local
  kibanadata:
    driver: local

I am not finding anything that stands out among the logs

4

but this is another thing

docker ps
CONTAINER ID   IMAGE                                                 COMMAND                  CREATED       STATUS                   PORTS                                        NAMES
b06275fd51f4   docker.elastic.co/kibana/kibana:8.5.0                 "/bin/tini -- /usr/l…"   2 hours ago   Up 2 hours (unhealthy)   0.0.0.0:5601->5601/tcp                       elk_kibana_1
f3fe8ef3c896   docker.elastic.co/logstash/logstash:8.5.0             "/usr/local/bin/dock…"   2 hours ago   Up 2 hours               5044/tcp, 9600/tcp, 0.0.0.0:5044->5044/udp   elk_logstash_1
eba9a3afef2a   docker.elastic.co/elasticsearch/elasticsearch:8.5.0   "/bin/tini -- /usr/l…"   2 hours ago   Up 2 hours (healthy)     9200/tcp, 9300/tcp                           elk_es03_1
f23356359140   docker.elastic.co/elasticsearch/elasticsearch:8.5.0   "/bin/tini -- /usr/l…"   2 hours ago   Up 2 hours (healthy)     9200/tcp, 9300/tcp                           elk_es02_1
0449df7c66bb   docker.elastic.co/elasticsearch/elasticsearch:8.5.0   "/bin/tini -- /usr/l…"   2 hours ago   Up 2 hours (healthy)     0.0.0.0:9200->9200/tcp, 9300/tcp             elk_es01_1
5

What is the value of MEM_LIMIT? Because it looks like you have 1.5GB of heap across 3 nodes, which is tiny.

6

where do i find this information?
by doing

GET /_nodes/stats/os

I receive

{
  "_nodes": {
    "total": 3,
    "successful": 3,
    "failed": 0
  },
  "cluster_name": "docker-cluster",
  "nodes": {
    "Y_g4Nks5RI6WVwDbEhN2Wg": {
      "timestamp": 1674121510426,
      "name": "es01",
      "transport_address": "172.24.0.3:9300",
      "host": "172.24.0.3",
      "ip": "172.24.0.3:9300",
      "roles": [
        "data",
        "data_cold",
        "data_content",
        "data_frozen",
        "data_hot",
        "data_warm",
        "ingest",
        "master",
        "ml",
        "remote_cluster_client",
        "transform"
      ],
      "attributes": {
        "ml.max_jvm_size": "536870912",
        "ml.allocated_processors_double": "8.0",
        "xpack.installed": "true",
        "ml.machine_memory": "1073741824",
        "ml.allocated_processors": "8"
      },
      "os": {
        "timestamp": 1674121510949,
        "cpu": {
          "percent": 19,
          "load_average": {
            "1m": 3.65,
            "5m": 3.5,
            "15m": 4.12
          }
        },
        "mem": {
          "total_in_bytes": 1073741824,
          "adjusted_total_in_bytes": 1073741824,
          "free_in_bytes": 2756608,
          "used_in_bytes": 1070985216,
          "free_percent": 0,
          "used_percent": 100
        },
        "swap": {
          "total_in_bytes": 1073741824,
          "free_in_bytes": 915169280,
          "used_in_bytes": 158572544
        },
        "cgroup": {
          "cpuacct": {
            "control_group": "/",
            "usage_nanos": 6520327463
          },
          "cpu": {
            "control_group": "/",
            "cfs_period_micros": 100000,
            "cfs_quota_micros": -1,
            "stat": {
              "number_of_elapsed_periods": 0,
              "number_of_times_throttled": 0,
              "time_throttled_nanos": 0
            }
          },
          "memory": {
            "control_group": "/",
            "limit_in_bytes": "1073741824",
            "usage_in_bytes": "1071087616"
          }
        }
      }
    },
    "UrMPFM4qS1S2WOO88f3KWQ": {
      "timestamp": 1674121510427,
      "name": "es03",
      "transport_address": "172.24.0.5:9300",
      "host": "172.24.0.5",
      "ip": "172.24.0.5:9300",
      "roles": [
        "data",
        "data_cold",
        "data_content",
        "data_frozen",
        "data_hot",
        "data_warm",
        "ingest",
        "master",
        "ml",
        "remote_cluster_client",
        "transform"
      ],
      "attributes": {
        "ml.allocated_processors_double": "8.0",
        "ml.machine_memory": "1073741824",
        "ml.max_jvm_size": "536870912",
        "xpack.installed": "true",
        "ml.allocated_processors": "8"
      },
      "os": {
        "timestamp": 1674121510610,
        "cpu": {
          "percent": 19,
          "load_average": {
            "1m": 3.65,
            "5m": 3.5,
            "15m": 4.12
          }
        },
        "mem": {
          "total_in_bytes": 1073741824,
          "adjusted_total_in_bytes": 1073741824,
          "free_in_bytes": 3072000,
          "used_in_bytes": 1070669824,
          "free_percent": 0,
          "used_percent": 100
        },
        "swap": {
          "total_in_bytes": 1073741824,
          "free_in_bytes": 883789824,
          "used_in_bytes": 189952000
        },
        "cgroup": {
          "cpuacct": {
            "control_group": "/",
            "usage_nanos": 1506654543
          },
          "cpu": {
            "control_group": "/",
            "cfs_period_micros": 100000,
            "cfs_quota_micros": -1,
            "stat": {
              "number_of_elapsed_periods": 0,
              "number_of_times_throttled": 0,
              "time_throttled_nanos": 0
            }
          },
          "memory": {
            "control_group": "/",
            "limit_in_bytes": "1073741824",
            "usage_in_bytes": "1070669824"
          }
        }
      }
    },
    "8yS8IJzxT-WcXaC1cmTlEw": {
      "timestamp": 1674121510427,
      "name": "es02",
      "transport_address": "172.24.0.4:9300",
      "host": "172.24.0.4",
      "ip": "172.24.0.4:9300",
      "roles": [
        "data",
        "data_cold",
        "data_content",
        "data_frozen",
        "data_hot",
        "data_warm",
        "ingest",
        "master",
        "ml",
        "remote_cluster_client",
        "transform"
      ],
      "attributes": {
        "ml.allocated_processors_double": "8.0",
        "ml.machine_memory": "1073741824",
        "ml.max_jvm_size": "536870912",
        "xpack.installed": "true",
        "ml.allocated_processors": "8"
      },
      "os": {
        "timestamp": 1674121510429,
        "cpu": {
          "percent": 19,
          "load_average": {
            "1m": 3.65,
            "5m": 3.5,
            "15m": 4.12
          }
        },
        "mem": {
          "total_in_bytes": 1073741824,
          "adjusted_total_in_bytes": 1073741824,
          "free_in_bytes": 2785280,
          "used_in_bytes": 1070956544,
          "free_percent": 0,
          "used_percent": 100
        },
        "swap": {
          "total_in_bytes": 1073741824,
          "free_in_bytes": 967667712,
          "used_in_bytes": 106074112
        },
        "cgroup": {
          "cpuacct": {
            "control_group": "/",
            "usage_nanos": 2825913908
          },
          "cpu": {
            "control_group": "/",
            "cfs_period_micros": 100000,
            "cfs_quota_micros": -1,
            "stat": {
              "number_of_elapsed_periods": 0,
              "number_of_times_throttled": 0,
              "time_throttled_nanos": 0
            }
          },
          "memory": {
            "control_group": "/",
            "limit_in_bytes": "1073741824",
            "usage_in_bytes": "1070956544"
          }
        }
      }
    }
  }
}```
7

sorry I was very tired, this is the value of MEM_LIMIT: 1073741824 (in bytes)

8

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.