High Load ELK on docker

Elastic Stack Elasticsearch
1

I've been having issues with my ELK deployment. Its on docker and the CPU/Load usage is always high for some reason I still cannot find.

image
image237×504 11.1 KB

Each ES node has 4GB memory assigned, total memory on the machine is 32GB.
I've tried disabling swapping, bootstrap memory lock and other solutions I've been able to find in the forums but none of them seems to work

the docker-compose file is

version: '2.2'
services:
  es-node01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.14.1
    container_name: es-node01
    environment:
      - node.name=es-node01
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es-node02,03
      - cluster.initial_master_nodes=es-node01,es-node02,es-node03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms4048m -Xmx4048m"
#      - xpack.security.enabled=true
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - nes-node01:/usr/share/elasticsearch/data
#      - /opt/ELK/elasticsearch/config/:/usr/share/elasticsearch/config/
    ports:
      - 9200:9200
    networks:
      - elastic
    logging:
      driver: "json-file"
      options:
        max-size: "100m"
        max-file: "3"

  es-node02:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.14.1
    container_name: es-node02
    environment:
      - node.name=es-node02
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es-node01,es-node03
      - cluster.initial_master_nodes=es-node01,es-node02,es-node03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms4048m -Xmx4048m"
#      - xpack.security.enabled=true
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - nes-node02:/usr/share/elasticsearch/data
#      - /opt/ELK/elasticsearch/config/:/usr/share/elasticsearch/config/
    ports:
      - 9200
    networks:
      - elastic
    logging:
      driver: "json-file"
      options:
        max-size: "100m"
        max-file: "3"

  es-node03:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.14.1
    container_name: es-node03
    environment:
      - node.name=es-node03
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es-node01,es-node02
      - cluster.initial_master_nodes=es-node01,es-node02,es-node03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms4048m -Xmx4048m"
#      - xpack.security.enabled=true
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - nes-node03:/usr/share/elasticsearch/data
#      - /opt/ELK/elasticsearch/config/:/usr/share/elasticsearch/config/
    ports:
      - 9200
    networks:
      - elastic
    logging:
      driver: "json-file"
      options:
        max-size: "100m"
        max-file: "3"

  es-node04:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.14.1
    container_name: es-node04
    environment:
      - node.name=es-node04
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es-node01,es-node02,es-node03
      - cluster.initial_master_nodes=es-node01,es-node02,es-node03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms4048m -Xmx4048m"
#      - xpack.security.enabled=true
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - nes-node04:/usr/share/elasticsearch/data
#      - /opt/ELK/elasticsearch/config/:/usr/share/elasticsearch/config/
    ports:
      - 9200
    networks:
      - elastic
    logging:
      driver: "json-file"
      options:
        max-size: "100m"
        max-file: "3"

  kib01:
    image: docker.elastic.co/kibana/kibana:7.14.1
    container_name: kib01
    ports:
      - 5601:5601
    environment:
      ELASTICSEARCH_HOSTS: '["http://es-node01:9200","http://es-node02:9200","http://es-node03:                                                                                                                                                             9200"]'
    volumes:
      - /opt/ELK/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
    networks:
      - elastic

  logstash:
    image: docker.elastic.co/logstash/logstash:7.14.1
    container_name: logstash-node01
    ports:
      - 9001:9001
      - "9002:9002/udp"
      - "9003:9003/udp"
    networks:
      - elastic
    volumes:
      - /opt/ELK/logstash/config/:/usr/share/logstash/config/

volumes:
  nes-node01:
    driver: local
  nes-node02:
    driver: local
  nes-node03:
    driver: local
  nes-node04:
    driver: local

networks:
  elastic:
    driver: bridge

Hot Threads:

2

Welcome to our community! :smiley:

What is the output from the _cluster/stats?pretty&human API?

3 
{
  "_nodes" : {
    "total" : 4,
    "successful" : 4,
    "failed" : 0
  },
  "cluster_name" : "es-docker-cluster-produccion",
  "cluster_uuid" : "JGD8e5nMRRi3rxJ-NEkF1Q",
  "timestamp" : 1635948932304,
  "status" : "green",
  "indices" : {
    "count" : 104,
    "shards" : {
      "total" : 534,
      "primaries" : 267,
      "replication" : 1.0,
      "index" : {
        "shards" : {
          "min" : 2,
          "max" : 8,
          "avg" : 5.134615384615385
        },
        "primaries" : {
          "min" : 1,
          "max" : 4,
          "avg" : 2.5673076923076925
        },
        "replication" : {
          "min" : 1.0,
          "max" : 1.0,
          "avg" : 1.0
        }
      }
    },
    "docs" : {
      "count" : 393148293,
      "deleted" : 438467
    },
    "store" : {
      "size" : "588.4gb",
      "size_in_bytes" : 631847054705,
      "total_data_set_size" : "588.4gb",
      "total_data_set_size_in_bytes" : 631847054705,
      "reserved" : "0b",
      "reserved_in_bytes" : 0
    },
    "fielddata" : {
      "memory_size" : "78.4mb",
      "memory_size_in_bytes" : 82286656,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "17.6mb",
      "memory_size_in_bytes" : 18498745,
      "total_count" : 96545291,
      "hit_count" : 2367562,
      "miss_count" : 94177729,
      "cache_size" : 12127,
      "cache_count" : 97804,
      "evictions" : 85677
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 6268,
      "memory" : "397.5mb",
      "memory_in_bytes" : 416845214,
      "terms_memory" : "335.1mb",
      "terms_memory_in_bytes" : 351461216,
      "stored_fields_memory" : "3.2mb",
      "stored_fields_memory_in_bytes" : 3360928,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "48.3mb",
      "norms_memory_in_bytes" : 50670464,
      "points_memory" : "0b",
      "points_memory_in_bytes" : 0,
      "doc_values_memory" : "10.8mb",
      "doc_values_memory_in_bytes" : 11352606,
      "index_writer_memory" : "604.2mb",
      "index_writer_memory_in_bytes" : 633618624,
      "version_map_memory" : "0b",
      "version_map_memory_in_bytes" : 0,
      "fixed_bit_set" : "108.8kb",
      "fixed_bit_set_memory_in_bytes" : 111448,
      "max_unsafe_auto_id_timestamp" : 1635945361842,
      "file_sizes" : { }
    },
    "mappings" : {
      "field_types" : [
        {
          "name" : "alias",
          "count" : 6,
          "index_count" : 2,
          "script_count" : 0
        },
        {
          "name" : "boolean",
          "count" : 186,
          "index_count" : 16,
          "script_count" : 0
        },
        {
          "name" : "byte",
          "count" : 2,
          "index_count" : 2,
          "script_count" : 0
        },
        {
          "name" : "date",
          "count" : 372,
          "index_count" : 97,
          "script_count" : 0
        },
        {
          "name" : "double",
          "count" : 398,
          "index_count" : 9,
          "script_count" : 0
        },
        {
          "name" : "float",
          "count" : 447,
          "index_count" : 13,
          "script_count" : 0
        },
        {
          "name" : "geo_point",
          "count" : 14,
          "index_count" : 2,
          "script_count" : 0
        },
        {
          "name" : "half_float",
          "count" : 10,
          "index_count" : 2,
          "script_count" : 0
        },
        {
          "name" : "integer",
          "count" : 116,
          "index_count" : 20,
          "script_count" : 0
        },
        {
          "name" : "ip",
          "count" : 114,
          "index_count" : 20,
          "script_count" : 0
        },
        {
          "name" : "keyword",
          "count" : 12557,
          "index_count" : 97,
          "script_count" : 0
        },
        {
          "name" : "long",
          "count" : 6120,
          "index_count" : 96,
          "script_count" : 0
        },
        {
          "name" : "nested",
          "count" : 7,
          "index_count" : 3,
          "script_count" : 0
        },
        {
          "name" : "object",
          "count" : 6322,
          "index_count" : 44,
          "script_count" : 0
        },
        {
          "name" : "scaled_float",
          "count" : 296,
          "index_count" : 2,
          "script_count" : 0
        },
        {
          "name" : "text",
          "count" : 10387,
          "index_count" : 90,
          "script_count" : 0
        }
      ],
      "runtime_field_types" : [ ]
    },
    "analysis" : {
      "char_filter_types" : [ ],
      "tokenizer_types" : [ ],
      "filter_types" : [ ],
      "analyzer_types" : [ ],
      "built_in_char_filters" : [ ],
      "built_in_tokenizers" : [ ],
      "built_in_filters" : [ ],
      "built_in_analyzers" : [ ]
    },
    "versions" : [
      {
        "version" : "7.14.2",
        "index_count" : 104,
        "primary_shard_count" : 267,
        "total_primary_size" : "294.4gb",
        "total_primary_bytes" : 316140010918
      }
    ]
  },
  "nodes" : {
    "count" : {
      "total" : 4,
      "coordinating_only" : 0,
      "data" : 4,
      "data_cold" : 4,
      "data_content" : 4,
      "data_frozen" : 4,
      "data_hot" : 4,
      "data_warm" : 4,
      "ingest" : 4,
      "master" : 4,
      "ml" : 4,
      "remote_cluster_client" : 4,
      "transform" : 4,
      "voting_only" : 0
    },
    "versions" : [
      "7.14.2"
    ],
    "os" : {
      "available_processors" : 16,
      "allocated_processors" : 16,
      "names" : [
        {
          "name" : "Linux",
          "count" : 4
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "CentOS Linux 8",
          "count" : 4
        }
      ],
      "architectures" : [
        {
          "arch" : "amd64",
          "count" : 4
        }
      ],
      "mem" : {
        "total" : "123.8gb",
        "total_in_bytes" : 132996890624,
        "free" : "957.2mb",
        "free_in_bytes" : 1003765760,
        "used" : "122.9gb",
        "used_in_bytes" : 131993124864,
        "free_percent" : 1,
        "used_percent" : 99
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 90
      },
      "open_file_descriptors" : {
        "min" : 1860,
        "max" : 2094,
        "avg" : 1973
      }
    },
    "jvm" : {
      "max_uptime" : "5.7d",
      "max_uptime_in_millis" : 499143897,
      "versions" : [
        {
          "version" : "16.0.2",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "16.0.2+7",
          "vm_vendor" : "Eclipse Foundation",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 4
        }
      ],
      "mem" : {
        "heap_used" : "8.9gb",
        "heap_used_in_bytes" : 9613691392,
        "heap_max" : "16gb",
        "heap_max_in_bytes" : 17179869184
      },
      "threads" : 377
    },
    "fs" : {
      "total" : "1.9tb",
      "total_in_bytes" : 2109484302336,
      "free" : "1.3tb",
      "free_in_bytes" : 1468857372672,
      "available" : "1.2tb",
      "available_in_bytes" : 1361416081408
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "security4" : 4
      },
      "http_types" : {
        "security4" : 4
      }
    },
    "discovery_types" : {
      "zen" : 4
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "docker",
        "count" : 4
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 2,
      "processor_stats" : {
        "gsub" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        },
        "script" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        }
      }
    }
  }
4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.