High disk read resulting in io wait on new Cluster 7.9

Hey hope for some help on where to optimize

We have just moved our cluster into new and bigger servers. so we went from 8 to 5 data nodes since the hardware should have improved a lot.
Once we started our application a very high disk wait occurred, and the hosting company moved us to NVMe disks to keep load from affecting performance, since we run 30GB a sec reading on the disks.

We tried lowering shards (still running) from 995 and down to 732 now.
We have checked slow logs
We set mlockall on data nodes.

Kinda hitting a wall on what to do next to make the cluster use memory better instead of disk, any ideas?
If more data is required please let me know.

Probably heap size is low and that might be causing high disk io.

What are your vm flavours , like RAM , cores and disk size?

What are your index sizes ?

How many indices you have ?

What is the heap size for es data nodes ?

What is the full output of the cluster stats API? What does you workload mix look like in terms of reads, writes and deletes?

How does the specification of the new and old nodes compare?

What does the hot threads API show?

12 CPU(s) Intel(R) Xeon(R) CPU E5-2650 v4 @ 2.20GHz
meminfo

MemTotal:       82769716 kB
MemFree:         2742864 kB
MemAvailable:   43521316 kB

disk specs i can not see after the hosting company switched to NVMe, but i know we have like 2 TB data with a replica so 4 TB Total data on the cluster. And only 50% disk space used.

index vi have 80 plus and varies in size: with replica
one 560gb
nine 30-40gb
27 40-50gb
20 50-60gb
five 60-65gb
18 1-10gb

and heap is set to -Xms31g

{
  "_nodes" : {
    "total" : 8,
    "successful" : 8,
    "failed" : 0
  },
  "cluster_name" : "Elastic1",
  "cluster_uuid" : "xIDXx92LRaKlPnj-Js1Uog",
  "timestamp" : 1646490767125,
  "status" : "green",
  "indices" : {
    "count" : 97,
    "shards" : {
      "total" : 1440,
      "primaries" : 732,
      "replication" : 0.9672131147540983,
      "index" : {
        "shards" : {
          "min" : 2,
          "max" : 24,
          "avg" : 14.845360824742269
        },
        "primaries" : {
          "min" : 1,
          "max" : 12,
          "avg" : 7.546391752577319
        },
        "replication" : {
          "min" : 0.0,
          "max" : 1.0,
          "avg" : 0.9690721649484536
        }
      }
    },
    "docs" : {
      "count" : 5543064811,
      "deleted" : 133299604
    },
    "store" : {
      "size_in_bytes" : 3974021645915,
      "reserved_in_bytes" : 0
    },
    "fielddata" : {
      "memory_size_in_bytes" : 3446046144,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size_in_bytes" : 1280777846,
      "total_count" : 708575392,
      "hit_count" : 121682137,
      "miss_count" : 586893255,
      "cache_size" : 831398,
      "cache_count" : 1425830,
      "evictions" : 594432
    },
    "completion" : {
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 35415,
      "memory_in_bytes" : 485103518,
      "terms_memory_in_bytes" : 309193888,
      "stored_fields_memory_in_bytes" : 40158616,
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory_in_bytes" : 23282496,
      "points_memory_in_bytes" : 0,
      "doc_values_memory_in_bytes" : 112468518,
      "index_writer_memory_in_bytes" : 47531452,
      "version_map_memory_in_bytes" : 1216360,
      "fixed_bit_set_memory_in_bytes" : 38462984,
      "max_unsafe_auto_id_timestamp" : 1646385856850,
      "file_sizes" : { }
    },
    "mappings" : {
      "field_types" : [
        {
          "name" : "binary",
          "count" : 9,
          "index_count" : 1
        },
        {
          "name" : "boolean",
          "count" : 1488,
          "index_count" : 87
        },
        {
          "name" : "date",
          "count" : 1301,
          "index_count" : 96
        },
        {
          "name" : "double",
          "count" : 136,
          "index_count" : 68
        },
        {
          "name" : "flattened",
          "count" : 9,
          "index_count" : 1
        },
        {
          "name" : "float",
          "count" : 3,
          "index_count" : 1
        },
        {
          "name" : "integer",
          "count" : 29,
          "index_count" : 2
        },
        {
          "name" : "keyword",
          "count" : 5838,
          "index_count" : 91
        },
        {
          "name" : "long",
          "count" : 3126,
          "index_count" : 94
        },
        {
          "name" : "nested",
          "count" : 44,
          "index_count" : 19
        },
        {
          "name" : "object",
          "count" : 1174,
          "index_count" : 32
        },
        {
          "name" : "text",
          "count" : 4507,
          "index_count" : 95
        }
      ]
    },
    "analysis" : {
      "char_filter_types" : [ ],
      "tokenizer_types" : [ ],
      "filter_types" : [ ],
      "analyzer_types" : [
        {
          "name" : "custom",
          "count" : 88,
          "index_count" : 88
        }
      ],
      "built_in_char_filters" : [ ],
      "built_in_tokenizers" : [
        {
          "name" : "keyword",
          "count" : 88,
          "index_count" : 88
        }
      ],
      "built_in_filters" : [
        {
          "name" : "lowercase",
          "count" : 88,
          "index_count" : 88
        }
      ],
      "built_in_analyzers" : [ ]
    }
  },
  "nodes" : {
    "count" : {
      "total" : 8,
      "coordinating_only" : 0,
      "data" : 5,
      "ingest" : 8,
      "master" : 3,
      "ml" : 8,
      "remote_cluster_client" : 8,
      "transform" : 5,
      "voting_only" : 0
    },
    "versions" : [
      "7.9.3"
    ],
    "os" : {
      "available_processors" : 72,
      "allocated_processors" : 72,
      "names" : [
        {
          "name" : "Linux",
          "count" : 8
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "AlmaLinux 8.5 (Arctic Sphynx)",
          "count" : 8
        }
      ],
      "mem" : {
        "total_in_bytes" : 447530594304,
        "free_in_bytes" : 5114134528,
        "used_in_bytes" : 442416459776,
        "free_percent" : 1,
        "used_percent" : 99
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 131
      },
      "open_file_descriptors" : {
        "min" : 567,
        "max" : 3833,
        "avg" : 2571
      }
    },
    "jvm" : {
      "max_uptime_in_millis" : 3158952459,
      "versions" : [
        {
          "version" : "15",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "15+36-1562",
          "vm_vendor" : "Oracle Corporation",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 8
        }
      ],
      "mem" : {
        "heap_used_in_bytes" : 86760326416,
        "heap_max_in_bytes" : 179314884608
      },
      "threads" : 782
    },
    "fs" : {
      "total_in_bytes" : 7987622051840,
      "free_in_bytes" : 3926288547840,
      "available_in_bytes" : 3926288547840
    },
    "plugins" : [
      {
        "name" : "repository-s3",
        "version" : "7.9.3",
        "elasticsearch_version" : "7.9.3",
        "java_version" : "1.8",
        "description" : "The S3 repository plugin adds S3 repositories",
        "classname" : "org.elasticsearch.repositories.s3.S3RepositoryPlugin",
        "extended_plugins" : [ ],
        "has_native_controller" : false
      }
    ],
    "network_types" : {
      "transport_types" : {
        "security4" : 8
      },
      "http_types" : {
        "security4" : 8
      }
    },
    "discovery_types" : {
      "zen" : 8
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "rpm",
        "count" : 8
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 2,
      "processor_stats" : {
        "gsub" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "script" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        }
      }
    }
  }
}

Thanks for taking a look.
We have continuously load of read and write, I dont have any numbers yet but can try to find if needed.

New setup should be better and i cant get the old specs any more since those a closed now.

did not know _nodes/hot_threads
can you help.on what it means?

::: {m1}{IyF4kZN3TnGckocWnky-JA}{tpeHKXKlR66-yc0Gld41Bw}{172.20.10.100}{172.20.10.100:9300}{ilmr}{ml.machine_memory=8117882880, xpack.installed=true, transform.node=false, ml.max_open_jobs=20}
   Hot threads at 2022-03-05T14:38:05.857Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
   
    3.5% (17.5ms out of 500ms) cpu usage by thread 'elasticsearch[m1][transport_worker][T#1]'
     8/10 snapshots sharing following 3 elements
       io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
       io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
       java.base@15/java.lang.Thread.run(Thread.java:832)

::: {d1}{mXpl5rJsRH-3EvYcKESmJA}{u3vd6mzwSkOg2UiejC_PZQ}{172.20.10.110}{172.20.10.110:9300}{dilrt}{ml.machine_memory=84756189184, ml.max_open_jobs=20, xpack.installed=true, transform.node=true}
   Hot threads at 2022-03-05T14:38:05.860Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

::: {d2}{NdH40pAdSHmflONpmT0Jvw}{g5_jEI3TTDCmYXGDne_rQQ}{172.20.10.111}{172.20.10.111:9300}{dilrt}{ml.machine_memory=84756189184, ml.max_open_jobs=20, xpack.installed=true, transform.node=true}
   Hot threads at 2022-03-05T14:38:05.860Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
   
    5.5% (27.3ms out of 500ms) cpu usage by thread 'elasticsearch[d2][write][T#8]'
     2/10 snapshots sharing following 10 elements
       app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:209)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:116)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:82)
       app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:161)
       app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:737)
       app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
       java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
       java.base@15/java.lang.Thread.run(Thread.java:832)

::: {m2}{upsB5lyOQIC3TeqFIqDjTg}{8ilTlpDyQtaV76nnOpi57g}{172.20.10.101}{172.20.10.101:9300}{ilmr}{ml.machine_memory=8117891072, ml.max_open_jobs=20, xpack.installed=true, transform.node=false}
   Hot threads at 2022-03-05T14:38:05.858Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

::: {m3}{fqwbWd4oStWezNHDj3__9Q}{VV0WrLB8R_GWM1AaznIaAw}{172.20.10.102}{172.20.10.102:9300}{ilmr}{ml.machine_memory=8117882880, ml.max_open_jobs=20, xpack.installed=true, transform.node=false}
   Hot threads at 2022-03-05T14:38:05.858Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

::: {d3}{ar08KCliQ5u-hs2tofyb1Q}{NTwUUAwSRWO4P-BLjqbYBw}{172.20.10.112}{172.20.10.112:9300}{dilrt}{ml.machine_memory=84756180992, ml.max_open_jobs=20, xpack.installed=true, transform.node=true}
   Hot threads at 2022-03-05T14:38:05.859Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
   
   65.6% (328ms out of 500ms) cpu usage by thread 'elasticsearch[d3][[events_2019_10][2]: Lucene Merge Thread #193]'
     2/10 snapshots sharing following 20 elements
       app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$TermsDict.seekExact(Lucene80DocValuesProducer.java:1124)
       app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$BaseSortedDocValues.lookupOrd(Lucene80DocValuesProducer.java:1006)
       app//org.apache.lucene.index.SingletonSortedSetDocValues.lookupOrd(SingletonSortedSetDocValues.java:92)
       app//org.apache.lucene.codecs.DocValuesConsumer$5$1.lookupOrd(DocValuesConsumer.java:886)
       app//org.apache.lucene.search.SortedSetSelector$MinValue.lookupOrd(SortedSetSelector.java:141)
       app//org.apache.lucene.index.SortedDocValuesTermsEnum.next(SortedDocValuesTermsEnum.java:83)
       app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addTermsDict(Lucene80DocValuesConsumer.java:622)
       app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.doAddSortedField(Lucene80DocValuesConsumer.java:603)
       app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:746)
       app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:804)
       app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:145)
       app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
       app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
       app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
       app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4586)
       app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4190)
       app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5741)
       app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
       app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:100)
       app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:682)
     8/10 snapshots sharing following 13 elements
       app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.doAddSortedField(Lucene80DocValuesConsumer.java:603)
       app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:746)
       app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:804)
       app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:145)
       app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
       app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
       app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
       app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4586)
       app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4190)
       app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5741)
       app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
       app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:100)
       app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:682)
   
    7.8% (38.7ms out of 500ms) cpu usage by thread 'elasticsearch[d3][write][T#7]'
     7/10 snapshots sharing following 2 elements
       java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
       java.base@15/java.lang.Thread.run(Thread.java:832)

::: {d5}{paw-pc_tTGS7y67cpGrKKw}{DrXqqz9wSOuMwY8MiU_Tpw}{172.20.10.114}{172.20.10.114:9300}{dilrt}{ml.machine_memory=84152188928, ml.max_open_jobs=20, xpack.installed=true, transform.node=true}
   Hot threads at 2022-03-05T14:38:05.858Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
   
    7.6% (37.8ms out of 500ms) cpu usage by thread 'elasticsearch[d5][write][T#3]'
     2/10 snapshots sharing following 15 elements
       app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:819)
       app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:791)
       app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:748)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:274)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:164)
       app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:209)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:116)
       app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:82)
       app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:161)
       app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:737)
       app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
       java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
       java.base@15/java.lang.Thread.run(Thread.java:832)
     5/10 snapshots sharing following 10 elements
       java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
       java.base@15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
       java.base@15/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:743)
       java.base@15/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
       java.base@15/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1366)
       app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165)
       java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
       java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
       java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
       java.base@15/java.lang.Thread.run(Thread.java:832)

::: {d4}{K-mGTGwLROSCZtsB9Ypphg}{YdGhxS5CSlmXFrZk1Pr-CA}{172.20.10.113}{172.20.10.113:9300}{dilrt}{ml.machine_memory=84756189184, ml.max_open_jobs=20, xpack.installed=true, transform.node=true}
   Hot threads at 2022-03-05T14:38:05.860Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:


guess elasticsearch[d3][[events_2019_10][2] is in there because a reindex is currently running.

FYI 7.9 is EOL and unsupported, you should really upgrade to latest 7.17, or 8.0.

1 Like

upgraded to 7.17 today and will monitor for a day or two and send results :slight_smile:

New stats taken while cluster workload is at lowest.

{
  "_nodes" : {
    "total" : 8,
    "successful" : 8,
    "failed" : 0
  },
  "cluster_name" : "Elastic1",
  "cluster_uuid" : "xIDXx92LRaKlPnj-Js1Uog",
  "timestamp" : 1648012978723,
  "status" : "green",
  "indices" : {
    "count" : 105,
    "shards" : {
      "total" : 1250,
      "primaries" : 625,
      "replication" : 1.0,
      "index" : {
        "shards" : {
          "min" : 2,
          "max" : 24,
          "avg" : 11.904761904761905
        },
        "primaries" : {
          "min" : 1,
          "max" : 12,
          "avg" : 5.9523809523809526
        },
        "replication" : {
          "min" : 1.0,
          "max" : 1.0,
          "avg" : 1.0
        }
      }
    },
    "docs" : {
      "count" : 5557740204,
      "deleted" : 32804974
    },
    "store" : {
      "size_in_bytes" : 4038916568550,
      "total_data_set_size_in_bytes" : 4038916568550,
      "reserved_in_bytes" : 0
    },
    "fielddata" : {
      "memory_size_in_bytes" : 3474736828,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size_in_bytes" : 1201372494,
      "total_count" : 190609439,
      "hit_count" : 25231204,
      "miss_count" : 165378235,
      "cache_size" : 542217,
      "cache_count" : 546740,
      "evictions" : 4523
    },
    "completion" : {
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 30976,
      "memory_in_bytes" : 425801444,
      "terms_memory_in_bytes" : 275149048,
      "stored_fields_memory_in_bytes" : 37897888,
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory_in_bytes" : 20758336,
      "points_memory_in_bytes" : 0,
      "doc_values_memory_in_bytes" : 91996172,
      "index_writer_memory_in_bytes" : 61598172,
      "version_map_memory_in_bytes" : 75168,
      "fixed_bit_set_memory_in_bytes" : 39459240,
      "max_unsafe_auto_id_timestamp" : 1647941291967,
      "file_sizes" : { }
    },
    "mappings" : {
      "field_types" : [
        {
          "name" : "boolean",
          "count" : 1481,
          "index_count" : 86,
          "script_count" : 0
        },
        {
          "name" : "constant_keyword",
          "count" : 3,
          "index_count" : 1,
          "script_count" : 0
        },
        {
          "name" : "date",
          "count" : 1261,
          "index_count" : 96,
          "script_count" : 0
        },
        {
          "name" : "double",
          "count" : 134,
          "index_count" : 67,
          "script_count" : 0
        },
        {
          "name" : "float",
          "count" : 2,
          "index_count" : 1,
          "script_count" : 0
        },
        {
          "name" : "ip",
          "count" : 1,
          "index_count" : 1,
          "script_count" : 0
        },
        {
          "name" : "keyword",
          "count" : 5581,
          "index_count" : 91,
          "script_count" : 0
        },
        {
          "name" : "long",
          "count" : 3113,
          "index_count" : 95,
          "script_count" : 0
        },
        {
          "name" : "nested",
          "count" : 34,
          "index_count" : 19,
          "script_count" : 0
        },
        {
          "name" : "object",
          "count" : 1066,
          "index_count" : 32,
          "script_count" : 0
        },
        {
          "name" : "text",
          "count" : 4403,
          "index_count" : 96,
          "script_count" : 0
        },
        {
          "name" : "version",
          "count" : 1,
          "index_count" : 1,
          "script_count" : 0
        }
      ],
      "runtime_field_types" : [ ]
    },
    "analysis" : {
      "char_filter_types" : [ ],
      "tokenizer_types" : [ ],
      "filter_types" : [ ],
      "analyzer_types" : [
        {
          "name" : "custom",
          "count" : 87,
          "index_count" : 87
        }
      ],
      "built_in_char_filters" : [ ],
      "built_in_tokenizers" : [
        {
          "name" : "keyword",
          "count" : 87,
          "index_count" : 87
        }
      ],
      "built_in_filters" : [
        {
          "name" : "lowercase",
          "count" : 87,
          "index_count" : 87
        }
      ],
      "built_in_analyzers" : [ ]
    },
    "versions" : [
      {
        "version" : "7.9.0",
        "index_count" : 6,
        "primary_shard_count" : 48,
        "total_primary_bytes" : 314947963586
      },
      {
        "version" : "7.9.3",
        "index_count" : 94,
        "primary_shard_count" : 572,
        "total_primary_bytes" : 1703845903226
      },
      {
        "version" : "7.17.1",
        "index_count" : 5,
        "primary_shard_count" : 5,
        "total_primary_bytes" : 57448334
      }
    ]
  },
  "nodes" : {
    "count" : {
      "total" : 8,
      "coordinating_only" : 0,
      "data" : 5,
      "data_cold" : 5,
      "data_content" : 5,
      "data_frozen" : 5,
      "data_hot" : 5,
      "data_warm" : 5,
      "ingest" : 8,
      "master" : 3,
      "ml" : 8,
      "remote_cluster_client" : 8,
      "transform" : 5,
      "voting_only" : 0
    },
    "versions" : [
      "7.17.1"
    ],
    "os" : {
      "available_processors" : 72,
      "allocated_processors" : 72,
      "names" : [
        {
          "name" : "Linux",
          "count" : 8
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "AlmaLinux 8.5 (Arctic Sphynx)",
          "count" : 8
        }
      ],
      "architectures" : [
        {
          "arch" : "amd64",
          "count" : 8
        }
      ],
      "mem" : {
        "total_in_bytes" : 447530594304,
        "free_in_bytes" : 4072624128,
        "used_in_bytes" : 443457970176,
        "free_percent" : 1,
        "used_percent" : 99
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 10
      },
      "open_file_descriptors" : {
        "min" : 697,
        "max" : 3612,
        "avg" : 2492
      }
    },
    "jvm" : {
      "max_uptime_in_millis" : 72727384,
      "versions" : [
        {
          "version" : "17.0.2",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "17.0.2+8",
          "vm_vendor" : "Eclipse Adoptium",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 8
        }
      ],
      "mem" : {
        "heap_used_in_bytes" : 74178322280,
        "heap_max_in_bytes" : 179314884608
      },
      "threads" : 871
    },
    "fs" : {
      "total_in_bytes" : 7987622051840,
      "free_in_bytes" : 3873329098752,
      "available_in_bytes" : 3873329098752
    },
    "plugins" : [
      {
        "name" : "repository-s3",
        "version" : "7.17.1",
        "elasticsearch_version" : "7.17.1",
        "java_version" : "1.8",
        "description" : "The S3 repository plugin adds S3 repositories",
        "classname" : "org.elasticsearch.repositories.s3.S3RepositoryPlugin",
        "extended_plugins" : [ ],
        "has_native_controller" : false,
        "licensed" : false,
        "type" : "isolated"
      }
    ],
    "network_types" : {
      "transport_types" : {
        "security4" : 8
      },
      "http_types" : {
        "security4" : 8
      }
    },
    "discovery_types" : {
      "zen" : 8
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "rpm",
        "count" : 8
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 2,
      "processor_stats" : {
        "gsub" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "script" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        }
      }
    }
  }
}

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.