That does indeed look like my issue. Here are the fs stats:
I truncated the second API call due to the 13k character limit here
{
"nodes" : {
"fs" : {
"total_in_bytes" : 30717450911744,
"free_in_bytes" : 28375317204992,
"available_in_bytes" : 28375317204992
}
}
}
{
"_nodes" : {
"total" : 16,
"successful" : 16,
"failed" : 0
},
"cluster_name" : "es-docker-cluster",
"nodes" : {
"9RZJAn2PS-6CxIPR9aKjyg" : {
"timestamp" : 1586375495430,
"name" : "hostname-01-es04",
"transport_address" : "x.x.5.250:9900",
"host" : "x.x.5.250",
"ip" : "x.x.5.250:9900",
"roles" : [
"ingest",
"data",
"ml"
],
"attributes" : {
"rack_id" : "rack_301",
"ml.machine_memory" : "404347047936",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
},
"fs" : {
"timestamp" : 1586375494624,
"total" : {
"total_in_bytes" : 7679362727936,
"free_in_bytes" : 5521810702336,
"available_in_bytes" : 5521810702336
},
"data" : [
{
"path" : "/usr/share/elasticsearch/data/nodes/0",
"mount" : "/usr/share/elasticsearch/data (/dev/nvme3n1p1)",
"type" : "xfs",
"total_in_bytes" : 7679362727936,
"free_in_bytes" : 5521810702336,
"available_in_bytes" : 5521810702336
}
],
"io_stats" : {
"devices" : [
{
"device_name" : "nvme3n1p1",
"operations" : 1619310,
"read_operations" : 764,
"write_operations" : 1618546,
"read_kilobytes" : 51720,
"write_kilobytes" : 1654367500
}
],
"total" : {
"operations" : 1619310,
"read_operations" : 764,
"write_operations" : 1618546,
"read_kilobytes" : 51720,
"write_kilobytes" : 1654367500
}
}
}
},
"DcIiPEdZRGq7El1ELJkDXg" : {
"timestamp" : 1586375495426,
"name" : "hostname-03-es01",
"transport_address" : "x.x.5.252:9300",
"host" : "x.x.5.252",
"ip" : "x.x.5.252:9300",
"roles" : [
"ingest",
"master",
"data",
"ml"
],
"attributes" : {
"rack_id" : "rack_301",
"ml.machine_memory" : "404347047936",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
},
"fs" : {
"timestamp" : 1586375494620,
"total" : {
"total_in_bytes" : 7679362727936,
"free_in_bytes" : 7581575376896,
"available_in_bytes" : 7581575376896
},
"data" : [
{
"path" : "/usr/share/elasticsearch/data/nodes/0",
"mount" : "/usr/share/elasticsearch/data (/dev/nvme0n1p1)",
"type" : "xfs",
"total_in_bytes" : 7679362727936,
"free_in_bytes" : 7581575376896,
"available_in_bytes" : 7581575376896
}
],
"io_stats" : {
"devices" : [
{
"device_name" : "nvme0n1p1",
"operations" : 12634139,
"read_operations" : 0,
"write_operations" : 12634139,
"read_kilobytes" : 0,
"write_kilobytes" : 625598279
}
],
"total" : {
"operations" : 12634139,
"read_operations" : 0,
"write_operations" : 12634139,
"read_kilobytes" : 0,
"write_kilobytes" : 625598279
}
}
}
},
"N9vzVnsRQGGO-IZOetdMTA" : {
"timestamp" : 1586375495426,
"name" : "hostname-03-es02",
"transport_address" : "x.x.5.252:9500",
"host" : "x.x.5.252",
"ip" : "x.x.5.252:9500",
"roles" : [
"ingest",
"data",
"ml"
],
"attributes" : {
"rack_id" : "rack_301",
"ml.machine_memory" : "404347047936",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
},
"fs" : {
"timestamp" : 1586375494621,
"total" : {
"total_in_bytes" : 7679362727936,
"free_in_bytes" : 7670782722048,
"available_in_bytes" : 7670782722048
},
"data" : [
{
"path" : "/usr/share/elasticsearch/data/nodes/0",
"mount" : "/usr/share/elasticsearch/data (/dev/nvme1n1p1)",
"type" : "xfs",
"total_in_bytes" : 7679362727936,
"free_in_bytes" : 7670782722048,
"available_in_bytes" : 7670782722048
}
],
"io_stats" : {
"devices" : [
{
"device_name" : "nvme1n1p1",
"operations" : 489997,
"read_operations" : 0,
"write_operations" : 489997,
"read_kilobytes" : 0,
"write_kilobytes" : 3273794
}
],
"total" : {
"operations" : 489997,
"read_operations" : 0,
"write_operations" : 489997,
"read_kilobytes" : 0,
"write_kilobytes" : 3273794
}
}
}
},
"uDljzeL5RnCIWffTv4eX3w" : {
"timestamp" : 1586375495431,
"name" : "hostname-01-es02",
"transport_address" : "x.x.5.250:9500",
"host" : "x.x.5.250",
"ip" : "x.x.5.250:9500",
"roles" : [
"ingest",
"data",
"ml"
],
"attributes" : {
"rack_id" : "rack_301",
"ml.machine_memory" : "404347047936",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
},
"fs" : {
"timestamp" : 1586375494624,
"total" : {
"total_in_bytes" : 7679362727936,
"free_in_bytes" : 7670853480448,
"available_in_bytes" : 7670853480448
},
"data" : [
{
"path" : "/usr/share/elasticsearch/data/nodes/0",
"mount" : "/usr/share/elasticsearch/data (/dev/nvme1n1p1)",
"type" : "xfs",
"total_in_bytes" : 7679362727936,
"free_in_bytes" : 7670853480448,
"available_in_bytes" : 7670853480448
}
],
"io_stats" : {
"devices" : [
{
"device_name" : "nvme1n1p1",
"operations" : 562998,
"read_operations" : 1413,
"write_operations" : 561585,
"read_kilobytes" : 106912,
"write_kilobytes" : 4189300
}
],
"total" : {
"operations" : 562998,
"read_operations" : 1413,
"write_operations" : 561585,
"read_kilobytes" : 106912,
"write_kilobytes" : 4189300
}
}
}
},
"aTgMdbTtQHurPUdWXSeV3w" : {
"timestamp" : 1586375495424,
"name" : "hostname-02-es01",
"transport_address" : "x.x.5.251:9300",
"host" : "x.x.5.251",
"ip" : "x.x.5.251:9300",
"roles" : [
"ingest",
"master",
"data",
"ml"
],
"attributes" : {
"rack_id" : "rack_301",
"ml.machine_memory" : "404347047936",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
},
"fs" : {
"timestamp" : 1586375494619,
"total" : {
"total_in_bytes" : 7679362727936,
"free_in_bytes" : 7659147415552,
"available_in_bytes" : 7659147415552
},
"data" : [
{
"path" : "/usr/share/elasticsearch/data/nodes/0",
"mount" : "/usr/share/elasticsearch/data (/dev/nvme0n1p1)",
"type" : "xfs",
"total_in_bytes" : 7679362727936,
"free_in_bytes" : 7659147415552,
"available_in_bytes" : 7659147415552
}
],
"io_stats" : {
"devices" : [
{
"device_name" : "nvme0n1p1",
"operations" : 63438,
"read_operations" : 1,
"write_operations" : 63437,
"read_kilobytes" : 16,
"write_kilobytes" : 940659
}
],
"total" : {
"operations" : 63438,
"read_operations" : 1,
"write_operations" : 63437,
"read_kilobytes" : 16,
"write_kilobytes" : 940659
}
}
}
},
"VOsjy6AiQZSO5TyanxQ6yQ" : {
"timestamp" : 1586375495425,
"name" : "hostname-04-es03",
"transport_address" : "x.x.5.253:9700",
"host" : "x.x.5.253",
"ip" : "x.x.5.253:9700",
"roles" : [
"ingest",
"data",
"ml"
],
"attributes" : {
"rack_id" : "rack_301",
"ml.machine_memory" : "404347027456",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
},
"fs" : {
"timestamp" : 1586375494620,
"total" : {
"total_in_bytes" : 7679362727936,
"free_in_bytes" : 7613507076096,
"available_in_bytes" : 7613507076096
},
"data" : [
{
"path" : "/usr/share/elasticsearch/data/nodes/0",
"mount" : "/usr/share/elasticsearch/data (/dev/nvme2n1p1)",
"type" : "xfs",
"total_in_bytes" : 7679362727936,
"free_in_bytes" : 7613507076096,
"available_in_bytes" : 7613507076096
}
],
"io_stats" : {
"devices" : [
{
"device_name" : "nvme2n1p1",
"operations" : 6110,
"read_operations" : 0,
"write_operations" : 6110,
"read_kilobytes" : 0,
"write_kilobytes" : 36320
}
],
"total" : {
"operations" : 6110,
"read_operations" : 0,
"write_operations" : 6110,
"read_kilobytes" : 0,
"write_kilobytes" : 36320
}
}
}
},
...
}
}
}
}
}
}