Many deleted indices, no recovery of disk space?

Elastic Stack Elasticsearch
1

I have a 1.2TB backup of Elasticsearch data that I have restored to a standalone machine running 7.14.0. The restore worked fine and I began grooming the data - about half of the indices in the snapshot are not needed for what I am doing next.

I deleted them using the simplest little command:

curl $ELKAUTH -XDELETE "$ELKHOST/$1?pretty"

Several hours later I happened to look and that file system still has 1.24TB of space in use. I am expecting the cull of indices to reduce utilization to about 800GB.

Is there something I need to do in order to reduce the amount of disk space claimed by the system?

2

What is the full output of the cluster stats API?

3 
{
  "_nodes" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "6E1IB7JDSyKZ2_EjN-dYDw",
  "timestamp" : 1630806073234,
  "status" : "yellow",
  "indices" : {
    "count" : 105,
    "shards" : {
      "total" : 271,
      "primaries" : 271,
      "replication" : 0.0,
      "index" : {
        "shards" : {
          "min" : 1,
          "max" : 5,
          "avg" : 2.580952380952381
        },
        "primaries" : {
          "min" : 1,
          "max" : 5,
          "avg" : 2.580952380952381
        },
        "replication" : {
          "min" : 0.0,
          "max" : 0.0,
          "avg" : 0.0
        }
      }
    },
    "docs" : {
      "count" : 512628859,
      "deleted" : 63390939
    },
    "store" : {
      "size_in_bytes" : 1093285505781,
      "total_data_set_size_in_bytes" : 1093285505781,
      "reserved_in_bytes" : 0
    },
    "fielddata" : {
      "memory_size_in_bytes" : 0,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size_in_bytes" : 5343240,
      "total_count" : 6546,
      "hit_count" : 6137,
      "miss_count" : 409,
      "cache_size" : 23,
      "cache_count" : 26,
      "evictions" : 3
    },
    "completion" : {
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 3409,
      "memory_in_bytes" : 771665418,
      "terms_memory_in_bytes" : 264765152,
      "stored_fields_memory_in_bytes" : 324143024,
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory_in_bytes" : 34933312,
      "points_memory_in_bytes" : 0,
      "doc_values_memory_in_bytes" : 147823930,
      "index_writer_memory_in_bytes" : 107578516,
      "version_map_memory_in_bytes" : 8372093,
      "fixed_bit_set_memory_in_bytes" : 123976,
      "max_unsafe_auto_id_timestamp" : 1627370417998,
      "file_sizes" : { }
    },
    "mappings" : {
      "field_types" : [
        {
          "name" : "boolean",
          "count" : 2239,
          "index_count" : 98,
          "script_count" : 0
        },
        {
          "name" : "date",
          "count" : 302,
          "index_count" : 98,
          "script_count" : 0
        },
        {
          "name" : "float",
          "count" : 221,
          "index_count" : 43,
          "script_count" : 0
        },
        {
          "name" : "geo_point",
          "count" : 91,
          "index_count" : 91,
          "script_count" : 0
        },
        {
          "name" : "half_float",
          "count" : 10,
          "index_count" : 2,
          "script_count" : 0
        },
        {
          "name" : "integer",
          "count" : 44,
          "index_count" : 2,
          "script_count" : 0
        },
        {
          "name" : "keyword",
          "count" : 13621,
          "index_count" : 99,
          "script_count" : 0
        },
        {
          "name" : "long",
          "count" : 9332,
          "index_count" : 98,
          "script_count" : 0
        },
        {
          "name" : "nested",
          "count" : 6,
          "index_count" : 2,
          "script_count" : 0
        },
        {
          "name" : "object",
          "count" : 6295,
          "index_count" : 98,
          "script_count" : 0
        },
        {
          "name" : "text",
          "count" : 13684,
          "index_count" : 99,
          "script_count" : 0
        }
      ],
      "runtime_field_types" : [ ]
    },
    "analysis" : {
      "char_filter_types" : [ ],
      "tokenizer_types" : [ ],
      "filter_types" : [ ],
      "analyzer_types" : [ ],
      "built_in_char_filters" : [ ],
      "built_in_tokenizers" : [ ],
      "built_in_filters" : [ ],
      "built_in_analyzers" : [ ]
    },
    "versions" : [
      {
        "version" : "6.5.4",
        "index_count" : 12,
        "primary_shard_count" : 34,
        "total_primary_bytes" : 828095263116
      },
      {
        "version" : "6.8.2",
        "index_count" : 8,
        "primary_shard_count" : 24,
        "total_primary_bytes" : 12476911908
      },
      {
        "version" : "6.8.6",
        "index_count" : 7,
        "primary_shard_count" : 21,
        "total_primary_bytes" : 92940424478
      },
      {
        "version" : "6.8.12",
        "index_count" : 51,
        "primary_shard_count" : 155,
        "total_primary_bytes" : 149821008315
      },
      {
        "version" : "6.8.14",
        "index_count" : 1,
        "primary_shard_count" : 3,
        "total_primary_bytes" : 92713
      },
      {
        "version" : "6.8.15",
        "index_count" : 4,
        "primary_shard_count" : 12,
        "total_primary_bytes" : 1470329120
      },
      {
        "version" : "7.12.1",
        "index_count" : 5,
        "primary_shard_count" : 5,
        "total_primary_bytes" : 2398169
      },
      {
        "version" : "7.13.1",
        "index_count" : 9,
        "primary_shard_count" : 9,
        "total_primary_bytes" : 6607096168
      },
      {
        "version" : "7.13.2",
        "index_count" : 3,
        "primary_shard_count" : 3,
        "total_primary_bytes" : 1478553912
      },
      {
        "version" : "7.14.0",
        "index_count" : 5,
        "primary_shard_count" : 5,
        "total_primary_bytes" : 393427882
      }
    ]
  },
  "nodes" : {
    "count" : {
      "total" : 1,
      "coordinating_only" : 0,
      "data" : 1,
      "data_cold" : 1,
      "data_content" : 1,
      "data_frozen" : 1,
      "data_hot" : 1,
      "data_warm" : 1,
      "ingest" : 1,
      "master" : 1,
      "ml" : 1,
      "remote_cluster_client" : 1,
      "transform" : 1,
      "voting_only" : 0
    },
    "versions" : [
      "7.14.0"
    ],
    "os" : {
      "available_processors" : 24,
      "allocated_processors" : 24,
      "names" : [
        {
          "name" : "Linux",
          "count" : 1
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "Ubuntu 20.04.3 LTS",
          "count" : 1
        }
      ],
      "architectures" : [
        {
          "arch" : "amd64",
          "count" : 1
        }
      ],
      "mem" : {
        "total_in_bytes" : 135016177664,
        "free_in_bytes" : 52989681664,
        "used_in_bytes" : 82026496000,
        "free_percent" : 39,
        "used_percent" : 61
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 0
      },
      "open_file_descriptors" : {
        "min" : 2634,
        "max" : 2634,
        "avg" : 2634
      }
    },
    "jvm" : {
      "max_uptime_in_millis" : 109961452,
      "versions" : [
        {
          "version" : "16.0.1",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "16.0.1+9",
          "vm_vendor" : "AdoptOpenJDK",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 1
        }
      ],
      "mem" : {
        "heap_used_in_bytes" : 7661955232,
        "heap_max_in_bytes" : 33285996544
      },
      "threads" : 227
    },
    "fs" : {
      "total_in_bytes" : 1759837683712,
      "free_in_bytes" : 664941494272,
      "available_in_bytes" : 664941494272
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "netty4" : 1
      },
      "http_types" : {
        "netty4" : 1
      }
    },
    "discovery_types" : {
      "single-node" : 1
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "deb",
        "count" : 1
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 2,
      "processor_stats" : {
        "gsub" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "rename" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "script" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "set" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        }
      }
    }
  }
}
4

How many indices did you restore from the snapshot? Have you deleted or updated individual documents since you restored the snapshot?

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.