I have indexed around 30-40 million documents to my ElasticSearch with the Bulk API and providing no _id field.
I noticed that the count of docs according to /_cat/indices is fixed at 20'000'000 even after a refresh. It also shows deleted documents and I have never deleted anything.
health status index pri rep docs.count docs.deleted store.size id
green open mydata 1 1 20000000 958297 1.4gb
If I add 10 documents the docs.count does not increase but the docs.deleted goes up:
health status index pri rep docs.count docs.deleted store.size id
green open mydata 1 1 20000000 958307 1.4gb
What is going on here?
This list from cerebro shows 40'000'000 docs but that may be adding up the 2 nodes/shards.
{
"_shards": {
"total": 2,
"successful": 2,
"failed": 0
},
"_all": {
"primaries": {
"docs": {
"count": 20000000,
"deleted": 958297
},
"store": {
"size": "764.3mb",
"size_in_bytes": 801523888
},
"indexing": {
"index_total": 8500000,
"index_time": "2.2m",
"index_time_in_millis": 134997,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time": "0s",
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time": "0s",
"throttle_time_in_millis": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size": "0b",
"current_size_in_bytes": 0,
"total": 2,
"total_time": "29.9s",
"total_time_in_millis": 29991,
"total_docs": 20078310,
"total_size": "744mb",
"total_size_in_bytes": 780158880,
"total_stopped_time": "0s",
"total_stopped_time_in_millis": 0,
"total_throttled_time": "10.8s",
"total_throttled_time_in_millis": 10881,
"total_auto_throttle": "16.5mb",
"total_auto_throttle_in_bytes": 17331834
},
"refresh": {
"total": 39,
"total_time": "33.8s",
"total_time_in_millis": 33846,
"external_total": 9,
"external_total_time": "1.6s",
"external_total_time_in_millis": 1691,
"listeners": 0
},
"flush": {
"total": 5,
"periodic": 3,
"total_time": "7.6s",
"total_time_in_millis": 7626
},
"warmer": {
"current": 0,
"total": 6,
"total_time": "0s",
"total_time_in_millis": 0
},
"segments": {
"count": 21,
"memory": "3.2mb",
"memory_in_bytes": 3441099,
"terms_memory": "2.9mb",
"terms_memory_in_bytes": 3055463,
"stored_fields_memory": "371.4kb",
"stored_fields_memory_in_bytes": 380344,
"term_vectors_memory": "0b",
"term_vectors_memory_in_bytes": 0,
"norms_memory": "1.3kb",
"norms_memory_in_bytes": 1344,
"points_memory": "0b",
"points_memory_in_bytes": 0,
"doc_values_memory": "3.8kb",
"doc_values_memory_in_bytes": 3948,
"index_writer_memory": "0b",
"index_writer_memory_in_bytes": 0,
"version_map_memory": "0b",
"version_map_memory_in_bytes": 0,
"fixed_bit_set": "0b",
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": {
}
},
"translog": {
"operations": 0,
"size": "55b",
"size_in_bytes": 55,
"uncommitted_operations": 0,
"uncommitted_size": "55b",
"uncommitted_size_in_bytes": 55,
"earliest_last_modified_age": 0
},
"request_cache": {
"memory_size": "0b",
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 0
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time": "17.1s",
"throttle_time_in_millis": 17160
}
},
"total": {
"docs": {
"count": 40000000,
"deleted": 1916594
},
"store": {
"size": "1.4gb",
"size_in_bytes": 1604927609
},
"indexing": {
"index_total": 8801000,
"index_time": "2.3m",
"index_time_in_millis": 140745,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time": "0s",
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time": "0s",
"throttle_time_in_millis": 0
},
"get": {
"total": 0,
"getTime": "0s",
"time_in_millis": 0,
"exists_total": 0,
"exists_time": "0s",
"exists_time_in_millis": 0,
"missing_total": 0,
"missing_time": "0s",
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 0,
"query_time": "0s",
"query_time_in_millis": 0,
"query_current": 0,
"fetch_total": 0,
"fetch_time": "0s",
"fetch_time_in_millis": 0,
"fetch_current": 0,
"scroll_total": 0,
"scroll_time": "0s",
"scroll_time_in_millis": 0,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time": "0s",
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size": "0b",
"current_size_in_bytes": 0,
"total": 2,
"total_time": "29.9s",
"total_time_in_millis": 29991,
"total_docs": 20078310,
"total_size": "744mb",
"total_size_in_bytes": 780158880,
"total_stopped_time": "0s",
"total_stopped_time_in_millis": 0,
"total_throttled_time": "10.8s",
"total_throttled_time_in_millis": 10881,
"total_auto_throttle": "36.5mb",
"total_auto_throttle_in_bytes": 38303354
},
"refresh": {
"total": 50,
"total_time": "37.8s",
"total_time_in_millis": 37825,
"external_total": 15,
"external_total_time": "1.7s",
"external_total_time_in_millis": 1798,
"listeners": 0
},
"flush": {
"total": 7,
"periodic": 3,
"total_time": "11s",
"total_time_in_millis": 11095
},
"fielddata": {
"memory_size": "0b",
"memory_size_in_bytes": 0,
"evictions": 0
},
"completion": {
"size": "0b",
"size_in_bytes": 0
},
"segments": {
"count": 43,
"memory": "6.5mb",
"memory_in_bytes": 6891815,
"terms_memory": "5.8mb",
"terms_memory_in_bytes": 6118587,
"stored_fields_memory": "744.4kb",
"stored_fields_memory_in_bytes": 762352,
"term_vectors_memory": "0b",
"term_vectors_memory_in_bytes": 0,
"norms_memory": "2.6kb",
"norms_memory_in_bytes": 2752,
"points_memory": "0b",
"points_memory_in_bytes": 0,
"doc_values_memory": "7.9kb",
"doc_values_memory_in_bytes": 8124,
"index_writer_memory": "0b",
"index_writer_memory_in_bytes": 0,
"version_map_memory": "0b",
"version_map_memory_in_bytes": 0,
"fixed_bit_set": "0b",
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": {
}
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time": "22.1s",
"throttle_time_in_millis": 22167
}
}
},
"indices": {
"mydata": {
"uuid": "hLp_HL8hTAKA51qlLraFOQ",
"primaries": {
"docs": {
"count": 20000000,
"deleted": 958297
},
"store": {
"size": "764.3mb",
"size_in_bytes": 801523888
},
"indexing": {
"index_total": 8500000,
"index_time": "2.2m",
"index_time_in_millis": 134997,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time": "0s",
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time": "0s",
"throttle_time_in_millis": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size": "0b",
"current_size_in_bytes": 0,
"total": 2,
"total_time": "29.9s",
"total_time_in_millis": 29991,
"total_docs": 20078310,
"total_size": "744mb",
"total_size_in_bytes": 780158880,
"total_stopped_time": "0s",
"total_stopped_time_in_millis": 0,
"total_throttled_time": "10.8s",
"total_throttled_time_in_millis": 10881,
"total_auto_throttle": "16.5mb",
"total_auto_throttle_in_bytes": 17331834
},
"refresh": {
"total": 39,
"total_time": "33.8s",
"total_time_in_millis": 33846,
"external_total": 9,
"external_total_time": "1.6s",
"external_total_time_in_millis": 1691,
"listeners": 0
},
"flush": {
"total": 5,
"periodic": 3,
"total_time": "7.6s",
"total_time_in_millis": 7626
},
"completion": {
"size": "0b",
"size_in_bytes": 0
},
},
"total": {
"docs": {
"count": 40000000,
"deleted": 1916594
},
"store": {
"size": "1.4gb",
"size_in_bytes": 1604927609
},
"indexing": {
"index_total": 8801000,
"index_time": "2.3m",
"index_time_in_millis": 140745,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time": "0s",
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time": "0s",
"throttle_time_in_millis": 0
},
"get": {
"total": 0,
"getTime": "0s",
"time_in_millis": 0,
"exists_total": 0,
"exists_time": "0s",
"exists_time_in_millis": 0,
"missing_total": 0,
"missing_time": "0s",
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 0,
"query_time": "0s",
"query_time_in_millis": 0,
"query_current": 0,
"fetch_total": 0,
"fetch_time": "0s",
"fetch_time_in_millis": 0,
"fetch_current": 0,
"scroll_total": 0,
"scroll_time": "0s",
"scroll_time_in_millis": 0,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time": "0s",
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size": "0b",
"current_size_in_bytes": 0,
"total": 2,
"total_time": "29.9s",
"total_time_in_millis": 29991,
"total_docs": 20078310,
"total_size": "744mb",
"total_size_in_bytes": 780158880,
"total_stopped_time": "0s",
"total_stopped_time_in_millis": 0,
"total_throttled_time": "10.8s",
"total_throttled_time_in_millis": 10881,
"total_auto_throttle": "36.5mb",
"total_auto_throttle_in_bytes": 38303354
},
"refresh": {
"total": 50,
"total_time": "37.8s",
"total_time_in_millis": 37825,
"external_total": 15,
"external_total_time": "1.7s",
"external_total_time_in_millis": 1798,
"listeners": 0
}
...