Max 20 Million Documents?

I have indexed around 30-40 million documents to my ElasticSearch with the Bulk API and providing no _id field.

I noticed that the count of docs according to /_cat/indices is fixed at 20'000'000 even after a refresh. It also shows deleted documents and I have never deleted anything.

health status index                    pri rep docs.count docs.deleted store.size id
green  open   mydata                     1   1   20000000       958297      1.4gb

If I add 10 documents the docs.count does not increase but the docs.deleted goes up:

health status index                    pri rep docs.count docs.deleted store.size id
green  open   mydata                     1   1   20000000       958307      1.4gb

What is going on here?

This list from cerebro shows 40'000'000 docs but that may be adding up the 2 nodes/shards.

{ 
  "_shards": { 
    "total": 2,
    "successful": 2,
    "failed": 0
  },
  "_all": { 
    "primaries": { 
      "docs": { 
        "count": 20000000,
        "deleted": 958297
      },
      "store": { 
        "size": "764.3mb",
        "size_in_bytes": 801523888
      },
      "indexing": { 
        "index_total": 8500000,
        "index_time": "2.2m",
        "index_time_in_millis": 134997,
        "index_current": 0,
        "index_failed": 0,
        "delete_total": 0,
        "delete_time": "0s",
        "delete_time_in_millis": 0,
        "delete_current": 0,
        "noop_update_total": 0,
        "is_throttled": false,
        "throttle_time": "0s",
        "throttle_time_in_millis": 0
      },
      "merges": { 
        "current": 0,
        "current_docs": 0,
        "current_size": "0b",
        "current_size_in_bytes": 0,
        "total": 2,
        "total_time": "29.9s",
        "total_time_in_millis": 29991,
        "total_docs": 20078310,
        "total_size": "744mb",
        "total_size_in_bytes": 780158880,
        "total_stopped_time": "0s",
        "total_stopped_time_in_millis": 0,
        "total_throttled_time": "10.8s",
        "total_throttled_time_in_millis": 10881,
        "total_auto_throttle": "16.5mb",
        "total_auto_throttle_in_bytes": 17331834
      },
      "refresh": { 
        "total": 39,
        "total_time": "33.8s",
        "total_time_in_millis": 33846,
        "external_total": 9,
        "external_total_time": "1.6s",
        "external_total_time_in_millis": 1691,
        "listeners": 0
      },
      "flush": { 
        "total": 5,
        "periodic": 3,
        "total_time": "7.6s",
        "total_time_in_millis": 7626
      },
      "warmer": { 
        "current": 0,
        "total": 6,
        "total_time": "0s",
        "total_time_in_millis": 0
      },
      "segments": { 
        "count": 21,
        "memory": "3.2mb",
        "memory_in_bytes": 3441099,
        "terms_memory": "2.9mb",
        "terms_memory_in_bytes": 3055463,
        "stored_fields_memory": "371.4kb",
        "stored_fields_memory_in_bytes": 380344,
        "term_vectors_memory": "0b",
        "term_vectors_memory_in_bytes": 0,
        "norms_memory": "1.3kb",
        "norms_memory_in_bytes": 1344,
        "points_memory": "0b",
        "points_memory_in_bytes": 0,
        "doc_values_memory": "3.8kb",
        "doc_values_memory_in_bytes": 3948,
        "index_writer_memory": "0b",
        "index_writer_memory_in_bytes": 0,
        "version_map_memory": "0b",
        "version_map_memory_in_bytes": 0,
        "fixed_bit_set": "0b",
        "fixed_bit_set_memory_in_bytes": 0,
        "max_unsafe_auto_id_timestamp": -1,
        "file_sizes": { 

        }
      },
      "translog": { 
        "operations": 0,
        "size": "55b",
        "size_in_bytes": 55,
        "uncommitted_operations": 0,
        "uncommitted_size": "55b",
        "uncommitted_size_in_bytes": 55,
        "earliest_last_modified_age": 0
      },
      "request_cache": { 
        "memory_size": "0b",
        "memory_size_in_bytes": 0,
        "evictions": 0,
        "hit_count": 0,
        "miss_count": 0
      },
      "recovery": { 
        "current_as_source": 0,
        "current_as_target": 0,
        "throttle_time": "17.1s",
        "throttle_time_in_millis": 17160
      }
    },
    "total": { 
      "docs": { 
        "count": 40000000,
        "deleted": 1916594
      },
      "store": { 
        "size": "1.4gb",
        "size_in_bytes": 1604927609
      },
      "indexing": { 
        "index_total": 8801000,
        "index_time": "2.3m",
        "index_time_in_millis": 140745,
        "index_current": 0,
        "index_failed": 0,
        "delete_total": 0,
        "delete_time": "0s",
        "delete_time_in_millis": 0,
        "delete_current": 0,
        "noop_update_total": 0,
        "is_throttled": false,
        "throttle_time": "0s",
        "throttle_time_in_millis": 0
      },
      "get": { 
        "total": 0,
        "getTime": "0s",
        "time_in_millis": 0,
        "exists_total": 0,
        "exists_time": "0s",
        "exists_time_in_millis": 0,
        "missing_total": 0,
        "missing_time": "0s",
        "missing_time_in_millis": 0,
        "current": 0
      },
      "search": { 
        "open_contexts": 0,
        "query_total": 0,
        "query_time": "0s",
        "query_time_in_millis": 0,
        "query_current": 0,
        "fetch_total": 0,
        "fetch_time": "0s",
        "fetch_time_in_millis": 0,
        "fetch_current": 0,
        "scroll_total": 0,
        "scroll_time": "0s",
        "scroll_time_in_millis": 0,
        "scroll_current": 0,
        "suggest_total": 0,
        "suggest_time": "0s",
        "suggest_time_in_millis": 0,
        "suggest_current": 0
      },
      "merges": { 
        "current": 0,
        "current_docs": 0,
        "current_size": "0b",
        "current_size_in_bytes": 0,
        "total": 2,
        "total_time": "29.9s",
        "total_time_in_millis": 29991,
        "total_docs": 20078310,
        "total_size": "744mb",
        "total_size_in_bytes": 780158880,
        "total_stopped_time": "0s",
        "total_stopped_time_in_millis": 0,
        "total_throttled_time": "10.8s",
        "total_throttled_time_in_millis": 10881,
        "total_auto_throttle": "36.5mb",
        "total_auto_throttle_in_bytes": 38303354
      },
      "refresh": { 
        "total": 50,
        "total_time": "37.8s",
        "total_time_in_millis": 37825,
        "external_total": 15,
        "external_total_time": "1.7s",
        "external_total_time_in_millis": 1798,
        "listeners": 0
      },
      "flush": { 
        "total": 7,
        "periodic": 3,
        "total_time": "11s",
        "total_time_in_millis": 11095
      },
      "fielddata": { 
        "memory_size": "0b",
        "memory_size_in_bytes": 0,
        "evictions": 0
      },
      "completion": { 
        "size": "0b",
        "size_in_bytes": 0
      },
      "segments": { 
        "count": 43,
        "memory": "6.5mb",
        "memory_in_bytes": 6891815,
        "terms_memory": "5.8mb",
        "terms_memory_in_bytes": 6118587,
        "stored_fields_memory": "744.4kb",
        "stored_fields_memory_in_bytes": 762352,
        "term_vectors_memory": "0b",
        "term_vectors_memory_in_bytes": 0,
        "norms_memory": "2.6kb",
        "norms_memory_in_bytes": 2752,
        "points_memory": "0b",
        "points_memory_in_bytes": 0,
        "doc_values_memory": "7.9kb",
        "doc_values_memory_in_bytes": 8124,
        "index_writer_memory": "0b",
        "index_writer_memory_in_bytes": 0,
        "version_map_memory": "0b",
        "version_map_memory_in_bytes": 0,
        "fixed_bit_set": "0b",
        "fixed_bit_set_memory_in_bytes": 0,
        "max_unsafe_auto_id_timestamp": -1,
        "file_sizes": { 

        }
      },
      "recovery": { 
        "current_as_source": 0,
        "current_as_target": 0,
        "throttle_time": "22.1s",
        "throttle_time_in_millis": 22167
      }
    }
  },
  "indices": { 
    "mydata": { 
      "uuid": "hLp_HL8hTAKA51qlLraFOQ",
      "primaries": { 
        "docs": { 
          "count": 20000000,
          "deleted": 958297
        },
        "store": { 
          "size": "764.3mb",
          "size_in_bytes": 801523888
        },
        "indexing": { 
          "index_total": 8500000,
          "index_time": "2.2m",
          "index_time_in_millis": 134997,
          "index_current": 0,
          "index_failed": 0,
          "delete_total": 0,
          "delete_time": "0s",
          "delete_time_in_millis": 0,
          "delete_current": 0,
          "noop_update_total": 0,
          "is_throttled": false,
          "throttle_time": "0s",
          "throttle_time_in_millis": 0
        },
        "merges": { 
          "current": 0,
          "current_docs": 0,
          "current_size": "0b",
          "current_size_in_bytes": 0,
          "total": 2,
          "total_time": "29.9s",
          "total_time_in_millis": 29991,
          "total_docs": 20078310,
          "total_size": "744mb",
          "total_size_in_bytes": 780158880,
          "total_stopped_time": "0s",
          "total_stopped_time_in_millis": 0,
          "total_throttled_time": "10.8s",
          "total_throttled_time_in_millis": 10881,
          "total_auto_throttle": "16.5mb",
          "total_auto_throttle_in_bytes": 17331834
        },
        "refresh": { 
          "total": 39,
          "total_time": "33.8s",
          "total_time_in_millis": 33846,
          "external_total": 9,
          "external_total_time": "1.6s",
          "external_total_time_in_millis": 1691,
          "listeners": 0
        },
        "flush": { 
          "total": 5,
          "periodic": 3,
          "total_time": "7.6s",
          "total_time_in_millis": 7626
        },
        "completion": { 
          "size": "0b",
          "size_in_bytes": 0
        },
      },
      "total": { 
        "docs": { 
          "count": 40000000,
          "deleted": 1916594
        },
        "store": { 
          "size": "1.4gb",
          "size_in_bytes": 1604927609
        },
        "indexing": { 
          "index_total": 8801000,
          "index_time": "2.3m",
          "index_time_in_millis": 140745,
          "index_current": 0,
          "index_failed": 0,
          "delete_total": 0,
          "delete_time": "0s",
          "delete_time_in_millis": 0,
          "delete_current": 0,
          "noop_update_total": 0,
          "is_throttled": false,
          "throttle_time": "0s",
          "throttle_time_in_millis": 0
        },
        "get": { 
          "total": 0,
          "getTime": "0s",
          "time_in_millis": 0,
          "exists_total": 0,
          "exists_time": "0s",
          "exists_time_in_millis": 0,
          "missing_total": 0,
          "missing_time": "0s",
          "missing_time_in_millis": 0,
          "current": 0
        },
        "search": { 
          "open_contexts": 0,
          "query_total": 0,
          "query_time": "0s",
          "query_time_in_millis": 0,
          "query_current": 0,
          "fetch_total": 0,
          "fetch_time": "0s",
          "fetch_time_in_millis": 0,
          "fetch_current": 0,
          "scroll_total": 0,
          "scroll_time": "0s",
          "scroll_time_in_millis": 0,
          "scroll_current": 0,
          "suggest_total": 0,
          "suggest_time": "0s",
          "suggest_time_in_millis": 0,
          "suggest_current": 0
        },
        "merges": { 
          "current": 0,
          "current_docs": 0,
          "current_size": "0b",
          "current_size_in_bytes": 0,
          "total": 2,
          "total_time": "29.9s",
          "total_time_in_millis": 29991,
          "total_docs": 20078310,
          "total_size": "744mb",
          "total_size_in_bytes": 780158880,
          "total_stopped_time": "0s",
          "total_stopped_time_in_millis": 0,
          "total_throttled_time": "10.8s",
          "total_throttled_time_in_millis": 10881,
          "total_auto_throttle": "36.5mb",
          "total_auto_throttle_in_bytes": 38303354
        },
        "refresh": { 
          "total": 50,
          "total_time": "37.8s",
          "total_time_in_millis": 37825,
          "external_total": 15,
          "external_total_time": "1.7s",
          "external_total_time_in_millis": 1798,
          "listeners": 0
        }
		...

How are you adding data?

Yep, it came to me in my sleep. I WAS using an _id and then I removed it to see if it helped performance which it didn't so I re-added it and forgot about it.

My _id was a sequence number which reset to 0 with every run. So each run I was overwriting the last ones.

Oooops!

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.