The count and index_total not same , why?


(andy_zhou) #1

with cerebro show the index stats .
i find the count and the index_total not same..

{
  "_shards": {
    "total": 84,
    "successful": 84,
    "failed": 0
  },
  "_all": {
    "primaries": {
      "docs": {
        "count": 643101511,
        "deleted": 0
      },
      "store": {
        "size": "115.7gb",
        "size_in_bytes": 124301239359,
        "throttle_time": "0s",
        "throttle_time_in_millis": 0
      },
      "indexing": {
        "index_total": 244966695,
        "index_time": "4.4d",
        "index_time_in_millis": 383165652,
        "index_current": 0,
        "index_failed": 0,
        "delete_total": 0,
        "delete_time": "0s",
        "delete_time_in_millis": 0,
        "delete_current": 0,
        "noop_update_total": 0,
        "is_throttled": false,
        "throttle_time": "0s",
        "throttle_time_in_millis": 0
      },
      "get": {
        "total": 0,
        "getTime": "0s",
        "time_in_millis": 0,
        "exists_total": 0,
        "exists_time": "0s",
        "exists_time_in_millis": 0,
        "missing_total": 0,
        "missing_time": "0s",
        "missing_time_in_millis": 0,
        "current": 0
      },
      "search": {
        "open_contexts": 11,
        "query_total": 49422,
        "query_time": "18.1m",
        "query_time_in_millis": 1089513,
        "query_current": 0,
        "fetch_total": 49420,
        "fetch_time": "1.9h",
        "fetch_time_in_millis": 7182579,
        "fetch_current": 0,
        "scroll_total": 8,
        "scroll_time": "2.6h",
        "scroll_time_in_millis": 9692070,
        "scroll_current": 11,
        "suggest_total": 0,
        "suggest_time": "0s",
        "suggest_time_in_millis": 0,
        "suggest_current": 0
      },
      "merges": {
        "current": 0,
        "current_docs": 0,
        "current_size": "0b",
        "current_size_in_bytes": 0,
        "total": 8895,
        "total_time": "16h",
        "total_time_in_millis": 57922826,
        "total_docs": 2417805565,
        "total_size": "442.4gb",
        "total_size_in_bytes": 475083196817,
        "total_stopped_time": "0s",
        "total_stopped_time_in_millis": 0,
        "total_throttled_time": "3h",
        "total_throttled_time_in_millis": 10843838,
        "total_auto_throttle": "361.7mb",
        "total_auto_throttle_in_bytes": 379286604
      },
      "refresh": {
        "total": 4552,
        "total_time": "1.9h",
        "total_time_in_millis": 6997405,
        "listeners": 0
      },
      "flush": {
        "total": 420,
        "total_time": "8.4m",
        "total_time_in_millis": 505639
      },
      "warmer": {
        "current": 0,
        "total": 4524,
        "total_time": "2.1m",
        "total_time_in_millis": 130547
      },
      "query_cache": {
        "memory_size": "125.9mb",
        "memory_size_in_bytes": 132070232,
        "total_count": 142901,
        "hit_count": 49350,
        "miss_count": 93551,
        "cache_size": 46,
        "cache_count": 46,
        "evictions": 0
      },
      "fielddata": {
        "memory_size": "0b",
        "memory_size_in_bytes": 0,
        "evictions": 0
      },
      "completion": {
        "size": "0b",
        "size_in_bytes": 0
      },
      "segments": {
        "count": 28,
        "memory": "221.8mb",
        "memory_in_bytes": 232643361,
        "terms_memory": "192.1mb",
        "terms_memory_in_bytes": 201519825,
        "stored_fields_memory": "12.5mb",
        "stored_fields_memory_in_bytes": 13134616,
        "term_vectors_memory": "0b",
        "term_vectors_memory_in_bytes": 0,
        "norms_memory": "5.2kb",
        "norms_memory_in_bytes": 5376,
        "points_memory": "17.1mb",
        "points_memory_in_bytes": 17958840,
        "doc_values_memory": "24.1kb",
        "doc_values_memory_in_bytes": 24704,
        "index_writer_memory": "0b",
        "index_writer_memory_in_bytes": 0,
        "version_map_memory": "0b",
        "version_map_memory_in_bytes": 0,
        "fixed_bit_set": "76.6mb",
        "fixed_bit_set_memory_in_bytes": 80389384,
        "max_unsafe_auto_id_timestamp": 1541088004630,
        "file_sizes": {

        }
      },
      "translog": {
        "operations": 0,
        "size": "1.1kb",
        "size_in_bytes": 1204
      },
      "request_cache": {
        "memory_size": "0b",
        "memory_size_in_bytes": 0,
        "evictions": 0,
        "hit_count": 0,
        "miss_count": 0
      },
      "recovery": {
        "current_as_source": 0,
        "current_as_target": 0,
        "throttle_time": "0s",
        "throttle_time_in_millis": 0
      }
    },
    "total": {
      "docs": {
        "count": 1929304533,
        "deleted": 0
      },
      "store": {
        "size": "347gb",
        "size_in_bytes": 372665507676,
        "throttle_time": "0s",
        "throttle_time_in_millis": 0
      },

(andy_zhou) #2
 "indexing": {
        "index_total": 734894178,
        "index_time": "11.2d",
        "index_time_in_millis": 971039982,
        "index_current": 0,
        "index_failed": 0,
        "delete_total": 0,
        "delete_time": "0s",
        "delete_time_in_millis": 0,
        "delete_current": 0,
        "noop_update_total": 0,
        "is_throttled": false,
        "throttle_time": "0s",
        "throttle_time_in_millis": 0
      },
      "get": {
        "total": 0,
        "getTime": "0s",
        "time_in_millis": 0,
        "exists_total": 0,
        "exists_time": "0s",
        "exists_time_in_millis": 0,
        "missing_total": 0,
        "missing_time": "0s",
        "missing_time_in_millis": 0,
        "current": 0
      },
      "search": {
        "open_contexts": 28,
        "query_total": 126168,
        "query_time": "50.3m",
        "query_time_in_millis": 3022389,
        "query_current": 0,
        "fetch_total": 126140,
        "fetch_time": "5.1h",
        "fetch_time_in_millis": 18533724,
        "fetch_current": 0,
        "scroll_total": 28,
        "scroll_time": "9.4h",
        "scroll_time_in_millis": 33985078,
        "scroll_current": 28,
        "suggest_total": 0,
        "suggest_time": "0s",
        "suggest_time_in_millis": 0,
        "suggest_current": 0
      },
      "merges": {
        "current": 0,
        "current_docs": 0,
        "current_size": "0b",
        "current_size_in_bytes": 0,
        "total": 23489,
        "total_time": "1.9d",
        "total_time_in_millis": 168320469,
        "total_docs": 7050107979,
        "total_size": "1.2tb",
        "total_size_in_bytes": 1375856763245,
        "total_stopped_time": "0s",
        "total_stopped_time_in_millis": 0,
        "total_throttled_time": "8.5h",
        "total_throttled_time_in_millis": 30943098,
        "total_auto_throttle": "1gb",
        "total_auto_throttle_in_bytes": 1093553497
      },
      "refresh": {
        "total": 13659,
        "total_time": "5.6h",
        "total_time_in_millis": 20495294,
        "listeners": 0
      },
      "flush": {
        "total": 1260,
        "total_time": "23.6m",
        "total_time_in_millis": 1417800
      },
      "warmer": {
        "current": 0,
        "total": 13630,
        "total_time": "6.4m",
        "total_time_in_millis": 386905
      },
      "query_cache": {
        "memory_size": "380.6mb",
        "memory_size_in_bytes": 399134519,
        "total_count": 364184,
        "hit_count": 125888,
        "miss_count": 238296,
        "cache_size": 139,
        "cache_count": 139,
        "evictions": 0
      },
      "fielddata": {
        "memory_size": "0b",
        "memory_size_in_bytes": 0,
        "evictions": 0
      },
      "completion": {
        "size": "0b",
        "size_in_bytes": 0
      },
      "segments": {
        "count": 84,
        "memory": "665.5mb",
        "memory_in_bytes": 697876006,
        "terms_memory": "576.5mb",
        "terms_memory_in_bytes": 604556972,
        "stored_fields_memory": "37.5mb",
        "stored_fields_memory_in_bytes": 39355728,
        "term_vectors_memory": "0b",
        "term_vectors_memory_in_bytes": 0,
        "norms_memory": "15.7kb",
        "norms_memory_in_bytes": 16128,
        "points_memory": "51.3mb",
        "points_memory_in_bytes": 53873066,
        "doc_values_memory": "72.3kb",
        "doc_values_memory_in_bytes": 74112,
        "index_writer_memory": "0b",
        "index_writer_memory_in_bytes": 0,
        "version_map_memory": "0b",
        "version_map_memory_in_bytes": 0,
        "fixed_bit_set": "229.9mb",
        "fixed_bit_set_memory_in_bytes": 241168152,
        "max_unsafe_auto_id_timestamp": 1541088004630,
        "file_sizes": {

        }
      },
      "translog": {
        "operations": 0,
        "size": "3.5kb",
        "size_in_bytes": 3612
      },
      "request_cache": {
        "memory_size": "0b",
        "memory_size_in_bytes": 0,
        "evictions": 0,
        "hit_count": 0,
        "miss_count": 0
      },
      "recovery": {
        "current_as_source": 0,
        "current_as_target": 0,
        "throttle_time": "0s",
        "throttle_time_in_millis": 0
      }
    }
  },
  "indices": {
    "xxx-xxx20181102": {
      "primaries": {
        "docs": {
          "count": 643101511,
          "deleted": 0
        },
        "store": {
          "size": "115.7gb",
          "size_in_bytes": 124301239359,
          "throttle_time": "0s",
          "throttle_time_in_millis": 0
        },
        "indexing": {
          "index_total": 244966695,
          "index_time": "4.4d",
          "index_time_in_millis": 383165652,
          "index_current": 0,
          "index_failed": 0,
          "delete_total": 0,
          "delete_time": "0s",
          "delete_time_in_millis": 0,
          "delete_current": 0,
          "noop_update_total": 0,
          "is_throttled": false,
          "throttle_time": "0s",
          "throttle_time_in_millis": 0
        },
        "get": {
          "total": 0,
          "getTime": "0s",
          "time_in_millis": 0,
          "exists_total": 0,
          "exists_time": "0s",
          "exists_time_in_millis": 0,
          "missing_total": 0,
          "missing_time": "0s",
          "missing_time_in_millis": 0,
          "current": 0
        },
        "search": {
          "open_contexts": 11,
          "query_total": 49422,
          "query_time": "18.1m",
          "query_time_in_millis": 1089513,
          "query_current": 0,
          "fetch_total": 49420,
          "fetch_time": "1.9h",
          "fetch_time_in_millis": 7182579,
          "fetch_current": 0,
          "scroll_total": 8,
          "scroll_time": "2.6h",
          "scroll_time_in_millis": 9692070,
          "scroll_current": 11,
          "suggest_total": 0,
          "suggest_time": "0s",
          "suggest_time_in_millis": 0,
          "suggest_current": 0

(andy_zhou) #3
 "merges": {
          "current": 0,
          "current_docs": 0,
          "current_size": "0b",
          "current_size_in_bytes": 0,
          "total": 8895,
          "total_time": "16h",
          "total_time_in_millis": 57922826,
          "total_docs": 2417805565,
          "total_size": "442.4gb",
          "total_size_in_bytes": 475083196817,
          "total_stopped_time": "0s",
          "total_stopped_time_in_millis": 0,
          "total_throttled_time": "3h",
          "total_throttled_time_in_millis": 10843838,
          "total_auto_throttle": "361.7mb",
          "total_auto_throttle_in_bytes": 379286604
        },
        "refresh": {
          "total": 4552,
          "total_time": "1.9h",
          "total_time_in_millis": 6997405,
          "listeners": 0
        },
        "flush": {
          "total": 420,
          "total_time": "8.4m",
          "total_time_in_millis": 505639
        },
        "warmer": {
          "current": 0,
          "total": 4524,
          "total_time": "2.1m",
          "total_time_in_millis": 130547
        },
        "query_cache": {
          "memory_size": "125.9mb",
          "memory_size_in_bytes": 132070232,
          "total_count": 142901,
          "hit_count": 49350,
          "miss_count": 93551,
          "cache_size": 46,
          "cache_count": 46,
          "evictions": 0
        },
        "fielddata": {
          "memory_size": "0b",
          "memory_size_in_bytes": 0,
          "evictions": 0
        },
        "completion": {
          "size": "0b",
          "size_in_bytes": 0
        },
        "segments": {
          "count": 28,
          "memory": "221.8mb",
          "memory_in_bytes": 232643361,
          "terms_memory": "192.1mb",
          "terms_memory_in_bytes": 201519825,
          "stored_fields_memory": "12.5mb",
          "stored_fields_memory_in_bytes": 13134616,
          "term_vectors_memory": "0b",
          "term_vectors_memory_in_bytes": 0,
          "norms_memory": "5.2kb",
          "norms_memory_in_bytes": 5376,
          "points_memory": "17.1mb",
          "points_memory_in_bytes": 17958840,
          "doc_values_memory": "24.1kb",
          "doc_values_memory_in_bytes": 24704,
          "index_writer_memory": "0b",
          "index_writer_memory_in_bytes": 0,
          "version_map_memory": "0b",
          "version_map_memory_in_bytes": 0,
          "fixed_bit_set": "76.6mb",
          "fixed_bit_set_memory_in_bytes": 80389384,
          "max_unsafe_auto_id_timestamp": 1541088004630,
          "file_sizes": {

          }
        },
        "translog": {
          "operations": 0,
          "size": "1.1kb",
          "size_in_bytes": 1204
        },
        "request_cache": {
          "memory_size": "0b",
          "memory_size_in_bytes": 0,
          "evictions": 0,
          "hit_count": 0,
          "miss_count": 0
        },
        "recovery": {
          "current_as_source": 0,
          "current_as_target": 0,
          "throttle_time": "0s",
          "throttle_time_in_millis": 0
        }
      },
      "total": {
        "docs": {
          "count": 1929304533,
          "deleted": 0
        },
        "store": {
          "size": "347gb",
          "size_in_bytes": 372665507676,
          "throttle_time": "0s",
          "throttle_time_in_millis": 0
        },
        "indexing": {
          "index_total": 734894178,
          "index_time": "11.2d",
          "index_time_in_millis": 971039982,
          "index_current": 0,
          "index_failed": 0,
          "delete_total": 0,
          "delete_time": "0s",
          "delete_time_in_millis": 0,
          "delete_current": 0,
          "noop_update_total": 0,
          "is_throttled": false,
          "throttle_time": "0s",
          "throttle_time_in_millis": 0
        },
        "get": {
          "total": 0,
          "getTime": "0s",
          "time_in_millis": 0,
          "exists_total": 0,
          "exists_time": "0s",
          "exists_time_in_millis": 0,
          "missing_total": 0,
          "missing_time": "0s",
          "missing_time_in_millis": 0,
          "current": 0
        },
        "search": {
          "open_contexts": 28,
          "query_total": 126168,
          "query_time": "50.3m",
          "query_time_in_millis": 3022389,
          "query_current": 0,
          "fetch_total": 126140,
          "fetch_time": "5.1h",
          "fetch_time_in_millis": 18533724,
          "fetch_current": 0,
          "scroll_total": 28,
          "scroll_time": "9.4h",
          "scroll_time_in_millis": 33985078,
          "scroll_current": 28,
          "suggest_total": 0,
          "suggest_time": "0s",
          "suggest_time_in_millis": 0,
          "suggest_current": 0
        },
        "merges": {
          "current": 0,
          "current_docs": 0,
          "current_size": "0b",
          "current_size_in_bytes": 0,
          "total": 23489,
          "total_time": "1.9d",
          "total_time_in_millis": 168320469,
          "total_docs": 7050107979,
          "total_size": "1.2tb",
          "total_size_in_bytes": 1375856763245,
          "total_stopped_time": "0s",
          "total_stopped_time_in_millis": 0,
          "total_throttled_time": "8.5h",
          "total_throttled_time_in_millis": 30943098,
          "total_auto_throttle": "1gb",
          "total_auto_throttle_in_bytes": 1093553497
        },
        "refresh": {
          "total": 13659,
          "total_time": "5.6h",
          "total_time_in_millis": 20495294,
          "listeners": 0
        },

(andy_zhou) #4
 "refresh": {
          "total": 13659,
          "total_time": "5.6h",
          "total_time_in_millis": 20495294,
          "listeners": 0
        },
        "flush": {
          "total": 1260,
          "total_time": "23.6m",
          "total_time_in_millis": 1417800
        },
        "warmer": {
          "current": 0,
          "total": 13630,
          "total_time": "6.4m",
          "total_time_in_millis": 386905
        },
        "query_cache": {
          "memory_size": "380.6mb",
          "memory_size_in_bytes": 399134519,
          "total_count": 364184,
          "hit_count": 125888,
          "miss_count": 238296,
          "cache_size": 139,
          "cache_count": 139,
          "evictions": 0
        },
        "fielddata": {
          "memory_size": "0b",
          "memory_size_in_bytes": 0,
          "evictions": 0
        },
        "completion": {
          "size": "0b",
          "size_in_bytes": 0
        },
        "segments": {
          "count": 84,
          "memory": "665.5mb",
          "memory_in_bytes": 697876006,
          "terms_memory": "576.5mb",
          "terms_memory_in_bytes": 604556972,
          "stored_fields_memory": "37.5mb",
          "stored_fields_memory_in_bytes": 39355728,
          "term_vectors_memory": "0b",
          "term_vectors_memory_in_bytes": 0,
          "norms_memory": "15.7kb",
          "norms_memory_in_bytes": 16128,
          "points_memory": "51.3mb",
          "points_memory_in_bytes": 53873066,
          "doc_values_memory": "72.3kb",
          "doc_values_memory_in_bytes": 74112,
          "index_writer_memory": "0b",
          "index_writer_memory_in_bytes": 0,
          "version_map_memory": "0b",
          "version_map_memory_in_bytes": 0,
          "fixed_bit_set": "229.9mb",
          "fixed_bit_set_memory_in_bytes": 241168152,
          "max_unsafe_auto_id_timestamp": 1541088004630,
          "file_sizes": {

          }
        },
        "translog": {
          "operations": 0,
          "size": "3.5kb",
          "size_in_bytes": 3612
        },
        "request_cache": {
          "memory_size": "0b",
          "memory_size_in_bytes": 0,
          "evictions": 0,
          "hit_count": 0,
          "miss_count": 0
        },
        "recovery": {
          "current_as_source": 0,
          "current_as_target": 0,
          "throttle_time": "0s",
          "throttle_time_in_millis": 0
        }
      }
    }
  }
}

(andy_zhou) #5

why ? it ok? how i can do about it ?
"_all": {
"primaries": {
"docs": {
"count": 643101511,
"deleted": 0
},
"store": {
"size": "115.7gb",
"size_in_bytes": 124301239359,
"throttle_time": "0s",
"throttle_time_in_millis": 0
},
"indexing": {
"index_total": 244966695,
"index_time": "4.4d",
"index_time_in_millis": 383165652,


(Christian Dahlqvist) #6

I suspect the reason might be that the count represents what is currently indexed on disk while the indexing counter (and many of the other stats) might just be kept in memory and therefore reset on a restart.


(Matija Bruncic) #7

Hi,

Are you maybe using nested or parent/child relationship? I have similar case since I use nested objects and that's why some documents are counted twice. There are indeed multiple Lucene documents for single Elasticsearch document. Cerebro will show you Lucene document count while Elasticsearch _count/_search API will return Elasticsearch document count.


(andy_zhou) #8

yes the doc something with nested .
but hao i can do ?


(system) #9

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.