Index does not accept docs from the start

I create indices hourly and bulk insert a lot of data. It usually works well, but I've experienced those spikes where the number of documents just drops to 0. That's when the index is created. What is this? =(


The screenshot is from Kibana. This index for 22:00-23:00 only starts accepting docs in the very end. The next index is fine.

Hourly indices is something I have never seen used successfully in practice. What is your use case? What is your retention period? What is the full output of the cluster stats API?

Our ES stores metadata of user requests (browser, etc) for ads. The retention period is, well, indefinite, as far as I know.

Output is:

{
	"_nodes": {
		"total": 3,
		"successful": 3,
		"failed": 0
	},
	"cluster_name": "inventory-cluster",
	"cluster_uuid": "N9PS1MEjRmGISyldE_6_8Q",
	"timestamp": 1643900486551,
	"status": "green",
	"indices": {
		"count": 780,
		"shards": {
			"total": 791,
			"primaries": 780,
			"replication": 0.014102564102564103,
			"index": {
				"shards": {
					"min": 1,
					"max": 2,
					"avg": 1.014102564102564
				},
				"primaries": {
					"min": 1,
					"max": 1,
					"avg": 1
				},
				"replication": {
					"min": 0,
					"max": 1,
					"avg": 0.014102564102564103
				}
			}
		},
		"docs": {
			"count": 85996056788,
			"deleted": 19434
		},
		"store": {
			"size_in_bytes": 6264857988008,
			"total_data_set_size_in_bytes": 6264857988008,
			"reserved_in_bytes": 0
		},
		"fielddata": {
			"memory_size_in_bytes": 64814320,
			"evictions": 0
		},
		"query_cache": {
			"memory_size_in_bytes": 3638343819,
			"total_count": 149964198,
			"hit_count": 5432947,
			"miss_count": 144531251,
			"cache_size": 37404,
			"cache_count": 171155,
			"evictions": 133751
		},
		"completion": {
			"size_in_bytes": 0
		},
		"segments": {
			"count": 20436,
			"memory_in_bytes": 159814200,
			"terms_memory_in_bytes": 100031776,
			"stored_fields_memory_in_bytes": 22120976,
			"term_vectors_memory_in_bytes": 0,
			"norms_memory_in_bytes": 9125952,
			"points_memory_in_bytes": 0,
			"doc_values_memory_in_bytes": 28535496,
			"index_writer_memory_in_bytes": 106816252,
			"version_map_memory_in_bytes": 0,
			"fixed_bit_set_memory_in_bytes": 63368,
			"max_unsafe_auto_id_timestamp": 1643900400952,
			"file_sizes": {}
		},
		"mappings": {
			"field_types": [
				{
					"name": "boolean",
					"count": 2,
					"index_count": 2,
					"script_count": 0
				},
				{
					"name": "date",
					"count": 791,
					"index_count": 773,
					"script_count": 0
				},
				{
					"name": "float",
					"count": 4,
					"index_count": 2,
					"script_count": 0
				},
				{
					"name": "keyword",
					"count": 5487,
					"index_count": 773,
					"script_count": 0
				},
				{
					"name": "long",
					"count": 779,
					"index_count": 773,
					"script_count": 0
				},
				{
					"name": "nested",
					"count": 2,
					"index_count": 2,
					"script_count": 0
				},
				{
					"name": "object",
					"count": 20,
					"index_count": 4,
					"script_count": 0
				},
				{
					"name": "text",
					"count": 5397,
					"index_count": 773,
					"script_count": 0
				}
			],
			"runtime_field_types": []
		},
		"analysis": {
			"char_filter_types": [],
			"tokenizer_types": [],
			"filter_types": [],
			"analyzer_types": [],
			"built_in_char_filters": [],
			"built_in_tokenizers": [],
			"built_in_filters": [],
			"built_in_analyzers": []
		},
		"versions": [
			{
				"version": "7.14.1",
				"index_count": 780,
				"primary_shard_count": 780,
				"total_primary_bytes": 6264727332030
			}
		]
	},
	"nodes": {
		"count": {
			"total": 3,
			"coordinating_only": 0,
			"data": 3,
			"data_cold": 3,
			"data_content": 3,
			"data_frozen": 3,
			"data_hot": 3,
			"data_warm": 3,
			"ingest": 3,
			"master": 3,
			"ml": 3,
			"remote_cluster_client": 3,
			"transform": 3,
			"voting_only": 0
		},
		"versions": [
			"7.14.1"
		],
		"os": {
			"available_processors": 56,
			"allocated_processors": 56,
			"names": [
				{
					"name": "Linux",
					"count": 3
				}
			],
			"pretty_names": [
				{
					"pretty_name": "Debian GNU/Linux 10 (buster)",
					"count": 3
				}
			],
			"architectures": [
				{
					"arch": "amd64",
					"count": 3
				}
			],
			"mem": {
				"total_in_bytes": 270079356928,
				"free_in_bytes": 1808535552,
				"used_in_bytes": 268270821376,
				"free_percent": 1,
				"used_percent": 99
			}
		},
		"process": {
			"cpu": {
				"percent": 13
			},
			"open_file_descriptors": {
				"min": 1742,
				"max": 4134,
				"avg": 3333
			}
		},
		"jvm": {
			"max_uptime_in_millis": 4159311377,
			"versions": [
				{
					"version": "16.0.2",
					"vm_name": "OpenJDK 64-Bit Server VM",
					"vm_version": "16.0.2+7",
					"vm_vendor": "Eclipse Foundation",
					"bundled_jdk": true,
					"using_bundled_jdk": true,
					"count": 3
				}
			],
			"mem": {
				"heap_used_in_bytes": 80659970856,
				"heap_max_in_bytes": 193273528320
			},
			"threads": 445
		},
		"fs": {
			"total_in_bytes": 16727295205376,
			"free_in_bytes": 10416860811264,
			"available_in_bytes": 9566801821696
		},
		"plugins": [],
		"network_types": {
			"transport_types": {
				"security4": 3
			},
			"http_types": {
				"security4": 3
			}
		},
		"discovery_types": {
			"zen": 3
		},
		"packaging_types": [
			{
				"flavor": "default",
				"type": "deb",
				"count": 3
			}
		],
		"ingest": {
			"number_of_pipelines": 2,
			"processor_stats": {
				"gsub": {
					"count": 0,
					"failed": 0,
					"current": 0,
					"time_in_millis": 0
				},
				"script": {
					"count": 0,
					"failed": 0,
					"current": 0,
					"time_in_millis": 0
				}
			}
		}
	}
}

For anyone having the same problem, I found the solution.

We had 1 hot node and 2 cold nodes with corresponding rules for shards. When the hot node got to the 90% high water mark, it could not allocate a new shard for a new index, so all of the data just was lost. Old shards did not move to cold nodes because of ILM rules.

Solution is, well, to get more free space (by changing the lifecycle, in my case) or add more hot nodes.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.