Thanks @stephenb for your help on this. Truly appreciate your time...
Yes, I have this cluster set up on elactic.co. I don't think there is anything out of the ordinary for this set up. We have a couple of continuous transforms happening and ILMs set up on some of the indices to manage disk space. The newly provisioned cluster has been set up using hot only architecture (as there wasn't a way to downgrade/remove the warm nodes in the hot/warm pattern I used initially).
I will provide the current outputs of the single node setup, then spin up another node and provide the same after...
GET /_cat/indices/.e*?v
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
green open .enrich-enrich_vessel-1609893468927 Fu5mGDdUSIO5PLdY-KTq2A 1 0 361842 0 24.7mb 24.7mb
GET _cat/aliases/.e*
.enrich-enrich_vessel .enrich-enrich_vessel-1609893468927 - - - -
GET _cat/shards/.e*?v
index shard prirep state docs store ip node
.enrich-enrich_vessel-1609893468927 0 p STARTED 361842 24.7mb 172.25.84.100 instance-0000000013
Enrich policy is:
{
"match": {
"indices": "vessel",
"match_field": "mmsi",
"enrich_fields": [
"callsign",
"shipname",
"shiptype",
"imo",
"mothership_mmsi",
"width",
"length",
"ais_msg_type",
"event_ts"
]
}
}
track pipeline:
{
"description": "Handles incoming position log events going to track index",
"processors": [
{
"pipeline": {
"name": "timestamps",
"tag": "timestamps_track"
}
},
{
"pipeline": {
"name": "enrich_vessel",
"tag": "vessel_enrich_track"
}
}
]
}
enrich_vessel pipeline:
{
"description": "This pipeline adds vessel information to position messages.",
"processors": [
{
"enrich": {
"policy_name": "enrich_vessel",
"field": "mmsi",
"target_field": "vessel"
}
}
]
}
I am using Filebeats to send events to ES. Here is the output of the config:
output.elasticsearch:
hosts: ["${ELASTIC_HOST}"]
username: "${ELASTIC_USERNAME}"
password: "${ELASTIC_PASSWORD}"
bulk_max_size: 1000
pipeline: "%{[@indexmeta.idx]}"
indices:
- index: "%{[@indexmeta.idx]}"
@indexmeta.idx in this case is track
Here are the settings of the current write index:
"track-000032" : {
"settings" : {
"index" : {
"lifecycle" : {
"name" : "ilm_track",
"rollover_alias" : "track"
},
"number_of_shards" : "1",
"provided_name" : "track-000032",
"creation_date" : "1609920008341",
"priority" : "100",
"number_of_replicas" : "1",
"uuid" : "JjLw6xnVRpqekY9okrFFyQ",
"version" : {
"created" : "7090399"
}
}
},
"defaults" : {
"index" : {
"flush_after_merge" : "512mb",
"final_pipeline" : "_none",
"max_inner_result_window" : "100",
"unassigned" : {
"node_left" : {
"delayed_timeout" : "1m"
}
},
"max_terms_count" : "65536",
"lifecycle" : {
"parse_origination_date" : "false",
"indexing_complete" : "false",
"origination_date" : "-1"
},
"routing_partition_size" : "1",
"force_memory_term_dictionary" : "false",
"max_docvalue_fields_search" : "100",
"merge" : {
"scheduler" : {
"max_thread_count" : "4",
"auto_throttle" : "true",
"max_merge_count" : "9"
},
"policy" : {
"reclaim_deletes_weight" : "2.0",
"floor_segment" : "2mb",
"max_merge_at_once_explicit" : "30",
"max_merge_at_once" : "10",
"max_merged_segment" : "5gb",
"expunge_deletes_allowed" : "10.0",
"segments_per_tier" : "10.0",
"deletes_pct_allowed" : "33.0"
}
},
"max_refresh_listeners" : "1000",
"max_regex_length" : "1000",
"load_fixed_bitset_filters_eagerly" : "true",
"number_of_routing_shards" : "1",
"write" : {
"wait_for_active_shards" : "1"
},
"verified_before_close" : "false",
"mapping" : {
"coerce" : "false",
"nested_fields" : {
"limit" : "50"
},
"depth" : {
"limit" : "20"
},
"field_name_length" : {
"limit" : "9223372036854775807"
},
"total_fields" : {
"limit" : "1000"
},
"nested_objects" : {
"limit" : "10000"
},
"ignore_malformed" : "false"
},
"source_only" : "false",
"soft_deletes" : {
"enabled" : "false",
"retention" : {
"operations" : "0"
},
"retention_lease" : {
"period" : "12h"
}
},
"max_script_fields" : "32",
"query" : {
"default_field" : [
"*"
],
"parse" : {
"allow_unmapped_fields" : "true"
}
},
"format" : "0",
"frozen" : "false",
"sort" : {
"missing" : [ ],
"mode" : [ ],
"field" : [ ],
"order" : [ ]
},
"codec" : "default",
"max_rescore_window" : "10000",
"max_adjacency_matrix_filters" : "100",
"analyze" : {
"max_token_count" : "10000"
},
"gc_deletes" : "60s",
"top_metrics_max_size" : "10",
"optimize_auto_generated_id" : "true",
"max_ngram_diff" : "1",
"hidden" : "false",
"translog" : {
"generation_threshold_size" : "64mb",
"flush_threshold_size" : "512mb",
"sync_interval" : "5s",
"retention" : {
"size" : "512MB",
"age" : "12h"
},
"durability" : "REQUEST"
},
"auto_expand_replicas" : "false",
"mapper" : {
"dynamic" : "true"
},
"recovery" : {
"type" : ""
},
"requests" : {
"cache" : {
"enable" : "true"
}
},
"data_path" : "",
"highlight" : {
"max_analyzed_offset" : "1000000"
},
"routing" : {
"rebalance" : {
"enable" : "all"
},
"allocation" : {
"enable" : "all",
"total_shards_per_node" : "-1"
}
},
"search" : {
"slowlog" : {
"level" : "TRACE",
"threshold" : {
"fetch" : {
"warn" : "-1",
"trace" : "-1",
"debug" : "-1",
"info" : "-1"
},
"query" : {
"warn" : "-1",
"trace" : "-1",
"debug" : "-1",
"info" : "-1"
}
}
},
"idle" : {
"after" : "30s"
},
"throttled" : "false"
},
"fielddata" : {
"cache" : "node"
},
"default_pipeline" : "_none",
"max_slices_per_scroll" : "1024",
"shard" : {
"check_on_startup" : "false"
},
"xpack" : {
"watcher" : {
"template" : {
"version" : ""
}
},
"version" : "",
"ccr" : {
"following_index" : "false"
}
},
"percolator" : {
"map_unmapped_fields_as_text" : "false"
},
"allocation" : {
"max_retries" : "5",
"existing_shards_allocator" : "gateway_allocator"
},
"refresh_interval" : "1s",
"indexing" : {
"slowlog" : {
"reformat" : "true",
"threshold" : {
"index" : {
"warn" : "-1",
"trace" : "-1",
"debug" : "-1",
"info" : "-1"
}
},
"source" : "1000",
"level" : "TRACE"
}
},
"compound_format" : "0.1",
"blocks" : {
"metadata" : "false",
"read" : "false",
"read_only_allow_delete" : "false",
"read_only" : "false",
"write" : "false"
},
"max_result_window" : "10000",
"store" : {
"stats_refresh_interval" : "10s",
"type" : "",
"fs" : {
"fs_lock" : "native"
},
"preload" : [ ]
},
"queries" : {
"cache" : {
"enabled" : "true"
}
},
"warmer" : {
"enabled" : "true"
},
"max_shingle_diff" : "3",
"query_string" : {
"lenient" : "false"
}
}
}
