Hi @stephenb much appreciated! Thanks!!
So posting the extra information that you requested.
We are at version 8.0.0, but we plan to update at least to version 8.9, we are not sure yet, if we can/should update to 8.12 due to some APM changes.
(also, update OS to Ubuntu 22.04.06 LTS, after)
GET .ds-logs-test-2024.01.10-000052
{
".ds-logs-test-2024.01.10-000052" : {
"aliases" : { },
"mappings" : {
"dynamic" : "true",
"_data_stream_timestamp" : {
"enabled" : true
},
"dynamic_date_formats" : [
"strict_date_optional_time",
"yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
],
"dynamic_templates" : [
{
"match_ip" : {
"match" : "ip",
"match_mapping_type" : "string",
"mapping" : {
"type" : "ip"
}
}
},
{
"match_message" : {
"match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "match_only_text"
}
}
},
{
"strings_as_keyword" : {
"match_mapping_type" : "string",
"mapping" : {
"ignore_above" : 1024,
"type" : "keyword"
}
}
}
],
"date_detection" : false,
"numeric_detection" : false,
"properties" : {
"@timestamp" : {
"type" : "date"
},
"@version" : {
"type" : "keyword",
"ignore_above" : 1024
},
"data_stream" : {
"properties" : {
"dataset" : {
"type" : "constant_keyword",
"value" : "generic"
},
"namespace" : {
"type" : "constant_keyword",
"value" : "default"
},
"type" : {
"type" : "constant_keyword",
"value" : "logs"
}
}
},
"ecs" : {
"properties" : {
"version" : {
"type" : "keyword",
"ignore_above" : 1024
}
}
},
"event" : {
"properties" : {
"original" : {
"type" : "keyword",
"ignore_above" : 1024
}
}
},
"host" : {
"type" : "object"
},
"message" : {
"type" : "match_only_text"
}
}
},
"settings" : {
"index" : {
"lifecycle" : {
"name" : "logs-generic",
"indexing_complete" : "true"
},
"codec" : "best_compression",
"routing" : {
"allocation" : {
"include" : {
"_tier_preference" : "data_cold,data_warm,data_hot"
}
}
},
"hidden" : "true",
"number_of_shards" : "1",
"provided_name" : ".ds-logs-test-2024.01.10-000052",
"query" : {
"default_field" : [
"message"
]
},
"creation_date" : "1704910916910",
"priority" : "0",
"number_of_replicas" : "1",
"uuid" : "GxucrpOARk-AiCOJOYgTtA",
"version" : {
"created" : "8000099"
}
}
},
"data_stream" : "logs-test-default"
}
}
I forgot to post this on the initial post, this result was one the reasons that led me to think, if maybe the setting replicas = 0 on Cold Phase is being ignored ?
GET /_cat/shards?h=index,node,shard,prirep,state,unassigned.reason
.ds-logs-test-2024.01.10-000052 coldnode01 0 p STARTED
.ds-logs-test-2024.01.10-000052 0 r UNASSIGNED PRIMARY_FAILED
...
.ds-logs-test-2024.01.18-000053 coldnode01 0 p STARTED
.ds-logs-test-2024.01.18-000053 0 r UNASSIGNED PRIMARY_FAILED
This is the policy, pretty much what I mentioned on initial brief description:
GET _ilm/policy/logs-generic
{
"logs-generic" : {
"version" : 6,
"modified_date" : "2024-02-01T11:49:48.384Z",
"policy" : {
"phases" : {
"warm" : {
"min_age" : "3d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"set_priority" : {
"priority" : 50
}
}
},
"cold" : {
"min_age" : "30d",
"actions" : {
"allocate" : {
"number_of_replicas" : 0,
"include" : { },
"exclude" : { },
"require" : { }
},
"set_priority" : {
"priority" : 0
}
}
},
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_primary_shard_size" : "50gb",
"max_age" : "30d"
},
"set_priority" : {
"priority" : 100
}
}
},
"delete" : {
"min_age" : "90d",
"actions" : {
"delete" : {
"delete_searchable_snapshot" : true
}
}
}
}
},
"in_use_by" : {
"indices" : [
".ds-logs-test-2024.02.07-000056",
".ds-logs-test-2024.01.31-000055",
".ds-logs-test-2024.01.18-000053",
".ds-logs-test-2024.01.10-000052",
".ds-logs-test-2024.01.24-000054"
],
"data_streams" : [
"logs-generic-default"
],
"composable_templates" : [
"logs-generic"
]
}
}
}
All the nodes in cluster orion:
GET _cat/nodes
10.11.23.67 38 99 5 0.14 0.15 0.10 irt - ingnode
10.11.23.50 57 98 0 0.00 0.00 0.00 m - master03
10.11.23.60 41 90 3 0.52 0.45 0.45 hs - hotnode01
10.11.23.56 52 88 2 0.98 0.50 0.41 hs - hotnode02
10.11.23.63 7 85 0 0.00 0.00 0.00 w - warmnode02
10.11.23.55 39 96 0 0.00 0.00 0.00 - - contrnode
10.11.23.61 22 84 0 0.00 0.02 0.00 w - warmnode01
10.11.23.52 12 96 0 0.00 0.00 0.00 mv - master02
10.11.23.62 33 98 2 0.05 0.06 0.05 c - coldnode1
10.11.23.51 41 98 0 0.00 0.00 0.00 m * master01
These are the node settings and now this made me realized that cold node is on version 8.0.1, which it shouldn't be (I only left data nodes settings to keep this shorter, if you need the other nodes settings, give me a heads up):
GET _nodes/settings
{
"_nodes" : {
"total" : 10,
"successful" : 10,
"failed" : 0
},
...
"z6uFC0iZQ_ifjX8j3OVe8w" : {
"name" : "hotnode02",
"transport_address" : "10.11.23.56:9300",
"host" : "10.11.23.56",
"ip" : "10.11.23.56",
"version" : "8.0.0",
"build_flavor" : "default",
"build_type" : "deb",
"build_hash" : "1b6a7ece17463df5ff54a3e1302d825889aa1161",
"roles" : [
"data_content",
"data_hot"
],
"attributes" : {
"xpack.installed" : "true"
},
"settings" : {
"cluster" : {
"name" : "orion-cluster",
"election" : {
"strategy" : "supports_voting_only"
}
},
"node" : {
"name" : "hotnode02",
"pidfile" : "/var/run/elasticsearch/elasticsearch.pid",
"attr" : {
"xpack" : {
"installed" : "true"
}
},
"roles" : [
"data_hot",
"data_content"
]
},
"path" : {
"data" : "/elk/lib/elasticsearch",
"logs" : "/elk/log/elasticsearch",
"home" : "/usr/share/elasticsearch"
},
"discovery" : {
"seed_hosts" : [
"master02",
"master03",
"master01"
]
},
"client" : {
"type" : "node"
},
"http" : {
"host" : [
"_local_",
"_site_"
],
"compression" : "false",
"type" : "security4",
"port" : "9200",
"type.default" : "netty4"
},
"transport" : {
"type" : "security4",
"type.default" : "netty4"
},
"xpack" : {
"security" : {
"http" : {
"ssl" : {
"enabled" : "true"
}
},
"transport" : {
"ssl" : {
"enabled" : "true"
}
},
"enabled" : "true",
"enrollment" : {
"enabled" : "true"
}
}
},
"network" : {
"host" : "0.0.0.0"
}
}
},
...
"EdabKikrQY-sQQpaWnWw6Q" : {
"name" : "warmnode01",
"transport_address" : "10.11.23.61:9300",
"host" : "10.11.23.61",
"ip" : "10.11.23.61",
"version" : "8.0.0",
"build_flavor" : "default",
"build_type" : "deb",
"build_hash" : "1b6a7ece17463df5ff54a3e1302d825889aa1161",
"roles" : [
"data_warm"
],
"attributes" : {
"xpack.installed" : "true"
},
"settings" : {
"cluster" : {
"name" : "orion-cluster",
"election" : {
"strategy" : "supports_voting_only"
}
},
"node" : {
"name" : "warmnode01",
"pidfile" : "/var/run/elasticsearch/elasticsearch.pid",
"attr" : {
"xpack" : {
"installed" : "true"
}
},
"roles" : [
"data_warm"
]
},
"path" : {
"data" : "/elk/lib/elasticsearch",
"logs" : "/elk/log/elasticsearch",
"home" : "/usr/share/elasticsearch"
},
"discovery" : {
"seed_hosts" : [
"master02",
"master03",
"master01"
]
},
"client" : {
"type" : "node"
},
"http" : {
"host" : [
"_local_",
"_site_"
],
"compression" : "false",
"type" : "security4",
"port" : "9200",
"type.default" : "netty4"
},
"transport" : {
"type" : "security4",
"type.default" : "netty4"
},
"xpack" : {
"security" : {
"http" : {
"ssl" : {
"enabled" : "true"
}
},
"transport" : {
"ssl" : {
"enabled" : "true"
}
},
"enabled" : "true",
"enrollment" : {
"enabled" : "true"
}
}
},
"network" : {
"host" : "0.0.0.0"
}
}
},
"Z7f4OHCXSyWtalnyiH4DcA" : {
"name" : "warmnode02",
"transport_address" : "10.11.23.63:9300",
"host" : "10.11.23.63",
"ip" : "10.11.23.63",
"version" : "8.0.0",
"build_flavor" : "default",
"build_type" : "deb",
"build_hash" : "1b6a7ece17463df5ff54a3e1302d825889aa1161",
"roles" : [
"data_warm"
],
"attributes" : {
"xpack.installed" : "true"
},
"settings" : {
"cluster" : {
"name" : "orion-cluster",
"election" : {
"strategy" : "supports_voting_only"
}
},
"node" : {
"name" : "warmnode02",
"pidfile" : "/var/run/elasticsearch/elasticsearch.pid",
"attr" : {
"xpack" : {
"installed" : "true"
}
},
"roles" : [
"data_warm"
]
},
"path" : {
"data" : "/elk/lib/elasticsearch",
"logs" : "/elk/log/elasticsearch",
"home" : "/usr/share/elasticsearch"
},
"discovery" : {
"seed_hosts" : [
"master02",
"master03",
"master01"
]
},
"client" : {
"type" : "node"
},
"http" : {
"host" : [
"_local_",
"_site_"
],
"compression" : "false",
"type" : "security4",
"port" : "9200",
"type.default" : "netty4"
},
"transport" : {
"type" : "security4",
"type.default" : "netty4"
},
"xpack" : {
"security" : {
"http" : {
"ssl" : {
"enabled" : "true"
}
},
"transport" : {
"ssl" : {
"enabled" : "true"
}
},
"enabled" : "true",
"enrollment" : {
"enabled" : "true"
}
}
},
"network" : {
"host" : "0.0.0.0"
}
}
},
...
"_Qi6Gp8tQu2nbaQzF8UHbg" : {
"name" : "hotnode01",
"transport_address" : "10.11.23.60:9300",
"host" : "10.11.23.60",
"ip" : "10.11.23.60",
"version" : "8.0.0",
"build_flavor" : "default",
"build_type" : "deb",
"build_hash" : "1b6a7ece17463df5ff54a3e1302d825889aa1161",
"roles" : [
"data_content",
"data_hot"
],
"attributes" : {
"xpack.installed" : "true"
},
"settings" : {
"cluster" : {
"name" : "orion-cluster",
"election" : {
"strategy" : "supports_voting_only"
}
},
"node" : {
"name" : "hotnode01",
"pidfile" : "/var/run/elasticsearch/elasticsearch.pid",
"attr" : {
"xpack" : {
"installed" : "true"
}
},
"roles" : [
"data_hot",
"data_content"
]
},
"path" : {
"data" : "/elk/lib/elasticsearch",
"logs" : "/elk/log/elasticsearch",
"home" : "/usr/share/elasticsearch"
},
"discovery" : {
"seed_hosts" : [
"master02",
"master03",
"master01"
]
},
"client" : {
"type" : "node"
},
"http" : {
"host" : [
"_local_",
"_site_"
],
"compression" : "false",
"type" : "security4",
"port" : "9200",
"type.default" : "netty4"
},
"transport" : {
"type" : "security4",
"type.default" : "netty4"
},
"xpack" : {
"security" : {
"http" : {
"ssl" : {
"enabled" : "true"
}
},
"transport" : {
"ssl" : {
"enabled" : "true"
}
},
"enabled" : "true",
"enrollment" : {
"enabled" : "true"
}
}
},
"network" : {
"host" : "0.0.0.0"
}
}
},
"-h6vvlcfQtKkeftVE9z2cw" : {
"name" : "coldnode01",
"transport_address" : "10.11.23.62:9300",
"host" : "10.11.23.62",
"ip" : "10.11.23.62",
"version" : "8.0.1",
"build_flavor" : "default",
"build_type" : "deb",
"build_hash" : "801d9ccc7c2ee0f2cb121bbe22ab5af77a902372",
"roles" : [
"data_cold"
],
"attributes" : {
"xpack.installed" : "true"
},
"settings" : {
"cluster" : {
"name" : "orion-cluster",
"election" : {
"strategy" : "supports_voting_only"
}
},
"node" : {
"name" : "coldnode01",
"pidfile" : "/var/run/elasticsearch/elasticsearch.pid",
"attr" : {
"xpack" : {
"installed" : "true"
}
},
"roles" : [
"data_cold"
]
},
"path" : {
"data" : "/elk/lib/elasticsearch",
"logs" : "/elk/log/elasticsearch",
"home" : "/usr/share/elasticsearch"
},
"discovery" : {
"seed_hosts" : [
"master02",
"master03",
"master01"
]
},
"client" : {
"type" : "node"
},
"http" : {
"host" : [
"_local_",
"_site_"
],
"compression" : "false",
"type" : "security4",
"port" : "9200",
"type.default" : "netty4"
},
"transport" : {
"type" : "security4",
"type.default" : "netty4"
},
"xpack" : {
"security" : {
"http" : {
"ssl" : {
"enabled" : "true"
}
},
"transport" : {
"ssl" : {
"enabled" : "true"
}
},
"enabled" : "true",
"enrollment" : {
"enabled" : "true"
}
}
},
"network" : {
"host" : "0.0.0.0"
}
}
}
}
}
Again, this made realize that one node is 8.0.1, instead of all being on 8.0.0.
But, since it's cold regardless, shouldn't be a problem ?
Since primary shard was moved there, just the replicas are not being deleted right ?
GET /_cluster/allocation/explain
{
"index": ".ds-logs-test-2024.01.10-000052",
"shard": 0,
"primary": true
}
{
"index" : ".ds-logs-test-2024.01.10-000052",
"shard" : 0,
"primary" : true,
"current_state" : "started",
"current_node" : {
"id" : "-h6vvlcfQtKkeftVE9z2cw",
"name" : "coldnode01",
"transport_address" : "10.11.23.62:9300",
"attributes" : {
"xpack.installed" : "true"
},
"weight_ranking" : 1
},
"can_remain_on_current_node" : "yes",
"can_rebalance_cluster" : "no",
"can_rebalance_cluster_decisions" : [
{
"decider" : "rebalance_only_when_active",
"decision" : "NO",
"explanation" : "rebalancing is not allowed until all replicas in the cluster are active"
},
{
"decider" : "cluster_rebalance",
"decision" : "NO",
"explanation" : "the cluster has unassigned shards and cluster setting [cluster.routing.allocation.allow_rebalance] is set to [indices_all_active]"
}
],
"can_rebalance_to_other_node" : "no",
"rebalance_explanation" : "rebalancing is not allowed",
"node_allocation_decisions" : [
{
"node_id" : "EdabKikrQY-sQQpaWnWw6Q",
"node_name" : "warmnode01",
"transport_address" : "10.11.23.61:9300",
"node_attributes" : {
"xpack.installed" : "true"
},
"node_decision" : "no",
"weight_ranking" : 2,
"deciders" : [
{
"decider" : "node_version",
"decision" : "NO",
"explanation" : "cannot relocate primary shard from a node with version [8.0.1] to a node with older version [8.0.0]"
},
{
"decider" : "data_tier",
"decision" : "NO",
"explanation" : "index has a preference for tiers [data_cold,data_warm,data_hot] and node does not meet the required [data_cold] tier"
}
]
},
{
"node_id" : "Z7f4OHCXSyWtalnyiH4DcA",
"node_name" : "warmnode02",
"transport_address" : "10.11.23.63:9300",
"node_attributes" : {
"xpack.installed" : "true"
},
"node_decision" : "no",
"weight_ranking" : 3,
"deciders" : [
{
"decider" : "node_version",
"decision" : "NO",
"explanation" : "cannot relocate primary shard from a node with version [8.0.1] to a node with older version [8.0.0]"
},
{
"decider" : "data_tier",
"decision" : "NO",
"explanation" : "index has a preference for tiers [data_cold,data_warm,data_hot] and node does not meet the required [data_cold] tier"
}
]
},
{
"node_id" : "z6uFC0iZQ_ifjX8j3OVe8w",
"node_name" : "hotnode02",
"transport_address" : "10.11.23.56:9300",
"node_attributes" : {
"xpack.installed" : "true"
},
"node_decision" : "no",
"weight_ranking" : 4,
"deciders" : [
{
"decider" : "node_version",
"decision" : "NO",
"explanation" : "cannot relocate primary shard from a node with version [8.0.1] to a node with older version [8.0.0]"
},
{
"decider" : "data_tier",
"decision" : "NO",
"explanation" : "index has a preference for tiers [data_cold,data_warm,data_hot] and node does not meet the required [data_cold] tier"
}
]
},
{
"node_id" : "_Qi6Gp8tQu2nbaQzF8UHbg",
"node_name" : "hotnode01",
"transport_address" : "10.11.23.60:9300",
"node_attributes" : {
"xpack.installed" : "true"
},
"node_decision" : "no",
"weight_ranking" : 5,
"deciders" : [
{
"decider" : "node_version",
"decision" : "NO",
"explanation" : "cannot relocate primary shard from a node with version [8.0.1] to a node with older version [8.0.0]"
},
{
"decider" : "data_tier",
"decision" : "NO",
"explanation" : "index has a preference for tiers [data_cold,data_warm,data_hot] and node does not meet the required [data_cold] tier"
}
]
}
]
}
Thanks once again for your help!
