I tried another approach. I scale the elastic sts to 0 replicas, then scale it to 3, but elastic pod still looking to the old IP addresses for nodes. That should be a problem.
[root@api.o1-729588 ~]# oc scale sts instana-es-default --replicas=0
statefulset.apps/instana-es-default scaled
[root@api.o1-729588 ~]# oc scale sts instana-es-default --replicas=3
statefulset.apps/instana-es-default scaled
[root@api.o1-729588 ~]#
[root@api.o1-729588 ~]# oc get po -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
curl 0/1 Completed 0 2d1h 10.254.16.27 worker2.o1-729588 <none> <none>
instana-es-default-0 1/1 Running 0 108s 10.254.32.62 worker3.o1-729588 <none> <none>
instana-es-default-1 1/1 Running 0 108s 10.254.12.58 worker0.o1-729588 <none> <none>
instana-es-default-2 1/1 Running 0 108s 10.254.28.58 worker4.o1-729588 <none> <none>
[root@api.o1-729588 ~]#
{"@timestamp":"2024-04-19T11:04:41.788Z", "log.level": "WARN", "message":"address [10.254.12.57:9300], node [unknown discovery result: [][10.254.12.57:9300] connect_timeout[30s]; for summary, see logs from org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper; for troubleshooting guidance, see https://www.elastic.co/guide/en/elasticsearch/reference/8.13/discovery-troubleshooting.html", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[instana-es-default-0][generic][T#3]","log.logger":"org.elasticsearch.discovery.PeerFinder","elasticsearch.node.name":"instana-es-default-0","elasticsearch.cluster.name":"instana"}
{"@timestamp":"2024-04-19T11:04:41.788Z", "log.level": "WARN", "message":"address [10.254.32.61:9300], node [unknown discovery result: [][10.254.32.61:9300] connect_timeout[30s]; for summary, see logs from org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper; for troubleshooting guidance, see https://www.elastic.co/guide/en/elasticsearch/reference/8.13/discovery-troubleshooting.html", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[instana-es-default-0][generic][T#2]","log.logger":"org.elasticsearch.discovery.PeerFinder","elasticsearch.node.name":"instana-es-default-0","elasticsearch.cluster.name":"instana"}
{"@timestamp":"2024-04-19T11:04:41.788Z", "log.level": "WARN", "message":"address [10.254.28.57:9300], node [unknown discovery result: [][10.254.28.57:9300] connect_timeout[30s]; for summary, see logs from org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper; for troubleshooting guidance, see https://www.elastic.co/guide/en/elasticsearch/reference/8.13/discovery-troubleshooting.html", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[instana-es-default-0][generic][T#5]","log.logger":"org.elasticsearch.discovery.PeerFinder","elasticsearch.node.name":"instana-es-default-0","elasticsearch.cluster.name":"instana"}
{"@timestamp":"2024-04-19T11:04:50.706Z", "log.level": "WARN", "message":"master not discovered or elected yet, an election requires at least 2 nodes with ids from [DXo4BIiJTRSUOiGCCDy9Zw, 8zZQRMjhQRq8zdzW7CEgoQ, l8imZI5dSOKbHNCIIo_i_A], have only discovered non-quorum [{instana-es-default-0}{l8imZI5dSOKbHNCIIo_i_A}{2pluVwLARqGKTcfo7P1Gvg}{instana-es-default-0}{10.254.32.62}{10.254.32.62:9300}{cdfhilmrstw}{8.13.0}{7000099-8503000}]; discovery will continue using [10.254.12.57:9300, 10.254.28.57:9300, 10.254.32.61:9300] from hosts providers and [{instana-es-default-0}{l8imZI5dSOKbHNCIIo_i_A}{2pluVwLARqGKTcfo7P1Gvg}{instana-es-default-0}{10.254.32.62}{10.254.32.62:9300}{cdfhilmrstw}{8.13.0}{7000099-8503000}] from last-known cluster state; node term 3, last-accepted version 58 in term 3; for troubleshooting guidance, see https://www.elastic.co/guide/en/elasticsearch/reference/8.13/discovery-troubleshooting.html", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[instana-es-default-0][cluster_coordination][T#1]","log.logger":"org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper","elasticsearch.node.name":"instana-es-default-0","elasticsearch.cluster.name":"instana"}