Hello Tim
Also, upon trying many methods, found a solution which is here.
./bin/elasticsearch-certutil cert -silent -pem -ca-cert ca.crt -ca-key ca.key -in new_instances.yml -out new_bundle.zip
This generated each node's key and crt but there was different issue which is
{"type": "server", "timestamp": "2021-04-23T15:04:06,398Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "Cluster", "node.name": "es-master-4", "message": "master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and this node must discover master-eligible nodes [172.38.32.9, 172.38.32.10, 172.38.32.15, 172.38.32.11, 172.38.32.16] to bootstrap a cluster: have discovered [{es-master-4}{lgQUHqC9RzO0kAQAvJL-1A}{yyPSRfpZRiyUUkzW1DEERA}{172.38.32.11}{172.38.32.11:9301}{cdhilmrstw}{ml.machine_memory=12429029376, xpack.installed=true, transform.node=true, ml.max_open_jobs=20}]; discovery will continue using [172.38.32.9:9301, 172.38.32.10:9301, 172.38.32.15:9301] from hosts providers and [{es-master-4}{lgQUHqC9RzO0kAQAvJL-1A}{yyPSRfpZRiyUUkzW1DEERA}{172.38.32.11}{172.38.32.11:9301}{cdhilmrstw}{ml.machine_memory=12429029376, xpack.installed=true, transform.node=true, ml.max_open_jobs=20}] from last-known cluster state; node term 0, last-accepted version 0 in term 0" }
{"type": "server", "timestamp": "2021-04-23T15:04:06,485Z", "level": "WARN", "component": "o.e.x.c.s.t.n.SecurityNetty4Transport", "cluster.name": "Cluster", "node.name": "es-master-4", "message": "client did not trust this server's certificate, closing connection Netty4TcpChannel{localAddress=/172.38.32.11:9301, remoteAddress=/172.38.32.16:52326}" }
{"type": "server", "timestamp": "2021-04-23T15:04:06,617Z", "level": "WARN", "component": "o.e.c.s.DiagnosticTrustManager", "cluster.name": "Cluster", "node.name": "es-master-4", "message": "failed to establish trust with server at [<unknown host>]; the server provided a certificate with subject name [CN=elk1] and fingerprint [226959ca2d25696cc3bd558ed049bcfb629b843c]; the certificate has subject alternative names [DNS:localhost,IP:127.0.0.1,IP:172.38.32.9,DNS:elk1]; the certificate is issued by [CN=Elastic Certificate Tool Autogenerated CA] but the server did not provide a copy of the issuing certificate in the certificate chain; this ssl context ([xpack.security.transport.ssl]) trusts [1] certificate with subject name [CN=Elastic Certificate Tool Autogenerated CA] and fingerprint [e5ac2583a3e148403f9296598909c68b0e675eb7] but the signatures do not match",
"stacktrace": ["sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: Path does not chain with any of the trust anchors",
"at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:369) ~[?:?]",
"at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:275) ~[?:?]",
"at sun.security.validator.Validator.validate(Validator.java:264) ~[?:?]",
"at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) ~[?:?]",
"at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) ~[?:?]",
"at org.elasticsearch.common.ssl.DiagnosticTrustManager.checkServerTrusted(DiagnosticTrustManager.java:110) [elasticsearch-ssl-config-7.10.0.jar:7.10.0]",
"at sun.security.ssl.CertificateMessage$T13CertificateConsumer.checkServerCerts(CertificateMessage.java:1335) [?:?]",
"at sun.security.ssl.CertificateMessage$T13CertificateConsumer.onConsumeCertificate(CertificateMessage.java:1232) [?:?]",
"at sun.security.ssl.CertificateMessage$T13CertificateConsumer.consume(CertificateMessage.java:1175) [?:?]",
"at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) [?:?]",
"at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) [?:?]",
"at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1267) [?:?]",
"at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1254) [?:?]",
"at java.security.AccessController.doPrivileged(AccessController.java:691) [?:?]",
"at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1199) [?:?]",
"at io.netty.handler.ssl.SslHandler.runAllDelegatedTasks(SslHandler.java:1542) [netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1556) [netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1440) [netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1267) [netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1314) [netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:501) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:440) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at java.lang.Thread.run(Thread.java:832) [?:?]",
"Caused by: java.security.cert.CertPathValidatorException: Path does not chain with any of the trust anchors",
"at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:158) ~[?:?]",
"at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:84) ~[?:?]",
"at java.security.cert.CertPathValidator.validate(CertPathValidator.java:309) ~[?:?]",
"at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:364) ~[?:?]",
"... 37 more"] }