Hi,
I was following all the instructions getting support for security addon under v7.5 and understood that all elastic<->kibna comm in to be TLS
i saw the certutil solution with csr but as lack of real documentation and examples i understood that i need the following :
- in elastic i need p12 cert+key for the signed to the proper elastic DNS name
- in kibana i need a proper pem cert+key and be able to set the signing CA auth to the certificateAuthorities config.
and used a public CA to generate both certificates for kibana and elastic
i created new cert+key for both kibana and elastic and generated PEM file consist of private + public crt (new entries + the ca crt) named elastic-certificate.pem then convert it to elastic-certificates.p12 (the format elastic support)
then generated the following secrets:
kubectl create secret generic elastic-certificates --from-file=elastic-certificates.p12 && \ kubectl create secret generic elastic-certificate-pem --from-file=elastic-certificate.pem && \
updating the DNS records to the new elastic server and its seems that the Cert is valid for the domain
Im double checking that the secret is ok elastic-certificate-pem and i can see its ok and consist of ca cert my cert and the private key
and as certificateAuthorities: /usr/share/kibana/config/certs/elastic-certificate.pem mapped to the pem file with the CA.crt it's should be ok verfiy it by certificate
Doing the same for Kibana and setting the elasticsearch address to be the DNS name of the elastic we just configured.
but kibana pod is failing due to 
{"type":"log","@timestamp":"2019-12-07T22:55:47Z","tags":["warning","elasticsearch","admin"],"pid":6,"message":"Unable to revive connection: https://elastic.xxxxx.com:9200/"}
{"type":"log","@timestamp":"2019-12-07T22:55:47Z","tags":["warning","elasticsearch","admin"],"pid":6,"message":"No living connections"}
so i was trying to change it back to the default svc name security-master:9200 on https but got the same.
so i ssh to the POD and then try to run curl -k -v https://security-master:9200 -k -u elastic:xxxxxxxxxx and it's working lovely .
< HTTP/1.1 200 OK
< content-type: application/json; charset=UTF-8
< content-length: 540
<
{
"name" : "security-master-0",
"cluster_name" : "security",
"cluster_uuid" : "8eit-sjfQFaS04RSrvfJRg",
"version" : {
"number" : "7.5.0",
"build_flavor" : "default",
"build_type" : "docker",
"build_hash" : "e9ccaed468e2fac2275a3761849cbee64b39519f",
"build_date" : "2019-11-26T01:06:52.518245Z",
"build_snapshot" : false,
"lucene_version" : "8.3.0",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}
so way i'm still getting this error
{"type":"log","@timestamp":"2019-12-07T23:35:07Z","tags":["debug","plugins","licensing"],"pid":12,"message":"Calling [data] Elasticsearch _xpack API. Polling frequency: 30001"}
{"type":"log","@timestamp":"2019-12-07T23:35:07Z","tags":["warning","elasticsearch","data"],"pid":12,"message":"Unable to revive connection: https://security-master:9200/"}
{"type":"log","@timestamp":"2019-12-07T23:35:07Z","tags":["warning","elasticsearch","data"],"pid":12,"message":"No living connections"}
{"type":"log","@timestamp":"2019-12-07T23:35:07Z","tags":["warning","plugins","licensing"],"pid":12,"message":"License information could not be obtained from Elasticsearch for the [data] cluster. Error: No Living connections"}
{"type":"log","@timestamp":"2019-12-07T23:35:07Z","tags":["debug","http"],"pid":12,"message":"Kibana server is not ready yet get:/app/kibana."}
{"type":"log","@timestamp":"2019-12-07T23:35:08Z","tags":["warning","elasticsearch","admin"],"pid":12,"message":"Unable to revive connection: https://security-master:9200/"}
{"type":"log","@timestamp":"2019-12-07T23:35:08Z","tags":["warning","elasticsearch","admin"],"pid":12,"message":"No living connections"}
security yaml for elastic
--- clusterName: "security" nodeGroup: "master" service: type: LoadBalancer replicas: 1 roles: master: "true" ingest: "true" data: "true" protocol: https esConfig: elasticsearch.yml: | xpack.security.enabled: true xpack.security.transport.ssl.enabled: true xpack.security.transport.ssl.verification_mode: certificate xpack.security.transport.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12 xpack.security.transport.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12 xpack.security.http.ssl.enabled: true xpack.security.http.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12 xpack.security.http.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12 extraEnvs: - name: ELASTIC_PASSWORD valueFrom: secretKeyRef: name: elastic-credentials key: password - name: ELASTIC_USERNAME valueFrom: secretKeyRef: name: elastic-credentials key: username secretMounts: - name: elastic-certificates secretName: elastic-certificates path: /usr/share/elasticsearch/config/certs
security yaml for Kibana
---
elasticsearchHosts: "https://external-dns-name.com:9200"
service:
type: LoadBalancer
extraEnvs:
- name: 'ELASTICSEARCH_USERNAME'
valueFrom:
secretKeyRef:
name: elastic-credentials
key: username
- name: 'ELASTICSEARCH_PASSWORD'
valueFrom:
secretKeyRef:
name: elastic-credentials
key: password
- name: 'KIBANA_ENCRYPTION_KEY'
valueFrom:
secretKeyRef:
name: kibana
key: encryptionkey
kibanaConfig:
kibana.yml: |
server.ssl:
enabled: true
key: /usr/share/kibana/config/certs/elastic-certificate.pem
certificate: /usr/share/kibana/config/certs/elastic-certificate.pem
xpack.security.encryptionKey: ${KIBANA_ENCRYPTION_KEY}
elasticsearch.ssl:
certificateAuthorities: /usr/share/kibana/config/certs/elastic-certificate.pem
verificationMode: certificate
protocol: https
secretMounts:
- name: elastic-certificate-pem
secretName: elastic-certificate-pem
path: /usr/share/kibana/config/certs