Hi,
It seems like I'm the only one who uses SASL_SSL with open-source Kafka in the market as i couldn't find anything related to my issue except from some unclear threads over the internet,
I'm trying to get Metricbeat send my kafka information to elastic,
using plaintext seems to be ok with my kafka version but as i try to activate SASL_SSL i keep on failing:
usually i create the TLS channel with the CAroot.pem ceritificate (with python or asp)
when i tried:
ssl.certificate_authorities: ["/home/kafka/ssl/CARoot.pem"]
it seemed to not be the configuration i needed,
i failied on SSL handshake.
using ssl.certificate: "/home/kafka/ssl/CARoot.pem"
required a key ,
so i created a key based on my client java store and it failed with parsing error,
i've read that i need to create it with -keyalg RSA
i've tried that as well and failed, im losing hope here..
so i'm pretty sure im failing on the ssl generating,
can anyone instruct me what to change in order to get the right pem and key?
this is my ansible flow before adding the -keyalg RSA :
- name: Creates the CA key and cert
shell: openssl req -new -passout pass:'{{ KEYSTORE_PASS }}' -x509 -keyout ca-key -out ca-cert -days 9999 -subj '/CN=localhost'
become: true
args:
chdir: /home/kafka/ssl
run_once: true
when: not ca_cert_file.stat.exists
- name: Copy the CA cert to build server
fetch:
src: /home/kafka/ssl/ca-cert
dest: /tmp/
flat: yes
run_once: true
- name: Check if ca-cert was already copied to remote server.
stat:
path: /home/kafka/ssl/ca-cert
register: ca_cert_copy
- name: Copy the CA cert from local server to kafka Brokers
copy:
src: /tmp/ca-cert
dest: /home/kafka/ssl/
when: not ca_cert_copy.stat.exists
- name: Check if local Keystore was already created.
stat:
path: /home/kafka/ssl/kafka.server.keystore.jks
register: keystore
- name: Create local Keystore
shell: "[ -f kafka.server.keystore.jks ] && echo 'Keystore exist' || keytool -keystore kafka.server.keystore.jks -alias localhost -validity 9999 -genkey -storepass {{ KEYSTORE_PASS }} -dname 'CN={{ inventory_hostname }}, OU=Unknown, O=Unknown, L=Unknown, ST=Unknown, C=Unknown' -keypass {{ KEYSTORE_PASS }}"
become: true
args:
chdir: /home/kafka/ssl
when: not keystore.stat.exists
- name: Check if local Truststore was already created.
stat:
path: /home/kafka/ssl/kafka.server.truststore.jks
register: truststore
- name: Create local Truststore
shell: "[ -f kafka.server.truststore.jks ] && echo 'Trustsore exist' || keytool -keystore kafka.server.truststore.jks -alias CARoot -import -file /home/kafka/ssl/ca-cert -storepass {{ KEYSTORE_PASS }} -noprompt "
become: true
args:
chdir: /home/kafka/ssl
when: not truststore.stat.exists
- name: Check if local cert was already exported.
stat:
path: /home/kafka/ssl/{{ inventory_hostname }}.cert-file
register: local_cert
- name: Exporting local Cert from keystore
shell: keytool -keystore kafka.server.keystore.jks -alias localhost -certreq -file {{ inventory_hostname }}.cert-file -storepass '{{ KEYSTORE_PASS }}'
become: true
args:
chdir: /home/kafka/ssl
when: not local_cert.stat.exists
- name: Copy the certs to build server
fetch:
src: /home/kafka/ssl/{{ inventory_hostname }}.cert-file
dest: /tmp/
flat: yes
- name: Copy the local brokers cert from build server to kafka CA
copy:
src: /tmp/{{ item }}.cert-file
dest: /home/kafka/ssl/
with_items:
- "{{groups.kafka}}"
run_once: true
- name: Sign the Brokers Certificates in the CA
shell: "openssl x509 -req -CA ca-cert -CAkey ca-key -in {{ item }}.cert-file -out cert-signed-{{ item }} -days 9999 -CAcreateserial -passin pass:'{{ KEYSTORE_PASS }}'"
become: true
args:
chdir: /home/kafka/ssl
with_items:
- "{{groups.kafka}}"
run_once: true
- name: Copy the signed certs to build server
fetch:
src: /home/kafka/ssl/cert-signed-{{ item }}
dest: /tmp/
flat: yes
with_items:
- "{{groups.kafka}}"
run_once: true
- name: Copy the signed certs back to the brokers
copy:
src: /tmp/cert-signed-{{ inventory_hostname }}
dest: /home/kafka/ssl/
- name: Import the Signed certs in the kafka brokers
shell: "keytool -keystore kafka.server.keystore.jks -alias CARoot -import -file ca-cert -storepass '{{ KEYSTORE_PASS }}' -noprompt;sudo keytool -keystore kafka.server.keystore.jks -alias localhost -import -file cert-signed-{{ inventory_hostname }} -storepass '{{ KEYSTORE_PASS }}'"
become: true
args:
chdir: /home/kafka/ssl
ignore_errors: yes
- name: make sure kafka is running
service:
name: kafka
state: restarted
enabled: true
- name: make sure zookeeper is running
service:
name: zookeeper
state: restarted
enabled: true
- name: Check if CAroot.pem exists or not.
stat:
path: /home/kafka/ssl/CARoot.pem
register: ca_root_pem
run_once: true
- name: Export client certificates
shell: "keytool -keystore kafka.client.keystore.jks -alias localhost -validity 9999 -genkey -storepass {{ KEYSTORE_PASS }} -dname 'CN=Unknown, OU=Unknown, O=Unknown, L=Unknown, ST=Unknown, C=Unknown' -keypass {{ KEYSTORE_PASS }}; keytool -keystore kafka.client.truststore.jks -alias CARoot -import -file ca-cert -storepass '{{ KEYSTORE_PASS }}' -noprompt; keytool -keystore kafka.client.keystore.jks -alias localhost -certreq -file client.cert-file -storepass '{{ KEYSTORE_PASS }}' -noprompt; openssl x509 -req -CA ca-cert -CAkey ca-key -in client.cert-file -out cert-signed-client -days 9999 -CAcreateserial -passin pass:'{{ KEYSTORE_PASS }}'; keytool -keystore kafka.client.keystore.jks -alias CARoot -import -file ca-cert -storepass {{ KEYSTORE_PASS }} -noprompt; keytool -keystore kafka.client.keystore.jks -alias localhost -import -file cert-signed-client -storepass {{ KEYSTORE_PASS }} -noprompt; keytool -exportcert -alias localhost -keystore kafka.client.keystore.jks -rfc -file certificate.pem -storepass {{ KEYSTORE_PASS }}; keytool -exportcert -alias CARoot -keystore kafka.client.keystore.jks -rfc -file CARoot.pem -storepass {{ KEYSTORE_PASS }}"
become: true
args:
chdir: /home/kafka/ssl
run_once: true
when: not ca_root_pem.stat.exists