Logstash doesnt work with ECC/ECDSA

[aubreybox]

Hello,
I tried to setup logstash and filebeat over selfmade ssl certificates using an own rootca and an intermediate ca.

The problem is: I cant connect to the running logstash

curl: (35) gnutls_handshake() failed: Handshake failed

Probably the solution is easy but I couldn't figure it out

Since I guess the problem is related to the certificates I tell you the steps I used to reproduce the problem:

## root ca
openssl ecparam -name secp521r1 -genkey -out private/ca.key.pem 

curl https://raw.githubusercontent.com/aubreybox/tmp/master/openssl.cnf openssl.cnf
 
# root ca
openssl req -config openssl.cnf \
      -key private/ca.key.pem \
      -new -x509 -days 7300 -sha256 -extensions v3_ca \
      -out certs/ca.cert.pem
 

## intermediate ca
curl https://raw.githubusercontent.com/aubreybox/tmp/master/intermediate/openssl.cnf  intermediate/openssl.cnf
openssl ecparam -name secp521r1 -genkey -out intermediate/private/intermediate.key.pem 
 
openssl req -config intermediate/openssl.cnf -new -sha256 \
      -key intermediate/private/intermediate.key.pem \
      -out intermediate/csr/intermediate.csr.pem
openssl ca -config openssl.cnf -extensions v3_intermediate_ca \
      -days 3650 -notext -md sha256 \
      -in intermediate/csr/intermediate.csr.pem \
      -out intermediate/certs/intermediate.cert.pem
cat intermediate/certs/intermediate.cert.pem \
      certs/ca.cert.pem intermediate/certs/ca-chain.cert.pem

## server
openssl ecparam -name secp521r1 -genkey -out /root/test/intermediate/private/logserver.tld.key.pem 

openssl req -config /root/test/intermediate/openssl.cnf -key /root/test/intermediate/private/logserver.tld.key.pem  -new -sha256 -out /root/test/intermediate/csr/logserver.tld.csr.pem -subj "/C=DE/ST=Test/L=Test/O=Test/CN=logserver.tld" 

openssl ca -config /root/test/intermediate/openssl.cnf -extensions server_cert -batch -days 375 -notext -md sha256 -in /root/test/intermediate/csr/logserver.tld.csr.pem -out /root/test/intermediate/certs/logserver.tld.cert.pem 
 
openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -in /root/test/intermediate/private/logserver.tld.key.pem -out /root/test/intermediate/private/logserver.tld.pkcs8.key```

/tmp/ls.conf

input { 
  beats { 
    port => 5045
    ssl => true
    ssl_certificate => "/root/test/intermediate/certs/logserver.tld.rsa.pem"
    ssl_key ="/root/test/intermediate/private/logserver.tld.rsa.pkcs8.key"
    ssl_certificate_authorities =["/root/test/intermediate/certs/intermediate.rsa.pem", "/root/test/intermediate/certs/ca-chain.rsa.pem", "/root/test/certs/ca.rsa.pem"]
    tls_min_version => 1.2
    ssl_verify_mode => "force_peer"
#    ssl_verify_mode => "none"
    cipher_suites => ["TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"]
}

}
input { 
  beats { 
    port => 5044
    ssl => true
    ssl_certificate => "/root/test/intermediate/certs/logserver.tld.ecc.pem"
    ssl_key ="/root/test/intermediate/private/logserver.tld.ecc.pkcs8.key"
    ssl_certificate_authorities =["/root/test/intermediate/certs/intermediate.ecc.pem", "/root/test/intermediate/certs/ca-chain.ecc.pem", "/root/test/certs/ca.ecc.pem"]
    tls_min_version => 1.2
#    ssl_verify_mode => "force_peer"
    ssl_verify_mode => "none"
#    cipher_suites => ["TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"]
  }

}

Problem:

/usr/share/logstash/bin/logstash  --config.debug --log.level=debug -f /tmp/ls.conf --path.config /tmp/ls.conf

/usr/share/logstash/bin/logstash  --version                                                                                                                                                                                                    
logstash 5.5.2

openssl version
OpenSSL 1.0.2g  1 Mar 2016

RSA:

# Works
curl https://logserver.tld:5045  --key /root/test/intermediate/private/logserver.tld.rsa.key.pem  --cert /root/test/intermediate/logserver.tld.rsa.cert.pem --cacert /root/test/intermediate/intermediate.rsa.cert.pem
curl: (52) Empty reply from server

# As expected: fails
curl https://logserver.tld:5045  
curl: (35) gnutls_handshake() failed: Handshake failed

# Shows Server Certificate
openssl s_client -connect logserver.tld:5045 -CAfile /root/test/intermediate/intermediate.rsa.cert.pem 2>/dev/null | grep Server
Server certificate
Server Temp Key: ECDH, P-256, 256 bits
Server public key is 2048 bit

ECC:

# Fails:
curl https://logserver.tld:5044  --key /root/test/intermediate/private/logserver.tld.ecc.key.pem  --cert /root/test/intermediate/logserver.tld.ecc.cert.pem --cacert /root/test/intermediate/intermediate.ecc.cert.pem
curl: (35) gnutls_handshake() failed: Handshake failed

# Fails too (note: ssl_verify_mode is none)
curl https://logserver.tld:5044  
curl: (35) gnutls_handshake() failed: Handshake failed

# Fails
openssl s_client -connect logserver.tld:5044 -CAfile /root/test/intermediate/intermediate.rsa.cert.pem
CONNECTED(00000003)
140231626094232:error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure:s23_clnt.c:769:
---
no peer certificate available
---
No client certificate CA names sent
---
SSL handshake has read 7 bytes and written 305 bytes
---
New, (NONE), Cipher is (NONE)
Secure Renegotiation IS NOT supported
Compression: NONE
Expansion: NONE
No ALPN negotiated
SSL-Session:
    Protocol  : TLSv1.2
    Cipher    : 0000
    Session-ID: 
    Session-ID-ctx: 
    Master-Key: 
    Key-Arg   : None
    PSK identity: None
    PSK identity hint: None
    SRP username: None
    Start Time: 1504696019
    Timeout   : 300 (sec)
    Verify return code: 0 (ok)
---

ECC via openssl direclty works:

openssl s_server -cert /root/test/intermediate/logserver.tld.ecc.cert.pem -key /root/test/intermediate/private/logserver.tld.ecc.key.pem -www -verify force_peer -CAfile /root/test/intermediate/intermediate.ecc.cert.pem

openssl s_client -connect logserver.tld:4433  2>/dev/null | grep Server
Server certificate
Server Temp Key: ECDH, P-256, 256 bits
Server public key is 521 bit

So I guess something fails silently in logstash?

[aubreybox]

I wrote a script that recreates the issue with more logs and handshake capture.

It creates root ca, intermediate ca and server cert for localhost, then it starts logstash and tcpdump an makes openssl client requests.
Afterwards it kills tcpdump and logstash.

It creates:

• hs.cap: capture of handshake
• ls.out: the logstash output (+ javax.net.debug)
• {ecc,rsa}.client.log: the openssl client output

I can't figure out the problem why ecc doesn't work with logstash
(https://raw.githubusercontent.com/aubreybox/tmp/master/test.sh)

#!/bin/bash

CURVE=secp521r1
PASS=test

## root ca
mkdir -p /root/test/ca/{certs,crl,newcerts,private}
cd /root/test/ca
touch index.{rsa,ecc}.txt
echo 1000 > serial

#create root ca keys
openssl genrsa -aes256 -passout pass:$PASS -out private/ca.rsa.key.pem 4096 
openssl ecparam -name $CURVE -genkey | openssl ec -aes256 -passout pass:$PASS -out private/ca.ecc.key.pem

#copy root ca configs
wget https://raw.githubusercontent.com/aubreybox/tmp/master/openssl.rsa.cnf -O openssl.rsa.cnf
wget https://raw.githubusercontent.com/aubreybox/tmp/master/openssl.ecc.cnf -O openssl.ecc.cnf

#sign root ca certs
openssl req -config openssl.rsa.cnf -key private/ca.rsa.key.pem -new -x509 -days 7300 -sha256 -extensions v3_ca -out certs/ca.rsa.cert.pem -passin file:<(echo -n "$PASS") -subj "/C=US/ST=test/L=test/O=test/CN=root ca rsa"
openssl req -config openssl.ecc.cnf -key private/ca.ecc.key.pem -new -x509 -days 7300 -sha256 -extensions v3_ca -out certs/ca.ecc.cert.pem -passin file:<(echo -n "$PASS") -subj "/C=US/ST=test/L=test/O=test/CN=root ca ecc"


## intermediate ca

mkdir -p /root/test/ca/intermediate/{certs,crl,csr,newcerts,private}
cd /root/test/ca/intermediate
touch index.{rsa,ecc}.txt
echo 1000 > serial
echo 1000 > crlnumber
cd /root/test/ca/

#create intermediate ca keys
openssl genrsa -aes256 -passout pass:$PASS -out intermediate/private/intermediate.rsa.key.pem 4096
openssl ecparam -name $CURVE -genkey  | openssl ec -aes256 -passout pass:$PASS -out intermediate/private/intermediate.ecc.key.pem

#copy intermediate ca configs
wget https://raw.githubusercontent.com/aubreybox/tmp/master/intermediate/openssl.rsa.cnf -O intermediate/openssl.rsa.cnf
wget https://raw.githubusercontent.com/aubreybox/tmp/master/intermediate/openssl.ecc.cnf -O intermediate/openssl.ecc.cnf

#sign intermediate ca certs
openssl req -config intermediate/openssl.rsa.cnf -new -sha256 -key intermediate/private/intermediate.rsa.key.pem -out intermediate/csr/intermediate.rsa.csr.pem -passin file:<(echo -n "$PASS") -subj "/C=US/ST=test/L=test/O=test/CN=intermediate ca rsa"
openssl req -config intermediate/openssl.ecc.cnf -new -sha256 -key intermediate/private/intermediate.ecc.key.pem -out intermediate/csr/intermediate.ecc.csr.pem -passin file:<(echo -n "$PASS") -subj "/C=US/ST=test/L=test/O=test/CN=intermediate ca ecc"

openssl ca -config openssl.rsa.cnf -extensions v3_intermediate_ca -days 3650 -notext -md sha256 -in intermediate/csr/intermediate.rsa.csr.pem -out intermediate/certs/intermediate.rsa.cert.pem -passin file:<(echo -n "$PASS") -batch
openssl ca -config openssl.ecc.cnf -extensions v3_intermediate_ca -days 3650 -notext -md sha256 -in intermediate/csr/intermediate.ecc.csr.pem -out intermediate/certs/intermediate.ecc.cert.pem -passin file:<(echo -n "$PASS") -batch


# server cert

# create server key
cd /root/test/ca/intermediate/
openssl genrsa -out private/localhost.rsa.key.pem 2048
openssl ecparam -name $CURVE -genkey -out private/localhost.ecc.key.pem 

#create csr
openssl req -config openssl.rsa.cnf -key private/localhost.rsa.key.pem -new -sha256 -out csr/localhost.rsa.csr.pem -subj "/C=US/ST=test/L=test/O=test/CN=localhost"
openssl req -config openssl.ecc.cnf -key private/localhost.ecc.key.pem -new -sha256 -out csr/localhost.ecc.csr.pem -subj "/C=US/ST=test/L=test/O=test/CN=localhost"

#sign csr
openssl ca -config openssl.rsa.cnf -extensions server_cert -batch -days 375 -notext -md sha256 -in csr/localhost.rsa.csr.pem -out certs/localhost.rsa.cert.pem -passin file:<(echo -n "$PASS") -batch
openssl ca -config openssl.ecc.cnf -extensions server_cert -batch -days 375 -notext -md sha256 -in csr/localhost.ecc.csr.pem -out certs/localhost.ecc.cert.pem -passin file:<(echo -n "$PASS") -batch

# check

# chain root / intermediate
cat certs/intermediate.rsa.cert.pem ../certs/ca.rsa.cert.pem > certs/ca-chain.rsa.cert.pem
cat certs/intermediate.ecc.cert.pem ../certs/ca.ecc.cert.pem > certs/ca-chain.ecc.cert.pem

openssl verify -CAfile certs/ca-chain.rsa.cert.pem  certs/localhost.rsa.cert.pem
if [ $? -ne 0 ]
then
    echo "certificates not verifiable"
    exit 1
fi
openssl verify -CAfile certs/ca-chain.ecc.cert.pem  certs/localhost.ecc.cert.pem
if [ $? -ne 0 ]
then
    echo "certificates not verifiable"
    exit 1
fi

openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -in private/localhost.rsa.key.pem -out private/localhost.rsa.pkcs8.key
openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -in private/localhost.ecc.key.pem -out private/localhost.ecc.pkcs8.key

## LS

#copy logstash config
cd /root/test
wget https://raw.githubusercontent.com/aubreybox/tmp/master/ls.conf -O ls.conf

#start logstash
LS_JAVA_OPTS="-Djavax.net.debug=all" /usr/share/logstash/bin/logstash --config.debug --log.level=debug -f /root/test/ls.conf -l /root/test/ &> /root/test/ls.out &
LS_PID=$!

#start tcpdump
tcpdump -w /root/test/hs.cap -i any port 5050 or port 5051 -U &
TD_PID=$!

#wait for logstash to be ready
until ss -nptl | grep -qE "\:505[01]"
do 
    sleep 1
    echo "waiting for logstash to be ready"
done

#connect to logstash via rsa certificate
echo | openssl s_client -CAfile /root/test/ca/intermediate/certs/ca-chain.rsa.cert.pem  -cert /root/test/ca/intermediate/certs/localhost.rsa.cert.pem -key /root/test/ca/intermediate/private/localhost.rsa.pkcs8.key  -servername localhost -state -tls1_2 -connect localhost:5050 2>&1 | tee /root/test/rsa.client.log

#connect to logstash via ecc certificate
echo | openssl s_client -CAfile /root/test/ca/intermediate/certs/ca-chain.ecc.cert.pem  -cert /root/test/ca/intermediate/certs/localhost.ecc.cert.pem -key /root/test/ca/intermediate/private/localhost.ecc.pkcs8.key  -servername localhost -state -tls1_2 -connect localhost:5051 2>&1 | tee /root/test/ecc.client.log

#kill logstash and tcpdump
kill $LS_PID 
kill -2 $TD_PID

[aubreybox]

I found out, that using "prime256v1" als CURVE seems to work - why is that? How can I choose another curve?