So I've been informed by our IT that we need to enable SSL on our setup. Currently, we are using logstash, Elasticsearch, and kibana running via 3 separate docker containers. I got a DoD SSL cert from our IT folks and I've added it to Elasticsearch and kibana.
Elasticsearch.yml
xpack.security.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.http.ssl.key: certs/ssl.key
xpack.security.http.ssl.certificate: certs/ssl.crt
xpack.security.http.ssl.certificate_authorities: certs/DOD_CA_60.cer
xpack.security.transport.ssl.key: certs/ssl.key
xpack.security.transport.ssl.certificate: certs/ssl.crt
xpack.security.transport.ssl.certificate_authorities: certs/DOD_CA_60.cer
node.name: 'elasticsearch'
cluster.name: "docker-cluster"
network.host: 0.0.0.0
cluster.initial_master_nodes:
- elasticsearch
Dockerfile for Elasticsearch
# https://github.com/elastic/elasticsearch-docker
ARG ELK_VERSION
FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
USER root
RUN groupadd -g 1370900540 mon
RUN usermod -a -G mon elasticsearch
RUN mkdir /usr/share/elasticsearch/config/certs
COPY certs/ssl.crt /usr/share/elasticsearch/config/certs/
COPY certs/ssl.key /usr/share/elasticsearch/config/certs/
COPY certs/DOD_CA_60.cer /usr/share/elasticsearch/config/certs/
RUN chown elasticsearch /usr/share/elasticsearch/config/certs
RUN chown elasticsearch /usr/share/elasticsearch/config/certs/*
RUN elasticsearch-users useradd elasticuser -p <INSERTPASSWORD> -r superuser
USER elasticsearch
# Add your elasticsearch plugins setup here
# Example: RUN elasticsearch-plugin install analysis-icu
kibana.yml
---
## Default Kibana configuration from kibana-docker.
## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml
#
server.name: kibana
server.host: "0.0.0.0"
xpack.reporting.roles.enabled: false
xpack.reporting.kibanaServer.hostname: localhost
elasticsearch.hosts: https://FULL_HOST_NAME:9200
elasticsearch.username: elasticuser
elasticsearch.password: "ELASTICUSERS PASSWORD"
elasticsearch.ssl.verificationMode: certificate
server.ssl.enabled: true
server.ssl.certificate: /etc/kibana/config/certs/ssl.crt
server.ssl.key: /etc/kibana/config/certs/ssl.key
server.ssl.certificateAuthorities: /etc/kibana/config/certs/DOD_CA_60.cer
xpack.encryptedSavedObjects.encryptionKey: /etc/kibana/config/certs/ssl.key
kibana Dockerfile
# https://github.com/elastic/kibana-docker
ARG ELK_VERSION
FROM docker.elastic.co/kibana/kibana:${ELK_VERSION}
COPY certs/* /etc/kibana/config/certs/
# Add your kibana plugins setup here
# Example: RUN kibana-plugin install <name|url>
docker-compose file:
version: '2.2'
networks:
elk:
driver: bridge
enable_ipv6: false
ipam:
driver: default
config:
- subnet: <SUBNET/26>
gateway: <GATEWAY_IP>
services:
elasticsearch:
build:
context: elasticsearch/
args:
ELK_VERSION: "$ELK_VERSION"
volumes:
- ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
- /logs/elk_esdata1:/usr/share/elasticsearch/data
ports:
- "9200:9200"
- "9300:9300"
environment:
ES_JAVA_OPTS: "-Xmx256m -Xms256m -Dlog4j2.formatMsgNoLookups=true"
networks:
- elk
restart: always
logstash:
build:
context: logstash/
args:
ELK_VERSION: "$ELK_VERSION"
volumes:
- ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
- ./logstash/config/pipelines.yml:/usr/share/logstash/config/pipelines.yml:ro
- ./logstash/pipeline:/usr/share/logstash/pipeline:ro
ports:
- "5000:5000"
- "5001:5001"
- "5002:5002"
environment:
LS_JAVA_OPTS: "-Xmx1g -Xms1g"
networks:
- elk
depends_on:
- elasticsearch
restart: always
kibana:
build:
context: kibana/
args:
ELK_VERSION: "$ELK_VERSION"
volumes:
- ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro
ports:
- "5601:5601"
networks:
- elk
depends_on:
- elasticsearch
restart: always
volumes:
esdata1:
driver: local
esdata2:
driver: local
Elasticsearch seems to be working fine:
> curl -u elasticuser:ELASTIC_PASSWORD -XGET 'FULL_HOSTNAME:9200/_cluster/health?pretty'
{
"cluster_name" : "docker-cluster",
"status" : "yellow",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"active_primary_shards" : 98,
"active_shards" : 98,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 35,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 73.68421052631578
}
I'm also able to access Elasticsearch via curl from within the kibana container.
Here are the logs from the kibana docker container:
[2022-05-19T16:33:04.977+00:00][INFO ][plugins-service] Plugin "cloudSecurityPosture" is disabled.
[2022-05-19T16:33:05.012+00:00][INFO ][plugins-service] Plugin "metricsEntities" is disabled.
[2022-05-19T16:33:05.109+00:00][INFO ][http.server.Preboot] http server running at https://0.0.0.0:5601
[2022-05-19T16:33:05.156+00:00][INFO ][plugins-system.preboot] Setting up [1] plugins: [interactiveSetup]
[2022-05-19T16:33:05.538+00:00][INFO ][plugins-system.standard] Setting up [117] plugins: [translations,monitoringCollection,licensing,globalSearch,globalSearchProviders,features,mapsEms,licenseApiGuard,usageCollection,taskManager,telemetryCollectionManager,telemetryCollectionXpack,kibanaUsageCollection,share,embeddable,uiActionsEnhanced,screenshotMode,screenshotting,banners,newsfeed,fieldFormats,expressions,eventAnnotation,dataViews,charts,esUiShared,bfetch,data,savedObjects,presentationUtil,expressionShape,expressionRevealImage,expressionRepeatImage,expressionMetric,expressionImage,customIntegrations,home,searchprofiler,painlessLab,grokdebugger,management,watcher,advancedSettings,spaces,security,savedObjectsTagging,reporting,lists,fileUpload,ingestPipelines,encryptedSavedObjects,dataEnhanced,cloud,snapshotRestore,telemetry,licenseManagement,eventLog,actions,alerting,triggersActionsUi,transform,stackAlerts,ruleRegistry,savedObjectsManagement,console,controls,graph,fleet,indexManagement,remoteClusters,crossClusterReplication,indexLifecycleManagement,visualizations,canvas,visTypeXy,visTypeVislib,visTypeVega,visTypeTimeseries,rollup,visTypeTimelion,visTypeTagcloud,visTypeTable,visTypeMetric,visTypeHeatmap,visTypeMarkdown,dashboard,dashboardEnhanced,expressionXY,expressionTagcloud,expressionPartitionVis,visTypePie,expressionMetricVis,expressionHeatmap,expressionGauge,visTypeGauge,dataViewFieldEditor,sharedUX,discover,lens,osquery,maps,dataVisualizer,ml,cases,timelines,sessionView,securitySolution,observability,uptime,infra,upgradeAssistant,monitoring,logstash,enterpriseSearch,apm,discoverEnhanced,dataViewManagement]
[2022-05-19T16:33:05.559+00:00][INFO ][plugins.taskManager] TaskManager is identified by the Kibana UUID: 953540db-3837-4605-91c3-65dcf50a356c
[2022-05-19T16:33:05.789+00:00][WARN ][plugins.security.config] Generating a random key for xpack.security.encryptionKey. To prevent sessions from being invalidated on restart, please set xpack.security.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.
[2022-05-19T16:33:05.809+00:00][WARN ][plugins.security.config] Generating a random key for xpack.security.encryptionKey. To prevent sessions from being invalidated on restart, please set xpack.security.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.
[2022-05-19T16:33:05.846+00:00][WARN ][plugins.reporting.config] Generating a random key for xpack.reporting.encryptionKey. To prevent sessions from being invalidated on restart, please set xpack.reporting.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.
[2022-05-19T16:33:05.937+00:00][INFO ][plugins.ruleRegistry] Installing common resources shared between all indices
[2022-05-19T16:33:07.048+00:00][INFO ][plugins.screenshotting.config] Chromium sandbox provides an additional layer of protection, and is supported for Linux Ubuntu 20.04 OS. Automatically enabling Chromium sandbox.
[2022-05-19T16:33:07.169+00:00][ERROR][elasticsearch-service] Unable to retrieve version information from Elasticsearch nodes. read ECONNRESET - Local: unknown:unknown, Remote: unknown:unknown
[2022-05-19T16:33:08.188+00:00][INFO ][plugins.screenshotting.chromium] Browser executable: /usr/share/kibana/x-pack/plugins/screenshotting/chromium/headless_shell-linux_x64/headless_shell
[2022-05-19T16:33:17.463+00:00][ERROR][elasticsearch-service] Unable to retrieve version information from Elasticsearch nodes. unable to verify the first certificate
[2022-05-19T16:53:05.937+00:00][ERROR][plugins.ruleRegistry] Error: Timeout: it took more than 1200000ms
at Timeout._onTimeout (/usr/share/kibana/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.js:54:20)
at listOnTimeout (node:internal/timers:559:17)
at processTimers (node:internal/timers:502:7)
[2022-05-19T16:53:05.941+00:00][ERROR][plugins.ruleRegistry] Error: Failure installing common resources shared between all indices. Timeout: it took more than 1200000ms
at ResourceInstaller.installWithTimeout (/usr/share/kibana/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.js:63:13)
at ResourceInstaller.installCommonResources (/usr/share/kibana/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.js:77:5)
Any help would be appreciated. I've been googling and flailing away at this for hours.