Error xpack monitoring elasticsearch password hasn't been registered and elastalert-kibana-plugin.serverHost and serverPort" settings were not applied

Elastic Stack Elasticsearch
1

Hi All,

I have all the yaml file and config file with me but i want to deploy through docker stack

like docker stack deploy -c docker-compose.yaml test

After deploy only elasticsearch container is working and logstash, kibana both are not connecting below is ERROR

  1. LOGSTASH ERROR

docker container logs monitoring_logstash.1.xmz06v4mp505gjncxh25wxyde

2021/05/17 15:40:02 Setting 'xpack.monitoring.elasticsearch.username' from environment.
2021/05/17 15:40:02 Setting 'xpack.monitoring.elasticsearch.password' from environment.
Sending Logstash logs to /usr/share/logstash/logs which is now configured via log4j2.properties
[2021-05-17T15:40:30,583][FATAL][logstash.runner ] An unexpected error occurred! {:error=>#<ArgumentError: Setting "xpack.monitoring.elasticsearch.password" hasn't been registered>, :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/settings.rb:36:in `get_setting'", "/usr/share/logstash/logstash-core/lib/logstash/settings.rb:69:in `set_value'", "/usr/share/logstash/logstash-core/lib/logstash/settings.rb:88:in `block in merge'", "org/jruby/RubyHash.java:1419:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/settings.rb:88:in `merge'", "/usr/share/logstash/logstash-core/lib/logstash/settings.rb:137:in `validate_all'", "/usr/share/logstash/logstash-core/lib/logstash/runner.rb:278:in `execute'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/clamp-0.6.5/lib/clamp/command.rb:67:in `run'", "/usr/share/logstash/logstash-core/lib/logstash/runner.rb:237:in `run'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/clamp-0.6.5/lib/clamp/command.rb:132:in `run'", "/usr/share/logstash/lib/bootstrap/environment.rb:73:in `<main>'"]}
[2021-05-17T15:40:30,602][ERROR][org.logstash.Logstash ] java.lang.IllegalStateException: Logstash stopped processing because of an error: (SystemExit) exit
  1. KIBANA ERROR

docker container logs monitoring_kibana.1.xzyf7047660rpxldiw5mv3fs7

{"type":"log","@timestamp":"2021-05-17T15:40:07Z","tags":["fatal","root"],"pid":1,"message":"{ Error: \"elastalert-kibana-plugin.serverHost\" and \"elastalert-kibana-plugin.serverPort\" settings were not applied. Check for spelling errors and ensure that expected plugins are installed.\n at KbnServer.exports.default (/usr/share/kibana/src/legacy/server/config/complete.js:88:17) code: 'InvalidConfig', processExitCode: 64 }"}

FATAL Error: "elastalert-kibana-plugin.serverHost" and "elastalert-kibana-plugin.serverPort" settings were not applied. Check for spelling errors and ensure that expected plugins are installed.
  1. This is my logstash.yaml file

/home/mohd/Music/Exception-Email-Notification/docker-elk/logstash/config/logstash.yml

http.host: 0.0.0.0
path.config: /usr/share/logstash/pipeline
xpack.monitoring.elasticsearch.password: xxxxx
xpack.monitoring.elasticsearch.url: http://elasticsearch:9200
xpack.monitoring.elasticsearch.username: elastic
  1. Dockerfile

/home/mohd/Music/Exception-Email-Notification/docker-elk/logstash/Dockerfile

ARG ELK_VERSION

# https://github.com/elastic/logstash-docker
#FROM docker.elastic.co/logstash/logstash-oss:${ELK_VERSION}
FROM docker.elastic.co/logstash/logstash:${ELK_VERSION}
# Add your logstash plugins setup here
# Example: RUN logstash-plugin install logstash-filter-json
  1. This is my docker-compose.yml file

/home/mohd/Music/Exception-Email-Notification/docker-elk/docker-compose.yml

version: '3.2'
services:
elasticsearch:
container_name: elasticsearch
build:
context: elasticsearch/
args:
ELK_VERSION: $ELK_VERSION
image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.1
volumes:
# - type: bind
# - ./elasticsearch:/usr/share/elasticsearch/data:rw
- ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
ports:
- "9200:9200"
- "9300:9300"
environment:
ES_JAVA_OPTS: "-Xmx256m -Xms256m"
# - ELASTIC_PASSWORD: changeme
# - discovery.type: single-node
# - cluster.name=docker-cluster
# - bootstrap.memory_lock: true
networks:
- elk

logstash:
container_name: logstash
build:
context: logstash/
args:
ELK_VERSION: $ELK_VERSION
image: docker.elastic.co/logstash/logstash-oss:7.0.1
volumes:
# - type: bind
- ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:rw
- ./logstash/pipeline:/usr/share/logstash/pipeline:ro
ports:
- "5000:5000"
- "9600:9600"
environment:
LS_JAVA_OPTS: "-Xmx256m -Xms256m"
xpack.monitoring.elasticsearch.url: "elasticsearch:9200"
xpack.monitoring.elasticsearch.username: "elastic" xpack.monitoring.elasticsearch.password: "xxxxxx"
networks:
- elk
depends_on:
- elasticsearch

kibana:
container_name: kibana
build:
context: kibana/
args:
ELK_VERSION: $ELK_VERSION
image: docker.elastic.co/kibana/kibana-oss:7.0.1
# image: fxlabs/kibana7:latest
volumes:
# - type: bind
- ./kibana/config/:/usr/share/kibana/config:ro
ports:
- "5601:5601"
networks:
- elk
depends_on:
- elasticsearch

networks:
elk:
# driver: bridge
driver: overlay

volumes:
elasticsearch:
  1. This is my docker-stack.yaml file

/home/mohd/Music/Exception-Email-Notification/docker-elk/docker-stack.yml

version: '3.3'

#networks:
# host:
# driver: overlay
# attachable: true

volumes:
elasticsearch:
logstash:
kibana:

services:

nginx:
image: nginx:latest
volumes:
- ./default.conf:/etc/nginx/conf.d/default.conf

elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.1
ports:
# - "9200:9200"
- "9300:9300"
configs:
- source: elastic_config
target: /usr/share/elasticsearch/config/elasticsearch.yml
environment:
# ES_JAVA_OPTS: "-Xmx256m -Xms256m"
ES_JAVA_OPTS: "-Xms4g -Xmx4g"
ELASTIC_USERNAME: "elastic"
ELASTIC_PASSWORD: "changeme"
# ELASTIC_USERNAME: "kibana"
# ELASTIC_PASSWORD: "admin"
# http.cors.enabled: "true"
# http.cors.allow-origin: "*"
# xpack.security.enabled: "true"
# networks:
# - host
volumes:
- elasticsearch:/usr/share/elasticsearch/data
deploy:
mode: replicated
replicas: 1

logstash:
image: docker.elastic.co/logstash/logstash-oss:7.0.1
ports:
# - "5000:5000"
- "5044:5044"
- "9600:9600"
configs:
- source: logstash_config
target: /usr/share/logstash/config/logstash.yml:rw
- source: logstash_pipeline
target: /usr/share/logstash/pipeline/logstash.conf
volumes:
- logstash:/usr/share/logstash/data
# - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:rw
# - ./logstash/pipeline:/usr/share/logstash/pipeline:ro

environment:
LS_JAVA_OPTS: "-Xmx256m -Xms256m"
# xpack.monitoring.elasticsearch.url: "elasticsearch:9200"
# xpack.monitoring.elasticsearch.username: "elastic"
# xpack.monitoring.elasticsearch.password: "xxxxxx"
# networks:
# - host
deploy:
mode: replicated
replicas: 1

kibana:
image: fxlabs/kibana7:latest
# ports:
# - "5601:5601"
volumes:
- kibana:/usr/share/kibana/data
configs:
- source: kibana_config
target: /usr/share/kibana/config/kibana.yml
# environment:
# ELASTIC_USERNAME: "elastic"
# ELASTIC_PASSWORD: "MyPw123"
# - "elasticsearch.username: elastic"
# - "elasticsearch.password: MyPw123"
# http.cors.enabled: "true"
# http.cors.allow-origin: "*"
# xpack.security.enabled: "true"
# networks:
# - host
deploy:
mode: replicated
replicas: 1
caddy:
image: openfaas/caddy
# image: fxlabs/caddy
# image: stefanprodan/caddy
# image: abiosoft/caddy
ports:
- "5601:5601"
- "9200:9200"
# - "5044:5044"
- "3030:3030"
# - "8080:8080"
# networks:
# - net
# environment:
# - ADMIN_USER= admin
# - ADMIN_PASSWORD= fxadmin@12345##
# configs:
# - source: caddy_config
# target: /etc/caddy/Caddyfile
volumes:
- /home/docker-elk/Caddyfile:/etc/caddy/Caddyfile
# - /home/ubuntu/docker-elk/caddy/cert.crt:/etc/pki/tls/certs/cert.crt
# - /home/ubuntu/docker-elk/caddy/cert.key:/etc/pki/tls/private/cert.key
deploy:
mode: replicated
replicas: 1
placement:
constraints:
- node.role == manager
resources:
limits:
memory: 128M
reservations:
memory: 64M

configs:

elastic_config:
file: ./elasticsearch/config/elasticsearch.yml
logstash_config:
file: ./logstash/config/logstash.yml
logstash_pipeline:
file: ./logstash/pipeline/logstash.conf
kibana_config:
file: ./kibana/config/kibana.yml

#networks:
# elk:
# driver: overlay
  1. This is my kibana.yaml file

/home/mohd/Music/Exception-Email-Notification/docker-elk/kibana/config/kibana.yml

---
## Default Kibana configuration from kibana-docker.
## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml
#
server.name: kibana
server.host: "0"
#elasticsearch.url: http://elasticsearch:9200
#elasticsearch.hosts: http://elasticsearch:9200
elasticsearch.hosts: http://nginx
#elasticsearch.username: "elastic"
elasticsearch.username: "test"
elasticsearch.password: "changeme"
#elastalert-kibana-plugin.serverHost: 123.0.0.1
#elastalert-kibana-plugin.serverPort: 9000
#timeout= 90000ms
#elasticsearch.timeout: "90000ms"
#readonly:
# cluster:
# - cluster:monitor/nodes/info
# - cluster:monitor/health
# indices:
# '*':
# privileges: indices:test/mappings/fields/get, indices:test/validate/query, indices:data/read/search, indices:data/read/msearch, indices:data/read/field_stats, indices:test/get
# '.kibana':
# privileges: indices:test/exists, indices:test/mappings/fields/get, indices:test/refresh, indices:test/validate/query, indices:data/read/get, indices:data/read/mget, indices:data/read/search

#elastalert.enabled:
elastalert-kibana-plugin.serverHost: elastalert
elastalert-kibana-plugin.serverPort: 3030

#sentinl:
# settings:
# email:
# active: true
# user: rashid777@gmail.com
# password: xxxxxx
# host: smtp.gmail.com
# port: 465
# ssl: true
# report:
# active: true
#
# slack:
# active: true
# token: xoxp-483014078565-482209664337-590134404704-38b035ea03ee31e25941db5e6f1957e3
#
  1. This is Dockerfile of kibana

/home/mohd/Music/Exception-Email-Notification/docker-elk/kibana/Dockerfile

ARG ELK_VERSION

# https://github.com/elastic/kibana-docker
#FROM docker.elastic.co/kibana/kibana-oss:${ELK_VERSION}
#FROM docker.elastic.co/kibana/kibana-oss:6.6.1

FROM docker.elastic.co/kibana/kibana-oss:7.0.1
# Add your kibana plugins setup here
# Example: RUN kibana-plugin install <name|url>
#RUN kibana-plugin install x-pack
#RUN ./bin/kibana-plugin install https://github.com/bitsensor/elastalert-kibana-plugin/releases/download/1.0.1/elastalert-kibana-plugin-1.0.1-${ELK_VERSION}.zip
#RUN ./bin/kibana-plugin install https://github.com/bitsensor/elastalert-kibana-plugin/releases/download/1.0.2/elastalert-kibana-plugin-1.0.2-${ELK_VERSION}.zip

#RUN ./bin/kibana-plugin install https://github.com/bitsensor/elastalert-kibana-plugin/releases/download/1.0.2/elastalert-kibana-plugin-1.0.2-6.6.1.zip

RUN ./bin/kibana-plugin install https://github.com/bitsensor/elastalert-kibana-plugin/releases/download/1.0.4/elastalert-kibana-plugin-1.0.4-7.0.1.zip
#RUN ./bin/kibana-plugin install https://releases.floragunn.com/search-guard-kibana-plugin-7/7.0.1-37.0.0/search-guard-kibana-plugin-7-7.0.1-37.0.0.zip
#RUN ./bin/kibana-plugin install https://github.com/sirensolutions/sentinl/releases/download/tag-6.6.0-0/sentinl-v6.6.1.zip
#RUN ./bin/kibana-plugin install https://github.com/sirensolutions/sentinl/releases/download/tag-7.0.0-0/sentinl-v7.0.1.zip
2

Why are you running 7.0, it's EOL. Please use a newer version.

It's also not clear why you have multiple dock files - docker-compose.yml and docker-stack.yml - which one are you using?

3

I want to use only 7 version not latest because in production we use that version

that only i dont know which one should i use docker-compose.yaml or docker-stack.yaml but anyway i have use docker-stack.yaml and i got another one error

ohd@master-node:~/Music/Exception-Email-Notification/docker-elk$ tail -f /var/log/syslog | grep kibana
May 18 16:55:40 master-node dockerd[24124]: time="2021-05-18T16:55:40.339808478+05:30" level=info msg="Attempting next endpoint for pull after error: Head https://registry-1.docker.io/v2/fxlabs/kibana7/manifests/latest: Get https://auth.docker.io/token?scope=repository%3Afxlabs%2Fkibana7%3Apull&service=registry.docker.io: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"
May 18 16:55:42 master-node dockerd[24124]: time="2021-05-18T16:55:42.195795459+05:30" level=error msg="pulling image failed" error="Head https://registry-1.docker.io/v2/fxlabs/kibana7/manifests/latest: Get https://auth.docker.io/token?scope=repository%3Afxlabs%2Fkibana7%3Apull&service=registry.docker.io: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)" module=node/agent/taskmanager node.id=kwpyr5ixo3xx6psp782zxtgre service.id=10h7ton3yj4y4xfqbcxkc2x7c task.id=94now9tz3qsron9ped4hmxzjr
May 18 16:56:48 master-node dockerd[24124]: time="2021-05-18T16:56:48.171941058+05:30" level=error msg="pulling image failed" error="pull access denied for fxlabs/kibana7, repository does not exist or may require 'docker login': denied: requested access to the resource is denied" module=node/agent/taskmanager node.id=kwpyr5ixo3xx6psp782zxtgre service.id=10h7ton3yj4y4xfqbcxkc2x7c task.id=z66hyorzth9oieyziqedblwl7
May 18 16:57:56 master-node dockerd[24124]: time="2021-05-18T16:57:56.669087844+05:30" level=error msg="pulling image failed" error="pull access denied for fxlabs/kibana7, repository does not exist or may require 'docker login': denied: requested access to the resource is denied" module=node/agent/taskmanager node.id=kwpyr5ixo3xx6psp782zxtgre service.id=10h7ton3yj4y4xfqbcxkc2x7c task.id=azbqxjx4w4u5oy2gwwa11kmh5

dont know how to solve it

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.