Kibana service stops after few seconds

[caiossouza (Caio)]

Hello,

My Kibana was working fine for months and it stopped working without apparently reason.
When I restart the service using "sudo service kibana start", it stops after few seconds.

1. I've checked the storage and memory of host and they are ok.
2. The /var/log/kibana.stdout and /var/log/kibana.stderr are empty (could be the logs anywhere else?).
3. Kibana Version: 6.2.3
4. Elastic service is active

Any idea why its stopping or any troubleshooting reference to help me?

Thank you very much

Ps: some lines of the /var/log/kibana.log (i think it doesn't show the problem)

{"type":"log","@timestamp":"2018-11-30T18:23:55Z","tags":["info","Sentinl","scheduler"],"pid":61615,"message":"scheduled watcher sentinl-watcher:af31eb90-85fb-11e8-be01-f337686e394a, to run every every 1 minute"}
{"type":"log","@timestamp":"2018-11-30T18:23:55Z","tags":["info","Sentinl","scheduler"],"pid":61615,"message":"scheduled watcher sentinl-watcher:76787c40-70d6-11e8-9025-8dad2c533d81, to run every every 10 seconds"}
{"type":"log","@timestamp":"2018-11-30T18:23:55Z","tags":["info","Sentinl","scheduler"],"pid":61615,"message":"scheduled watcher sentinl-watcher:de1301f0-860b-11e8-be01-f337686e394a, to run every every 10 seconds"}
{"type":"log","@timestamp":"2018-11-30T18:24:00Z","tags":["info","Sentinl","scheduler"],"pid":61615,"message":"watcher sentinl-watcher:abbaf040-85da-11e8-be01-f337686e394a: executing"}
{"type":"log","@timestamp":"2018-11-30T18:24:00Z","tags":["info","Sentinl","watcher sentinl-watcher:abbaf040-85da-11e8-be01-f337686e394a"],"pid":61615,"message":"executing"}
{"type":"log","@timestamp":"2018-11-30T18:24:00Z","tags":["warning","Sentinl","watcher sentinl-watcher:abbaf040-85da-11e8-be01-f337686e394a"],"pid":61615,"message":""elasticsearch.plugins" not available when running from kibana"}
{"type":"log","@timestamp":"2018-11-30T18:24:00Z","tags":["info","Sentinl","scheduler"],"pid":61615,"message":"watcher sentinl-watcher:daea63d0-85e1-11e8-be01-f337686e394a: executing"}
{"type":"log","@timestamp":"2018-11-30T18:24:00Z","tags":["info","Sentinl","watcher sentinl-watcher:daea63d0-85e1-11e8-be01-f337686e394a"],"pid":61615,"message":"executing"}
{"type":"log","@timestamp":"2018-11-30T18:24:00Z","tags":["warning","Sentinl","watcher sentinl-watcher:daea63d0-85e1-11e8-be01-f337686e394a"],"pid":61615,"message":""elasticsearch.plugins" not available when running from kibana"}
{"type":"log","@timestamp":"2018-11-30T18:24:00Z","tags":["info","Sentinl","scheduler"],"pid":61615,"message":"watcher sentinl-watcher:0375afd0-85fb-11e8-be01-f337686e394a: executing"}
{"type":"log","@timestamp":"2018-11-30T18:24:00Z","tags":["info","Sentinl","watcher sentinl-watcher:0375afd0-85fb-11e8-be01-f337686e394a"],"pid":61615,"message":"executing"}
{"type":"log","@timestamp":"2018-11-30T18:24:00Z","tags":["warning","Sentinl","watcher sentinl-watcher:0375afd0-85fb-11e8-be01-f337686e394a"],"pid":61615,"message":""elasticsearch.plugins" not available when running from kibana"}
{"type":"log","@timestamp":"2018-11-30T18:24:00Z","tags":["info","Sentinl","scheduler"],"pid":61615,"message":"watcher sentinl-watcher:af31eb90-85fb-11e8-be01-f337686e394a: executing"}
{"type":"log","@timestamp":"2018-11-30T18:24:00Z","tags":["info","Sentinl","watcher sentinl-watcher:af31eb90-85fb-11e8-be01-f337686e394a"],"pid":61615,"message":"executing"}
{"type":"log","@timestamp":"2018-11-30T18:24:00Z","tags":["warning","Sentinl","watcher sentinl-watcher:af31eb90-85fb-11e8-be01-f337686e394a"],"pid":61615,"message":""elasticsearch.plugins" not available when running from kibana"}

[nickpeihl]

You may want to configure the logging.dest and logging.verbose parameters in your kibana.yml file. This will write debug logs to the file location you choose. It may give you additional details on why Kibana is stopping. For more info, see the Configuring Kibana documentation.

Let me know if you still need help.

[caiossouza (Caio)]

Thank you @nickpeihl

It seems a problem with a sentinl plugin.

Regards,

{"type":"log","@timestamp":"2018-12-04T17:56:10Z","tags":["warning","Sentinl","watcher sentinl-watcher:de1301f0-860b-11e8-be01-f337686e394a"],"pid":31230,"message":""elasticsearch.plugins" not av
ailable when running from kibana"}

{"type":"log","@timestamp":"2018-12-04T17:56:06Z","tags":["reporting","warning"],"pid":12336,"message":"Generating a random key for xpack.reporting.encryptionKey. To prevent pending reports from fa
iling on restart, please set xpack.reporting.encryptionKey in kibana.yml"}

{"type":"log","@timestamp":"2018-12-04T17:56:05Z","tags":["reporting","debug","exportTypes"],"pid":12336,"message":"Found exportType at /usr/share/kibana/plugins/x-pack/plugins/reporting/export_typ
es/printable_pdf/server/index.js"}

{"type":"log","@timestamp":"2018-12-04T17:56:05Z","tags":["plugins","debug"],"pid":12336,"plugin":{"author":"Lorenzo Mangani lorenzo.mangani@gmail.com","repository":{"type":"git","url":"http://gi
thub.com/sirensolutions/sentinl.git"},"name":"sentinl","version":"6.0.0","kibana":{"version":"6.2.3"},"description":"Kibana Alert & Report App for Elasticsearch","main":"index.js","scripts":{"test"
:"gulp test","testdev":"gulp testdev","testserver":"gulp testserver","test:coverage":"gulp coverage","precommit":"gulp lint","sentinl-install":"/opt/kibana/bin/kibi plugin --install sentinl -u file
://pwd/sentinl-latest.tar.gz","sentinl-remove":"/opt/kibana/bin/kibi plugin --remove sentinl","package":"gulp package","archive":"tar cfz ./sentinl-latest.tar.gz --exclude .git --exclude .gitigno
re --exclude .github --exclude gradle --exclude build --exclude target --exclude sentinl-latest.tar.gz ./* && ls -alF sentinl-latest*"},"dependencies":{"@elastic/datemath":"2.3.0","angular":"1.4.8"
,"angular-ui-bootstrap":"2.5.0","anomaly-finder":"1.0.1","bluebird":"3.4.7","boom":"7.1.1","bootstrap":"3.3.7","decompress":"4.2.0","elasticsearch":"13.2.0","emailjs":"1.0.8","klaw-sync":"4.0.0","l
ater":"1.2.0","mustache":"2.3.0","node-horseman":"3.3.0","node-slack":"0.0.7","puppeteer":"1.3.0","sum-time":"1.0.0","uuid":"3.1.0"},"devDependencies":{"babel-eslint":"6.1.2","babel-plugin-angularj
s-annotate":"0.8.2","eslint":"3.14.1","eslint-plugin-mocha":"4.8.0","gulp":"3.9.1","gulp-eslint":"1.1.1","gulp-util":"3.0.8","gulp-zip":"3.1.0","husky":"0.10.2","lodash":"3.10.1","minimist":"1.2.0"
,"mkdirp":"0.5.1","rimraf":"2.5.4","rsync":"0.4.0"},"license":"Apache-2.0"},"message":"Initializing plugin sentinl@6.0.0"}

{"type":"log","@timestamp":"2018-12-04T17:56:05Z","tags":["status","plugin:searchguard@6.2.3","info"],"pid":12336,"state":"yellow","message":"Status changed from yellow to yellow - 'searchguard.coo
kie.secure' is set to false, cookies are transmitted over unsecure HTTP connection. Consider using HTTPS and set this key to 'true'","prevState":"yellow","prevMsg":"Default cookie password detected
, please set a password in kibana.yml by setting 'searchguard.cookie.password' (min. 32 characters)."}

{"type":"log","@timestamp":"2018-12-04T17:56:05Z","tags":["status","plugin:searchguard@6.2.3","info"],"pid":12336,"state":"yellow","message":"Status changed from yellow to yellow - Default cookie p
assword detected, please set a password in kibana.yml by setting 'searchguard.cookie.password' (min. 32 characters).","prevState":"yellow","prevMsg":"Initialising Search Guard authentication plugin
."}

{"type":"log","@timestamp":"2018-12-04T17:56:04Z","tags":["plugins","debug"],"pid":12336,"plugin":{"name":"searchguard","version":"6.2.3","description":"Search Guard features for kibana","main":"in
dex.js","homepage":"https://floragunn.com","license":"Apache-2.0","repository":{"type":"git","url":"https://github.com/floragunncom/search-guard-kibana-plugin"},"dependencies":{"@elastic/kibana-ui-
framework":"0.0.11","boom":"^3.2.2","hapi":"^16.0.1","hapi-async-handler":"^1.0.3","hapi-auth-cookie":"^3.1.0","hapi-authorization":"^3.0.2","joi":"^6.6.1","js-yaml":"^3.7.0","requirefrom":"^0.2.0"
},"devDependencies":{"ui-select":"^0.19.8"}},"message":"Initializing plugin searchguard@6.2.3"}

{"type":"log","@timestamp":"2018-12-04T17:56:04Z","tags":["plugins","debug"],"pid":12336,"plugin":{"name":"x-pack","version":"6.2.3","author":"Elastic","dependencies":{"@elastic/datemath":"4.0.2","
@elastic/node-crypto":"0.1.2","@elastic/node-phantom-simple":"2.2.4","angular-paging":"2.2.1","angular-resource":"1.4.9","angular-sanitize":"1.4.9","angular-ui-ace":"0.2.3","angular-ui-bootstrap":"
1.2.5","babel-core":"6.21.0","babel-preset-es2015":"6.22.0","babel-runtime":"5.8.38","bluebird":"3.1.1","boom":"3.1.1","chrome-remote-interface":"0.24.2","clarinet":"0.11.0","concat-stream":"1.5.1"
,"d3":"3.5.6","d3-scale":"1.0.6","dragselect":"1.7.17","elasticsearch":"13.0.1","extract-zip":"1.5.0","flot-charts":"0.8.3","font-awesome":"4.4.0","get-port":"2.1.0","glob":"6.0.4","hapi-auth-cooki
e":"6.1.1","history":"4.7.2","humps":"2.0.1","icalendar":"0.7.1","isomorphic-fetch":"2.2.1","joi":"6.10.1","lodash":"3.10.1","lodash.orderby":"4.6.0","moment":"2.13.0","moment-duration-format":"1.3
.0","moment-timezone":"0.5.4","object-hash":"1.2.0","path-match":"1.2.4","pdfmake":"0.1.33","pluralize":"3.1.0","prop-types":"^15.6.0","pui-react-overlay-trigger":"8.3.3","pui-react-tooltip":"8.3.3
","puid":"1.0.5","react":"16.0.0","react-clipboard.js":"1.1.2","react-dom":"16.0.0","react-markdown-renderer":"1.4.0","react-portal":"3.2.0","react-redux":"5.0.5","react-router-dom":"4.2.2","react-
sticky":"6.0.1","react-syntax-highlighter":"5.7.0","react-vis":"1.8.1","redux":"3.7.2","redux-actions":"2.2.1","redux-thunk":"2.2.0","reselect":"3.0.1","rison-node":"0.3.1","rxjs":"5.3.0","semver":
"5.1.0","styled-components":"2.3.2","tar-fs":"1.13.0","tinycolor2":"1.3.0","ui-select":"0.19.4","unbzip2-stream":"1.0.9","uuid":"3.0.1","venn.js":"0.2.9","webcola":"3.3.6","xregexp":"3.2.0"},"resol
utions":{"moment":"2.13.0","react":"16.0.0"},"engines":{"yarn":"^1.3.2"},"kibana":{"version":"6.2.3"},"build":{"git":{"count":"3894","sha":"1ab54ec","date":"Fri, 9 Mar 2018 09:05:45 -0800"},"date":
"Tue Mar 13 2018 11:16:36 GMT+0000 (UTC)"}},"message":"Initializing plugin tilemap@6.2.3"}

{"type":"log","@timestamp":"2018-12-04T17:56:00Z","tags":["plugin","debug"],"pid":12336,"path":"/usr/share/kibana/src/core_plugins/region_map","message":"Found plugin at /usr/share/kibana/src/core_
plugins/region_map"}

{"type":"log","@timestamp":"2018-12-04T17:56:00Z","tags":["plugin","debug"],"pid":12336,"path":"/usr/share/kibana/src/core_plugins/metrics","message":"Found plugin at /usr/share/kibana/src/core_plu
gins/metrics"}

{"type":"log","@timestamp":"2018-12-04T17:56:00Z","tags":["plugin","debug"],"pid":12336,"path":"/usr/share/kibana/src/core_plugins/metric_vis","message":"Found plugin at /usr/share/kibana/src/core_
plugins/metric_vis"}