part2
-- Subject: Unit elasticsearch.service has begun start-up
-- Defined-By: systemd
-- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel
--
-- Unit elasticsearch.service has begun starting up.
Jun 02 16:04:40 AU01ELS00V01 filebeat[127642]: 2020-06-02T16:04:40.372+1000 INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":9393070,"time":{"ms":44}},"total":{"ticks":26635320,"time":{"ms":107},"value":26635320},"user":{"ticks":17242250,"time":{"ms":63}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":13},"info":{"ephemeral_id":"fd66180f-8ea8-4d23-be51-e42d9ba20262","uptime":{"ms":3722850126}},"memstats":{"gc_next":16438448,"memory_alloc":9738664,"memory_total":2048297097056},"runtime":{"goroutines":80}},"filebeat":{"events":{"added":84,"done":84},"harvester":{"open_files":3,"running":3}},"libbeat":{"config":{"module":{"running":0},"reloads":3},"output":{"events":{"acked":84,"batches":8,"total":84},"read":{"bytes":3365},"write":{"bytes":87743}},"pipeline":{"clients":5,"events":{"active":0,"published":84,"retry":15,"total":84},"queue":{"acked":84}}},"registrar":{"states":{"current":41,"update":84},"writes":{"success":8,"total":8}},"system":{"load":{"1":0.47,"15":0.54,"5":0.55,"norm":{"1":0.0294,"15":0.0338,"5":0.0344}}}}}}
Jun 02 16:04:43 AU01ELS00V01 filebeat[127642]: 2020-06-02T16:04:43.765+1000 ERROR pipeline/output.go:100 Failed to connect to backoff(elasticsearch(http://10.150.120.9:9200)): Get http://10.150.120.9:9200: dial tcp 10.150.120.9:9200: connect: connection refused
Jun 02 16:04:43 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:43.815+1000 DEBUG [cfgfile] cfgfile/reload.go:197 Scan for new config files
Jun 02 16:04:43 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:43.816+1000 DEBUG [cfgfile] cfgfile/cfgfile.go:193 Load config from file: /etc/metricbeat/modules.d/system.yml
Jun 02 16:04:43 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:43.816+1000 DEBUG [cfgfile] cfgfile/reload.go:216 Number of module configs found: 3
Jun 02 16:04:43 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:43.816+1000 DEBUG [reload] cfgfile/list.go:62 Starting reload procedure, current runners: 3
Jun 02 16:04:43 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:43.816+1000 DEBUG [reload] cfgfile/list.go:80 Start list: 0, Stop list: 0
Jun 02 16:04:48 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:48.999+1000 INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":96562290,"time":{"ms":10}},"total":{"ticks":150952800,"time":{"ms":10},"value":150952800},"user":{"ticks":54390510}},"handles":{"limit":{"hard":4096,"soft":1024},"open":7},"info":{"ephemeral_id":"e0ca6585-ec39-48a2-b83f-8d949ab1c068","uptime":{"ms":19357263113}},"memstats":{"gc_next":13172112,"memory_alloc":10431136,"memory_total":11699445229296},"runtime":{"goroutines":38}},"libbeat":{"config":{"module":{"running":0},"reloads":3},"pipeline":{"clients":3,"events":{"active":0}}},"system":{"load":{"1":0.48,"15":0.54,"5":0.55,"norm":{"1":0.03,"15":0.0338,"5":0.0344}}}}}}
Jun 02 16:04:51 AU01ELS00V01 filebeat[127642]: 2020-06-02T16:04:51.391+1000 INFO log/harvester.go:253 Harvester started for file: /var/log/elasticsearch/gc.log
Jun 02 16:04:52 AU01ELS00V01 elasticsearch[231919]: OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
Jun 02 16:04:53 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:53.817+1000 DEBUG [cfgfile] cfgfile/reload.go:197 Scan for new config files
Jun 02 16:04:53 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:53.817+1000 DEBUG [cfgfile] cfgfile/cfgfile.go:193 Load config from file: /etc/metricbeat/modules.d/system.yml
Jun 02 16:04:53 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:53.817+1000 DEBUG [cfgfile] cfgfile/reload.go:216 Number of module configs found: 3
Jun 02 16:04:53 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:53.817+1000 DEBUG [reload] cfgfile/list.go:62 Starting reload procedure, current runners: 3
Jun 02 16:04:53 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:53.817+1000 DEBUG [reload] cfgfile/list.go:80 Start list: 0, Stop list: 0
Jun 02 16:04:59 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:59.000+1000 DEBUG [processors] processing/processors.go:183 Publish event: {
...
Jun 02 16:04:59 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:59.009+1000 DEBUG [system.fsstat] fsstat/fsstat.go:86 filesystem: /mnt/nfs_snapshots total=2083241590784, used=1592164089856, free=491077500928
Jun 02 16:04:59 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:59.009+1000 DEBUG [system.fsstat] fsstat/fsstat.go:86 filesystem: / total=53660876800, used=17173524480, free=36487352320
Jun 02 16:04:59 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:59.009+1000 DEBUG [system.fsstat] fsstat/fsstat.go:86 filesystem: /boot total=1063256064, used=183042048, free=880214016
Jun 02 16:04:59 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:59.009+1000 DEBUG [system.fsstat] fsstat/fsstat.go:86 filesystem: /mnt/data total=3170527019008, used=364502388736, free=2806024630272
Jun 02 16:04:59 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:59.009+1000 DEBUG [system.fsstat] fsstat/fsstat.go:86 filesystem: /home total=41851293696, used=33832960, free=41817460736
Jun 02 16:04:59 AU01ELS00V01 metricbeat[29305]: 2020-06-02T16:04:59.009+1000 DEBUG [processors] processing/processors.go:183 Publish event: {
...
Jun 02 16:05:01 AU01ELS00V01 filebeat[127642]: 2020-06-02T16:05:01.392+1000 INFO log/harvester.go:253 Harvester started for file: /var/log/elasticsearch/gc.log
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: Exception in thread "main" 2020-06-02 16:05:03,152 main ERROR No Log4j 2 configuration file found. Using default configuration (logging only errors to the console), or user programmatically provided configurations. Set system property 'log4j2.debug' to show Log4j 2 internal initialization logging. See https://logging.apache.org/log4j/2.x/manual/configuration.html for instructions on how to configure Log4j 2
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: ElasticsearchParseException[null-valued setting found for key [xpack.security.authc.realms] found at line number [109], column number [29]]
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: at org.elasticsearch.common.settings.Settings.validateValue(Settings.java:694)
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: at org.elasticsearch.common.settings.Settings.fromXContent(Settings.java:670)
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: at org.elasticsearch.common.settings.Settings.fromXContent(Settings.java:620)
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: at org.elasticsearch.common.settings.Settings.access$400(Settings.java:82)
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: at org.elasticsearch.common.settings.Settings$Builder.loadFromStream(Settings.java:1093)
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: at org.elasticsearch.common.settings.Settings$Builder.loadFromPath(Settings.java:1070)
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: at org.elasticsearch.node.InternalSettingsPreparer.prepareEnvironment(InternalSettingsPreparer.java:83)
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: at org.elasticsearch.cli.EnvironmentAwareCommand.createEnv(EnvironmentAwareCommand.java:100)
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: at org.elasticsearch.cli.EnvironmentAwareCommand.createEnv(EnvironmentAwareCommand.java:91)
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:86)
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:125)
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: at org.elasticsearch.cli.Command.main(Command.java:90)
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:126)
Jun 02 16:05:03 AU01ELS00V01 elasticsearch[231919]: at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:92)
Jun 02 16:05:03 AU01ELS00V01 systemd[1]: elasticsearch.service: main process exited, code=exited, status=1/FAILURE
Jun 02 16:05:03 AU01ELS00V01 systemd[1]: Failed to start Elasticsearch.
-- Subject: Unit elasticsearch.service has failed
-- Defined-By: systemd
-- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel
--
-- Unit elasticsearch.service has failed.
--
-- The result is failed.
Jun 02 16:05:03 AU01ELS00V01 systemd[1]: Unit elasticsearch.service entered failed state.
Jun 02 16:05:03 AU01ELS00V01 systemd[1]: elasticsearch.service failed.
Jun 02 16:05:03 AU01ELS00V01 polkitd[97204]: Unregistered Authentication Agent for unix-process:231913:1981383374 (system bus name :1.24807, object path /org/freedesktop/PolicyKit1/AuthenticationAgent, locale en_US.UTF-8) (disconnected from bus)
Jun 02 16:05:09 AU01ELS00V01 filebeat[127642]: 2020-06-02T16:05:09.394+1000 INFO pipeline/output.go:93 Attempting to reconnect to backoff(elasticsearch(http://10.150.120.9:9200)) with 4 reconnect attempt(s)
Jun 02 16:05:10 AU01ELS00V01 filebeat[127642]: 2020-06-02T16:05:10.374+1000 INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":9393120,"time":{"ms":47}},"total":{"ticks":26635400,"time":{"ms":86},"value":26635400},"user":{"ticks":17242280,"time":{"ms":39}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":15},"info":{"ephemeral_id":"fd66180f-8ea8-4d23-be51-e42d9ba20262","uptime":{"ms":3722880127}},"memstats":{"gc_next":16436784,"memory_alloc":13946856,"memory_total":2048302280008,"rss":266240},"runtime":{"goroutines":90}},"filebeat":{"events":{"added":32,"done":32},"harvester":{"open_files":5,"running":5,"started":2},"input":{"log":{"files":{"truncated":2}}}},"libbeat":{"config":{"module":{"running":0},"reloads":3},"output":{"events":{"acked":30,"batches":5,"total":30},"read":{"bytes":1947},"write":{"bytes":30995}},"pipeline":{"clients":5,"events":{"active":0,"filtered":2,"published":30,"retry":1,"total":32},"queue":{"acked":30}}},"registrar":{"states":{"current":41,"update":32},"writes":{"success":7,"total":7}},"system":{"load":{"1":0.47,"15":0.54,"5":0.54,"norm":{"1":0.0294,"15":0.0338,"5":0.0338}}}}}}
Jun 02 16:05:13 AU01ELS00V01 systemd-journal[3553]: Suppressed 728 messages from /system.slice/metricbeat.service
-- Subject: Messages from a service have been suppressed
-- Defined-By: systemd
-- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel
-- Documentation: man:journald.conf(5)
--
-- A service has logged too many messages within a time period. Messages
-- from the service have been dropped.
--
-- Note that only messages from the service in question have been
-- dropped, other services' messages are unaffected.
--