Since you want me to post here before opening an issue, here we go:
Today one of our servers' disk was full. The situation has been resolved, but journalbeat does not resume operation:
root@mainvm:/services/EnoEngine# date
Tue Jul 2 22:21:44 UTC 2019
root@mainvm:/services/EnoEngine# journalctl -u journalbeat | tail
Jul 02 20:53:34 mainvm journalbeat[66930]: 2019-07-02 20:53:34.108453103 +0000 UTC m=+15599.314296376 write error: failed to write to file: write /var/log/journalbeat/journalbeat: no space left on device
Jul 02 20:53:35 mainvm journalbeat[66930]: 2019-07-02 20:53:35.011908545 +0000 UTC m=+15600.217751928 write error: failed to write to file: write /var/log/journalbeat/journalbeat: no space left on device
Jul 02 20:53:35 mainvm journalbeat[66930]: 2019-07-02 20:53:35.112646966 +0000 UTC m=+15600.318490222 write error: failed to write to file: write /var/log/journalbeat/journalbeat: no space left on device
Jul 02 20:53:36 mainvm journalbeat[66930]: 2019-07-02 20:53:36.115377123 +0000 UTC m=+15601.321220341 write error: failed to write to file: write /var/log/journalbeat/journalbeat: no space left on device
Jul 02 20:53:37 mainvm journalbeat[66930]: 2019-07-02 20:53:37.117972354 +0000 UTC m=+15602.323815582 write error: failed to write to file: write /var/log/journalbeat/journalbeat: no space left on device
Jul 02 20:53:38 mainvm journalbeat[66930]: 2019-07-02 20:53:38.205371149 +0000 UTC m=+15603.411214544 write error: failed to write to file: write /var/log/journalbeat/journalbeat: no space left on device
Jul 02 20:53:39 mainvm journalbeat[66930]: 2019-07-02 20:53:39.20838299 +0000 UTC m=+15604.414226269 write error: failed to write to file: write /var/log/journalbeat/journalbeat: no space left on device
Jul 02 20:53:40 mainvm journalbeat[66930]: 2019-07-02 20:53:40.21170599 +0000 UTC m=+15605.417549217 write error: failed to write to file: write /var/log/journalbeat/journalbeat: no space left on device
Jul 02 20:53:41 mainvm journalbeat[66930]: 2019-07-02 20:53:41.214972553 +0000 UTC m=+15606.420815811 write error: failed to write to file: write /var/log/journalbeat/journalbeat: no space left on device
Jul 02 20:53:42 mainvm journalbeat[66930]: 2019-07-02 20:53:42.219536085 +0000 UTC m=+15607.425379591 write error: failed to write to file: write /var/log/journalbeat/journalbeat: no space left on device
root@mainvm:/services/EnoEngine# tail /var/log/journalbeat/journalbeat
2019-07-02T20:54:56.655Z INFO [input] input/input.go:133 journalbeat successfully published 2 events {"id": "b6eeff36-b97c-448c-91ff-6e4e4572d452"}
2019-07-02T20:55:04.973Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":28720},"total":{"ticks":333750,"time":{"ms":345},"value":333750},"user":{"ticks":305030,"time":{"ms":345}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":33},"info":{"ephemeral_id":"fad06d78-6258-4f69-8867-40b2ebb09437","uptime":{"ms":15690040}},"memstats":{"gc_next":12083776,"memory_alloc":9268864,"memory_total":21603363840}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":20,"batches":8,"total":20},"read":{"bytes":48},"write":{"bytes":7924}},"pipeline":{"clients":1,"events":{"active":0,"published":20,"total":20},"queue":{"acked":20}}},"system":{"load":{"1":0.9,"15":2.63,"5":1.41,"norm":{"1":0.007,"15":0.0205,"5":0.011}}}}}}
2019-07-02T20:55:06.277Z INFO [input] input/input.go:133 journalbeat successfully published 4 events {"id": "b6eeff36-b97c-448c-91ff-6e4e4572d452"}
2019-07-02T20:55:11.666Z INFO [input] input/input.go:133 journalbeat successfully published 3 events {"id": "b6eeff36-b97c-448c-91ff-6e4e4572d452"}
2019-07-02T20:55:12.703Z INFO [input] input/input.go:133 journalbeat successfully published 1 events {"id": "b6eeff36-b97c-448c-91ff-6e4e4572d452"}
2019-07-02T20:55:20.022Z INFO [input] input/input.go:133 journalbeat successfully published 2 events {"id": "b6eeff36-b97c-448c-91ff-6e4e4572d452"}
2019-07-02T20:55:21.358Z INFO [input] input/input.go:133 journalbeat successfully published 4 events {"id": "b6eeff36-b97c-448c-91ff-6e4e4572d452"}
2019-07-02T20:55:26.765Z INFO [input] input/input.go:133 journalbeat successfully published 2 events {"id": "b6eeff36-b97c-448c-91ff-6e4e4572d452"}
2019-07-02T20:55:29.560Z INFO [input] input/input.go:133 journalbeat successfully published 2 events {"id": "b6eeff36-b97c-448c-91ff-6e4e4572d452"}
2019-07-02T20:55:34.968Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":28720},"total":{"ticks":334120,"time":{"ms":367},"value":334120},"user":{"ticks":305400,"time":{"ms":367}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":33},"info":{"ephemeral_id":"fad06d78-6258-4f69-8867-40b2ebb09437","uptime":{"ms":15720040}},"memstats":{"gc_next":12024896,"memory_alloc":9240656,"memory_total":21614791984}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":18,"batches":7,"total":18},"read":{"bytes":42},"write":{"bytes":7414}},"pipeline":{"clients":1,"events":{"active":0,"published":18,"total":18},"queue":{"acked":18}}},"system":{"load":{"1":0.81,"15":2.57,"5":1.34,"norm":{"1":0.0063,"15":0.0201,"5":0.0105}}}}}}
root@mainvm:/services/EnoEngine# df -h
Filesystem Size Used Avail Use% Mounted on
[...]
/dev/xvda1 7.7G 6.4G 1.4G 83% /
[...]
As you can see, there are 1.4 GB free again, but journalbeat is stuck for almost 90 minutes. Should I open an issue on github?