Auditbeat randomly shuts down

Elastic Stack Beats
21

[root@auditbeat-testing auditbeat]# auditbeat -d '*'
fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0xd pc=0x7f416d796568]

runtime stack:
runtime.throw(0x17e346f, 0x2a)
/usr/local/go/src/runtime/panic.go:616 +0x81
runtime.sigpanic()
/usr/local/go/src/runtime/signal_unix.go:372 +0x28e

goroutine 95 [syscall, locked to thread]:
runtime.cgocall(0x14bcd60, 0xc42056f598, 0xc400000000)
/usr/local/go/src/runtime/cgocall.go:128 +0x64 fp=0xc42056f568 sp=0xc42056f530 pc=0xa096c4
github.com/elastic/beats/x-pack/auditbeat/module/system/package._Cfunc_my_rpmReadConfigFiles(0x7f416e7cb1b0, 0x7f4100000000)
_cgo_gotypes.go:147 +0x49 fp=0xc42056f598 sp=0xc42056f568 pc=0x144c209
github.com/elastic/beats/x-pack/auditbeat/module/system/package.listRPMPackages.func4(0x7f416e7cb1b0, 0x18153c8)
/go/src/github.com/elastic/beats/x-pack/auditbeat/module/system/package/rpm_linux.go:292 +0x56 fp=0xc42056f5d0 sp=0xc42056f598 pc=0x144de26
github.com/elastic/beats/x-pack/auditbeat/module/system/package.listRPMPackages(0x0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/elastic/beats/x-pack/auditbeat/module/system/package/rpm_linux.go:292 +0x106 fp=0xc42056f670 sp=0xc42056f5d0 pc=0x144d036
github.com/elastic/beats/x-pack/auditbeat/module/system/package.getPackages(0x17b8772, 0x6, 0x0, 0x21a9ea8, 0x0, 0x0, 0x0)
/go/src/github.com/elastic/beats/x-pack/auditbeat/module/system/package/package.go:475 +0x16b fp=0xc42056f700 sp=0xc42056f670 pc=0x1449f9b
github.com/elastic/beats/x-pack/auditbeat/module/system/package.(*MetricSet).reportChanges(0xc4200ee000, 0x7f417ebc7d48, 0xc4206b0540, 0x79b09d984b, 0xa1551d)
/go/src/github.com/elastic/beats/x-pack/auditbeat/module/system/package/package.go:318 +0x56 fp=0xc42056faa8 sp=0xc42056f700 pc=0x14487f6
github.com/elastic/beats/x-pack/auditbeat/module/system/package.(*MetricSet).Fetch(0xc4200ee000, 0x7f417ebc7d48, 0xc4206b0540)
/go/src/github.com/elastic/beats/x-pack/auditbeat/module/system/package/package.go:273 +0x320 fp=0xc42056fb78 sp=0xc42056faa8 pc=0x1448130
github.com/elastic/beats/metricbeat/mb/module.(*metricSetWrapper).fetch(0xc420467c80, 0x18a9e00, 0xc4206b0540)
/go/src/github.com/elastic/beats/metricbeat/mb/module/wrapper.go:238 +0x2ac fp=0xc42056fc18 sp=0xc42056fb78 pc=0x13ed9dc
github.com/elastic/beats/metricbeat/mb/module.(*metricSetWrapper).startPeriodicFetching(0xc420467c80, 0x18a9e00, 0xc4206b0540)
/go/src/github.com/elastic/beats/metricbeat/mb/module/wrapper.go:219 +0x117 fp=0xc42056fd88 sp=0xc42056fc18 pc=0x13ed5f7
github.com/elastic/beats/metricbeat/mb/module.(*metricSetWrapper).run(0xc420467c80, 0xc4202a55c0, 0xc42051b1a0)
/go/src/github.com/elastic/beats/metricbeat/mb/module/wrapper.go:196 +0x5ca fp=0xc42056ff58 sp=0xc42056fd88 pc=0x13ed0ea
github.com/elastic/beats/metricbeat/mb/module.(*Wrapper).Start.func1(0xc4205fd520, 0xc4202a55c0, 0xc42051b1a0, 0xc420467c80)
/go/src/github.com/elastic/beats/metricbeat/mb/module/wrapper.go:137 +0x249 fp=0xc42056ffc0 sp=0xc42056ff58 pc=0x13efd39
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc42056ffc8 sp=0xc42056ffc0 pc=0xa62d41
created by github.com/elastic/beats/metricbeat/mb/module.(*Wrapper).Start
/go/src/github.com/elastic/beats/metricbeat/mb/module/wrapper.go:125 +0x145

goroutine 1 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4205fd21c)
/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc4205fd210)
/usr/local/go/src/sync/waitgroup.go:129 +0x72
github.com/elastic/beats/metricbeat/beater.(*Metricbeat).Run(0xc420132690, 0xc4204b4480, 0xc420138360, 0x1)
/go/src/github.com/elastic/beats/metricbeat/beater/metricbeat.go:251 +0x3c8
github.com/elastic/beats/libbeat/cmd/instance.(*Beat).launch(0xc4204b4480, 0x17bbd2b, 0x9, 0x17bbd2b, 0x9, 0x0, 0x0, 0x0, 0x0, 0xc4200c20f0, ...)
/go/src/github.com/elastic/beats/libbeat/cmd/instance/beat.go:412 +0x4f6
github.com/elastic/beats/libbeat/cmd/instance.Run.func1(0x17bbd2b, 0x9, 0x17bbd2b, 0x9, 0x0, 0x0, 0x17bbd2b, 0x9, 0x17bbd2b, 0x9, ...)
/go/src/github.com/elastic/beats/libbeat/cmd/instance/beat.go:183 +0x611
github.com/elastic/beats/libbeat/cmd/instance.Run(0x17bbd2b, 0x9, 0x17bbd2b, 0x9, 0x0, 0x0, 0x0, 0x0, 0xc4200c20f0, 0x0, ...)
/go/src/github.com/elastic/beats/libbeat/cmd/instance/beat.go:184 +0xbe
github.com/elastic/beats/libbeat/cmd.genRunCmd.func1(0xc4203dc000, 0xc4203dfc20, 0x0, 0xa)
/go/src/github.com/elastic/beats/libbeat/cmd/run.go:37 +0x4f
github.com/elastic/beats/vendor/github.com/spf13/cobra.(*Command).execute(0xc4203dc000, 0xc42003a0c0, 0xa, 0xa, 0xc4203dc000, 0xc42003a0c0)
/go/src/github.com/elastic/beats/vendor/github.com/spf13/cobra/command.go:704 +0x2c6
github.com/elastic/beats/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0xc4203dc000, 0x0, 0x1813898, 0x0)
/go/src/github.com/elastic/beats/vendor/github.com/spf13/cobra/command.go:785 +0x2e4
github.com/elastic/beats/vendor/github.com/spf13/cobra.(*Command).Execute(0xc4203dc000, 0xc42023df78, 0xc420092058)
/go/src/github.com/elastic/beats/vendor/github.com/spf13/cobra/command.go:738 +0x2b
main.main()
/go/src/github.com/elastic/beats/x-pack/auditbeat/main.go:21 +0x2f

22

[root@auditbeat-testing Desktop]# /etc/init.d/auditbeat start
Starting auditbeat: 2019-05-02T08:59:02.231-0500 INFO instance/beat.go:611 Home path: [/usr/share/auditbeat] Config path: [/etc/auditbeat] Data path: [/var/lib/auditbeat] Logs path: [/var/log/auditbeat]
2019-05-02T08:59:02.232-0500 INFO instance/beat.go:618 Beat UUID: 5b853b88-7972-4d17-956c-4949c793cbce
2019-05-02T08:59:02.232-0500 INFO [beat] instance/beat.go:931 Beat info {"system_info": {"beat": {"path": {"config": "/etc/auditbeat", "data": "/var/lib/auditbeat", "home": "/usr/share/auditbeat", "logs": "/var/log/auditbeat"}, "type": "auditbeat", "uuid": "5b853b88-7972-4d17-956c-4949c793cbce"}}}
2019-05-02T08:59:02.232-0500 INFO [beat] instance/beat.go:940 Build info {"system_info": {"build": {"commit": "1d55b4bd9dbf106a4ad4bc34fe9ee425d922363b", "libbeat": "6.7.1", "time": "2019-04-02T15:20:46.000Z", "version": "6.7.1"}}}
2019-05-02T08:59:02.232-0500 INFO [beat] instance/beat.go:943 Go runtime info {"system_info": {"go": {"os":"linux","arch":"amd64","max_procs":1,"version":"go1.10.8"}}}
2019-05-02T08:59:02.242-0500 INFO [beat] instance/beat.go:947 Host info {"system_info": {"host": {"architecture":"x86_64","boot_time":"2019-05-02T08:54:46-05:00","containerized":true,"name":"auditbeat-testing.local","ip":["127.0.0.1/8","::1/128","192.168.153.184/24","fe80::20c:29ff:fec1:5f6/64"],"kernel_version":"2.6.32-754.el6.x86_64","mac":["00:0c:29:c1:05:f6","3e:de:48:7c:4a:a3"],"os":{"family":"redhat","platform":"centos","name":"CentOS","version":"6.10 (Final)","major":6,"minor":10,"patch":0,"codename":"Final"},"timezone":"CDT","timezone_offset_sec":-18000}}}
2019-05-02T08:59:02.246-0500 INFO [beat] instance/beat.go:976 Process info {"system_info": {"process": {"capabilities": {"inheritable":null,"permitted":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"effective":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"bounding":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"ambient":null}, "cwd": "/home/auditbeat/Desktop", "exe": "/usr/share/auditbeat/bin/auditbeat", "name": "auditbeat", "pid": 22848, "ppid": 22847, "seccomp": {"mode":""}, "start_time": "2019-05-02T08:59:01.640-0500"}}}
2019-05-02T08:59:02.246-0500 INFO instance/beat.go:280 Setup Beat: auditbeat; Version: 6.7.1
2019-05-02T08:59:02.247-0500 INFO elasticsearch/client.go:164 Elasticsearch url: http://192.168.153.177:9200
2019-05-02T08:59:02.251-0500 INFO [publisher] pipeline/module.go:110 Beat name: auditbeat-testing.local
2019-05-02T08:59:02.252-0500 INFO [auditd] auditd/audit_linux.go:104 auditd module is running as euid=0 on kernel=2.6.32-754.el6.x86_64
2019-05-02T08:59:02.303-0500 INFO [auditd] auditd/audit_linux.go:131 socket_type=unicast will be used.
2019-05-02T08:59:02.305-0500 WARN [cfgwarn] host/host.go:163 EXPERIMENTAL: The system/host dataset is experimental
2019-05-02T08:59:02.307-0500 WARN [cfgwarn] login/login.go:95 EXPERIMENTAL: The system/login dataset is experimental
2019-05-02T08:59:02.308-0500 WARN [cfgwarn] package/package.go:184 EXPERIMENTAL: The system/package dataset is experimental
2019-05-02T08:59:02.309-0500 WARN [cfgwarn] process/process.go:127 EXPERIMENTAL: The system/process dataset is experimental
2019-05-02T08:59:02.309-0500 WARN [cfgwarn] socket/socket.go:210 EXPERIMENTAL: The system/socket dataset is experimental
2019-05-02T08:59:02.416-0500 WARN [cfgwarn] user/user.go:205 EXPERIMENTAL: The system/user dataset is experimental
Config OK
[ OK ]

[root@auditbeat-testing Desktop]# *** glibc detected *** /usr/share/auditbeat/bin/auditbeat: realloc(): invalid next size: 0x00007fa220044210 ***
======= Backtrace: =========
/lib64/libc.so.6(+0x75e5e)[0x7fa2434eee5e]
/lib64/libc.so.6(+0x7bd3a)[0x7fa2434f4d3a]
/lib64/libc.so.6(realloc+0x158)[0x7fa2434f4f78]
/usr/lib64/liblua-5.1.so(+0xfd2c)[0x7fa238588d2c]
/usr/lib64/liblua-5.1.so(+0xbb99)[0x7fa238584b99]
/usr/lib64/liblua-5.1.so(+0x1010c)[0x7fa23858910c]
/usr/lib64/liblua-5.1.so(+0x10169)[0x7fa238589169]
/usr/lib64/liblua-5.1.so(+0xaebb)[0x7fa238583ebb]
/usr/lib64/liblua-5.1.so(+0x1439f)[0x7fa23858d39f]
/usr/lib64/liblua-5.1.so(+0x14972)[0x7fa23858d972]
/usr/lib64/liblua-5.1.so(+0x14723)[0x7fa23858d723]
/usr/lib64/liblua-5.1.so(+0x159c5)[0x7fa23858e9c5]
/usr/lib64/liblua-5.1.so(lua_settable+0x20)[0x7fa238581090]
/usr/lib64/liblua-5.1.so(luaL_findtable+0x113)[0x7fa238591683]
/usr/lib64/liblua-5.1.so(luaL_openlib+0x5f)[0x7fa238591adf]
/usr/lib64/librpm.so.1(rpmReadConfigFiles+0x201)[0x7fa2395c23b1]
/usr/share/auditbeat/bin/auditbeat(_cgo_616c53f371e2_Cfunc_my_rpmReadConfigFiles+0x26)[0x14bcd86]
/usr/share/auditbeat/bin/auditbeat(runtime.asmcgocall+0x70)[0xa61a10]
======= Memory map: ========
00400000-01f2d000 r-xp 00000000 fd:00 939196 /usr/share/auditbeat/bin/auditbeat
0212c000-02188000 rw-p 01b2c000 fd:00 939196 /usr/share/auditbeat/bin/auditbeat

Hi,
Sorry about the first paste. Didn't format like I was expecting.

Thank you for taking a look and if there is anything else I can provide, please let me know.

Thank you.

23

Thanks for your report @Salty, we're looking at the issue and hopefully provide a fix soon

24

I've submitted a pull request with the fix https://github.com/elastic/beats/pull/12168