Auditbeat crashes few seconds after start

Elastic Stack Beats
1

My conf

Redhat EL7 x64
auditbeat 7.1 (tar.gz)

Any idea ?
/
2019-05-23T14:51:03.739+0200 INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"auditd":{"received_msgs":12},"beat":{"cpu":{"system":{"ticks":3020,"time":{"ms":374}},"total":{"ticks":26210,"time":{"ms":3473},"value":26210},"user":{"ticks":23190,"time":{"ms":3099}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":17},"info":{"ephemeral_id":"13f580c1-1a97-4f7b-916a-0eaaeff43ee8","uptime":{"ms":210107}},"memstats":{"gc_next":5666112,"memory_alloc":4420168,"memory_total":348415096,"rss":331776}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":58,"batches":6,"total":58},"read":{"bytes":36},"write":{"bytes":10495}},"pipeline":{"clients":3,"events":{"active":0,"published":58,"total":58},"queue":{"acked":58}}},"metricbeat":{"auditd":{"auditd":{"events":12,"success":12}},"system":{"socket":{"events":46,"success":46}}},"system":{"load":{"1":0.07,"15":0.29,"5":0.17,"norm":{"1":0.035,"15":0.145,"5":0.085}}}}}}
fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0xd pc=0x7fd12ea16630]

runtime stack:
runtime.throw(0x1843fbf, 0x2a)
/usr/local/go/src/runtime/panic.go:608 +0x72
runtime.sigpanic()
/usr/local/go/src/runtime/signal_unix.go:374 +0x2f2

goroutine 68 [syscall, locked to thread]:
runtime.cgocall(0x150bd30, 0xc0004675b8, 0x0)
/usr/local/go/src/runtime/cgocall.go:128 +0x5e fp=0xc000467588 sp=0xc000467550 pc=0xa1e20e
github.com/elastic/beats/x-pack/auditbeat/module/system/package._Cfunc_my_rpmReadConfigFiles(0x7fd144443850, 0x7fd100000000)
_cgo_gotypes.go:134 +0x49 fp=0xc0004675b8 sp=0xc000467588 pc=0x149a1c9
github.com/elastic/beats/x-pack/auditbeat/module/system/package.listRPMPackages.func4(0x7fd144443850, 0x187a2b0)
/go/src/github.com/elastic/beats/x-pack/auditbeat/module/system/package/rpm_linux.go:274 +0x56 fp=0xc0004675f0 sp=0xc0004675b8 pc=0x149bb86
github.com/elastic/beats/x-pack/auditbeat/module/system/package.listRPMPackages(0x0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/elastic/beats/x-pack/auditbeat/module/system/package/rpm_linux.go:274 +0x104 fp=0xc000467688 sp=0xc0004675f0 pc=0x149adf4
github.com/elastic/beats/x-pack/auditbeat/module/system/package.(*MetricSet).getPackages(0xc000188000, 0x1818f45, 0x6, 0x251e330, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/elastic/beats/x-pack/auditbeat/module/system/package/package.go:479 +0x64 fp=0xc000467728 sp=0xc000467688 pc=0x1497d14
github.com/elastic/beats/x-pack/auditbeat/module/system/package.(*MetricSet).reportChanges(0xc000188000, 0x7fd14bf5c440, 0xc000720300, 0x13dfe82d298, 0x2)
/go/src/github.com/elastic/beats/x-pack/auditbeat/module/system/package/package.go:322 +0x5b fp=0xc000467b08 sp=0xc000467728 pc=0x149661b
github.com/elastic/beats/x-pack/auditbeat/module/system/package.(*MetricSet).Fetch(0xc000188000, 0x7fd14bf5c440, 0xc000720300)
/go/src/github.com/elastic/beats/x-pack/auditbeat/module/system/package/package.go:277 +0xb1 fp=0xc000467bc8 sp=0xc000467b08 pc=0x1495d71
github.com/elastic/beats/metricbeat/mb/module.(*metricSetWrapper).fetch(0xc00047c400, 0x196aa40, 0xc000720300)
/go/src/github.com/elastic/beats/metricbeat/mb/module/wrapper.go:238 +0x2b0 fp=0xc000467c68 sp=0xc000467bc8 pc=0x143a0d0
github.com/elastic/beats/metricbeat/mb/module.(*metricSetWrapper).startPeriodicFetching(0xc00047c400, 0x196aa40, 0xc000720300)
/go/src/github.com/elastic/beats/metricbeat/mb/module/wrapper.go:219 +0x11a fp=0xc000467dc0 sp=0xc000467c68 pc=0x1439d1a
github.com/elastic/beats/metricbeat/mb/module.(*metricSetWrapper).run(0xc00047c400, 0xc000091aa0, 0xc000493e00)
/go/src/github.com/elastic/beats/metricbeat/mb/module/wrapper.go:196 +0x5b7 fp=0xc000467f58 sp=0xc000467dc0 pc=0x1439827
github.com/elastic/beats/metricbeat/mb/module.(*Wrapper).Start.func1(0xc00053ed40, 0xc000091aa0, 0xc000493e00, 0xc00047c400)
/go/src/github.com/elastic/beats/metricbeat/mb/module/wrapper.go:137 +0x249 fp=0xc000467fc0 sp=0xc000467f58 pc=0x143c339
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc000467fc8 sp=0xc000467fc0 pc=0xa78081
created by github.com/elastic/beats/metricbeat/mb/module.(*Wrapper).Start
/go/src/github.com/elastic/beats/metricbeat/mb/module/wrapper.go:125 +0x140
/

2

Hi @Olivier_JUDITH

Thanks for your report. We are aware of this issue and will be fixed in the upcoming Auditbeat 7.1.1.

3

I would like to note that auditbeat 6.8.0 can crash as well. i wouldnt say immediatly but not too long afterwards. restarting it seems to fix it for a time.

including a portion of the snippet since its too big.

*** Error in `/usr/share/auditbeat/bin/auditbeat': malloc(): smallbin double linked list corrupted: 0x00007fdbfc027e70 ***
======= Backtrace: =========
/lib64/libc.so.6(+0x7f644)[0x7fdc16820644]
/lib64/libc.so.6(+0x82d80)[0x7fdc16823d80]
/lib64/libc.so.6(__libc_malloc+0x4c)[0x7fdc1682621c]
/usr/lib64/librpmio.so.3(rstrdup+0x1a)[0x7fdc0c37852a]
/usr/lib64/librpm.so.3(+0x3c7dc)[0x7fdc0c5ce7dc]
/usr/lib64/librpm.so.3(+0x3ce8b)[0x7fdc0c5cee8b]
/usr/lib64/librpm.so.3(+0x3cec6)[0x7fdc0c5ceec6]
/usr/lib64/librpm.so.3(+0x3cec6)[0x7fdc0c5ceec6]
/usr/lib64/librpm.so.3(+0x3cec6)[0x7fdc0c5ceec6]
/usr/lib64/librpm.so.3(+0x3cec6)[0x7fdc0c5ceec6]
/usr/lib64/librpm.so.3(+0x3cfda)[0x7fdc0c5cefda]
/usr/lib64/librpm.so.3(+0x3e039)[0x7fdc0c5d0039]
/usr/lib64/librpm.so.3(rpmReadConfigFiles+0x49)[0x7fdc0c5d06e9]
/usr/share/auditbeat/bin/auditbeat(_cgo_58ab88efd0e9_Cfunc_my_rpmReadConfigFiles+0x26)[0x14bd046]
/usr/share/auditbeat/bin/auditbeat(runtime.asmcgocall+0x70)[0xa61af0]

fdc067fd000-7fdc067fe000 ---p 00000000 00:00 0 SIGABRT: abort
PC=0x7fdc167d7277 m=18 sigcode=18446744073709551610
signal arrived during cgo execution

goroutine 243 [syscall, locked to thread]:
runtime.cgocall(0x14bd020, 0xc4208a3598, 0x0)
/usr/local/go/src/runtime/cgocall.go:128 +0x64 fp=0xc4208a3568 sp=0xc4208a3530 pc=0xa097a4
github.com/elastic/beats/x-pack/auditbeat/module/system/package._Cfunc_my_rpmReadConfigFiles(0x7fdc0c5d06a0, 0x7fdc00000000)
_cgo_gotypes.go:134 +0x49 fp=0xc4208a3598 sp=0xc4208a3568 pc=0x144c7a9
github.com/elastic/beats/x-pack/auditbeat/module/system/package.listRPMPackages.func4(0x7fdc0c5d06a0, 0x18156d0)
/go/src/github.com/elastic/beats/x-pack/auditbeat/module/system/package/rpm_linux.go:274 +0x56 fp=0xc4208a35d0 sp=0xc4208a3598 pc=0x144e196
github.com/elastic/beats/x-pack/auditbeat/module/system/package.listRPMPackages(0x0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/elastic/beats/x-pack/auditbeat/module/system/package/rpm_linux.go:274 +0x106 fp=0xc4208a3670 sp=0xc4208a35d0 pc=0x144d3c6
github.com/elastic/beats/x-pack/auditbeat/module/system/package.(*MetricSet).getPackages(0xc4201d6140, 0x17b8a7a, 0x6, 0x21aa228, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/elastic/beats/x-pack/auditbeat/module/system/package/package.go:479 +0x66 fp=0xc4208a3710 sp=0xc4208a3670 pc=0x144a346
github.com/elastic/beats/x-pack/auditbeat/module/system/package.(*MetricSet).reportChanges(0xc4201d6140, 0x7fdc1457e3c8, 0xc42017d6e0, 0x4a81772ac, 0x163c980)
/go/src/github.com/elastic/beats/x-pack/auditbeat/module/system/package/package.go:322 +0x5b fp=0xc4208a3ab8 sp=0xc4208a3710 pc=0x1448c9b
github.com/elastic/beats/x-pack/auditbeat/module/system/package.(*MetricSet).Fetch(0xc4201d6140, 0x7fdc1457e3c8, 0xc42017d6e0)
/go/src/github.com/elastic/beats/x-pack/auditbeat/module/system/package/package.go:277 +0xb1 fp=0xc4208a3b78 sp=0xc4208a3ab8 pc=0x14483e1
github.com/elastic/beats/metricbeat/mb/module.(*metricSetWrapper).fetch(0xc420609720, 0x18aa0e0, 0xc42017d6e0)
/go/src/github.com/elastic/beats/metricbeat/mb/module/wrapper.go:238 +0x2ac fp=0xc4208a3c18 sp=0xc4208a3b78 pc=0x13edc4c
github.com/elastic/beats/metricbeat/mb/module.(*metricSetWrapper).startPeriodicFetching(0xc420609720, 0x18aa0e0, 0xc42017d6e0)
/go/src/github.com/elastic/beats/metricbeat/mb/module/wrapper.go:219 +0x117 fp=0xc4208a3d88 sp=0xc4208a3c18 pc=0x13ed867
github.com/elastic/beats/metricbeat/mb/module.(*metricSetWrapper).run(0xc420609720, 0xc420196a80, 0xc4203e8c00)
/go/src/github.com/elastic/beats/metricbeat/mb/module/wrapper.go:196 +0x5ca fp=0xc4208a3f58 sp=0xc4208a3d88 pc=0x13ed35a
github.com/elastic/beats/metricbeat/mb/module.(*Wrapper).Start.func1(0xc420663590, 0xc420196a80, 0xc4203e8c00, 0xc420609720)
/go/src/github.com/elastic/beats/metricbeat/mb/module/wrapper.go:137 +0x249 fp=0xc4208a3fc0 sp=0xc4208a3f58 pc=0x13effa9
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc4208a3fc8 sp=0xc4208a3fc0 pc=0xa62e21
created by github.com/elastic/beats/metricbeat/mb/module.(*Wrapper).Start
/go/src/github.com/elastic/beats/metricbeat/mb/module/wrapper.go:125 +0x145

goroutine 1 [semacquire, 15 minutes]:
sync.runtime_Semacquire(0xc42066352c)
/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc420663520)
/usr/local/go/src/sync/waitgroup.go:129 +0x72
github.com/elastic/beats/metricbeat/beater.(*Metricbeat).Run(0xc4201ee1c0, 0xc4201f2b40, 0xc420442510, 0x1)
/go/src/github.com/elastic/beats/metricbeat/beater/metricbeat.go:251 +0x3c8
github.com/elastic/beats/libbeat/cmd/instance.(*Beat).launch(0xc4201f2b40, 0x17bc03b, 0x9, 0x17bc03b, 0x9, 0x0, 0x0, 0x0, 0x0, 0xc42048e000, ...)
/go/src/github.com/elastic/beats/libbeat/cmd/instance/beat.go:412 +0x4f6
github.com/elastic/beats/libbeat/cmd/instance.Run.func1(0x17bc03b, 0x9, 0x17bc03b, 0x9, 0x0, 0x0, 0x17bc03b, 0x9, 0x17bc03b, 0x9, ...)
/go/src/github.com/elastic/beats/libbeat/cmd/instance/beat.go:183 +0x611
github.com/elastic/beats/libbeat/cmd/instance.Run(0x17bc03b, 0x9, 0x17bc03b, 0x9, 0x0, 0x0, 0x0, 0x0, 0xc42048e000, 0x0, ...)
/go/src/github.com/elastic/beats/libbeat/cmd/instance/beat.go:184 +0xbe
github.com/elastic/beats/libbeat/cmd.genRunCmd.func1(0xc42046cc80, 0xc420199a40, 0x0, 0xa)
/go/src/github.com/elastic/beats/libbeat/cmd/run.go:37 +0x4f
github.com/elastic/beats/vendor/github.com/spf13/cobra.(*Command).execute(0xc42046cc80, 0xc4200dc010, 0xa, 0xa, 0xc42046cc80, 0xc4200dc010)
/go/src/github.com/elastic/beats/vendor/github.com/spf13/cobra/command.go:704 +0x2c6
github.com/elastic/beats/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0xc42046cc80, 0x0, 0x1813ba0, 0x0)
/go/src/github.com/elastic/beats/vendor/github.com/spf13/cobra/command.go:785 +0x2e4
github.com/elastic/beats/vendor/github.com/spf13/cobra.(*Command).Execute(0xc42046cc80, 0xc4204f7f78, 0xc4200be058)
/go/src/github.com/elastic/beats/vendor/github.com/spf13/cobra/command.go:738 +0x2b
main.main()
/go/src/github.com/elastic/beats/x-pack/auditbeat/main.go:21 +0x2f

4

Yes, it's the same issue. It will be fixed in 6.8.1.

5

This topic was automatically closed 21 days after the last reply. New replies are no longer allowed.