Filebeat restart repeatedly with throwing out of memory in runtime

log shows the following:


client mode check start 2017-10-10 07:40:58.303980858 +0000 UTC
client mode check start 2017-10-10 07:41:01.305266634 +0000 UTC
client mode check start 2017-10-10 07:41:04.506058547 +0000 UTC
fatal error: runtime: out of memory

runtime stack:
runtime.throw(0xb5637a, 0x16)
	/usr/local/go/src/runtime/panic.go:596 +0x95
runtime.sysMap(0xc554320000, 0x2f120000, 0xf77200, 0xf8fdf8)
	/usr/local/go/src/runtime/mem_linux.go:216 +0x1d0
runtime.(*mheap).sysAlloc(0xf76a80, 0x2f120000, 0x422741)
	/usr/local/go/src/runtime/malloc.go:440 +0x374
runtime.(*mheap).grow(0xf76a80, 0x1788e, 0x0)
	/usr/local/go/src/runtime/mheap.go:774 +0x62
runtime.(*mheap).allocSpanLocked(0xf76a80, 0x1788e, 0x200)
	/usr/local/go/src/runtime/mheap.go:678 +0x44f
runtime.(*mheap).alloc_m(0xf76a80, 0x1788e, 0x100000000, 0xc400000000)
	/usr/local/go/src/runtime/mheap.go:562 +0xe2
runtime.(*mheap).alloc.func1()
	/usr/local/go/src/runtime/mheap.go:627 +0x4b
runtime.systemstack(0x7fe77911fda8)
	/usr/local/go/src/runtime/asm_amd64.s:343 +0xab
runtime.(*mheap).alloc(0xf76a80, 0x1788e, 0x100000000, 0xf79710)
	/usr/local/go/src/runtime/mheap.go:628 +0xa0
runtime.largeAlloc(0x2f11c000, 0x7fe77b116900, 0xc40000003a)
	/usr/local/go/src/runtime/malloc.go:807 +0x93
runtime.mallocgc.func1()
	/usr/local/go/src/runtime/malloc.go:702 +0x3e
runtime.systemstack(0xc42001b300)
	/usr/local/go/src/runtime/asm_amd64.s:327 +0x79
runtime.mstart()
	/usr/local/go/src/runtime/proc.go:1132

goroutine 335 [running]:
runtime.systemstack_switch()
	/usr/local/go/src/runtime/asm_amd64.s:281 fp=0xc430e1aa18 sp=0xc430e1aa10
runtime.mallocgc(0x2f11c000, 0x0, 0x0, 0x0)
	/usr/local/go/src/runtime/malloc.go:703 +0x930 fp=0xc430e1aab8 sp=0xc430e1aa18
runtime.growslice(0xa443a0, 0xc508e20000, 0x25a7c000, 0x25a7c000, 0x25a80000, 0xc430e1ab88, 0x479c02, 0x59dc7990)
	/usr/local/go/src/runtime/slice.go:140 +0x23e fp=0xc430e1ab40 sp=0xc430e1aab8
git.tap4fun.com/platform-server/beats/libbeat/common/streambuf.(*Buffer).doAppend(0xc420642820, 0xc4405cc000, 0x4000, 0x4000, 0xc42062a101, 0xffffffffffffffff, 0x1, 0xffffffffffffffff)
	/pf-dept/src/git.tap4fun.com/platform-server/beats/libbeat/common/streambuf/streambuf.go:143 +0x4ad fp=0xc430e1abd0 sp=0xc430e1ab40
git.tap4fun.com/platform-server/beats/libbeat/common/streambuf.(*Buffer).Append(0xc420642820, 0xc4405cc000, 0x4000, 0x4000, 0x4000, 0x0)
	/pf-dept/src/git.tap4fun.com/platform-server/beats/libbeat/common/streambuf/streambuf.go:158 +0x57 fp=0xc430e1ac20 sp=0xc430e1abd0
git.tap4fun.com/platform-server/beats/filebeat/harvester/reader.(*Line).advance(0xc420140380, 0x9, 0x9)
	/pf-dept/src/git.tap4fun.com/platform-server/beats/filebeat/harvester/reader/line.go:115 +0x160 fp=0xc430e1ace8 sp=0xc430e1ac20
git.tap4fun.com/platform-server/beats/filebeat/harvester/reader.(*Line).Next(0xc420140380, 0xa2b3a0, 0xc421cbb200, 0xc4202245e8, 0x77ad2d, 0xc420013100, 0xa2b3a0)
	/pf-dept/src/git.tap4fun.com/platform-server/beats/filebeat/harvester/reader/line.go:57 +0x35 fp=0xc430e1ad70 sp=0xc430e1ace8
git.tap4fun.com/platform-server/beats/filebeat/harvester/reader.Encode.Next(0xc420140380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x13, ...)
	/pf-dept/src/git.tap4fun.com/platform-server/beats/filebeat/harvester/reader/encode.go:30 +0x5e fp=0xc430e1ae28 sp=0xc430e1ad70
git.tap4fun.com/platform-server/beats/filebeat/harvester/reader.(*StripNewline).Next(0xc42062a1c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420224750, ...)
	/pf-dept/src/git.tap4fun.com/platform-server/beats/filebeat/harvester/reader/strip_newline.go:16 +0x66 fp=0xc430e1aed0 sp=0xc430e1ae28
git.tap4fun.com/platform-server/beats/filebeat/harvester/reader.(*Timeout).Next.func1(0xc420124080)
	/pf-dept/src/git.tap4fun.com/platform-server/beats/filebeat/harvester/reader/timeout.go:51 +0x52 fp=0xc430e1afd8 sp=0xc430e1aed0
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc430e1afe0 sp=0xc430e1afd8
created by git.tap4fun.com/platform-server/beats/filebeat/harvester/reader.(*Timeout).Next
	/pf-dept/src/git.tap4fun.com/platform-server/beats/filebeat/harvester/reader/timeout.go:57 +0x24b

goroutine 1 [chan receive]:
git.tap4fun.com/platform-server/beats/filebeat/beater.(*Filebeat).Run(0xc420376d40, 0xc4201a6180, 0x0, 0x0)
	/pf-dept/src/git.tap4fun.com/platform-server/beats/filebeat/beater/filebeat.go:138 +0xc1b
git.tap4fun.com/platform-server/beats/libbeat/beat.(*Beat).launch(0xc4201a6180, 0xb73558, 0x0, 0x0)
	/pf-dept/src/git.tap4fun.com/platform-server/beats/libbeat/beat/beat.go:210 +0x706
git.tap4fun.com/platform-server/beats/libbeat/beat.Run(0xb4a0bd, 0x8, 0x0, 0x0, 0xb73558, 0xc4200001a0, 0xc4200001a0)
	/pf-dept/src/git.tap4fun.com/platform-server/beats/libbeat/beat/beat.go:135 +0x65
main.main()
	/app/filebeat/main.go:22 +0x57

goroutine 17 [syscall, 2 minutes, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1

goroutine 23 [syscall, 2 minutes]:
os/signal.signal_recv(0x0)
	/usr/local/go/src/runtime/sigqueue.go:116 +0x104
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.1
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 7 [sleep]:
time.Sleep(0xb2d05e00)
	/usr/local/go/src/runtime/time.go:59 +0xf9
git.tap4fun.com/platform-server/beats/libbeat/cfgfile.ClientMode(0x1000004d2, 0x0, 0x0, 0x0)
	/pf-dept/src/git.tap4fun.com/platform-server/beats/libbeat/cfgfile/config_check.go:167 +0x2ff
created by git.tap4fun.com/platform-server/beats/libbeat/cfgfile.CheckConfigViaRPC
	/pf-dept/src/git.tap4fun.com/platform-server/beats/libbeat/cfgfile/config_check.go:81 +0x1d2

goroutine 47 [IO wait]:
net.runtime_pollWait(0x7fe7788e90d0, 0x72, 0xe98400)
	/usr/local/go/src/runtime/netpoll.go:164 +0x59
net.(*pollDesc).wait(0xc420218068, 0x72, 0xe926c8, 0xc425caa060)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x38
net.(*pollDesc).waitRead(0xc420218068, 0xffffffffffffffff, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x34
net.(*netFD).accept(0xc420218000, 0x0, 0xe96500, 0xc425caa060)
	/usr/local/go/src/net/fd_unix.go:430 +0x1e5
net.(*TCPListener).accept(0xc4201381d0, 0xc425ccc080, 0xa6dec0, 0xffffffffffffffff)
	/usr/local/go/src/net/tcpsock_posix.go:136 +0x2e

my config:

filebeat:
    spool_size: 20
    idle_timeout: "10s"
filebeat.registry_file: "/data/platform/gold/servers/beats/data/registry"

filebeat.prospectors:
- input_type: log
  paths:
    - /data/log/apps/platform/*.log
  scan_frequency: "1s"

  scan_frequency: 10s
  ignore_older: 24h
  close_inactive: 25h
  close_renamed: true
  close_removed: true

  clean_inactive: 26h
  clean_removed: true

  max_bytes: 15000
  multiline:    # 多行合并
    pattern: '^[[:space:]]'
    negate: false
    match: after
    max_lines: 1000

    timeout: 1s
  • Which filebeat version are you using?
  • What is your OS?
  • How much memory does filebeat have available?

How long are the lines in your log files?

version: 5.6.X(and i extend some other functions in output module)

AWS server:
Linux pf-us-app-4.or.aws.tap4fun.com 3.2.0-60-virtual #91-Ubuntu SMP Wed Feb 19 04:13:28 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux

memory usage

i start filebeat at about 6:30~7:30 ,and the problem happens at that time.
21

each line is about 1000 byte, not exact . some is short ,other is too long maybe.

usual time


$> free -m
             total       used       free     shared    buffers     cached
Mem:          7225       7107        118          0         20       5091
-/+ buffers/cache:       1994       5230
Swap:            0          0          0

hello,
the issue occur again, could you help me with it please ?
it looks like memory leak ,the memory usage is so strange.

range 6:22~6:30 , it crashes.

Could you share the output part of your config file and share the full logs from filebeat? How many log files are in the platform directory?

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.