Hello,
We are facing an issue on our production server. We are logging into a file using Log4Net with a rolling file appender.
Sometimes, when the file is rolling the file is flagged as inactive and nor more logs are send to our logstash instance.
Here is the filebeat configuration :
filebeat.inputs:
- type: log
enabled: true
paths:- C:\Gaming1\APR\BettingSlipManagement\Logs\BSM.txt
fields:
service: bsm
multiline.pattern: ^[?\d{2,4}-\d{2}-\d{2,4} \d{2}:\d{2}:\d{2}[,.]\d{3}]?
multiline.negate: true
multiline.match: after
- C:\Gaming1\APR\BettingSlipManagement\Logs\BSM.txt
- type: log
enabled: true
paths:- C:**\Logs\serv1.txt
fields:
service: serv1
multiline.pattern: ^[?\d{2,4}-\d{2}-\d{2,4} \d{2}:\d{2}:\d{2}[,.]\d{3}]?
multiline.negate: true
multiline.match: after
- C:**\Logs\serv1.txt
- type: log
enabled: true
paths:- C:**\Logs\serv2.txt
fields:
service: serv2
multiline.pattern: ^[?\d{2,4}-\d{2}-\d{2,4} \d{2}:\d{2}:\d{2}[,.]\d{3}]?
multiline.negate: true
multiline.match: after
filebeat.config.modules:
path: ${path.config}/modules.d/.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
fields:
environment: production
country: pt
partner: estoril
output.logstash:
hosts: [ "" ]
ssl.certificate_authorities: [ "*" ]
ssl.certificate: ***/beat.crt
ssl.key: ***/beat.key
processors: - add_host_metadata: ~
- add_cloud_metadata: ~
- C:**\Logs\serv2.txt
Here is the filebeat logs :
2019-11-17T23:59:50.003Z INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":4977796,"time":{"ms":343}},"total":{"ticks":6638202,"time":{"ms":343},"value":6638202},"user":{"ticks":1660406}},"handles":{"open":465},"info":{"ephemeral_id":"1de5841e-625d-4ef3-a266-56f141950438","uptime":{"ms":466710119}},"memstats":{"gc_next":8175824,"memory_alloc":5838712,"memory_total":396617788864,"rss":-888832},"runtime":{"goroutines":46}},"filebeat":{"events":{"active":3,"added":111,"done":108},"harvester":{"open_files":2,"running":2}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":108,"batches":18,"total":108},"read":{"bytes":630},"write":{"bytes":25998}},"pipeline":{"clients":3,"events":{"active":4,"published":111,"total":111},"queue":{"acked":108}}},"registrar":{"states":{"current":3,"update":108},"writes":{"success":18,"total":18}}}}}
2019-11-18T00:00:04.958Z INFO log/harvester.go:253 Harvester started for file: C:**\Logs\serv1.txt
2019-11-18T00:00:09.910Z INFO log/harvester.go:253 Harvester started for file: C:**\Logs\serv2.txt
2019-11-18T00:00:19.995Z INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":4978250,"time":{"ms":454}},"total":{"ticks":6638656,"time":{"ms":454},"value":6638656},"user":{"ticks":1660406}},"handles":{"open":469},"info":{"ephemeral_id":"1de5841e-625d-4ef3-a266-56f141950438","uptime":{"ms":466740108}},"memstats":{"gc_next":8162800,"memory_alloc":4209992,"memory_total":396642895056,"rss":995328},"runtime":{"goroutines":56}},"filebeat":{"events":{"active":2,"added":104,"done":102},"harvester":{"open_files":4,"running":4,"started":2}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":100,"batches":15,"total":100},"read":{"bytes":525},"write":{"bytes":19993}},"pipeline":{"clients":3,"events":{"active":6,"filtered":2,"published":102,"total":104},"queue":{"acked":100}}},"registrar":{"states":{"current":5,"update":102},"writes":{"success":17,"total":17}}}}}
...
2019-11-18T00:04:50.000Z INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":4981484,"time":{"ms":266}},"total":{"ticks":6642874,"time":{"ms":406},"value":6642874},"user":{"ticks":1661390,"time":{"ms":140}}},"handles":{"open":469},"info":{"ephemeral_id":"1de5841e-625d-4ef3-a266-56f141950438","uptime":{"ms":467010109}},"memstats":{"gc_next":8141232,"memory_alloc":4103880,"memory_total":396880342912,"rss":647168},"runtime":{"goroutines":56}},"filebeat":{"events":{"added":84,"done":84},"harvester":{"open_files":4,"running":4}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":84,"batches":15,"total":84},"read":{"bytes":525},"write":{"bytes":18272}},"pipeline":{"clients":3,"events":{"active":0,"published":84,"total":84},"queue":{"acked":84}}},"registrar":{"states":{"current":5,"update":84},"writes":{"success":15,"total":15}}}}}
2019-11-18T00:05:01.003Z INFO log/harvester.go:278 File is inactive: C:**\Logs\serv1.txt. Closing because close_inactive of 5m0s reached.
2019-11-18T00:05:08.020Z INFO log/harvester.go:278 File is inactive: C:**\Logs\serv2.txt. Closing because close_inactive of 5m0s reached.
2019-11-18T00:05:19.999Z INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":4981750,"time":{"ms":266}},"total":{"ticks":6643453,"time":{"ms":579},"value":6643453},"user":{"ticks":1661703,"time":{"ms":313}}},"handles":{"open":467},"info":{"ephemeral_id":"1de5841e-625d-4ef3-a266-56f141950438","uptime":{"ms":467040108}},"memstats":{"gc_next":5209232,"memory_alloc":2696552,"memory_total":396903572888,"rss":131072},"runtime":{"goroutines":46}},"filebeat":{"events":{"active":12,"added":96,"done":84},"harvester":{"closed":2,"open_files":2,"running":2}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":80,"batches":14,"total":80},"read":{"bytes":490},"write":{"bytes":15483}},"pipeline":{"clients":3,"events":{"active":12,"filtered":4,"published":92,"total":96},"queue":{"acked":80}}},"registrar":{"states":{"cleanup":2,"current":3,"update":84},"writes":{"success":16,"total":16}}}}}