excerpt from log file:
2017-11-06 11:54:43,333; [LOG_LEVEL=ALWAYS, CMPNT_NM=com.fmr.ifeb.alayer.cache.ehcache.ApplicationCacheImpl, MESSAGE=Initialized cache named 'oscarJdbcDaxCache']
2017-11-06 11:54:43,344; [LOG_LEVEL=ALWAYS, CMPNT_NM=com.fmr.ifeb.alayer.cache.ehcache.ApplicationCacheImpl, MESSAGE=Initialized cache named 'beneaggrJdbcDaxCache']
2017-11-06 11:54:53,509; [LOG_LEVEL=ALWAYS, CMPNT_NM=com.fmr.ifeb.alayer.cache.ehcache.ApplicationCacheImpl, MESSAGE=Initialized cache named 'footerDaxCache']
Filebeat.yml
filebeat.prospectors:
- input_type: log
paths:
- C:\Users\a617744\NewData11\data8.log
document_type: springlog
#tail_files: true
multiline.pattern: '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
multiline.negate: true
multiline.match: after
multiline.max_lines: 500
multiline.timeout: 1m
backoff: 5m
max_backoff: 10m
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 3
output.logstash:
# The Logstash hosts
hosts: ["localhost:5044"]
logging.level: debug
logstash config file:
input {
beats {
port => 5044
}
}
filter {
mutate{
gsub=>["message","\r",""]
}
grok {
id => "myspringlogfilter"
match => { "message" => [ "%{TIMESTAMP_ISO8601:timestamp}; \[LOG_LEVEL=%{WORD:log-level}, CMPNT_NM=%{GREEDYDATA:component_name}, MESSAGE=%{GREEDYDATA:message}\]"]}
overwrite => ["message"]
}
}
output {
elasticsearch {
hosts => "localhost:9200"
#manage_template => false
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
#index => "filebeat"
document_type => "%{[@metadata][type]}"
}
stdout {
codec => rubydebug
}
}
logstaconfig file:
input {
beats {
port => 5044
}
}
filter {
mutate{
gsub=>["message","\r",""]
}
grok {
id => "myspringlogfilter"
match => { "message" => [ "%{TIMESTAMP_ISO8601:timestamp}; \[LOG_LEVEL=%{WORD:log-level}, CMPNT_NM=%{GREEDYDATA:component_name}, MESSAGE=%{GREEDYDATA:message}\]"]}
overwrite => ["message"]
}
}
output {
elasticsearch {
hosts => "localhost:9200"
#manage_template => false
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
#index => "filebeat"
document_type => "%{[@metadata][type]}"
}
stdout {
codec => rubydebug
}
}
filebeat log:
2017-12-10T03:49:47+05:30 INFO Stopping 1 prospectors
2017-12-10T03:49:47+05:30 INFO Dynamic config reloader stopped
2017-12-10T03:49:47+05:30 INFO Prospector ticker stopped
2017-12-10T03:49:47+05:30 INFO Stopping Prospector: 500517288269846499
2017-12-10T03:49:47+05:30 DBG Multiline event flushed because timeout reached.
output looks like:
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 3,
"successful": 3,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 1,
"hits": [
{
"_index": "filebeat-6.0.0-2017.12.09",
"_type": "doc",
"_id": "99xYPWABfws4q82EV5qr",
"_score": 1,
"_source": {
"@timestamp": "2017-12-09T22:13:46.070Z",
"offset": 329,
"log-level": "ALWAYS",
"component_name": "com.fmr.ifeb.alayer.cache.ehcache.ApplicationCacheImpl, MESSAGE=Initialized cache named 'oscarJdbcDaxCache']\n2017-11-06 11:54:43,344; [LOG_LEVEL=ALWAYS, CMPNT_NM=com.fmr.ifeb.alayer.cache.ehcache.ApplicationCacheImpl",
"@version": "1",
"beat": {
"name": "PCCV050921",
"hostname": "PCCV050921",
"version": "6.0.0"
},
"host": "INDV050921",
"source": """C:\Users\a617744\NewData11\data8.log""",
"message": "Initialized cache named 'beneaggrJdbcDaxCache'",
"tags": [
"beats_input_codec_plain_applied"
],
"timestamp": "2017-11-06 11:54:43,333"
}
}
]
}
}