filebeat.input:
- type: log
enabled: true
paths:
- /var/log/nginx/*log
tags: ["server-log"]
fields: {log_type: Server-log}
- type: log
enabled: true
paths:
- /var/www/laravel/storage/logs/*.log
tags: ["laravel-log"]
feilds: {log_type: laravel-log}
setup.kibana:
host: "****:5601"
output.logstash:
# The Logstash hosts
hosts: ["*****:5044"]
input {
beats {
port => 5044
}
}
filter {
if ([fields][log_type] == "server-log") {
grok {
match => { message => "%{TIMESTAMP_ISO8601:timestamp} \[%{DATA:thread}] %{LOGLEVEL:log-level}%{DATA:class}- %{GREEDYDATA:message}"}
}
kv {
source => "message"
remove_field => "kv"
field_split => " "
value_split => ":"
include_brackets => "false"
remove_char_key => "{,"
recursive => "true"
}
}
else if ([fields][log_type] == "laravel-log") {
grok {
match => { message => "%{TIMESTAMP_ISO8601:timestamp} \[%{DATA:thread}] %{LOGLEVEL:log-level}%{DATA:class}- %{GREEDYDATA:message}"}
}
kv {
source => "message"
remove_field => "kv"
field_split => " "
value_split => ":"
include_brackets => "false"
remove_char_key => "{,"
recursive => "true"
}
}
}
output {
elasticsearch {
hosts => ["*****:9200"]
manage_template => false
index => "%{[fields][log_type]}-index"
}
Logstash Log >>>>
[2022-07-24T05:35:17,242][INFO ][logstash.filters.kv ][main] ECS compatibility is enabled but `target` option was not specified. This may cause fields to be set at the top-level of the event where they are likely to clash with the Elastic Common Schema. It is recommended to set the `target` option to avoid potential schema conflicts (if your data is ECS compliant or non-conflicting, feel free to ignore this message)
[2022-07-24T05:35:17,409][INFO ][logstash.javapipeline ][main] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>1, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>125, "pipeline.sources"=>["/etc/logstash/conf.d/app-server-1.conf"], :thread=>"#<Thread:0x3975ef97 run>"}
[2022-07-24T05:35:19,026][INFO ][logstash.javapipeline ][main] Pipeline Java execution initialization time {"seconds"=>1.61}
[2022-07-24T05:35:19,082][INFO ][logstash.inputs.beats ][main] Starting input listener {:address=>"0.0.0.0:5044"}
[2022-07-24T05:35:19,166][INFO ][logstash.javapipeline ][main] Pipeline started {"pipeline.id"=>"main"}
[2022-07-24T05:35:19,328][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
[2022-07-24T05:35:19,470][INFO ][org.logstash.beats.Server][main][91e605f9a7d96973e54e5cafaf46c8b7e263dfc362a8c48083b4ef0acbaab0ea] Starting server on port: 5044
filebeat log >>>>
{"log.level":"info","@timestamp":"2022-07-24T05:58:17.026Z","log.logger":"monitoring","log.origin":{"file.name":"log/log.go","file.line":185},"message":"Non-zero metrics in the last 30s","service.name":"filebeat","monitoring":{"metrics":{"beat":{"cpu":{"system":{"ticks":20,"time":{"ms":20}},"total":{"ticks":130,"time":{"ms":130},"value":0},"user":{"ticks":110,"time":{"ms":110}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"7eb60458-bc50-4556-b157-6f50e63c85e0","name":"filebeat","uptime":{"ms":30088},"version":"8.3.2"},"memstats":{"gc_next":20439800,"memory_alloc":13086720,"memory_sys":32850952,"memory_total":51808360,"rss":83009536},"runtime":{"goroutines":15}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"active":0},"type":"logstash"},"pipeline":{"clients":0,"events":{"active":0},"queue":{"max_events":4096}}},"registrar":{"states":{"current":0}},"system":{"cpu":{"cores":1},"load":{"1":0,"15":0.05,"5":0.01,"norm":{"1":0,"15":0.05,"5":0.01}}}},"ecs.version":"1.6.0"}}
EDITED by Moderator to be readable, please properly format your text going forward