Hallo, I have a configuration like the following, but elastic can't retrieve the logs. And the plain-log .log also doesn't show the activation log. How else can I confirm that logstash and filebeat are connected and transmitting data. Thank you
The Filebeat
filebeat.inputs:
- type: log
enabled: true
paths:
- :X\xxxx\xxx\xxx\log\xxxdriver.log
multiline.pattern: ^R10\.97389
multiline.negate: true
multiline.match: after
fields:
data_source: XXXDriverxx
# ============================== Filebeat modules ==============================
filebeat.config.modules:
# Glob pattern for configuration loading
path: ${path.config}/modules.d/*.yml
# Set to true to enable config reloading
reload.enabled: false
# ------------------------------ Logstash Output -------------------------------
output.logstash:
# The Logstash hosts
hosts: ["xxx.xxx.xxx:5046"]
# ================================= Processors =================================
processors:
- add_host_metadata:
when.not.contains.tags: forwarded
- add_cloud_metadata: ~
- add_docker_metadata: ~
- add_kubernetes_metadata: ~
The Logstash
input {
beats {
port => 5046
}
}
filter {
if [fields][data_source] == "global"
{
grok {
match => { "rawMessage" => "%{TIMESTAMP_ISO8601:datetime}\s\[%{GREEDYDATA:thread}\]\s%{LOGLEVEL:logLevel}\s%{GREEDYDATA:category1}\s\s\-\s%{GREEDYDATA:message}"}
}
date {
match => ["datetime", "yyyy-MM-dd HH:mm:ss,SSS"]
target => "@timestamp"
}
mutate
{
add_field => { "logtype" => "Temenos_global" }
}
}
else
{
mutate {
rename => ["message", "rawMessage" ]
}
mutate {
gsub => ["rawMessage", "[\r\n\t]", ""]
gsub => ["rawMessage", "\\r", ""]
gsub => ["rawMessage", "\\n", ""]
gsub => ["rawMessage", "\\t", ""]
}
ruby
{
code => " event.set('dashgetter', event.get('rawMessage').scan(/-/).length )"
}
if [dashgetter] > 6
{
mutate
{
gsub => ["rawMessage", "-(?!.*-)", ""]
}
}
grok {
pattern_definitions => { "F_DATETIME_XML" => "%{DAY:day}\s*%{MONTH:month}\s*%{MONTHDAY:monthday}\s*%{TIME:time}" }
match => { "rawMessage" => "%{GREEDYDATA:nama_produk_dan_versi}\s-\s%{F_DATETIME_XML:tanggal}%{GREEDYDATA:category3}\s-\s%{GREEDYDATA:table}\s-\s%{GREEDYDATA:alias}\s-\s%{GREEDYDATA:message}"}
}
date {
match => [ "category3" , "EEE MMM dd HH:mm:ss" ]
target => "@timestamp"
}
if "ERROR" in [rawMessage]
{
mutate {
add_field => { "LogLevel" => "ERROR" }
}
}
else if "deadlock" in [rawMessage] or "WARNING" in [rawMessage]
{
mutate {
add_field => { "LogLevel" => "WARN" }
}
}
else
{
mutate {
add_field => { "LogLevel" => "INFO"}
}
}
mutate
{
remove_field => ["category3"]
remove_field => ["dashgetter"]
remove_field => ["day","month","monthday","time","HOUR","MINUTE","SECOND"]
add_field => { "logtype" => "Temenos_XMLdriver" }
add_field => {"table_alias" => "%{table} - %{alias}"}
add_field => {"hostname" => "CBS-APP1-JKT.ibsm.net"}
}
if ![alias] and ![table]
{
mutate{
remove_field => ["table_alias"]
}
}
else if ![table]
{
mutate {
gsub => ["table_alias", "%{table} -", ""]
}
}
else if ![alias]
{
mutate{
gsub => ["table_alias", " - %{alias}", ""]
}
}
}
}
output {
if "ERROR" in [LogLevel] or "WARN" in [LogLevel]
{
dynatrace {
ingest_endpoint_url => "xxxxx.com"
api_key => "xxxxxxx"
ssl_verify_none => true
}
}
else {
else {
elasticsearch {
hosts => ["xxx.xxx.xxx:9200"]
index => "xxxxxxlogs-10.xx.x.xx"
user => "elastic"
password => "xxxx123"
}
}
}
