I can't get a directory name added as a field in Logstash and Kibana


#1

Hi again.

I have a structure within Logstash running in Docker of /opt/logs/CUSTOMERNAME/SERVERNAME/SystemOut*.log

I want to have SERVERNAME and CUSTOMERNAME as fields in Kibana but it doesn't work for me.

Below is the config as it is now

input {
file {
path => [ "/opt/logs///SystemOut*.log" ]
start_position => "beginning"
type => "websphere"
# important! logstash read only logs from files touched the last 24 hours
# 8640000 = 100 days
sincedb_path => "/dev/null"
ignore_older => "8640000"
}
}
filter {
if [type] =~ "websphere" {
grok {
match => ["source", "%{GREEDYDATA}/%{GREEDYDATA:server_name}/SystemOut.log"]
}
grok {
match => ["message", "[%{DATA:wastimestamp} %{WORD:tz}] %{BASE16NUM:was_threadID} (?<was_shortname>\b[A-Za-z0-9$]{2,}\b) %{SPACE}%{WORD:was_loglevel}%{SPACE} %{GREEDYDATA:was_
msg}"]
}
grok {
match => ["was_msg", "(?<was_errcode>[A-Z0-9]{9,10})[:,\s\s]%{GREEDYDATA:was_msg}"]
overwrite => [ "was_msg" ]
tag_on_failure => [ ]
}
translate {
field => "tz"
destination => "tz_num"
dictionary => [
"CET", "+0100",
"CEST", "+0200",
"EDT", "-0400",
"GMT", "+0000"
]
}
translate {
field => "was_errcode"
destination => "was_application"
regex => "true"
exact => "true"
dictionary => [
"CLFRW", "Search",
"CLFRA", "Activities",
"CLFRS", "Blogs",
"CLFRL", "Bookmarks",
"CLFRK", "Common",
"CLFRM", "Communities",
"EJPVJ", "Files",
"CLFRV", "Forums",
"CLFRQ", "Homepage",
"CLFRP", "Installer",
"CLFRO", "Configuration",
"CLFRR", "Notifications",
"CLFNF", "Portlet",
"CLFRT", "FedSearch",
"CLFWX", "News",
"CLFWY", "Event",
"CLFWZ", "Widget",
"CLFRN", "Profiles",
"CLFWY", "User",
"EJPIC", "Portal",
"EJPVJ", "Wikis",
"ADMS", "Websphere",
"SECJ", "Security"
]
}
mutate {
replace => ['timestamp', '%{wastimestamp} %{tz_num}']
}
date{
match => ["timestamp", "MM/dd/YY HH:mm:ss:SSS Z", "M/d/YY HH:mm:ss:SSS Z"]
tag_on_failure => [ ]
}
mutate {
remove_field => [ 'tz', 'tz_num', 'wastimestamp' ]
}
}
}
output {
elasticsearch {
hosts => ["elasticsearch:9200"]
}
stdout { codec => rubydebug }
}

I have tried

}
filter {
if [type] =~ "websphere" {
grok {
match => ["source", "/opt/logs/%{GREEDYDATA:customer_name}/%{GREEDYDATA:server_name}/SystemOut.log"]
}

As well as

}
filter {
if [type] =~ "websphere" {
grok {
match => ["path", "/opt/logs/%{GREEDYDATA:customer_name}/%{GREEDYDATA:server_name}/SystemOut.log"]
}

And

}
filter {
if [type] =~ "websphere" {
grok {
match => ["source", "/opt/logs/%{DATA:customer_name}/%{DATA:server_name}/SystemOut.log"]
}

and

}
filter {
if [type] =~ "websphere" {
grok {
match => ["path", "/opt/logs/%{WORD:customer_name}/%{WORD:server_name}/SystemOut.log"]
}

Since I'm not sure whether it should be WORD or DATA or path or source I'm trying things a bit randomly.

Can anyone help with what the syntax should be please?

This is what is in Logstash with the current config

{
"was_loglevel" => "O",
"was_msg" => " lzwcompression=enabled, showhiddentext=no, gridcols=100, suppressfontcolor=no, strictdtd=no, collapsewhitespace=no, processgeneratedtext=yes, defaultinputcharset=utf8, charset=utf8, pagesize=0, extractembeddedformat=off, javascripttabs=no, exportendpage=1, charbyteorder=template, jpegquality=100, graphicsizemethod=smooth, fiflags=SCCUT_FI_NORMAL, flavor=msie40, gifinterlace=yes, graphicskipsize=5, labelwpcells=no, wellformed=no, jpegcompression=enabled, exportstartpage=1, noxmldeclarationflag=no, graphicbuffersize=0, gridwrap=TRUE}",
"message" => "[1/9/17 14:57:25:448 GMT] 00000511 SystemOut O {fallbackformat=FI_UTF8, pstylenamesflag=no, preferoitrendering=true, gridrows=5000, graphictype=jpeg, simplestylenames=no, suppressfontsize=no, embeddingsflag=yes, graphicwidthlimit=1000, unmappablechar=0x002A, nosourceformatting=yes, suppressfontface=no, separategraphicsbuffer=yes, textbuffersize=0, graphicheightlimit=1000, genbulletsandnums=yes, graphicoutputdpi=96, labelssdbcells=no, exepath=/opt/IBM/Connections/data/local/search/stellent/dcs/oiexport/exporter, xmldefmethod=NONE, fallbackfont=Arial, maxurllength=0, whattoexport=all, graphicsizelimit=0, preventgraphicoverlap=no, gridadvance=down, outputid=FI_SEARCHTEXT, lzwcompression=enabled, showhiddentext=no, gridcols=100, suppressfontcolor=no, strictdtd=no, collapsewhitespace=no, processgeneratedtext=yes, defaultinputcharset=utf8, charset=utf8, pagesize=0, extractembeddedformat=off, javascripttabs=no, exportendpage=1, charbyteorder=template, jpegquality=100, graphicsizemethod=smooth, fiflags=SCCUT_FI_NORMAL, flavor=msie40, gifinterlace=yes, graphicskipsize=5, labelwpcells=no, wellformed=no, jpegcompression=enabled, exportstartpage=1, noxmldeclarationflag=no, graphicbuffersize=0, gridwrap=TRUE}",
"type" => "websphere",
"was_shortname" => "SystemOut",
"tags" => [
[0] "_grokparsefailure"
],
"was_threadID" => "00000511",
"path" => "/opt/logs/portal/InfraCluster_server1/SystemOut_17.01.11_22.39.17.log",
"@timestamp" => 2017-01-09T14:57:25.448Z,
"@version" => "1",
"host" => "f2730df8227c",
"was_errcode" => "SEARCHTEXT",
"timestamp" => "1/9/17 14:57:25:448 +0000"
}


(Magnus B├Ąck) #2

The field with the path is named path but you're attempting to parse source.

Unrelated, but don't use multiple DATA or GREEDYDATA patterns in the same expression. In this case you can just drop the first use of GREEDYDATA as you don't need it.


#3

I fixed this using the following configuration. I have left in the remmed out lines to show what I tried.

input {
file {
path => [ "/opt/logs///SystemOut*.log" ]
start_position => "beginning"
type => "websphere"
# important! logstash read only logs from files touched the last 24 hours
# 8640000 = 100 days
sincedb_path => "/dev/null"
ignore_older => "8640000"
}
}
filter {
if [type] =~ "websphere" {
grok {
#match => [ "path", "%{GREEDYDATA}/%{GREEDYDATA:server_name}/SystemOut*.log" ]
#match => [ "path", "/opt/logs//%{GREEDYDATA:server_name}/SystemOut.log" ]
#match => [ "path", "/opt/logs//(?<server_name>[^/]+)/SystemOut.log" ]
match => [ "path", "/opt/logs/%{WORD:customer}/%{WORD:server_name}" ]
}


(system) #4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.