Hi again.
I have a structure within Logstash running in Docker of /opt/logs/CUSTOMERNAME/SERVERNAME/SystemOut*.log
I want to have SERVERNAME and CUSTOMERNAME as fields in Kibana but it doesn't work for me.
Below is the config as it is now
input {
file {
path => [ "/opt/logs///SystemOut*.log" ]
start_position => "beginning"
type => "websphere"
# important! logstash read only logs from files touched the last 24 hours
# 8640000 = 100 days
sincedb_path => "/dev/null"
ignore_older => "8640000"
}
}
filter {
if [type] =~ "websphere" {
grok {
match => ["source", "%{GREEDYDATA}/%{GREEDYDATA:server_name}/SystemOut.log"]
}
grok {
match => ["message", "[%{DATA:wastimestamp} %{WORD:tz}] %{BASE16NUM:was_threadID} (?<was_shortname>\b[A-Za-z0-9$]{2,}\b) %{SPACE}%{WORD:was_loglevel}%{SPACE} %{GREEDYDATA:was_
msg}"]
}
grok {
match => ["was_msg", "(?<was_errcode>[A-Z0-9]{9,10})[:,\s\s]%{GREEDYDATA:was_msg}"]
overwrite => [ "was_msg" ]
tag_on_failure => [ ]
}
translate {
field => "tz"
destination => "tz_num"
dictionary => [
"CET", "+0100",
"CEST", "+0200",
"EDT", "-0400",
"GMT", "+0000"
]
}
translate {
field => "was_errcode"
destination => "was_application"
regex => "true"
exact => "true"
dictionary => [
"CLFRW", "Search",
"CLFRA", "Activities",
"CLFRS", "Blogs",
"CLFRL", "Bookmarks",
"CLFRK", "Common",
"CLFRM", "Communities",
"EJPVJ", "Files",
"CLFRV", "Forums",
"CLFRQ", "Homepage",
"CLFRP", "Installer",
"CLFRO", "Configuration",
"CLFRR", "Notifications",
"CLFNF", "Portlet",
"CLFRT", "FedSearch",
"CLFWX", "News",
"CLFWY", "Event",
"CLFWZ", "Widget",
"CLFRN", "Profiles",
"CLFWY", "User",
"EJPIC", "Portal",
"EJPVJ", "Wikis",
"ADMS", "Websphere",
"SECJ", "Security"
]
}
mutate {
replace => ['timestamp', '%{wastimestamp} %{tz_num}']
}
date{
match => ["timestamp", "MM/dd/YY HH:mm:ss:SSS Z", "M/d/YY HH:mm:ss:SSS Z"]
tag_on_failure => [ ]
}
mutate {
remove_field => [ 'tz', 'tz_num', 'wastimestamp' ]
}
}
}
output {
elasticsearch {
hosts => ["elasticsearch:9200"]
}
stdout { codec => rubydebug }
}
I have tried
}
filter {
if [type] =~ "websphere" {
grok {
match => ["source", "/opt/logs/%{GREEDYDATA:customer_name}/%{GREEDYDATA:server_name}/SystemOut.log"]
}
As well as
}
filter {
if [type] =~ "websphere" {
grok {
match => ["path", "/opt/logs/%{GREEDYDATA:customer_name}/%{GREEDYDATA:server_name}/SystemOut.log"]
}
And
}
filter {
if [type] =~ "websphere" {
grok {
match => ["source", "/opt/logs/%{DATA:customer_name}/%{DATA:server_name}/SystemOut.log"]
}
and
}
filter {
if [type] =~ "websphere" {
grok {
match => ["path", "/opt/logs/%{WORD:customer_name}/%{WORD:server_name}/SystemOut.log"]
}
Since I'm not sure whether it should be WORD or DATA or path or source I'm trying things a bit randomly.
Can anyone help with what the syntax should be please?
This is what is in Logstash with the current config
{
"was_loglevel" => "O",
"was_msg" => " lzwcompression=enabled, showhiddentext=no, gridcols=100, suppressfontcolor=no, strictdtd=no, collapsewhitespace=no, processgeneratedtext=yes, defaultinputcharset=utf8, charset=utf8, pagesize=0, extractembeddedformat=off, javascripttabs=no, exportendpage=1, charbyteorder=template, jpegquality=100, graphicsizemethod=smooth, fiflags=SCCUT_FI_NORMAL, flavor=msie40, gifinterlace=yes, graphicskipsize=5, labelwpcells=no, wellformed=no, jpegcompression=enabled, exportstartpage=1, noxmldeclarationflag=no, graphicbuffersize=0, gridwrap=TRUE}",
"message" => "[1/9/17 14:57:25:448 GMT] 00000511 SystemOut O {fallbackformat=FI_UTF8, pstylenamesflag=no, preferoitrendering=true, gridrows=5000, graphictype=jpeg, simplestylenames=no, suppressfontsize=no, embeddingsflag=yes, graphicwidthlimit=1000, unmappablechar=0x002A, nosourceformatting=yes, suppressfontface=no, separategraphicsbuffer=yes, textbuffersize=0, graphicheightlimit=1000, genbulletsandnums=yes, graphicoutputdpi=96, labelssdbcells=no, exepath=/opt/IBM/Connections/data/local/search/stellent/dcs/oiexport/exporter, xmldefmethod=NONE, fallbackfont=Arial, maxurllength=0, whattoexport=all, graphicsizelimit=0, preventgraphicoverlap=no, gridadvance=down, outputid=FI_SEARCHTEXT, lzwcompression=enabled, showhiddentext=no, gridcols=100, suppressfontcolor=no, strictdtd=no, collapsewhitespace=no, processgeneratedtext=yes, defaultinputcharset=utf8, charset=utf8, pagesize=0, extractembeddedformat=off, javascripttabs=no, exportendpage=1, charbyteorder=template, jpegquality=100, graphicsizemethod=smooth, fiflags=SCCUT_FI_NORMAL, flavor=msie40, gifinterlace=yes, graphicskipsize=5, labelwpcells=no, wellformed=no, jpegcompression=enabled, exportstartpage=1, noxmldeclarationflag=no, graphicbuffersize=0, gridwrap=TRUE}",
"type" => "websphere",
"was_shortname" => "SystemOut",
"tags" => [
[0] "_grokparsefailure"
],
"was_threadID" => "00000511",
"path" => "/opt/logs/portal/InfraCluster_server1/SystemOut_17.01.11_22.39.17.log",
"@timestamp" => 2017-01-09T14:57:25.448Z,
"@version" => "1",
"host" => "f2730df8227c",
"was_errcode" => "SEARCHTEXT",
"timestamp" => "1/9/17 14:57:25:448 +0000"
}