Hi.
I can not view the data in recovering logstash -> elasticsearch . system windows srv 2012+IIS
Unfortunately, the application creates one file = one event
Each file has a size of 20Kb. My logstash config:
input
{
file {
type => "IISLog"
sincedb_path => "D:\loghost\logstash\sincedb\IISLog"
path => "C:/inetpub/logs/LogFiles/W3SVC*/*.log"
}
############ G2 test log ########################
file {
type => "G2Dir"
path => "D:/TempDir/1/*.txt"
sincedb_path => "D:\loghost\logstash\sincedb\G2Dir"
stat_interval => 10
start_position => "beginning"
codec => multiline
{
pattern => "duration="
negate => true
what => "previous"
}
}
}
filter
{
if [type] == "IISLog" {
grok {
match => ["message", "%{TIMESTAMP_ISO8601:log_timestamp} %{WORD:iisSite} %{IPORHOST:site} %{WORD:method} %{URIPATH:page} %{NOTSPACE:querystring} %{NUMBER:port} %{NOTSPACE:username} %{IPORHOST:clienthost} %{NOTSPACE:useragent} %{NOTSPACE:referer} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:scstatus} %{NUMBER:bytes:int} %{NUMBER:timetaken:int}"]
}
#Set the Event Timesteamp from the log
date {
match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss" ]
timezone => "Etc/UCT"
}
useragent {
source=> "useragent"
prefix=> "browser"
}
mutate {
remove_field => [ "log_timestamp"]
}
}
if [type] == "G2Dir"
{
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:log_timestamp}" }
}
date {
match => [ "timestamp", "YYYY-MM-dd HH:mm:ss.SSS"]
}
grok { match => ["message", "((?<Username>((?<=Username=)(.*?))),)" ]}
grok { match => ["message", "(?<requestTypeName=>(?<=requestTypeName=)(.*))" ]}
grok { match => ["message", "(<.*?:Message>(?<Message>.*?)(\r\n|</.*?:Message>))" ]}
grok { match => ["message", "<.*?:Code>(?<Code>.*?)</.*?:Code>" ]}
}
}
output {
if [type] == "IISLog" {
elasticsearch {
index => "iisloglocal-%{+YYYY.MM.dd}"
host => "10.77.200.214"
protocol => "http"
port => 9200
}
}
if [type] == "G2Dir"{
elasticsearch {
index_type => "custom_log"
index => "global2-%{+YYYY.MM.dd}"
host => "10.77.200.214"
port => 9200
protocol => "http"
}
}
I can make out two types of logs. One way of example and test IIS
The second log is already working G2Dir
The test configuration shows the correct analysis of my file
stdout {
codec => rubydebug
}
Sample debug logstash output
{
"@timestamp" => "2015-05-12T03:23:56.977Z",
"message" => "\r\n2014-12-30 15:03:48.025\r\n <e:CipherData>\r\n
\r\n\r\nUsername=bancadeeconomi_g2, requestTypeName=CreatePersonRequestMessage\r\n\r\n\r\n2014-12-30 15:03:48.212\r\nInvoking. .. invocationID=i15:03:48.0093s1, connection.U1Login=bancadeeconomi_g2, actualCert=1A5C, AppId=BancaDeEconomiApplication\r\n <d4p1:Code >UnknownError</d4p1:Code>\r\n <d4p1:ID>2014-12-30 17:51:13.446s1</d4p1:ID>\r\n <d4p1:Message>(Can't map transfer1 id=37598427, sTransferCode=289125778489, msg=transfer id=37598427 does not contain Amount with type=TransferAmountFixedInRC)</d4p1:Message >\r\n\r\n\r\n <s:Header>\r\n\t<a:Username>bancadeeconomi_g2</a:Username>\r\n </Key Info>\r\n <e:CipherData>\r\n <e:CipherValue></e:CipherValue>\r\n </e:Cipher
Data>\r\n </e:EncryptedKey>\r\nduration=0,515625 seconds.\r\n\r\n\n\r\n",
"@version" => "1",
"tags" => [
[0] "multiline"
],
"host" => "SR-ELK01-S01-01",
"log_timestamp" => "2014-12-30 15:03:48.025",
"Username" => "bancadeeconomi_g2",
"requestTypeName=" => "CreatePersonRequestMessage\r",
"Message" => "(Can't map transfer1 id=37598427, sTransferCode=289125778489, m
sg=transfer id=37598427 does not contain Amount with type=TransferAmountFixedInRC)",
"Code" => "UnknownError"
}
looking at Kibina? I'm do not find the index global2.
any idea?