How to make sure receive data from logstash to elasticsearch

Elastic Stack Elasticsearch
1

Hi.
I can not view the data in recovering logstash -> elasticsearch . system windows srv 2012+IIS
Unfortunately, the application creates one file = one event
Each file has a size of 20Kb. My logstash config:

input 
{ 
	file 	{
		type => "IISLog"
		sincedb_path => "D:\loghost\logstash\sincedb\IISLog"
		path => "C:/inetpub/logs/LogFiles/W3SVC*/*.log"
		}

############	G2 test log 	########################
	file 	{
		type => "G2Dir"
		path => "D:/TempDir/1/*.txt"
		sincedb_path => "D:\loghost\logstash\sincedb\G2Dir"
		stat_interval => 10
		start_position => "beginning"
			 codec => multiline 
			{
	      			pattern => "duration="
	      			negate => true
	      			what => "previous"
		    	}
		}

}

filter 
{
 
	if [type] == "IISLog" { 
		grok 	{
			match => ["message", "%{TIMESTAMP_ISO8601:log_timestamp} %{WORD:iisSite} %{IPORHOST:site} %{WORD:method} %{URIPATH:page} %{NOTSPACE:querystring} %{NUMBER:port} %{NOTSPACE:username} %{IPORHOST:clienthost} %{NOTSPACE:useragent} %{NOTSPACE:referer} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:scstatus} %{NUMBER:bytes:int} %{NUMBER:timetaken:int}"]
  			}
  
  			#Set the Event Timesteamp from the log
   		date 	{
     	 		match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss" ]
			timezone => "Etc/UCT"
			} 
  		useragent {
    			source=> "useragent"
    			prefix=> "browser"
  			  }
  
  		mutate 	{
      			remove_field => [ "log_timestamp"]
    			}
		}

	if [type] == "G2Dir" 
		{
		 grok { 
		    match => { "message" => "%{TIMESTAMP_ISO8601:log_timestamp}" } 
		  }

		  date { 
		    match => [ "timestamp", "YYYY-MM-dd HH:mm:ss.SSS"]
			  }

			grok { match => ["message", "((?<Username>((?<=Username=)(.*?))),)" ]}
			grok { match => ["message", "(?<requestTypeName=>(?<=requestTypeName=)(.*))"  ]}
			grok { match => ["message", "(<.*?:Message>(?<Message>.*?)(\r\n|</.*?:Message>))" ]}
			grok { match => ["message", "<.*?:Code>(?<Code>.*?)</.*?:Code>" ]}
		}

}

output {
		if [type] == "IISLog" { 
		 elasticsearch {
			index => "iisloglocal-%{+YYYY.MM.dd}"
			host => "10.77.200.214"
			protocol => "http"
			port => 9200
			}
		}
		
		if [type] == "G2Dir"{
		 elasticsearch {
			index_type => "custom_log"
			index => "global2-%{+YYYY.MM.dd}"
			host => "10.77.200.214"
			port => 9200
			protocol => "http"
			}
	       }

I can make out two types of logs. One way of example and test IIS
The second log is already working G2Dir

The test configuration shows the correct analysis of my file

stdout { 
codec => rubydebug 
}

Sample debug logstash output

{
          "@timestamp" => "2015-05-12T03:23:56.977Z",
             "message" => "\r\n2014-12-30 15:03:48.025\r\n        <e:CipherData>\r\n
    \r\n\r\nUsername=bancadeeconomi_g2,    requestTypeName=CreatePersonRequestMessage\r\n\r\n\r\n2014-12-30 15:03:48.212\r\nInvoking.    .. invocationID=i15:03:48.0093s1, connection.U1Login=bancadeeconomi_g2, actualCert=1A5C, AppId=BancaDeEconomiApplication\r\n          <d4p1:Code    >UnknownError</d4p1:Code>\r\n          <d4p1:ID>2014-12-30 17:51:13.446s1</d4p1:ID>\r\n           <d4p1:Message>(Can't map transfer1 id=37598427, sTransferCode=289125778489, msg=transfer id=37598427 does not contain Amount with type=TransferAmountFixedInRC)</d4p1:Message    >\r\n\r\n\r\n  <s:Header>\r\n\t<a:Username>bancadeeconomi_g2</a:Username>\r\n        </Key    Info>\r\n        <e:CipherData>\r\n          <e:CipherValue></e:CipherValue>\r\n        </e:Cipher
Data>\r\n      </e:EncryptedKey>\r\nduration=0,515625 seconds.\r\n\r\n\n\r\n",
            "@version" => "1",
                "tags" => [
        [0] "multiline"
    ],
                "host" => "SR-ELK01-S01-01",
       "log_timestamp" => "2014-12-30 15:03:48.025",
            "Username" => "bancadeeconomi_g2",
    "requestTypeName=" => "CreatePersonRequestMessage\r",
             "Message" => "(Can't map transfer1 id=37598427, sTransferCode=289125778489, m
sg=transfer id=37598427 does not contain Amount with type=TransferAmountFixedInRC)",
                "Code" => "UnknownError"
}

looking at Kibina? I'm do not find the index global2.
any idea?

2

I don't think this is an Elasticsearch problem.

Can you post the full output config?

3

Full output section. Log type == "IISLog" it works completely correctly
The second section == "G2Dir" of the config is not working fully

output {
		if [type] == "IISLog" { 
		 elasticsearch {
			index => "iisloglocal-%{+YYYY.MM.dd}"
			host => "10.77.200.214"
			protocol => "http"
			port => 9200
			}
		}
		
		if [type] == "G2Dir"{
		 elasticsearch {
			index_type => "custom_log"
			index => "global2-%{+YYYY.MM.dd}"
			host => "10.77.200.214"
			port => 9200
			protocol => "http"
			}
	       }