Logstash combining all the grok pattern returns

Elastic Stack Logstash
1

Hi,

I have a logfile like this

2016-03-10 04:49:41,586 p=8250 u=root |  TASK [filetester : Display messages] *******************************************
2016-03-10 04:49:41,619 p=8250 u=root |  skipping: [MYLABSERVER]
2016-03-10 04:49:41,621 p=8250 u=root |  TASK [filetester : Copying files] **********************************************
2016-03-10 04:49:43,467 p=8250 u=root |  changed: [MYLABSERVER]
2016-03-10 04:49:43,551 p=8250 u=root |  PLAY RECAP *********************************************************************
2016-03-10 04:49:43,551 p=8250 u=root |  MYLABSERVER         : ok=2    changed=1    unreachable=0    failed=0

My intended output is,

MYLABSERVER , filetester, ok

But I am getting all of them in different patterns

This is my logstash file,

input {
  file {
    path => ["ansible.log"]
    start_position => "beginning"
  }
}

filter {
        if "PLAY" in [message] {
         grok {
				match => { "message" => "%{TIMESTAMP_ISO8601:date} p=%{INT:process} u=%{USER:user} \|  %{WORD:action} %{GREEDYDATA:message}" }
				add_tag => ["Play"]
			   }
		 }
		 
		else if "TASK"  in [message] {
		  grok {
			match => { "message" =>"%{TIMESTAMP_ISO8601:date} p=%{INT:process} u=%{USER:user} \|  %{WORD:action} \[%{WORD:role} : %{GREEDYDATA:message}\]" }
			add_tag => ["_Action"]
			    }
			}
		else if "skipping" in [message] {
			
				drop {}
			 }
	    
		else {
		    grok {
			  match => { "message" => "%{TIMESTAMP_ISO8601:time} p=%{INT:pid} u=%{WORD:user} \|  %{HOSTNAME:host}         : %{WORD:status}=%{INT:change}    %{WORD:changes}=%{INT:NumberofChanges}    %{WORD:Reachable}=%{INT:Failure}    %{WORD:Fails}=%{INT:failednumbers}" }
			  add_tag => ["PlayBookRun_Status"]
			  }
			}
	    }
		
output {
         
    stdout { codec => rubydebug}
        }

Regards,

A

2

Look into the aggregate and collate filters since you, from what I understand, need to collect information from multiple input events and create a single output event. What you're currently trying just won't work.

3

Thanks for pointing me to the right direction :slightly_smiling:

I used the aggregate filter and it is working now. the code may not be elegant, but it is doing the job

input {
  file {
    path => ["/etc/logstash/ansible.log"]
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
}

filter {
        if "PLAY" in [message] {
                grok {
                        match => { "message" => "%{TIMESTAMP_ISO8601:date} p=%{INT:process} u=%{USER:user} \|  %{WORD:action} %{GREEDYDATA:message}" }
                        }
                }

        else if "TASK"  in [message] {
                grok {
                        match => { "message" =>"%{TIMESTAMP_ISO8601:date} p=%{INT:process} u=%{USER:user} \|  %{WORD:action} \[%{WORD:role} : %{GREEDYDATA:message}\]" }
                        }
                aggregate {
                        task_id => "%{process}"
                        code => "map ['playbook_role'] = event['role']"
                        map_action => "create"
                                        }
                        }
        else {
                grok {
               match => { "message" => "%{TIMESTAMP_ISO8601:time} p=%{INT:process} u=%{WORD:user} \|  %{HOSTNAME:host}         : %{WORD:status}=%{INT:change}    %{WORD:changes}=%{INT:NumberofChanges}    %{WORD:Reachable}=%{INT:Failure}    %{WORD:Fails}=%{INT:failednumbers}" }
                 add_tag => ["_ansible_PlayBookRun_Status"]
               }
        aggregate {
                task_id => "%{process}"
                code => "event ['role_playbook'] = map ['playbook_role']"
                map_action => "update"
                }
                }
        }

output {
  

        stdout { codec => rubydebug}
        }

Regards,

A