Logstash log says "Pushing flushed events" but I can't see any output in ES

tomer · September 27, 2017, 1:38pm

Hi,

I am getting the following log in Logstash, but from some reason in Elasticsearch I can't see any output. How can this be?

[2017-09-27T16:38:40,961][DEBUG][logstash.pipeline ] Pushing flushed events {:event=>2017-09-27T13:38:40.956Z %{host} %{message}}
[2017-09-27T16:38:40,962][DEBUG][logstash.pipeline ] output received {"event"=>{"msgSentToJMS_MT"=>"1506519335020", "reason"=>"", "lnpInfoGot"=>"1506519334976", "MedGotMsgFromApi"=>"1506519334971", "medSentRespToApi"=>"1506519335021", "blackListCBGot"=>"1506519334978", "FSM_got"=>"1506519508985", "transactionId"=>"10529843-c76d-4f20-b367-b97286e2a34f", "accountInfoGot"=>"1506519334974", "SRI_sent"=>"1506519505363", "tags"=>["aggregate", "_dateparsefailure"], "concatenatedFromTotal"=>"", "medGotAck_MT"=>"1506519341405", "SRI_got"=>"1506519507287", "@timestamp"=>2017-09-27T13:38:40.956Z, "authCBGot"=>"1506519334973", "terminationCause"=>"QUEUED", "cdrType"=>"aggregated", "@version"=>"1", "FSM_sent"=>"1506519507510"}}

magnusbaeck · September 27, 2017, 8:08pm

Show your configuration.

tomer · September 28, 2017, 9:00am

Hi @magnusbaeck disregard the config file I sent since:

When checking the logstash output by:

output {
	stdout {
		codec => rubydebug
	}
}

I see that the logstash does post info. So my guess is that since my ES is in a yellow state from somereason it does not read these certain logs.

magnusbaeck · September 28, 2017, 10:14am

A yellow state shouldn't be a problem. If ES doesn't accept what Logstash is sending there should be stuff in the log about that.

tomer · September 28, 2017, 11:33am

so here is my config file, maybe there I have a problem?:

filter {
  json{
        source => "message"
  }

  aggregate {
		task_id => "%{transactionId}"
		code => "

			if (map['authCBGot'])
				event.set('authCBGot', (map['authCBGot']))
			else
				if (event.get('authCBGot').eql? '0')
					# do nothing
				else
					map['authCBGot'] = event.get('authCBGot')
				end
			end
			if (map['blackListCBGot'])
				event.set('blackListCBGot', (map['blackListCBGot']))
			else
				if (event.get('blackListCBGot').eql? '0')
					# do nothing
				else
					map['blackListCBGot'] = event.get('blackListCBGot')
				end
			end
			if (map['medGotFromUser_MO'])
				event.set('medGotFromUser_MO', (map['medGotFromUser_MO']))
			else
				if (event.get('medGotFromUser_MO').eql? '0')
					# do nothing
				else
					map['medGotFromUser_MO'] = event.get('medGotFromUser_MO')
				end
			end
			if (map['MedGotMsgFromApi'])
				event.set('MedGotMsgFromApi', (map['MedGotMsgFromApi']))
			else
				if (event.get('MedGotMsgFromApi').eql? '0')
					# do nothing
				else
					map['MedGotMsgFromApi'] = event.get('MedGotMsgFromApi')
				end
			end
			if (map['lnpInfoGot'])
				event.set('lnpInfoGot', (map['lnpInfoGot']))
			else
				if (event.get('lnpInfoGot').eql? '0')
					# do nothing
				else
					map['lnpInfoGot'] = event.get('lnpInfoGot')
				end
			end
			if (map['msgSentToJMS_MT'])
				event.set('msgSentToJMS_MT', (map['msgSentToJMS_MT']))
			else
				if (event.get('msgSentToJMS_MT').eql? '0')
					# do nothing
				else
					map['msgSentToJMS_MT'] = event.get('msgSentToJMS_MT')
				end
			end
			if (map['medGotAck_MT'])
				event.set('medGotAck_MT', (map['medGotAck_MT']))
			else
				if (event.get('medGotAck_MT').eql? '0')
					# do nothing
				else
					map['medGotAck_MT'] = event.get('medGotAck_MT')
				end
			end
			if (map['medSentToJMS_MO'])
				event.set('medSentToJMS_MO', (map['medSentToJMS_MO']))
			else
				if (event.get('medSentToJMS_MO').eql? '0')
					# do nothing
				else
					map['medSentToJMS_MO'] = event.get('medSentToJMS_MO')
				end
			end
			if (map['medSentDLR_MT'])
				event.set('medSentDLR_MT', (map['medSentDLR_MT']))
			else
				if (event.get('medSentDLR_MT').eql? '0')
					# do nothing
				else
					map['medSentDLR_MT'] = event.get('medSentDLR_MT')
				end
			end
			if (map['accountInfoGot'])
				event.set('accountInfoGot', (map['accountInfoGot']))
			else
				if (event.get('accountInfoGot').eql? '0')
					# do nothing
				else
					map['accountInfoGot'] = event.get('accountInfoGot')
				end
			end
			if (map['smscGotAck_MO'])
				event.set('smscGotAck_MO', (map['smscGotAck_MO']))
			else
				if (event.get('smscGotAck_MO').eql? '0')
					# do nothing
				else
					map['smscGotAck_MO'] = event.get('smscGotAck_MO')
				end
			end
			if (map['SMPP_got'])
				event.set('SMPP_got', (map['SMPP_got']))
			else
				if (event.get('SMPP_got').eql? '0')
					# do nothing
				else
					map['SMPP_got'] = event.get('SMPP_got')
				end
			end
			if (map['SMPP_sent'])
				event.set('SMPP_sent', (map['SMPP_sent']))
			else
				if (event.get('SMPP_sent').eql? '0')
					# do nothing
				else
					map['SMPP_sent'] = event.get('SMPP_sent')
				end
			end
			if (map['medGotMsg_MO'])
				event.set('medGotMsg_MO', (map['medGotMsg_MO']))
			else
				if (event.get('medGotMsg_MO').eql? '0')
					# do nothing
				else
					map['medGotMsg_MO'] = event.get('medGotMsg_MO')
				end
			end
			if (map['smscSentAck_MO'])
				event.set('smscSentAck_MO', (map['smscSentAck_MO']))
			else
				if (event.get('smscSentAck_MO').eql? '0')
					# do nothing
				else
					map['smscSentAck_MO'] = event.get('smscSentAck_MO')
				end
			end
			if (map['medSentRespToApi'])
				event.set('medSentRespToApi', (map['medSentRespToApi']))
			else
				if (event.get('medSentRespToApi').eql? '0')
					# do nothing
				else
					map['medSentRespToApi'] = event.get('medSentRespToApi')
				end
			end
			if (map['medGotDLR_MT'])
				event.set('medGotDLR_MT', (map['medGotDLR_MT']))
			else
				if (event.get('medGotDLR_MT').eql? '0')
					# do nothing
				else
					map['medGotDLR_MT'] = event.get('medGotDLR_MT')
				end
			end
			if (map['medSentToUser_MO'])
				event.set('medSentToUser_MO', (map['medSentToUser_MO']))
			else
				if (event.get('medSentToUser_MO').eql? '0')
					# do nothing
				else
					map['medSentToUser_MO'] = event.get('medSentToUser_MO')
				end
			end
		    if (map['SRI_sent'])
				event.set('SRI_sent', (map['SRI_sent']))
			else
				if (event.get('SRI_sent').eql? '0')
					# do nothing
				else
					map['SRI_sent'] = event.get('SRI_sent')
				end
			end
			if (map['SRI_got'])
				event.set('SRI_got', (map['SRI_got']))
			else
				if (event.get('SRI_got').eql? '0')
					# do nothing
				else
					map['SRI_got'] = event.get('SRI_got')
				end
			end
			if (map['FSM_sent'])
				event.set('FSM_sent', (map['FSM_sent']))
			else
				if (event.get('FSM_sent').eql? '0')
					# do nothing
				else
					map['FSM_sent'] = event.get('FSM_sent')
				end
			end
			if (map['FSM_got'])
				event.set('FSM_got', (map['FSM_got']))
			else
				if (event.get('FSM_got').eql? '0')
					# do nothing
				else
					map['FSM_got'] = event.get('FSM_got')
				end
			end
      if (map['terminationCause'])
          event.set('terminationCause', (map['terminationCause']))
      else
				if (event.get('terminationCause').eql? '0')
					# do nothing
				else
					map['terminationCause'] = event.get('terminationCause')
				end
			end
      if (map['reason'])
          event.set('reason', (map['reason']))
      else
				if (event.get('reason').eql? '0')
					# do nothing
				else
					map['reason'] = event.get('reason')
				end
			end

			"
		push_map_as_event_on_timeout => true
    timeout_task_id_field => "transactionId"
		timeout => 3600
    inactivity_timeout => 2
    timeout_tags => ["aggregate"]
		timeout_code => "event.set('cdrType', 'aggregated')"

}
if "aggregate" not in [tags] {
		drop{}
	}

tomer · September 28, 2017, 11:35am

and the second part that looks a lot less important, since if I erase the "drop" part I will get logs to ES, but not the aggregated ones.

May I mention that this worked for someone else on his ELK

  date {
      match => [ "eventTimestamp", "UNIX" ]
      target => "eventTimestamp"
    }
    date {
      match => [ "FSM_got", "UNIX" ]
      target => "FSM_got"
    }
    date {
      match => [ "FSM_sent", "UNIX" ]
      target => "FSM_sent"
    }
    date {
      match => [ "authCBGot", "UNIX" ]
      target => "authCBGot"
    }
    date {
      match => [ "MedGotMsgFromApi", "UNIX" ]
      target => "MedGotMsgFromApi"
    }
    date {
      match => [ "accountInfoGot", "UNIX" ]
      target => "accountInfoGot"
    }
    date {
      match => [ "lnpInfoGot", "UNIX" ]
      target => "lnpInfoGot"
    }
    date {
      match => [ "blackListCBGot", "UNIX" ]
      target => "blackListCBGot"
    }
    date {
      match => [ "msgSentToJMS_MT", "UNIX" ]
      target => "msgSentToJMS_MT"
    }
    date {
      match => [ "alertSC_got", "UNIX" ]
      target => "alertSC_got"
    }
    date {
      match => [ "SRI_sent", "UNIX" ]
      target => "SRI_sent"
    }
    date {
      match => [ "SRI_got", "UNIX" ]
      target => "SRI_got"
    }
    date {
      match => [ "medGotMsg_MO", "UNIX" ]
      target => "medGotMsg_MO"
    }
    date {
      match => [ "medSentToUser_MO", "UNIX" ]
      target => "medSentToUser_MO"
    }
    date {
      match => [ "medGotDLR_MT", "UNIX" ]
      target => "medGotDLR_MT"
    }
    date {
      match => [ "medSentDLR_MT", "UNIX" ]
      target => "medSentDLR_MT"
    }
    date {
      match => [ "medGotAck_MT", "UNIX" ]
      target => "medGotAck_MT"
    }
    date {
      match => [ "SMPP_sent", "UNIX" ]
      target => "SMPP_sent"
    }
    date {
      match => [ "SMPP_got", "UNIX" ]
      target => "SMPP_got"
    }
    date {
      match => [ "medGotFromUser_MO", "UNIX" ]
      target => "medGotFromUser_MO"
    }
    date {
      match => [ "medSentToJMS_MO", "UNIX" ]
      target => "medSentToJMS_MO"
    }
    date {
      match => [ "smscGotAck_MO", "UNIX" ]
      target => "smscGotAck_MO"
    }
    date {
      match => [ "smscSentAck_MO", "UNIX" ]
      target => "smscSentAck_MO"
    }
    date {
      match => [ "medSentRespToApi", "UNIX" ]
      target => "medSentRespToApi"
    }
     date {
      match => [ "asrFrom", "UNIX" ]
      target => "asrFrom"
    }

}
#output {
#	stdout {
#		codec => rubydebug
#	}
#}
output {
    elasticsearch {
      hosts => ["localhost:9200"]
      manage_template => false
      index => "aggregated"
      document_type => "aggregatedtype"
    }
}

system · October 26, 2017, 11:36am

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.