Grok log

Elastic Stack Logstash
1
  • Version: logstash 6.4.2
  • Operating System: Ubuntu
    I have log:
    2018-10-30 11:45:02 act_id: 76698461 st:SED_AGENT_SIGN_ACCEPTED request:c01df4fe1cd62d867934d1903456bfc1

My grok

filter {
if [type] == "act-process" {
grok {
match => { "message" => "%{DATESTAMP:timestamp} act_id: %{NUMBER:act_id} st:%{WORD:stage} %{GREEDYDATA:message}" }
}
}
}

I see in Elastic

@timestamp October 30th 2018, 12:07:17.003
t _id sAg6xGYBD5P5ZfC_KuxY
t _index filebeat-6.4.2-2018.10.30
# _score -
t _type doc
t beat.hostname vlab-agent-app
t beat.name vlab-agent-app
t beat.version 6.4.2
t host.name vlab-agent-app
t input.type log
t message 2018-10-30 11:45:02 act_id: 76698461 st:SED_AGENT_SIGN_ACCEPTED request:c01df4fe1cd62d867934d1903456bfc1
# offset 126
t prospector.type log
t source /mnt/log/act-sign/logs/process/process2.log
t type act-process

All my data is placed in the message string....

What is the problem?

2

Are you sending the data through Logstash? What does your full Filebeat and Logstash config look like?

3

root@vlab-elk-redis:~# cat /etc/logstash/conf.d/02-beats-input.conf
input {
beats {
port => 5044
type => "act-process"
ssl => false

ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt"

ssl_key => "/etc/pki/tls/private/logstash-beats.key"

}
}
root@vlab-elk-redis:~# cat /etc/logstash/conf.d/
02-beats-input.conf 12-process.conf 30-output.conf
root@vlab-elk-redis:~# cat /etc/logstash/conf.d/12-process.conf
filter {
grok {
match => { "message" => "%{DATESTAMP:timestamp} act_id: %{NUMBER:act_id} st:%{WORD:stage} %{GREEDYDATA:message}" }
add_tag => [ "zdes_bil_vasya" ]
}
}

root@vlab-elk-redis:~# cat /etc/logstash/conf.d/30-output.conf
output {
elasticsearch {
hosts => ["localhost"]
manage_template => false
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
root@vlab-agent-app:/home/alfa_agent# grep -v '^ *#|^ *$' /etc/filebeat/filebeat.yml
filebeat.inputs:

  • type: log
    enabled: true
    paths:
    • /mnt/log/act-sign/logs/process/.log
      fields:
      type: act-process
      fields_under_root: true
      scan_frequency: 5s
      filebeat.config.modules:
      path: ${path.config}/modules.d/      .yml
      reload.enabled: false
      setup.template.settings:
      index.number_of_shards: 3
      setup.kibana:
      output.elasticsearch:
      hosts: ["172.16.91.43:9200"]