Logstash Parsing for tags

Elastic Stack Logstash
1

I am using rsyslog to send selected logs to a server with ELK.
It is hard to manage the list of logs through rsyslog.d conf, thus I want to send all logs from a certain directory in /var/log.
The problem I encounter is that I cannot select the Tag Name for each log if I use asterisk to select all logs from a directory.

I.E. (etc/rsyslog.d/example.conf)
####HOW IT IS RIGHT NOW####
$InputFileName /var/log/example/example1.log
$InputFileTag example1
$InputFileStateFile example1-status
$InputRunFileMonitor

$InputFileName /var/log/example/example2.log
$InputFileTag example2
$InputFileStateFile example2-status
$InputRunFileMonitor

####HOW I WANT IT LATER####
$InputFileName /var/log/example/*.log
$InputFileTag example
$InputFileStateFile example-status
$InputRunFileMonitor

with the new configuration in rsyslog.d, Is there a way in logstash.conf to parse each log with a certain tag to be distinguished in Kibana?
In example, if example1.log is parsed, it will have example1 tag as syslog_program.

2

There are a few ways to do this in Logstash, for example you can add tags using the grok filter or filter the inputs based in some condition and send the filtered result to different outputs.

What is your Logstash configuration?

How are you ingesting the log files? With logstash or using Filebeat?

3

I am using logstash to ingest the logs into elasticsearch.
Below is the logstash.conf that we have.
Thanks.

input {
tcp {
port => 5000
type => syslog
}
udp {
port => 5000
type => syslog
}
}

filter {
if [type] == "syslog" {
grok {
match => [
"message","^<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{SYSLOGPROG:syslog_program}:? %{GREEDYDATA:syslog_message}$",
"message","^<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}:? %{GREEDYDATA:syslog_message}$",
"message","^<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{GREEDYDATA:syslog_message}$"
]
add_field => {
"received_at" => "%{@timestamp}"
"received_from" => "%{host}"
}
}
mutate {
gsub => ["syslog_program", ":", ""]
remove_field => [ "program" ]
}
grok {
match => [
"syslog_hostname","^%{WORD:server_rack}-%{WORD:server_dc}-%{INT:server_pos}$",
"syslog_hostname","^%{INT}-%{INT}-%{INT}-%{WORD:server_rack}-%{WORD:server_pos}$",
"syslog_hostname","%{GREEDYDATA}"
]
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}

    if [syslog_program] == "apache_access" {
        grok {
            match => [
                "syslog_message", "%{COMBINEDAPACHELOG} %{POSINT:resp_micros}",
                "syslog_message", "%{COMBINEDAPACHELOG}"
            ]
        }
        date {
            match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss,SSS Z", "dd/MMM/YYYY:HH:mm:ss Z" ]
        }
        geoip {
            source => "clientip"
            target => "geoip"
            add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
            add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
        }
        mutate {
            gsub => [ "resp_micros", "^$", "-1" ]
            remove_field => [ "syslog_timestamp", "syslog_program", "syslog_facility", "syslog_facility_code", "syslog_message", "syslog_severity", "syslog_severity_code", "syslog_timestamp" ]
            replace => [ "type", "apacheaccess" ]
            convert => [ "[geoip][coordinates]", "float", "bytes", "integer", "resp_micros", "integer" ]
        }
    # Temporary add post work because lots of fields are inconsistant
    #if [beans_user] {
    #    filter {
    #        mutate {
    #            uppercase => [ "beans_user" ]
    #        }
    #    }
    #}
}

}

output {
#stdout { codec => rubydebug
elasticsearch {
hosts => "{{ansible_bond0.ipv4.address}}:9200"
}

4

Your reply is not properly formatted, so maybe I didn't understood it correctly.

But to do what you want you need to add tags, for example:

 if [syslog_program] == "apache_access" {
            grok {
             match => [
                "syslog_message", "%{COMBINEDAPACHELOG} %{POSINT:resp_micros}",
                "syslog_message", "%{COMBINEDAPACHELOG}"
                ]
              add_tag => "apache-access"
            }
}

The add_tag => "apache-access" parameter above will add a tag called apache-access to each entry that grok parses with success for the program apache_access, is something like that that you want to do?

5

Yes, I want to parse it like it but if I use asterisk on the rsyslog conf, I should have same syslog_program tag for all the logs under same directory. that is my only concern because I want all logs to have different tag for it.

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.