Logstash: how to include/modify facility/priority

Hi Folks

I m trying to insert/modify via logstash priority and facility.

Using debian wheezy + rsyslog + logstash 1.4.2 and elasticsearch 1.1.1.

part of my rsyslog to undersand that I wanna:

/etc/rsyslog.conf:
.....

auth

auth.=emerg -/var/log/auth/auth_emerg.log
auth.=alert -/var/log/auth/auth_alert.log
auth.=crit -/var/log/auth/auth_crit.log
auth.=err -/var/log/auth/auth_err.log
auth.=warning -/var/log/auth/auth_warning.log
auth.=notice -/var/log/auth/auth_notice.log
auth.=info -/var/log/auth/auth_info.log
auth.=debug -/var/log/auth/auth_debug.log

authpriv

authpriv.=emerg -/var/log/authpriv/authpriv_emerg.log
authpriv.=alert -/var/log/authpriv/authpriv_alert.log
authpriv.=crit -/var/log/authpriv/authpriv_crit.log
......

/etc/logstash/conf.d/syslog.conf
input {
file {
path => "/var/log/auth/auth_.log"
type => "syslog"
}
file {
path => "/var/log/authpriv/authpriv_
.log"
type => "syslog"
}
file {
path => "/var/log/cron/cron_*.log"
type => "syslog"

.....

filter {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp}
%{SYSLOGHOST:syslog_hostname}
%{DATA:syslog_program}(?:[%{POSINT:syslog_pid}])?:
%{GREEDYDATA:syslog_message}" }
}
date {
locale => "en"
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd
HH:mm:ss", "ISO8601" ]
}
}

output {
elasticsearch { host => localhost }
stdout { codec => rubydebug }
}

json of one os syslog entries:

{
"_index": "logstash-2014.10.20",
"_type": "syslog",
"_id": "57KDKSXKSeCy9VFDr1Arlw",
"_score": null,
"_source": {
"message": "Oct 20 18:10:01 wheezy CRON[5576]: pam_unix(cron:session): session closed for user www-data",
"@version": "1",
"@timestamp": "2014-10-20T20:10:01.000Z",
"type": "syslog",
"host": "wheezy",
"path": "/var/log/authpriv/authpriv_info.log",
"tags": [
"_grokparsefailure"
],
"syslog_timestamp": "Oct 20 18:10:01",
"syslog_hostname": "wheezy",
"syslog_program": "CRON",
"syslog_pid": "5576",
"syslog_message": "pam_unix(cron:session): session closed for user www-data"
},
"sort": [
1413835801000,
1413835801000
]
}

How can I include facility priority knowing that at PATH I already have this information?
explo: path => "/var/log/auth/auth_emerg.log, /var/log/auth/auth_crit.log..."

best regards

BTW is there a book or another doc to read ? I've been reading logstah.net/docs but it is not enough to me...80)

--
You received this message because you are subscribed to the Google Groups "elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email to elasticsearch+unsubscribe@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/elasticsearch/ad3fe421-0027-4986-99b4-a10b8ae1741b%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

Hi

For whoever is interested below the solutions that I found 80)

Em segunda-feira, 20 de outubro de 2014 18h50min28s UTC-2, paulo bruck
escreveu:

Hi Folks

I m trying to insert/modify via logstash priority and facility.

Using debian wheezy + rsyslog + logstash 1.4.2 and elasticsearch 1.1.1.

part of my rsyslog to undersand that I wanna:

/etc/rsyslog.conf:
.....

auth

auth.=emerg -/var/log/auth/auth_emerg.log
auth.=alert -/var/log/auth/auth_alert.log
auth.=crit -/var/log/auth/auth_crit.log
auth.=err -/var/log/auth/auth_err.log
auth.=warning -/var/log/auth/auth_warning.log
auth.=notice -/var/log/auth/auth_notice.log
auth.=info -/var/log/auth/auth_info.log
auth.=debug -/var/log/auth/auth_debug.log

authpriv

authpriv.=emerg -/var/log/authpriv/authpriv_emerg.log
authpriv.=alert -/var/log/authpriv/authpriv_alert.log
authpriv.=crit -/var/log/authpriv/authpriv_crit.log
......

/etc/logstash/conf.d/syslog.conf
input {
file {
path => "/var/log/auth/auth_.log"
type => "syslog"
}
file {
path => "/var/log/authpriv/authpriv_
.log"
type => "syslog"
}
file {
path => "/var/log/cron/cron_*.log"
type => "syslog"

.....

filter {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp}
%{SYSLOGHOST:syslog_hostname}
%{DATA:syslog_program}(?:[%{POSINT:syslog_pid}])?:
%{GREEDYDATA:syslog_message}" }
}
date {
locale => "en"
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd
HH:mm:ss", "ISO8601" ]
}
}

  •  syslog_pri { }               if [path] =~ "auth" { mutate { replace 
    

=> { "syslog_facility" => "auth" }}} if [path] =~ "auth" { mutate {
replace => { "syslog_facility_code" => "4" }}} if [path] =~
"authpriv" { mutate { replace => { "syslog_facility" => "authpriv" }}
} if [path] =~ "authpriv" { mutate { replace => {
"syslog_facility_code" => "4" }}} if [path] =~ "cron" { mutate {
replace => { "syslog_facility" => "cron" }}}......... if [path] =~
"emerg" { mutate { replace => { "syslog_severity" => "emerg" }}} if
[path] =~ "emerg" { mutate { replace => { "syslog_severity_code" => "0" }}}
if [path] =~ "alert" { mutate { replace => { "syslog_severity" =>
"alert" }}} if [path] =~ "alert" { mutate { replace => {
"syslog_severity_code" => "1" }}} if [path] =~ "crit" { mutate {
replace => { "syslog_severity" => "crit" }}} ....*

output {
elasticsearch { host => localhost }
stdout { codec => rubydebug }
}

json of one os syslog entries:

{
"_index": "logstash-2014.10.20",
"_type": "syslog",
"_id": "57KDKSXKSeCy9VFDr1Arlw",
"_score": null,
"_source": {
"message": "Oct 20 18:10:01 wheezy CRON[5576]: pam_unix(cron:session): session closed for user www-data",
"@version": "1",
"@timestamp": "2014-10-20T20:10:01.000Z",
"type": "syslog",
"host": "wheezy",
"path": "/var/log/authpriv/authpriv_info.log",
"tags": [
"_grokparsefailure"
],
"syslog_timestamp": "Oct 20 18:10:01",
"syslog_hostname": "wheezy",
"syslog_program": "CRON",
"syslog_pid": "5576",
"syslog_message": "pam_unix(cron:session): session closed for user www-data"
},
"sort": [
1413835801000,
1413835801000
]
}

How can I include facility priority knowing that at PATH I already have this information?
explo: path => "/var/log/auth/auth_emerg.log, /var/log/auth/auth_crit.log..."

best regards

BTW is there a book or another doc to read ? I've been reading logstah.net/docs but it is not enough to me...80)

Em segunda-feira, 20 de outubro de 2014 18h50min28s UTC-2, paulo bruck
escreveu:

Hi Folks

I m trying to insert/modify via logstash priority and facility.

Using debian wheezy + rsyslog + logstash 1.4.2 and elasticsearch 1.1.1.

part of my rsyslog to undersand that I wanna:

/etc/rsyslog.conf:
.....

auth

auth.=emerg -/var/log/auth/auth_emerg.log
auth.=alert -/var/log/auth/auth_alert.log
auth.=crit -/var/log/auth/auth_crit.log
auth.=err -/var/log/auth/auth_err.log
auth.=warning -/var/log/auth/auth_warning.log
auth.=notice -/var/log/auth/auth_notice.log
auth.=info -/var/log/auth/auth_info.log
auth.=debug -/var/log/auth/auth_debug.log

authpriv

authpriv.=emerg -/var/log/authpriv/authpriv_emerg.log
authpriv.=alert -/var/log/authpriv/authpriv_alert.log
authpriv.=crit -/var/log/authpriv/authpriv_crit.log
......

/etc/logstash/conf.d/syslog.conf
input {
file {
path => "/var/log/auth/auth_.log"
type => "syslog"
}
file {
path => "/var/log/authpriv/authpriv_
.log"
type => "syslog"
}
file {
path => "/var/log/cron/cron_*.log"
type => "syslog"

.....

filter {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp}
%{SYSLOGHOST:syslog_hostname}
%{DATA:syslog_program}(?:[%{POSINT:syslog_pid}])?:
%{GREEDYDATA:syslog_message}" }
}
date {
locale => "en"
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd
HH:mm:ss", "ISO8601" ]
}
}

output {
elasticsearch { host => localhost }
stdout { codec => rubydebug }
}

json of one os syslog entries:

{
"_index": "logstash-2014.10.20",
"_type": "syslog",
"_id": "57KDKSXKSeCy9VFDr1Arlw",
"_score": null,
"_source": {
"message": "Oct 20 18:10:01 wheezy CRON[5576]: pam_unix(cron:session): session closed for user www-data",
"@version": "1",
"@timestamp": "2014-10-20T20:10:01.000Z",
"type": "syslog",
"host": "wheezy",
"path": "/var/log/authpriv/authpriv_info.log",
"tags": [
"_grokparsefailure"
],
"syslog_timestamp": "Oct 20 18:10:01",
"syslog_hostname": "wheezy",
"syslog_program": "CRON",
"syslog_pid": "5576",
"syslog_message": "pam_unix(cron:session): session closed for user www-data"
},
"sort": [
1413835801000,
1413835801000
]
}

How can I include facility priority knowing that at PATH I already have this information?
explo: path => "/var/log/auth/auth_emerg.log, /var/log/auth/auth_crit.log..."

best regards

BTW is there a book or another doc to read ? I've been reading logstah.net/docs but it is not enough to me...80)

Em segunda-feira, 20 de outubro de 2014 18h50min28s UTC-2, paulo bruck
escreveu:

Hi Folks

I m trying to insert/modify via logstash priority and facility.

Using debian wheezy + rsyslog + logstash 1.4.2 and elasticsearch 1.1.1.

part of my rsyslog to undersand that I wanna:

/etc/rsyslog.conf:
.....

auth

auth.=emerg -/var/log/auth/auth_emerg.log
auth.=alert -/var/log/auth/auth_alert.log
auth.=crit -/var/log/auth/auth_crit.log
auth.=err -/var/log/auth/auth_err.log
auth.=warning -/var/log/auth/auth_warning.log
auth.=notice -/var/log/auth/auth_notice.log
auth.=info -/var/log/auth/auth_info.log
auth.=debug -/var/log/auth/auth_debug.log

authpriv

authpriv.=emerg -/var/log/authpriv/authpriv_emerg.log
authpriv.=alert -/var/log/authpriv/authpriv_alert.log
authpriv.=crit -/var/log/authpriv/authpriv_crit.log
......

/etc/logstash/conf.d/syslog.conf
input {
file {
path => "/var/log/auth/auth_.log"
type => "syslog"
}
file {
path => "/var/log/authpriv/authpriv_
.log"
type => "syslog"
}
file {
path => "/var/log/cron/cron_*.log"
type => "syslog"

.....

filter {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp}
%{SYSLOGHOST:syslog_hostname}
%{DATA:syslog_program}(?:[%{POSINT:syslog_pid}])?:
%{GREEDYDATA:syslog_message}" }
}
date {
locale => "en"
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd
HH:mm:ss", "ISO8601" ]
}
}

output {
elasticsearch { host => localhost }
stdout { codec => rubydebug }
}

json of one os syslog entries:

{
"_index": "logstash-2014.10.20",
"_type": "syslog",
"_id": "57KDKSXKSeCy9VFDr1Arlw",
"_score": null,
"_source": {
"message": "Oct 20 18:10:01 wheezy CRON[5576]: pam_unix(cron:session): session closed for user www-data",
"@version": "1",
"@timestamp": "2014-10-20T20:10:01.000Z",
"type": "syslog",
"host": "wheezy",
"path": "/var/log/authpriv/authpriv_info.log",
"tags": [
"_grokparsefailure"
],
"syslog_timestamp": "Oct 20 18:10:01",
"syslog_hostname": "wheezy",
"syslog_program": "CRON",
"syslog_pid": "5576",
"syslog_message": "pam_unix(cron:session): session closed for user www-data"
},
"sort": [
1413835801000,
1413835801000
]
}

How can I include facility priority knowing that at PATH I already have this information?
explo: path => "/var/log/auth/auth_emerg.log, /var/log/auth/auth_crit.log..."

best regards

BTW is there a book or another doc to read ? I've been reading logstah.net/docs but it is not enough to me...80)

Em segunda-feira, 20 de outubro de 2014 18h50min28s UTC-2, paulo bruck
escreveu:

Hi Folks

I m trying to insert/modify via logstash priority and facility.

Using debian wheezy + rsyslog + logstash 1.4.2 and elasticsearch 1.1.1.

part of my rsyslog to undersand that I wanna:

/etc/rsyslog.conf:
.....

auth

auth.=emerg -/var/log/auth/auth_emerg.log
auth.=alert -/var/log/auth/auth_alert.log
auth.=crit -/var/log/auth/auth_crit.log
auth.=err -/var/log/auth/auth_err.log
auth.=warning -/var/log/auth/auth_warning.log
auth.=notice -/var/log/auth/auth_notice.log
auth.=info -/var/log/auth/auth_info.log
auth.=debug -/var/log/auth/auth_debug.log

authpriv

authpriv.=emerg -/var/log/authpriv/authpriv_emerg.log
authpriv.=alert -/var/log/authpriv/authpriv_alert.log
authpriv.=crit -/var/log/authpriv/authpriv_crit.log
......

/etc/logstash/conf.d/syslog.conf
input {
file {
path => "/var/log/auth/auth_.log"
type => "syslog"
}
file {
path => "/var/log/authpriv/authpriv_
.log"
type => "syslog"
}
file {
path => "/var/log/cron/cron_*.log"
type => "syslog"

.....

filter {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp}
%{SYSLOGHOST:syslog_hostname}
%{DATA:syslog_program}(?:[%{POSINT:syslog_pid}])?:
%{GREEDYDATA:syslog_message}" }
}
date {
locale => "en"
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd
HH:mm:ss", "ISO8601" ]
}
}

output {
elasticsearch { host => localhost }
stdout { codec => rubydebug }
}

json of one os syslog entries:

{
"_index": "logstash-2014.10.20",
"_type": "syslog",
"_id": "57KDKSXKSeCy9VFDr1Arlw",
"_score": null,
"_source": {
"message": "Oct 20 18:10:01 wheezy CRON[5576]: pam_unix(cron:session): session closed for user www-data",
"@version": "1",
"@timestamp": "2014-10-20T20:10:01.000Z",
"type": "syslog",
"host": "wheezy",
"path": "/var/log/authpriv/authpriv_info.log",
"tags": [
"_grokparsefailure"
],
"syslog_timestamp": "Oct 20 18:10:01",
"syslog_hostname": "wheezy",
"syslog_program": "CRON",
"syslog_pid": "5576",
"syslog_message": "pam_unix(cron:session): session closed for user www-data"
},
"sort": [
1413835801000,
1413835801000
]
}

How can I include facility priority knowing that at PATH I already have this information?
explo: path => "/var/log/auth/auth_emerg.log, /var/log/auth/auth_crit.log..."

best regards

BTW is there a book or another doc to read ? I've been reading logstah.net/docs but it is not enough to me...80)

--
You received this message because you are subscribed to the Google Groups "elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email to elasticsearch+unsubscribe@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/elasticsearch/2f756f58-a7f7-4e0b-a9c8-66072a845b4b%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.