Converting old logstash file to new syntax

Elastic Stack Logstash
1

Hello all! I would like to use the following configuration to parse through my Juniper logs.

## Grok filter SRX ##
input {
  syslog{
  host => "10.25.2.200"
  port => "10514"
  type => "syslog"
}
}
filter {
if [type] == "syslog" {
if [message] =~ "x.x.x.x" or [message] =~ "x.x.x.x" {
if [message] =~ "RT_FLOW_SESSION_CREATE" {
grok {
match => [ "message", ".* %{SYSLOGTIMESTAMP:event} .*_CREATE: session created %{IP:src-ip}/%{DATA:src-port}->%{IP:dst-ip}/%{DATA:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{DATA:nat-src-port}->%{IP:nat-dst-ip}/%{DATA:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} .*" ]
}
date { match => [ "event", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] }
geoip {
source => "src-ip"
target => "geoip"
database => "/usr/share/logstash/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float" ]
}
mutate {
replace => [ "type", "fw-create-log" ]
}
}else if [message] =~ "RT_FLOW_SESSION_CLOSE" {
grok {
match => [ "message", ".* %{SYSLOGTIMESTAMP:event} .*_CLOSE: %{GREEDYDATA:close-reason}: %{IP:src-ip}/%{DATA:src-port}->%{IP:dst-ip}/%{DATA:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{DATA:nat-src-port}->%{IP:nat-dst-ip}/%{DATA:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} \d+\(%{DATA:sent}\) \d+\(%{DATA:received}\) %{INT:elapsed-time} .*" ]
}
date { match => [ "event", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] }
geoip {
source => "src-ip"
target => "geoip"
database => "/usr/share/logstash/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float" ]
convert => [ "sent", "integer" ]
convert => [ "received", "integer" ]
}
mutate {
replace => [ "type", "fw-close-log" ]
}
}else if [message] =~ "RT_FLOW_SESSION_DENY" {
grok {
match => [ "message", ".* %{SYSLOGTIMESTAMP:event} .*_DENY: session denied %{IP:src-ip}/%{DATA:src-port}->%{IP:dst-ip}/%{DATA:dst-port} %{DATA:service} %{INT:protocol-id}\(\d\) %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} .*" ]
}
date { match => [ "event", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] }
geoip {
source => "src-ip"
target => "geoip"
database => "/usr/share/logstash/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float" ]
}
mutate {
replace => [ "type", "fw-deny-log" ]
}
} else if [message] =~ "RT_IDP" {
grok {
match => ["message", ".* %{SYSLOGTIMESTAMP:event} .*ATTACK_LOG_EVENT: %{DATA:log-type}: at %{DATA:INT}, SIG Attack log <%{IP:src-ip}/%{DATA:src-port}->%{IP:dst-ip}/%{DATA:dst-port}> %{GREEDYDATA} .*" ]
}
date { match => [ "event", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] }
geoip {
source => "src-ip"
target => "geoip"
database => "/usr/share/logstash/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float" ]
}
mutate {
replace => [ "type", "fw-idp-log" ]
}
} else {
grok {
match => [ "message", "<%{DATA:facility}>Original Address=%{IP:src-host} %{DATA:level} %{TIMESTAMP_ISO8601:event} %{GREEDYDATA:syslog_message}" ]
}
date { match => [ "event", "YYYY-MM-dd;HH:mm:ss.SSS", "ISO8601" ] }
mutate {
replace => [ "type", "sys-log" ]
}
}
}
}
}
output {
if [type] == "fw-create-log" {
elasticsearch {
hosts => ["10.25.2.200:9200"]
index => "fw-create-%{+YYYY.MM.dd}"
}
}else if [type] == "fw-close-log" {
elasticsearch {
hosts => ["10.25.2.200:9200"]
index => "fw-close-%{+YYYY.MM.dd}"
}
}else if [type] == "fw-deny-log" {
elasticsearch {
hosts => ["10.25.2.200:9200"]
index => "fw-deny-%{+YYYY.MM.dd}"
}
}else if [type] == "fw-idp-log" {
elasticsearch {
hosts => ["10.25.2.200:9200"]
index => "fw-idp-%{+YYYY.MM.dd}"
}
} else if [type] == "sys-log" {
elasticsearch {
hosts => ["10.25.2.200:9200"]
index => "sys-%{+YYYY.MM.dd}"
}
}
}

Looking through logs it appears that the type field is no longer utilized what was this replaced by?

2

What does the log say?

3

Detected a 6.x and above cluster: thetypeevent field won't be used to determine the document _type {:es_version=>6}

4

There is also another log that makes it seem like things go to sleep.

[2018-05-20T15:04:08,137][INFO ][logstash.pipeline        ] Pipeline started successfully {:pipeline_id=>"main", :thread=>"#<Thread:0x4f7037c1@/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:247 sleep>"}
[2018-05-20T15:04:08,167][INFO ][logstash.agent           ] Pipelines running {:count=>1, :pipelines=>["main"]}
[2018-05-20T15:04:08,181][INFO ][logstash.inputs.syslog   ] Starting syslog udp listener {:address=>"10.25.2.200:10514"}
[2018-05-20T15:04:08,183][INFO ][logstash.inputs.syslog   ] Starting syslog tcp listener {:address=>"10.25.2.200:10514"}
5

That relates to https://www.elastic.co/guide/en/logstash/6.2/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-document_type

6

Forgive the noobness, I assume after reading that, that I should just replace type with doc?

so
doc => "syslog"
if [doc] == "syslog" {

7

What do your Elasticsearch outputs look like?

8

All listed above in that config dump :slight_smile:

output {
if [type] == "fw-create-log" {
elasticsearch {
hosts => ["10.25.2.200:9200"]
index => "fw-create-%{+YYYY.MM.dd}"
}
}else if [type] == "fw-close-log" {
elasticsearch {
hosts => ["10.25.2.200:9200"]
index => "fw-close-%{+YYYY.MM.dd}"
}
}else if [type] == "fw-deny-log" {
elasticsearch {
hosts => ["10.25.2.200:9200"]
index => "fw-deny-%{+YYYY.MM.dd}"
}
}else if [type] == "fw-idp-log" {
elasticsearch {
hosts => ["10.25.2.200:9200"]
index => "fw-idp-%{+YYYY.MM.dd}"
}
} else if [type] == "sys-log" {
elasticsearch {
hosts => ["10.25.2.200:9200"]
index => "sys-%{+YYYY.MM.dd}"
9

Bumping in case you forgot :slight_smile:

10

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.