Hi, everyone
I'm a beginner on ELK, and I'm making a conf file that processes snort logs.
when i run "./logstash -f /etc/logstash/conf.d/selk.conf", i got error
[ERROR] 2019-12-07 16:19:57.858 [[main]-pipeline-manager] pipeline - Error registering plugin {:pipeline_id=>"main", :plugin=>"#<LogStash::Filter Delegator:0x7edc062f @metric_events_out=org.jruby.proxy.org.logstash.instrument.metrics.counter.LongCounter$Proxy2 - name: out value:0, @metric_ events_in=org.jruby.proxy.org.logstash.instrument.metrics.counter.LongCounter$Proxy2 - name: in value:0, @metric_events_time=org.jruby.proxy.org .logstash.instrument.metrics.counter.LongCounter$Proxy2 - name: duration_in_millis value:0, @id=\"12c0bfcaaee4be1c4a56ad1f9cc40ed09adcff251150a1 c3fe5405805041b339\", @klass=LogStash::Filters::Grok, @metric_events=#<LogStash::Instrument::NamespacedMetric:0x4809b3cf @metric=#<LogStash::Inst rument::Metric:0xec22e5b @collector=#<LogStash::Instrument::Collector:0x65c858c0 @agent=nil, @metric_store=#<LogStash::Instrument::MetricStore:0x 7bbbd728 @store=#<Concurrent::Map:0x00000000000fbc entries=3 default_proc=nil>, @structured_lookup_mutex=#<Mutex:0x7ea9845b>, @fast_lookup=#<Conc urrent::Map:0x00000000000fc0 entries=132 default_proc=nil>>>>, @namespace_name=[:stats, :pipelines, :main, :plugins, :filters, :\"12c0bfcaaee4be1 c4a56ad1f9cc40ed09adcff251150a1c3fe5405805041b339\", :events]>, @filter=<LogStash::Filters::Grok add_tag=>[\"httpd\"], match=>{\"message\"=>\"%{I PORHOST:clientip} %{USER:ident} %{USER:auth} \\\\[%{HTTPDATE:timestamp}\\\\] \\\\\\\"(?:%{WORD:verb} %{NOTSPACE:request}(?:HTTP/%{NUMBER:httpvers ion})?|%{DATA:rawrequest})\\\\\\\" %{NUMBER:response}(?:%{NUMBER:bytes}|-) %{OS:referrer} %{OS:agent}\"}, id=>\"12c0bfcaaee4be1c4a56ad1f9cc40ed09 adcff251150a1c3fe5405805041b339\", enable_metric=>true, periodic_flush=>false, patterns_files_glob=>\"*\", break_on_match=>true, named_captures_o nly=>true, keep_empty_captures=>false, tag_on_failure=>[\"_grokparsefailure\"], timeout_millis=>30000, tag_on_timeout=>\"_groktimeout\">>", :erro r=>"pattern %{OS:referrer} not defined", :thread=>"#<Thread:0x46af979b@/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:246 run>"}
[ERROR] 2019-12-07 16:19:57.911 [[main]-pipeline-manager] pipeline - Pipeline aborted due to error {:pipeline_id=>"main", :exception=>#<Grok::Pat ternError: pattern %{OS:referrer} not defined>, :backtrace=>["/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/jls-grok-0.11.4/lib/grok-pure.rb :123:in `block in compile'", "org/jruby/RubyKernel.java:1292:in `loop'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/jls-grok-0.11.4/lib/ grok-pure.rb:93:in `compile'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-filter-grok-4.0.2/lib/logstash/filters/grok.rb:281:in `block in register'", "org/jruby/RubyArray.java:1734:in `each'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-filter-grok-4.0.2/ lib/logstash/filters/grok.rb:275:in `block in register'", "org/jruby/RubyHash.java:1343:in `each'", "/usr/share/logstash/vendor/bundle/jruby/2.3. 0/gems/logstash-filter-grok-4.0.2/lib/logstash/filters/grok.rb:270:in `register'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:34 1:in `register_plugin'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:352:in `block in register_plugins'", "org/jruby/RubyArray.ja va:1734:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:352:in `register_plugins'", "/usr/share/logstash/logstash-core/li b/logstash/pipeline.rb:736:in `maybe_setup_out_plugins'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:362:in `start_workers'", "/ usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:289:in `run'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:249:in `bloc k in start'"], :thread=>"#<Thread:0x46af979b@/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:246 run>"}
[ERROR] 2019-12-07 16:19:57.940 [Ruby-0-Thread-1: /usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:22] agent - Fai led to execute action {:id=>:main, :action_type=>LogStash::ConvergeResult::FailedAction, :message=>"Could not execute action: LogStash::PipelineA ction::Create/pipeline_id:main, action_result: false", :backtrace=>nil}
here is the "selk.conf"
1 input
2 {
3 file {
4 path => "/var/log/snort/alert"
5 type => "snort_tcp"
6 start_position => beginning
7 ignore_older => 0
8 sincedb_path => "/dev/null"
9 }
10 file {
11 path => "var/log/httpd/access_log"
12 type => "access_log"
13 start_position => beginning
14 ignore_older => 0
15 sincedb_path => "/dev/null"
16 }
17 }
18
19 filter
20 {
21 if [type] == "snort_tcp" {
22 grok {
23 add_tag => ["IDS"]
24 match => ["message",
25 "%{SNORTIME:snort_time}\s+\[\*\*\]\s+\[%{INT:ids_gid}\:%{INT:ids_sid}\:%{INT:ids_ rev}\]\s+\[%{DATA:Attk_Category}\]\s+\[%{DATA:Attk_Level}\]\s+%{DATA:Attk_Name}\s +\[\*\*\]\s+\[Classification:\s+%{DATA:ids_classification}\]\s+\[Priority:\s+%{IN T:priority}\]\s+\{%{WORD:ids_proto}\}\s+%{IP:src_ip}\:%{INT:src_port}\s+\-\>\s+%{ IP:dst_ip}\:%{INT:dst_port}"]
26 }
27 }
28 date {
29 match => ["snort_time", "MM/dd-HH:mm:ss.SSSSSS"]
30 }
31 geoip {
32 source => "src_ip"
33 target => "geoip_snort_src"
34 }
35 geoip {
36 source => "dst_ip"
37 target => "geoip_snort_dst"
38 }
39 if [priority] == "1" {
40 mutate {
41 add_field => { "severity" => "High & Medium & Low"} } }
42 if [priority] == "2" {
43 mutate {
44 add_field => { "severity" => "High & Medium" } } }
45 if [priority] == "3" {
46 mutate {
47 add_field => { "severity" => "High & Low" } } }
48 if [priority] == "4" {
49 mutate {
50 add_field => { "severity" => "Medium & Low" } } }
51 if [priority] == "5" {
52 mutate {
53 add_field => { "severity" => "High" } } }
54 if [priority] == "6" {
55 mutate {
56 add_field => { "severity" => "Medium" } } }
57 if [priority] == "7" {
58 mutate {
59 add_field => { "severity" => "Low" } } }
60 if [priority] == "8" {
61 mutate {
62 add_field => { "severity" => "ETC" } } }
63 }
64
65 filter {
66 if [type] == "access_log" {
67 grok {
68 add_tag => ["httpd"]
69 match => ["message","%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?:HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response}(?:%{NUMBER:bytes}|-) %{OS:referrer} %{OS:agent}" ] }
70 date { match => ["timestamp", "dd/MMM/YYYY/:HH:mm:ss Z"] }
71 mutate { convert => {"bytes" => "integer"} }
72 geoip { source => "clientip" }
73 mutate { convert => {"response" => "integer"} } } }
74
75 output
76 {
77 if [type] == "snort_tcp" {
78 elasticsearch {
79 hosts => ["localhost:9200"]
80 #manage_template => true
81 index => "logstash-snort"
82 }
83 }
84 if [type] == "access_log" {
85 elasticsearch {
86 hosts => ["localhost:9200"]
87 index => "logstash-httpd"
88 }
89 }
90 }
91
I am making a reference to the book, but I can't proceed anymore due to an error.
If you know how to fix this, please help this newbie.
Any help is appreciated. Thanks in advance.
have a nice day.