Parsing logs from audit.log from /var/log/audit/audit.log

ranjini · March 29, 2020, 7:39pm

Logfile contents type=USER_END msg=audit(1585509001.049:14007): pid=15642 uid=0 auid=0 ses=1100 msg='op=PAM:session_close grantors=pam_loginuid,pam_keyinit,pam_limits,pam_systemd acct="root" exe="/usr/sbin/crond" hostname=? addr=? terminal=cron res=success'

input {
  file {
    path => " /var/log/audit/audit.log"
    start_position => "beginning"
        type => "audit"
    start_position => beginning
    sincedb_path => "/dev/null"
    ignore_older => 0
  }
}
filter {
if [type] == "audit"{
  grok {
        match => { "message" => "type=%{WORD:audit_type} msg=audit\(%{NUMBER:audit_epoch}:%{NUMBER:audit_counter}\): pid=%{NUMBER:audit_pid} uid=%{NUMBER:audit_uid} auid=%{NUMBER:audit_audid} ses=%{NUMBER:ses} msg=\'op=%{WORD:operation}:%{WORD:detail_operation} grantors=%{WORD:pam_login},%{WORD:pam_key},%{WORD:pam_limit},%{WORD:pam_system} acct=\"%{WORD:acct_user}\" exe=\"%{GREEDYDATA:exec}\" hostname=%{GREEDYDATA:hostname} addr=%{GREEDYDATA:ipaddr} terminal=%{WORD:terminal} res=%{WORD:result}\'" }
    }
    date {
        match => [ "audit_epoch", "UNIX_MS" ]
    }
mutate {
        split => ["host", "."]
        add_field => { "hostname" => "%{[host][0]}" }
        add_field => { "podName" => "%{[host][1]}" }
        add_field => { "ignore" => "%{[host][2]}" }
        remove_field => ["ignore", "host"]
    }
	}
	}


output {
    if [type] == "audit" {
     stdout {
      codec  => rubydebug
     }
}
}

error : [ERROR] 2020-03-29 19:34:58.462 [[main]-pipeline-manager] javapipeline - Pipeline aborted due to error {:pipeline_id=>"main", :exception=>#<ArgumentError: File paths must be absolute, relative path specified: /var/log/audit.log>, :backtrace=>["/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-file-4.1.16/lib/logstash/inputs/file.rb:275:in block in register'", "org/jruby/RubyArray.java:1792:in each'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-file-4.1.16/lib/logstash/inputs/file.rb:273:in register'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:191:in block in register_plugins'", "org/jruby/RubyArray.java:1792:in each'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:190:in register_plugins'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:280:in start_inputs'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:244:in start_workers'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:145:in run'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:104:in block in start'"], :thread=>"#<Thread:0x1436a724 run>"}
[ERROR] 2020-03-29 19:34:58.486 [Converge PipelineAction::Create] agent - Failed to execute action {:id=>:main, :action_type=>LogStash::ConvergeResult::FailedAction, :message=>"Could not execute action: PipelineAction::Create, action_result: false", :backtrace=>nil}
[DEBUG] 2020-03-29 19:34:58.552 [LogStash::Runner] os - Stopping

Please help to resolve the error.

Badger · March 29, 2020, 11:14pm

There is a leading space in your path, so this is effectively "./ /var/log/audit/audit.log" and that results in the error 'ArgumentError: File paths must be absolute, relative path specified'.

ranjini · March 30, 2020, 5:21am

Thank you for the solution, yes it worked when removing the space as you mentioned.
Now I have encountered date parse failure.

This is the log file :

filter:

grok {
        match => { "message" => "type=%{WORD:audit_type} msg=audit\(%{NUMBER:audit_epoch}:%{NUMBER:audit_counter}\): pid=%{NUMBER:audit_pid} uid=%{NUMBER:audit_uid} auid=%{NUMBER:audit_audid} ses=%{NUMBER:ses} msg=\'op=%{WORD:operation}:%{WORD:detail_operation} grantors=%{WORD:pam_login},%{WORD:pam_key},%{WORD:pam_limit},%{WORD:pam_system} acct=\"%{WORD:acct_user}\" exe=\"%{GREEDYDATA:exec}\" hostname=%{GREEDYDATA:hostname} addr=%{GREEDYDATA:ipaddr} terminal=%{WORD:terminal} res=%{WORD:result}" }
    }
    date {
        match => [ "audit_epoch", "UNIX_MS" ]
    }
mutate {
        split => ["host", "."]
        add_field => { "hostname" => "%{[host][0]}" }
        add_field => { "myName" => "%{[host][1]}" }
        add_field => { "ignore" => "%{[host][2]}" }
        remove_field => ["ignore", "host"]
    }

ranjini · March 30, 2020, 5:22am

log file contents type=USER_START msg=audit(1585543849.155:15332): pid=9606 uid=0 auid=1000 ses=1198 msg='op=PAM:session_open grantors=pam_keyinit,pam_keyinit,pam_limits,pam_systemd,pam_unix,pam_xauth acct="root" exe="/usr/bin/su" hostname=abc-de-01234edsf.ohai-new7.local addr=? terminal=pts/1 res=success'

Robo · March 30, 2020, 9:40am

try to use:

grok {
    match => { "message" => "type=%{WORD:audit_type} msg=audit\(%{NUMBER:audit_epoch:timestamp}:%{NUMBER:audit_counter}\): pid=%{NUMBER:audit_pid} uid=%{NUMBER:audit_uid} auid=%{NUMBER:audit_audid} ses=%{NUMBER:ses} msg=\'op=%{WORD:operation}:%{WORD:detail_operation} grantors=%{WORD:pam_login},%{WORD:pam_key},%{WORD:pam_limit},%{WORD:pam_system} acct=\"%{WORD:acct_user}\" exe=\"%{GREEDYDATA:exec}\" hostname=%{GREEDYDATA:hostname} addr=%{GREEDYDATA:ipaddr} terminal=%{WORD:terminal} res=%{WORD:result}" }
}

ranjini · March 30, 2020, 11:05am

There are date parse failure and grok failure please tell me how to resolve
tags:_dateparsefailure terminal:cron pam_login:pam_loginuid pam_system:pam_systemd acct_user:root myname:abc_g1 pam_key:pam_keyinit audit_pid:26122 detail_operation:session_close operation:PAM hostname:?, abc-123-aw3ed exec:/usr/sbin/crond audit_epoch:1585302001.531 result:success audit_audid:0 message:type=USER_END msg=audit(1585302001.531:6789): pid=26122 uid=0 auid=0 ses=537 msg='op=PAM:session_close grantors=pam_loginuid,pam_keyinit,pam_limits,pam_systemd acct="root" exe="/usr/sbin/crond" hostname=? addr=? terminal=cron res=success' path:/var/log/audit/audit.log audit_counter:6789 ipaddr:? audit_type:USER_END ses:537 pam_limit:pam_limits audit_uid:0 type:audit _id:SeoWK3EBMRsGEO0Fx1EX _type:_doc _index:abc-123-aw70-audit-final- _score:2.045

tags:_grokparsefailure myname:abc_g1 message:type=SERVICE_STOP msg=audit(1585296872.086:6648): pid=1 uid=0 auid=4294967295 ses=4294967295 msg='unit=systemd-hostnamed comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' hostname:abc-123-aw3ed path:/var/log/audit/audit.log type:audit _id:vOoWK3EBMRsGEO0Fx1AO _type:_doc _index:abc-123-aw70-audit-final- _score:0.16

filter used

filter {
if [type] == "audit"{
  grok {
        match => { "message" => "type=%{WORD:audit_type} msg=audit\(%{NUMBER:audit_epoch:timestamp}:%{NUMBER:audit_counter}\): pid=%{NUMBER:audit_pid} uid=%{NUMBER:audit_uid} auid=%{NUMBER:audit_audid} ses=%{NUMBER:ses} msg=\'op=%{WORD:operation}:%{WORD:detail_operation} grantors=%{WORD:pam_login},%{WORD:pam_key},%{WORD:pam_limit},%{WORD:pam_system} acct=\"%{WORD:acct_user}\" exe=\"%{GREEDYDATA:exec}\" hostname=%{GREEDYDATA:hostname} addr=%{GREEDYDATA:ipaddr} terminal=%{WORD:terminal} res=%{WORD:result}" }
    }
    date {
        match => [ "audit_epoch", "UNIX_MS" ]
    }
mutate {
remove_field => ["@timestamp", "@version"]
}	
mutate {
        split => ["host", "."]
        add_field => { "hostname" => "%{[host][0]}" }
        add_field => { "podName" => "%{[host][1]}" }
        add_field => { "ignore" => "%{[host][2]}" }
        remove_field => ["ignore", "host"]
    }
	```

ranjini · March 31, 2020, 3:26pm

There are date parse failure and grok failure please tell me how to resolve
tags:_dateparsefailure terminal:cron pam_login:pam_loginuid pam_system:pam_systemd acct_user:root myname:abc_g1 pam_key:pam_keyinit audit_pid:26122 detail_operation:session_close operation:PAM hostname:?, abc-123-aw3ed exec:/usr/sbin/crond audit_epoch:1585302001.531 result:success audit_audid:0 message:type=USER_END msg=audit(1585302001.531:6789): pid=26122 uid=0 auid=0 ses=537 msg='op=PAM:session_close grantors=pam_loginuid,pam_keyinit,pam_limits,pam_systemd acct="root" exe="/usr/sbin/crond" hostname=? addr=? terminal=cron res=success' path:/var/log/audit/audit.log audit_counter:6789 ipaddr:? audit_type:USER_END ses:537 pam_limit:pam_limits audit_uid:0 type:audit _id:SeoWK3EBMRsGEO0Fx1EX _type:_doc _index:abc-123-aw70-audit-final- _score:2.045

tags:_grokparsefailure myname:abc_g1 message:type=SERVICE_STOP msg=audit(1585296872.086:6648): pid=1 uid=0 auid=4294967295 ses=4294967295 msg='unit=systemd-hostnamed comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' hostname:abc-123-aw3ed path:/var/log/audit/audit.log type:audit _id:vOoWK3EBMRsGEO0Fx1AO _type:_doc _index:abc-123-aw70-audit-final- _score:0.16

filter used

filter {
if [type] == "audit"{
  grok {
        match => { "message" => "type=%{WORD:audit_type} msg=audit\(%{NUMBER:audit_epoch:timestamp}:%{NUMBER:audit_counter}\): pid=%{NUMBER:audit_pid} uid=%{NUMBER:audit_uid} auid=%{NUMBER:audit_audid} ses=%{NUMBER:ses} msg=\'op=%{WORD:operation}:%{WORD:detail_operation} grantors=%{WORD:pam_login},%{WORD:pam_key},%{WORD:pam_limit},%{WORD:pam_system} acct=\"%{WORD:acct_user}\" exe=\"%{GREEDYDATA:exec}\" hostname=%{GREEDYDATA:hostname} addr=%{GREEDYDATA:ipaddr} terminal=%{WORD:terminal} res=%{WORD:result}" }
    }
    date {
        match => [ "audit_epoch", "UNIX_MS" ]
    }
mutate {
remove_field => ["@timestamp", "@version"]
}	
mutate {
        split => ["host", "."]
        add_field => { "hostname" => "%{[host][0]}" }
        add_field => { "podName" => "%{[host][1]}" }
        add_field => { "ignore" => "%{[host][2]}" }
        remove_field => ["ignore", "host"]
    }
	```

ranjini · April 1, 2020, 4:17am

@magnusbaeck Any help here , greatly appreciated.

system · April 29, 2020, 4:17am

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.