How can I tell if Auditbeat is grouping syscall messages?

Elastic Stack Beats
1

Just a quick question to make sure I'm not missing something.

Thanks to @andrewkroh I have my auditbeat working.

My understanding of the auditd module in auditbeat is that it groups syscalls together so. for example, a rename is sent as one message and not 8 of them.

At the moment I can see multiple matching time stamps/messages that seems to indicate that this is not happening.

Is there something I need to set in the auditd module to group them or is that the default and I'm still too new to all this to pick it up correctly!?

Many thanks.

2

Hi @nlh, could you please share some of the JSON events where the data wasn't grouped together. You can copy the raw JSON for the events out of Kibana.

3

Hi @andrewkroh, Thanks again for your help. Just need some help in finding my feet with all of this.

All have "@timestamp": "2018-02-28T14:08:20.804Z",

Message 1

{
  "_index": "auditbeat-6.2.2-2018.02.28",
  "_type": "logs",
  "_id": "AWHcvhsql410dLnM6zqn",
  "_version": 1,
  "_score": null,
  "_source": {
    "process": {
      "name": "ntpd",
      "pid": "16097",
      "exe": "/usr/sbin/ntpd",
      "ppid": "1"
    },
    "@timestamp": "2018-02-28T14:08:20.804Z",
    "beat": {
      "name": "ptc-desk",
      "hostname": "soptct61-01.hursley.ibm.com",
      "version": "6.2.2"
    },
    "@version": "1",
    "host": "soptct61-01.hursley.ibm.com",
    "auditd": {
      "summary": {
        "actor": {
          "secondary": "ntp",
          "primary": "nhopper"
        },
        "how": "/usr/sbin/ntpd"
      },
      "result": "success",
      "sequence": 38316124,
      "data": {
        "a1": "7fff4468f400",
        "syscall": "select",
        "a2": "0",
        "exit": "1",
        "a3": "0",
        "tty": "(none)",
        "arch": "x86_64",
        "items": "0",
        "a0": "18"
      },
      "session": "8675"
    },
    "event": {
      "action": "",
      "category": "audit-rule",
      "type": "syscall",
      "module": "auditd"
    },
    "user": {
      "fsuid": "38",
      "uid": "38",
      "name_map": {
        "fsuid": "ntp",
        "uid": "ntp",
        "egid": "ntp",
        "auid": "nhopper",
        "gid": "ntp",
        "euid": "ntp",
        "fsgid": "ntp",
        "sgid": "ntp",
        "suid": "ntp"
      },
      "auid": "507",
      "egid": "38",
      "gid": "38",
      "euid": "38",
      "fsgid": "38",
      "sgid": "38",
      "suid": "38"
    },
    "tags": [
      "b64_call",
      "beats_input_raw_event",
      "_grokparsefailure"
    ]
  },
  "fields": {
    "@timestamp": [
      1519826900804
    ]
  },
  "sort": [
    1519826900804
  ]
}

Message 2

{
  "_index": "auditbeat-6.2.2-2018.02.28",
  "_type": "logs",
  "_id": "AWHcvhsql410dLnM6zqp",
  "_version": 1,
  "_score": null,
  "_source": {
    "process": {
      "name": "ntpd",
      "pid": "16097",
      "exe": "/usr/sbin/ntpd",
      "ppid": "1"
    },
    "@timestamp": "2018-02-28T14:08:20.804Z",
    "beat": {
      "name": "ptc-desk",
      "hostname": "soptct61-01.hursley.ibm.com",
      "version": "6.2.2"
    },
    "@version": "1",
    "host": "soptct61-01.hursley.ibm.com",
    "auditd": {
      "summary": {
        "actor": {
          "secondary": "ntp",
          "primary": "nhopper"
        },
        "how": "/usr/sbin/ntpd",
        "object": {
          "type": "socket"
        }
      },
      "result": "fail",
      "sequence": 38316126,
      "data": {
        "a1": "7fff4468f480",
        "syscall": "recvmsg",
        "exit": "EAGAIN",
        "a2": "0",
        "a3": "0",
        "tty": "(none)",
        "arch": "x86_64",
        "items": "0",
        "a0": "13"
      },
      "session": "8675"
    },
    "event": {
      "action": "received-from",
      "category": "audit-rule",
      "type": "syscall",
      "module": "auditd"
    },
    "user": {
      "fsuid": "38",
      "uid": "38",
      "auid": "507",
      "egid": "38",
      "name_map": {
        "fsuid": "ntp",
        "uid": "ntp",
        "auid": "nhopper",
        "egid": "ntp",
        "gid": "ntp",
        "euid": "ntp",
        "fsgid": "ntp",
        "sgid": "ntp",
        "suid": "ntp"
      },
      "gid": "38",
      "euid": "38",
      "fsgid": "38",
      "sgid": "38",
      "suid": "38"
    },
    "tags": [
      "b64_call",
      "beats_input_raw_event",
      "_grokparsefailure"
    ]
  },
  "fields": {
    "@timestamp": [
      1519826900804
    ]
  },
  "sort": [
    1519826900804
  ]
}

Message 3

{
  "_index": "auditbeat-6.2.2-2018.02.28",
  "_type": "logs",
  "_id": "AWHcvhsql410dLnM6zqo",
  "_version": 1,
  "_score": null,
  "_source": {
    "process": {
      "name": "ntpd",
      "pid": "16097",
      "exe": "/usr/sbin/ntpd",
      "ppid": "1"
    },
    "@timestamp": "2018-02-28T14:08:20.804Z",
    "beat": {
      "name": "ptc-desk",
      "hostname": "soptct61-01.hursley.ibm.com",
      "version": "6.2.2"
    },
    "@version": "1",
    "host": "soptct61-01.hursley.ibm.com",
    "source": {
      "port": "123",
      "ip": "85.199.214.101"
    },
    "auditd": {
      "result": "success",
      "summary": {
        "actor": {
          "secondary": "ntp",
          "primary": "nhopper"
        },
        "how": "/usr/sbin/ntpd",
        "object": {
          "secondary": "123",
          "type": "socket",
          "primary": "85.199.214.101"
        }
      },
      "sequence": 38316125,
      "data": {
        "a1": "7fff4468f480",
        "syscall": "recvmsg",
        "a2": "0",
        "exit": "48",
        "a3": "0",
        "tty": "(none)",
        "socket": {
          "family": "ipv4",
          "addr": "85.199.214.101",
          "port": "123"
        },
        "arch": "x86_64",
        "a0": "13"
      },
      "session": "8675"
    },
    "event": {
      "action": "received-from",
      "category": "audit-rule",
      "type": "syscall",
      "module": "auditd"
    },
    "user": {
      "fsuid": "38",
      "uid": "38",
      "egid": "38",
      "name_map": {
        "fsuid": "ntp",
        "uid": "ntp",
        "egid": "ntp",
        "auid": "nhopper",
        "gid": "ntp",
        "euid": "ntp",
        "fsgid": "ntp",
        "sgid": "ntp",
        "suid": "ntp"
      },
      "auid": "507",
      "gid": "38",
      "euid": "38",
      "fsgid": "38",
      "sgid": "38",
      "suid": "38"
    },
    "network": {
      "direction": "incoming"
    },
    "tags": [
      "b64_call",
      "beats_input_raw_event",
      "_grokparsefailure"
    ]
  },
  "fields": {
    "@timestamp": [
      1519826900804
    ]
  },
  "sort": [
    1519826900804
  ]
}
4

It appears to be working as designed. Those are three different syscall events for select, recvmsg, and recvmsg again (each with a unique sequence number). If you temporarily set include_raw_message: true you will see all of the related audit messages that were coalesced into the single event.

It also looks like you have a misconfigured Logstash pipeline that's adding _grokparsefailure to the events.

Based on the config you posted previously, I would also recommend being more selective in what you audit. The config was setup to audit every syscall and this will likely slow down your kernel. If you want some inspiration take a look at the examples in https://github.com/linux-audit/audit-userspace/tree/master/rules.

5

Hi @andrewkroh. Excellent, thank you for reviewing it. I will get there eventually.

Bit of background, we have a system that we need to monitor for certain activities, but some are very 'grey'. We also have to monitor what the system administrators are doing. So someone signing in as root is a red flag, but someone signing in as themselves and su'ing to root is OK. Reading stuff in certain directories is OK, but copying it is not. Changing any of the auditing configuration is a red flag and so on.

No-one in the department has done something like this so I volunteered and it has been an interesting learning curve. New to Docker, Linux Audit, ELK, etc, etc. Reasonably OK on Linux, but certainly not an expert! So often a case of 2 steps forward one step back. To cap it all we need notifications and so not using X-Pack (commercial) and will try Sentinl.

So at the moment I have set up a test server that is roughly like the production one just to see what is being sent across. I can see that Nagios is sending 88% of the data that I am seeing so that will be one I will look to exclude one way or another.

But unless there is a significant performance impact, I've been asked to let it all across and deal with it on the ELK server. So as to Nagios, I will first try to exclude it in Logstash. It is a case of balancing performance verses missing information.

It appears easier to look at these things in Kibana than it is from the server sending it all.

I know about the grok parsing failure, that is the other thing I am working on. (The next learning curve!)

I need to be able to interrogate specific fields in the messages in Logstash, but cannot believe I need to work out a grok 'pattern' for these messages. I would have thought one already existed. (Asked under the Logstash Thread).

I can see in Auditbeat, there is an auditbeat.json file that looks to be the template that is deployed to Elasticsearch that allows Kibana and ES to work with the messages.

What I'd like to know is how to get Logstash to use it and for me to exclude nagios messages (user.name_map.auid(?) = nagios).

So it appears that (with your help!) I have auditbeat sending everything across, but just need to get Logstash to filter things out. Once the basics are in place, I can build on that to get a system that will, hopefully, do what we are looking for.

As always, any help gratefully received! N

6

OK I won't repeat the dumb comment, just "still learning".

The message being output from Auditbeat defaults to json? as in if I send it to the screen with pretty:true I get.

{
  "@timestamp": "2018-03-02T11:21:54.695Z",
  "@metadata": {
    "beat": "auditbeat",
    "type": "doc",
    "version": "6.2.2"
  },
  "auditd": {
    "session": "unset",
    "data": {
      "tty": "(none)",
      "a0": "5",
      "items": "0",
      "syscall": "write",
      "a3": "8030",
      "exit": "12",
      "a2": "c",
      "arch": "x86_64",
      "a1": "7ffe90cd71c0"
    },
    "summary": {
      "how": "/usr/libexec/postfix/pickup",
      "actor": {
        "primary": "unset",
        "secondary": "postfix"
      }
    },
    "sequence": 49842409,
    "result": "success"
  },
  "event": {
    "module": "auditd",
    "category": "audit-rule",
    "type": "syscall",
    "action": ""
  },
  "user": {
    "gid": "89",
    "egid": "89",
    "fsuid": "89",
    "name_map": {
      "uid": "postfix",
      "egid": "postfix",
      "euid": "postfix",
      "fsgid": "postfix",
      "fsuid": "postfix",
      "gid": "postfix",
      "sgid": "postfix",
      "suid": "postfix"
    },
    "uid": "89",
    "auid": "unset",
    "euid": "89",
    "fsgid": "89",
    "sgid": "89",
    "suid": "89"
  },
  "process": {
    "ppid": "2186",
    "name": "pickup",
    "exe": "/usr/libexec/postfix/pickup",
    "pid": "30606"
  },
  "beat": {
    "name": "ptc-desk",
    "hostname": "soptct61-01.hursley.ibm.com",
    "version": "6.2.2"
  },
  "tags": [
    "b64_call"
  ]
}

So will try logstash with the json filter and user.name_map.auid != nagios

and try and drop it. Sigh.... :wink:

7

Guessing something like:

filter {
  json {
    source => "message"
    if [user.name_map.auid] == "nagios" {
      drop { }
    }
  }
}
8

Turned out to be so so easy once you know

filter {
  if [user][name_map][auid] == "nagios" {
    drop { }
  }
}

Disappointed this is not documented better.