How to extract the time stamp from

Elastic Stack Logstash
1

Hi All,

Please I need help on how to extract the timestamp from "type=SYSCALL msg=audit(1701877882.123:5786893): " in the below code using grok filter

{
  "_index": "auditbeat-2023.12.06",
  "_type": "_doc",
  "_id": "0lnTP4wBwVgAs3eEtizy",
  "_version": 1,
  "_score": null,
  "_source": {
    "source.port": "0000",
    "event": {
      "action": "accepted-connection-from",
      "original": [
        "type=SYSCALL msg=audit(1701877882.123:5786893): arch=c000003e syscall=43 success=yes exit=3 a0=c a1=0 a2=0 a3=0 items=0 ppid=1 pid=2836 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 ses=4294967295 tty=(none) comm=\"wazuh-modulesd\" exe=\"/var/ossec/bin/wazuh-modulesd\" key=\"external-access\""
      ],
      "outcome": "success",
      "category": "audit-rule"
    },
    "service": {
      "type": "auditd"
    },
    "destination.ip": "127.0.0.4",
    "@timestamp": "2023-12-06T15:51:22.123Z",
    "tag_3": "beats_input_raw_event",
    "auditd": {
      "message_type": "syscall",
      "sequence": 5786893,
      "summary": {
        "actor": {
          "secondary": "root",
          "primary": "unset"
        },
        "object": {
          "type": "socket"
        },
        "how": "/var/ossec/bin/wazuh-modulesd"
      },
      "data": {
        "tty": "(none)",
        "arch": "x86_64",
        "a3": "0",
        "a0": "c",
        "a1": "0",
        "items": "0",
        "exit": "3",
        "a2": "0",
        "syscall": "accept"
      },
      "result": "success"
    },
    "tags": [
      "_grokparsefailure"
    ],
    "host": {
      "name": "userB",
      "architecture": "x86_64",
      "containerized": false,
      "os": {
        "version": "14.04.6 LTS, Trusty Tahr",
        "codename": "trusty",
        "platform": "ubuntu",
        "name": "Ubuntu",
        "family": "debian",
        "kernel": "3.13.0-24-generic"
      },
      "id": "26b1ceca54c197067320f0bf634024c6",
      "hostname": "userB"
    },
    "user": {
      "name": "root",
      "group": {
        "id": "0",
        "name": "root"
      },
      "effective": {
        "name": "root",
        "group": {
          "id": "0",
          "name": "root"
        },
        "id": "0"
      },
      "filesystem": {
        "name": "root",
        "group": {
          "id": "0",
          "name": "root"
        },
        "id": "0"
      },
      "saved": {
        "name": "root",
        "group": {
          "id": "0",
          "name": "root"
        },
        "id": "0"
      },
      "id": "0"
    },
    "signature_id": "777777",
    "@version": "1",
    "signature": "auditbeat",
    "destination.port": "0000",
    "agent": {
      "version": "7.4.1",
      "ephemeral_id": "e1e84631-bf14-4834-8205-30f01c94246c",
      "id": "1b2ffabb-287b-4a21-8e9e-7886e49673b0",
      "type": "auditbeat",
      "hostname": "userB"
    },
    "tag_1": "external-access",
    "peter.field": "auditd",
    "application": "auditbeat",
    "process": {
      "ppid": 1,
      "name": "wazuh-modulesd",
      "pid": 2836,
      "executable": "/var/ossec/bin/wazuh-modulesd"
    },
    "ecs": {
      "version": "1.1.0"
    },
    "source.ip": "127.0.0.2",
    "tag_2": "beats123",
    "network.transport": "udp"
  },
  "fields": {
    "@timestamp": [
      "2023-12-06T15:51:22.123Z"
    ]
  },
  "sort": [
    1701877882123
  ]
}
2

Try

    grok { match => { "message" => "\(%{NUMBER:[@metadata][timestamp]}" } }
    date { match => [ "[@metadata][timestamp]", "UNIX" ] }
3

Hi Badger,
Thanks for responding so quick. Below is what I did, but it did not work


filter {
  if [application] == "auditbeat" { 
    if [event][module] == "auditd" {
       grok {
        match => { "message" => "\(%{NUMBER:[@metadata][timestamp]}" }
       }
      date {
        match => [ "[@metadata][timestamp]", "UNIX" ]
        target => "newtime"
      }
    }
  }
}
4

Your sample event does not have that field, so neither filter will execute.

5

Hi Badger,
I have another filter after this that takes away the [event][module] field and replaces it with "peter.field". Hence you can't see it in the final document

6

Add add_tag => [ "randomText" ] to the grok filter and check whether the document ends up having that tag. That will tell you whether the grok is executing.

7

Hi Badger,
The document has the tag as

"event": "%{{ event.original }}",

see below the json document after adding the tag.

filter {
  if [application] == "auditbeat" { 
    if [event][module] == "auditd" {
       grok {
        match => { "message" => "\(%{NUMBER:[@metadata][timestamp]}" }
       }
      date {
        match => [ "[@metadata][timestamp]", "UNIX" ]
        target => "newtime"
        add_tag => [ "randonText" ]
      }
    }
  }
}

Json Document

{
  "_index": "auditbeat-2023.12.07",
  "_type": "_doc",
  "_id": "e19pQ4wBwVgAs3eEtAUD",
  "_version": 1,
  "_score": null,
  "_source": {
    "user": {
      "name": "dirmngr",
      "filesystem": {
        "name": "dirmngr",
        "group": {
          "id": "111",
          "name": "dirmngr"
        },
        "id": "105"
      },
      "saved": {
        "name": "dirmngr",
        "group": {
          "id": "111",
          "name": "dirmngr"
        },
        "id": "105"
      },
      "effective": {
        "name": "dirmngr",
        "group": {
          "id": "111",
          "name": "dirmngr"
        },
        "id": "105"
      },
      "group": {
        "id": "111",
        "name": "dirmngr"
      },
      "id": "105"
    },
    "auditd": {
      "sequence": 3506072,
      "summary": {
        "object": {
          "type": "socket"
        },
        "how": "/usr/bin/dirmngr",
        "actor": {
          "primary": "unset",
          "secondary": "dirmngr"
        }
      },
      "result": "fail",
      "message_type": "syscall",
      "data": {
        "tty": "(none)",
        "a3": "7fffc24315c0",
        "syscall": "accept",
        "exit": "EAGAIN",
        "arch": "x86_64",
        "a1": "7fffc2431820",
        "a0": "6",
        "items": "0",
        "a2": "7fffc243163c"
      }
    },
    "@version": "1",
    "tag_2": "beats123",
    "agent": {
      "version": "7.4.1",
      "ephemeral_id": "7bdc663e-b192-469b-83ed-990ab4be761b",
      "hostname": "userC",
      "id": "e1485ca7-0a14-4e80-8aa9-176f4fc3c73d",
      "type": "auditbeat"
    },
    "signature": "auditbeat",
    "ecs": {
      "version": "1.1.0"
    },
    "application": "auditbeat",
    "signature_id": "343434",
    "service": {
      "type": "auditd"
    },
    "tag_1": "external-access",
    "host": {
      "name": "userC",
      "containerized": false,
      "architecture": "x86_64",
      "hostname": "userC",
      "id": "ba8745c1e300bd09c4a045ac6340252c",
      "os": {
        "version": "14.04.6 LTS, Trusty Tahr",
        "family": "debian",
        "name": "Ubuntu",
        "platform": "ubuntu",
        "kernel": "3.13.0-24-generic",
        "codename": "trusty"
      }
    },
    "event": "%{{ event.original }}",
    "@timestamp": "2023-12-07T08:34:05.843Z",
    "process": {
      "ppid": 1,
      "name": "dirmngr",
      "executable": "/usr/bin/dirmngr",
      "pid": 30082
    },
    "tag_3": "beats_input_raw_event"
  },
  "fields": {
    "@timestamp": [
      "2023-12-07T08:34:05.843Z"
    ]
  },
  "sort": [
    1701938045843
  ]
}
8

The event does not have the tag you are adding in the date filter, so that suggests the date filter is not being executed, which means either the grok is not matching (which should result in a _grokparsefailure tag) or else the conditionals evaluate to false.

9

Hi Badger,
I can see in the first post I did, there was "tags": [ "_grokparsefailure". Any suggestion on how i can extract the timestamp then, since the grok is not matching?
I will really appreciate any help to extract the timestamp

10

You could try what I suggested before.

11

Hi Badger,

Since the grok filter is not working can I use dissect and if yes how do I do it please
Thanks

12

You could try

dissect { mapping => { "message" => "%{}(%{[@metadata][timestamp]}:%{}" } }
13

Hi Badger,
This would not work as well, so my conclusion is that there is a limit to which logstash filters can actually handle Auditd logs and this may be due to their security and compliance features. is there a way to escalate/flag this, because I consider it as a downside of logstash application, especially when you cannot extract certain information form the logs.

Regards

Peter

14

Have you tested it, or are you just asserting that it would not work? If you have tested it then please provide a reproducible example with a complete logstash configuration (I suggest using a generator input) and rubydebug output.

15

Hi Badger,
I meant to say the dissect did not work in my earlier correspondence. I have tested it.

16

Hi Badger,

I have created the filter below

filter {
   if [application] == "auditbeat" {
      if [event][module] == "auditd" and [auditd][result] == "success" {
        mutate {
          add_field => {
               "[service][source]"=> '{"bytes": "36547"}, {"ip": "10.0.2.21"}, {"port": "0"}, {"packets": "201"}' 
               "[service][destination]" => '{"bytes": "35619"}, {"ip": "10.0.2.25}, {port: "0"}, {"packet": "192"}'
               "[service][network]" => '{"bytes": "72166"}, {"transport": "tcp"}, {"community_id": "1:oSVCaMUYTHBGHtfBHGFBHnkioKYTRV="}, { "packets": "393"}, {"direction": "outbound"}, {"type": "ipv4"}'
               "signature_id" => "777777"

          }
      }
    }
  }
}

The result is as follows

"service":{"source":"{\"bytes\": \"36547\"}, {\"ip\": \"10.0.2.21\"}, {\"port\": \"0\"}, {\"packets\": \"201\"}","destination":"{\"bytes\": \"35619\"}, {\"ip\": \"10.0.2.25}, {port: \"0\"}, {\"packet\": \"192\"}","network":"{\"bytes\": \"72166\"}, {\"transport\": \"tcp\"}, {\"community_id\": \"1:oSVCaMUYTHBGHtfBHGFBHnkioKYTRV=\"}, { \"packets\": \"393\"}, {\"direction\": \"outbound\"}, {\"type\": \"ipv4\"}"},

but I want the result to be structured as below

"source": {"bytes": "36547", "ip": "10.0.2.21", "port": "0,  "packets": "201"}","destination":{"bytes": "35619", "ip": "10.0.2.25, port: "0", "packet": "192", } "network":{"bytes": "72166", "transport": "tcp", "community_id": "1:oSVCaMUYTHBGHtfBHGFBHnkioKYTRV=", "packets": "393", "direction": "outbound", "type": "ipv4"}"}

How do I do this please. I will be very grateful if you can help

Thanks for your help

17

If you want [source] and [destination] to be at the top-level and not nested inside [service] then just update your add_field to be

add_field => {
    "[source]"=> '{"bytes": "36547"}, {"ip": "10.0.2.21"}, {"port": "0"}, {"packets": "201"}' 
    "[destination]" => '{"bytes": "35619"}, {"ip": "10.0.2.25}, {port: "0"}, {"packet": "192"}'
    ....
18

Hi Badger,
The result I want to get is

"source": {
              "bytes": "36547" 
               "ip": "10.0.2.21"
                "port": "0
               "packets": "201" 
          }
     "destination": {
             "bytes": "35619"
             "ip": "10.0.2.25" 
               "port": "0"
              "packet": "192"
             } 
       "network": {
          "bytes": "72166"
          "transport": "tcp"
          "community_id": "1:oSVCaMUYTHBGHtfBHGFBHnkioKYTRV="
           "packets": "393"
           "direction": "outbound"
           "type": "ipv4" 
      }
 }

As against a single string that encompasses the fields in source, destination and the network
like this

"[source]"=> '{"bytes": "36547"}, {"ip": "10.0.2.21"}, {"port": "0"}, {"packets": "201"}'
 "[destination]" => '{"bytes": "35619"}, {"ip": "10.0.2.25}, {port: "0"}, {"packet": "192"}'
19

If you use

    mutate { add_field => { "source" => '{"bytes": "36547"}, {"ip": "10.0.2.21"}, {"port": "0"}, {"packets": "201"}' } }
    mutate {
        gsub => [
            "source", "[{}]", "",
            "source", "^", "{",
            "source", "$", "}"
        ]
    }
    json { source => "source" target => "source" }

then that will produce

    "source" => {
    "packets" => "201",
         "ip" => "10.0.2.21",
       "port" => "0",
      "bytes" => "36547"
},
21

Hi Badger,

I have managed to extract the timestamp with the result as follows

 "my_time":"2024-01-10T15:44:38.795Z"

I need to take out the character 'Z'.
I tried this

filter {
  date {
    match => ["my_time", "ISO8601"]
    target => "my_time"
  }
}

but unsuccessful
Please any other way to do this.

Regards

Pero