Extract timestamp from multiple records

Elastic Stack Logstash
1

Hi,

Below is my config file,

input {
file{
	path => "/Users/.../Work/Projects/ELK/Logstash/Input/*.txt"
    start_position => beginning
    codec => "json"
    type => "data"
    sincedb_path => "NUL"
   }
}

filter { 
	grok { match => { "message" => "<%{NUMBER}>%{SYSLOGTIMESTAMP} %{IPV4} %{HOSTNAME:nodeName}: %{NUMBER} %{WORD} \[%{DATA}\]: %{GREEDYDATA:[anotherField]} \n" } }
	    # Create array of strings
	    mutate { split => { "[anotherField]" => "| " } }
	    # Create a separate event for each array entry
	    split { field => "[anotherField]" }

	mutate {
			remove_field => ["@version"]
			remove_field => [ "message" ]
			remove_field => ["[event][original]"]
	
			add_field => { 
	            "record" => "%{nodeName} %{anotherField}"
	        } 
	        remove_field => ["anotherField"]

		}

	}

output {
 stdout{}	
 file {
   path => "/Users/shanthkumar/Work/Projects/ELK/Logstash/test.csv"
   codec => line { format => "%{record}"}
 	}
 elasticsearch { hosts => ["localhost:9200"] 
					index => "natlog" }
	
}

My input data is below,

{"@timestamp":"2022-12-01T13:30:00.004Z","message":"<190>Dec  1 14:29:59 10.62.161.199 AA-AMG3U: 0950198238 NN [MDA 8/4]: LN44 SA 2022 Dec  1 14:29:59:87 CET 17 4001 10.XX.133.XX 56560 401 91.235.10.25 15179 2400160261XXXXX_467000XXXX1_35292011220XXXXX_string1 | LN44 SD 2022 Dec  1 14:29:59:89 CET 17 4001 10.XX.133.XX 56560 401 91.235.10.25 15179 2400160261XXXXX_467000XXXX1_35292011220XXXXX_string2 | LN44 SA 2022 Dec  1 14:29:59:87 CET 17 4001 10.XX.133.XX 56560 401 91.235.10.25 15179 2400160261XXXXX_467679XXXX2_35292011220XXXXX_string1 \n","@version":"1","host":"100.62.161.11"}

Current output is as below

{
          "host" => "100.62.161.11",
      "nodeName" => "AA-AMG3U",
           "log" => {
        "file" => {
            "path" => "/Users/shanthkumar/Work/Projects/ELK/Logstash/Input/sample.txt"
        }
    },
         "event" => {},
    "@timestamp" => 2022-12-01T13:30:00.004Z,
          "type" => "data",
        "record" => "AA-AMG3U LN44 SA 2022 Dec  1 14:29:59:87 CET 17 4001 10.XX.133.XX 56560 401 91.235.10.25 15179 2400160261XXXXX_467000XXXX1_35292011220XXXXX_string1 "
}
{
          "host" => "100.62.161.11",
      "nodeName" => "AA-AMG3U",
           "log" => {
        "file" => {
            "path" => "/Users/shanthkumar/Work/Projects/ELK/Logstash/Input/sample.txt"
        }
    },
         "event" => {},
    "@timestamp" => 2022-12-01T13:30:00.004Z,
          "type" => "data",
        "record" => "AA-AMG3U LN44 SD 2022 Dec  1 14:29:59:89 CET 17 4001 10.XX.133.XX 56560 401 91.235.10.25 15179 2400160261XXXXX_467000XXXX1_35292011220XXXXX_string2 "
}
{
          "host" => "100.62.161.11",
      "nodeName" => "AA-AMG3U",
           "log" => {
        "file" => {
            "path" => "/Users/shanthkumar/Work/Projects/ELK/Logstash/Input/sample.txt"
        }
    },
         "event" => {},
    "@timestamp" => 2022-12-01T13:30:00.004Z,
          "type" => "data",
        "record" => "AA-AMG3U LN44 SA 2022 Dec  1 14:29:59:87 CET 17 4001 10.XX.133.XX 56560 401 91.235.10.25 15179 2400160261XXXXX_467679XXXX2_35292011220XXXXX_string1"
}

I need the timestamp to be extracted from the records, that's the field after SA/SD in the "record".
Also, should the "event" field be empty? Timestamp examples are "2022 Dec 1 14:29:59:87 CET" and "2022 Dec 11 14:29:59:87 CET" - it's 2 blank spaces when single digit date and 1 blank space when double digit date.
I need to search for specific period and specific IP address (that's in the record field) in Elasticsearch.

Please advice!

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.