Send syslogs in batches/an array of events records?

Is there a way to group linux syslogs events together in to an array with a logstash filter or output plugin?

Here is the example schema I need to get the logs into for sending to ADX:

[
  {
    "records": [
		{
			r1
		},
		{
			r2
		},		
		...
		{
			rn
		}
	]
  }
]

Here's my current Grok filter that parses the logs, but doesn't put them into an array and send several at once, just one at a time:

filter {
    # Grok Pattern to parse Ubuntu syslog schema into Log Analytics Syslog Table
       if [type] == "syslog" {
    grok {
  match => { "message" => "%{SYSLOGTIMESTAMP:EventTime} %{HOSTNAME:HostName} %{WORD:ProcessName}\[%{POSINT:ProcessID}\]: %{GREEDYDATA:SyslogMessage}"}
  overwrite => [ "message" ]
    }
  }
}

My input plugin:

input {
    # This input reads syslog as a file
  file {
    path => ["/var/log/syslog"] # Syslog file path
    type => "syslog"    # Define type as "syslog" 
    tags => "_config3"  # Shows which config generated results in Event Hub for testing
    start_position => "beginning"   # Start at beginning of syslog file
  }
}

My output plugin:

output { 
    # Azure Event Hub Plugin (GitHub: https://github.com/bryanklewis/logstash-output-azure_event_hubs)
    azure_event_hubs {
        service_namespace => "########" 
        event_hub => "############"
        sas_key_name => "#################"
        sas_key => '######################"
       }
}

Here is the current output, 2 events sampled:

[
  {
    "event": {
      "original": "Nov  8 20:01:01 ubuntu-ls-test CRON[3958]: (sshadmin) CMD (/home/sshadmin/test.sh)"
    },
    "SyslogMessage": "(sshadmin) CMD (/home/sshadmin/test.sh)",
    "message": "Nov  8 20:01:01 ubuntu-ls-test CRON[3958]: (sshadmin) CMD (/home/sshadmin/test.sh)",
    "host": {
      "name": "ubuntu-ls-test"
    },
    "EventTime": "Nov  8 20:01:01",
    "log": {
      "file": {
        "path": "/var/log/syslog"
      }
    },
    "@version": "1",
    "ProcessID": "3958",
    "tags": [
      "_config3"
    ],
    "ProcessName": "CRON",
    "HostName": "ubuntu-ls-test",
    "@timestamp": "2022-11-08T20:01:01.736899697Z",
    "type": "syslog",
    "EventProcessedUtcTime": "2022-11-08T20:01:17.6750269Z",
    "PartitionId": 0,
    "EventEnqueuedUtcTime": "2022-11-08T20:01:01.8750000Z"
  },

  {
    "event": {
      "original": "Nov  8 20:01:01 ubuntu-ls-test CRON[3957]: (CRON) info (No MTA installed, discarding output)"
    },
    "SyslogMessage": "(CRON) info (No MTA installed, discarding output)",
    "message": "Nov  8 20:01:01 ubuntu-ls-test CRON[3957]: (CRON) info (No MTA installed, discarding output)",
    "host": {
      "name": "ubuntu-ls-test"
    },
    "EventTime": "Nov  8 20:01:01",
    "log": {
      "file": {
        "path": "/var/log/syslog"
      }
    },
    "@version": "1",
    "ProcessID": "3957",
    "tags": [
      "_config3"
    ],
    "ProcessName": "CRON",
    "HostName": "ubuntu-ls-test",
    "@timestamp": "2022-11-08T20:01:01.737340599Z",
    "type": "syslog",
    "EventProcessedUtcTime": "2022-11-08T20:01:16.5968985Z",
    "PartitionId": 0,
    "EventEnqueuedUtcTime": "2022-11-08T20:01:01.8440000Z"
  },
]

Should I be using a different input, filter, or output plugin type to put these together into an array before sending to Azure Event Hub? Thanks in advance for any support.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.