Improve filtering with control fields

Hello Community,

I'm working of a kind of project to visulaze the mail flow. (see also here: Issue in Controls - #19 by moep ). The main problem is, that my content is not in the same line for example a mail flow looks like that:

2023-01-10 13:04:42 24HKTd-0007I3-QZ <= jira@evil.corp H=server1.evil.corp [1.1.1.1] P=esmtps X=TLS1.3:ECDHE_RSA_AES_256_GCM_SHA384:256 CV=no S=10000 id=CONFLUENCE 
2023-01-10 13:04:42 24HkTd-0007I3-QZ => /var/spool/vmail/evil.corp/max.headroom/Maildir (max.headroom@evil.corp) <max.headroom@evil.corp> R=virtual_domains T=dovecot_virtual_delivery

What I want to do:

Search in the control fields for sender and recipient and get one or more MSGID in my chart below. Basicly it works,when I search for one of both. Very import to keep in mind is the case, that when you write an email to an email list with e.g. 5 entrys, each email have the same MSGID, so I can't search just for the last entry.
I added into my sample a field action_id and added values, like 1 and 2 for successful delivered and recivied.

Do you have any suggestions to do some in contrls?

here is my current doc:

# field example, where exim_flags: **
{
  "_index": "logstash-exim-001234",
  "_id": "MmWk2oUB1Y2dhY6YBV85",
  "_version": 1,
  "_score": 0,
  "_source": {
    "exim_msg_state": "routing defer",
    "tags": [
      "beats__codec_plain_applied"
    ],
    "next_grok": " input@mail.net routing defer (-51): retry time not reached",
    "exim_year": "2023",
    "event": {
      "timezone": "+01:00"
    },
    "@version": "1",
    "exim_time": "19:00:52",
    "state": "mainlog",
    "@timestamp": "2023-01-10T18:00:59.389Z",
    "real_exim_date": "2023-01-10 19:00:52",
    "host": {
      "mac": [
        "XXX"
      ],
      "os": {
        "version": "X",
        "codename": "X",
        "family": "X",
        "kernel": "X",
        "type": "linux",
        "name": "X",
        "platform": "X"
      },
      "containerized": false,
      "architecture": "x86_64",
      "id": "528e142b0f17415ab7f1efe333385a85",
      "ip": [
        "1.2.3.4"
      ],
      "name": "SERVER1",
      "hostname": "SERVER1"
    },
    "exim_day": "22",
    "exim_flags": "==",
    "agent": {
      "version": "7.17.6",
      "ephemeral_id": "92978d0c-30c8-4aac-9bbe-86fc2c7ccea2",
      "type": "filebeat",
      "id": "866641e8-1fa9-4a7e-a33f-b2ee251bc205",
      "name": "SERVER1",
      "hostname": "SERVER1"
    },
    "exim_month": "01",
    "exim_recipient": "input@mail.net",
    "exim_msg_id": "1pIlON-0006A7-iA",
    "ecs": {
      "version": "1.12.0"
    },
    "": {
      "type": "filestream"
    },
    "log": {
      "offset": 10561162,
      "file": {
        "path": "/var/log/exim4/mainlog"
      }
    },
    "message": "1pIlON-0006A7-iA == input@mail.net routing defer (-51): retry time not reached"
  },
  "fields": {
    "agent.version.keyword": [
      "7.17.6"
    ],
    "exim_month.keyword": [
      "01"
    ],
    "exim_recipient.keyword": [
      "input@mail.net"
    ],
    "exim_flags": [
      "=="
    ],
    "host.architecture.keyword": [
      "x86_64"
    ],
    "host.name.keyword": [
      "SERVER1"
    ],
    "exim_msg_id.keyword": [
      "1pIlON-0006a6-iA"
    ],
    "exim_year": [
      "2023"
    ],
    "host.hostname": [
      "SERVER1"
    ],
    "host.mac": [
      "X"
    ],
    "agent.hostname.keyword": [
      "SERVER1"
    ],
    "exim_msg_state.keyword": [
      "routing defer"
    ],
    "real_exim_date": [
      "2023-01-10 19:00:52"
    ],
    "ecs.version.keyword": [
      "1.12.0"
    ],
    "host.ip.keyword": [
      "1.2.3.4"
    ],
    "host.os.version": [
      "X"
    ],
    "host.os.name": [
      "X"
    ],
    "exim_month": [
      "01"
    ],
    "exim_time": [
      "19:00:52"
    ],
    "host.id.keyword": [
      "X"
    ],
    "agent.name": [
      "SERVER1"
    ],
    "state": [
      "mainlog"
    ],
    "host.name": [
      "SERVER1"
    ],
    "real_exim_date.keyword": [
      "2023-01-10 19:00:52"
    ],
    "host.os.version.keyword": [
      "X"
    ],
    "exim_time.keyword": [
      "19:00:52"
    ],
    "host.os.type": [
      "linux"
    ],
    "agent.id.keyword": [
      "X"
    ],
    "@version.keyword": [
      "1"
    ],
    ".type": [
      "filestream"
    ],
    "log.offset": [
      X
    ],
    "agent.hostname": [
      "SERVER1"
    ],
    "tags": [
      "beats__codec_plain_applied"
    ],
    "host.architecture": [
      "x86_64"
    ],
    "agent.id": [
      "X"
    ],
    "ecs.version": [
      "1.12.0"
    ],
    "host.containerized": [
      false
    ],
    "message.keyword": [
      "1pIlON-0006a6-iA == input@mail.net routing defer (-51): retry time not reached"
    ],
    "state.keyword": [
      "mainlog"
    ],
    "host.hostname.keyword": [
      "SERVER1"
    ],
    "agent.version": [
      "7.17.6"
    ],
    "exim_recipient": [
      "input@mail.net"
    ],
    "exim_year.keyword": [
      "2023"
    ],
    "host.os.family": [
      "X"
    ],
    "event.timezone.keyword": [
      "+01:00"
    ],
    ".type.keyword": [
      "filestream"
    ],
    "tags.keyword": [
      "beats__codec_plain_applied"
    ],
    "exim_msg_id": [
      "1pIlON-0006a6-iA"
    ],
    "host.ip": [
      "1.2.3.4"
    ],
    "agent.type": [
      "filebeat"
    ],
    "next_grok": [
      " @ routing defer (-51): retry time not reached"
    ],
    "host.os.kernel.keyword": [
      "X"
    ],
    "host.os.kernel": [
      "X"
    ],
    "exim_msg_state": [
      "routing defer"
    ],
    "@version": [
      "1"
    ],
    "host.os.name.keyword": [
      "X"
    ],
    "host.id": [
      "528e142b0f17415affe5d262385a85"
    ],
    "log.file.path.keyword": [
      "/var/log/exim4/mainlog"
    ],
    "event.timezone": [
      "+01:00"
    ],
    "agent.type.keyword": [
      "filebeat"
    ],
    "agent.ephemeral_id.keyword": [
      "X"
    ],
    "host.os.codename.keyword": [
      "X"
    ],
    "host.mac.keyword": [
      "X"
    ],
    "agent.name.keyword": [
      "SERVER1"
    ],
    "exim_flags.keyword": [
      "=="
    ],
    "host.os.codename": [
      "X"
    ],
    "exim_day": [
      "10"
    ],
    "message": [
      "1pIlON-0006a6-iA == input@mail.net routing defer (-51): retry time not reached"
    ],
    "exim_day.keyword": [
      "10"
    ],
    "host.os.family.keyword": [
      "x"
    ],
    "host.os.type.keyword": [
      "linux"
    ],
    "@timestamp": [
      "2023-01-10T18:00:59.389Z"
    ],
    "host.os.platform.keyword": [
      "X"
    ],
    "host.os.platform": [
      "X"
    ],
    "log.file.path": [
      "/var/log/exim4/mainlog"
    ],
    "agent.ephemeral_id": [
      "9sdrrtc-3rr8-4aac-9bbe-98fc2c7ccea2"
    ],
    "next_grok.keyword": [
      " input@mail.net routing defer (-51): retry time not reached"
    ]
  }
}

# field example, where exim_flags:  =>
{
  "_index": "logstash-exim-000017",
  "_id": "lV5f2oUBUskX8assdfU2qk",
  "_version": 1,
  "_score": 0,
  "_source": {
    "state": "mainlog",
    "real_exim_date": "2023-01-10 17:45:53",
    "exim_day": "22",
    "exim_flags": "=>",
    "agent": {
      "hostname": "SERVER1",
      "ephemeral_id": "92978d0c-30c8-4aac-9bbe-96fc2c7ccea2",
      "type": "filebeat",
      "id": "866641e8-1fa9-4a7e-a33f-a3ee251bc205",
      "name": "SERVER1",
      "version": "7.17.6"
    },
    "exim_month": "01",
    "exim_recipient": "foo@bar.net",
    "exim_msg_id": "1pJdTc-0002AR-Ma",
    "exim_ciphers": "ECDHE_RSA_AES_256_GCM_SHA384:256",
    "": {
      "type": "filestream"
    },
    "log": {
      "offset": 10459499,
      "file": {
        "path": "/var/log/exim4/mainlog"
      }
    },
    "exim_remote_smtp": "remote_smtp",
    "exim_msg_state": "delivered",
    "tags": [
      "beats__codec_plain_applied"
    ],
    "next_grok": " foo@bar.net R=dnslookup T=remote_smtp H=mail.bar.net [198.27.92.8] X=TLS1.0:ECDHE_RSA_AES_256_GCM_SHA384:256 CV=no C=\"250 OK id=1pJdTd-0007FL-1Z\"",
    "exim_year": "2023",
    "remote_host": "198.27.92.8",
    "event": {
      "timezone": "+01:00"
    },
    "@version": "1",
    "exim_time": "17:45:53",
    "@timestamp": "2023-01-10T16:45:55.104Z",
    "host": {
      "mac": [
        "X"
      ],
      "os": {
        "version": "X",
        "codename": "X",
        "family": "X",
        "kernel": "X",
        "type": "linux",
        "name": "X",
        "platform": "X"
      },
      "containerized": false,
      "architecture": "x86_64",
      "id": "528e142b0rtrassddt8899efd262385a85",
      "ip": [
        "1.10.1.2"
      ],
      "name": "SERVER1",
      "hostname": "SERVER1"
    },
    "exim_router": "dnslookup",
    "exim_cv_value": "no",
    "ecs": {
      "version": "1.12.0"
    },
    "remote_hostname": "mail.bar.net",
    "exim_tls": "TLS1.0",
    "message": " => foo@bar.net R=dnslookup T=remote_smtp H=mail.bar.net [198.27.92.8] X=TLS1.0:ECDHE_RSA_AES_256_GCM_SHA384:256 CV=no C=\"250 OK id=1pJdTd-0007FL-1Z\""p
  },
  "fields": {
    "agent.version.keyword": [
      "7.17.6"
    ],
    "exim_month.keyword": [
      "01"
    ],
    "exim_recipient.keyword": [
      "foo@bar.net"
    ],
    "exim_flags": [
      "=>"
    ],
    "host.architecture.keyword": [
      "x86_64"
    ],
    "remote_hostname.keyword": [
      "mail.bar.net"
    ],
    "host.name.keyword": [
      "SERVER1"
    ],
    "exim_msg_id.keyword": [
      "1pJdTd-0007FL-1Z"
    ],
    "exim_year": [
      "2023"
    ],
    "host.hostname": [
      "SERVER1"
    ],
    "exim_cv_value.keyword": [
      "no"
    ],
    "host.mac": [
      "X"
    ],
    "exim_ciphers": [
      "ECDHE_RSA_AES_256_GCM_SHA384:256"
    ],
    "agent.hostname.keyword": [
      "SERVER1"
    ],
    "exim_msg_state.keyword": [
      "delivered"
    ],
    "real_exim_date": [
      "2023-01-10 17:45:53"
    ],
    "ecs.version.keyword": [
      "1.12.0"
    ],
    "host.ip.keyword": [
      "10.23.24.25"
    ],
    "host.os.version": [
      "X"
    ],
    "exim_router": [
      "dnslookup"
    ],
    "exim_tls.keyword": [
      "TLS1.0"
    ],
    "host.os.name": [
      "X"
    ],
    "exim_month": [
      "01"
    ],
    "exim_time": [
      "17:45:53"
    ],
    "host.id.keyword": [
      "528e142b046ajgj7f1efd262385a85"
    ],
    "agent.name": [
      "SERVER1"
    ],
    "state": [
      "mainlog"
    ],
    "host.name": [
      "SERVER1"
    ],
    "real_exim_date.keyword": [
      "2023-01-10 17:45:53"
    ],
    "host.os.version.keyword": [
      "x"
    ],
    "exim_time.keyword": [
      "17:45:53"
    ],
    "host.os.type": [
      "linux"
    ],
    "exim_cv_value": [
      "no"
    ],
    "agent.id.keyword": [
      "861234-1wa9-4a7e-a33f-b2ee251bc205"
    ],
    "@version.keyword": [
      "1"
    ],
    ".type": [
      "filestream"
    ],
    "remote_hostname": [
      "mail.bar.net"
    ],
    "log.offset": [
      10459499
    ],
    "agent.hostname": [
      "SERVER1"
    ],
    "remote_host": [
      "198.27.92.8"
    ],
    "tags": [
      "beats__codec_plain_applied"
    ],
    "remote_host.keyword": [
      "198.27.92.8"
    ],
    "host.architecture": [
      "x86_64"
    ],
    "agent.id": [
      "866641erafffed-4a7e-a33f-b2ee251bc205"
    ],
    "ecs.version": [
      "1.12.0"
    ],
    "host.containerized": [
      false
    ],
    "message.keyword": [
      "1pJdTc-0001AR-Ma => foo@bar.net R=dnslookup T=remote_smtp H=mail.bar.net [198.27.92.8] X=TLS1.0:ECDHE_RSA_AES_256_GCM_SHA384:256 CV=no C=\"250 OK id=1pJdTd-0007FL-1Z\""
    ],
    "state.keyword": [
      "mainlog"
    ],
    "host.hostname.keyword": [
      "SERVER1"
    ],
    "agent.version": [
      "7.17.6"
    ],
    "exim_recipient": [
      "foo@bar.net"
    ],
    "exim_year.keyword": [
      "2023"
    ],
    "host.os.family": [
      "x"
    ],
    "event.timezone.keyword": [
      "+01:00"
    ],
    "exim_remote_smtp.keyword": [
      "remote_smtp"
    ],
    ".type.keyword": [
      "filestream"
    ],
    "tags.keyword": [
      "beats__codec_plain_applied"
    ],
    "exim_msg_id": [
      "1pJdTd-0007FL-1Z"
    ],
    "exim_tls": [
      "TLS1.0"
    ],
    "exim_remote_smtp": [
      "remote_smtp"
    ],
    "host.ip": [
      "10.21.22.24"
    ],
    "agent.type": [
      "filebeat"
    ],
    "next_grok": [
      " foo@bar.net R=dnslookup T=remote_smtp H=mail.bar.net [198.27.92.8] X=TLS1.0:ECDHE_RSA_AES_256_GCM_SHA384:256 CV=no C=\"250 OK id=1pJdTd-0007FL-1Z\""
    ],
    "host.os.kernel.keyword": [
      "x"
    ],
    "host.os.kernel": [
      "x"
    ],
    "exim_msg_state": [
      "delivered"
    ],
    "@version": [
      "1"
    ],
    "host.os.name.keyword": [
      "x"
    ],
    "host.id": [
      "52344b0f17415cdfetefd262385a85"
    ],
    "log.file.path.keyword": [
      "/var/log/exim4/mainlog"
    ],
    "event.timezone": [
      "+01:00"
    ],
    "agent.type.keyword": [
      "filebeat"
    ],
    "agent.ephemeral_id.keyword": [
      "98878d0c-30c8-4aac-affe-86fc2c7ccea2"
    ],
    "host.os.codename.keyword": [
      "xr"
    ],
    "host.mac.keyword": [
      "x"
    ],
    "agent.name.keyword": [
      "SERVER1"
    ],
    "exim_flags.keyword": [
      "=>"
    ],
    "host.os.codename": [
      "x"
    ],
    "exim_day": [
      "22"
    ],
    "message": [
      "1pJdTd-0007FL-1Z => foo@bar.net R=dnslookup T=remote_smtp H=mail.bar.net [198.27.92.8] X=TLS1.0:ECDHE_RSA_AES_256_GCM_SHA384:256 CV=no C=\"250 OK id=1pJdTd-0007FL-1Z\""
    ],
    "exim_day.keyword": [
      "22"
    ],
    "host.os.family.keyword": [
      "x"
    ],
    "exim_ciphers.keyword": [
      "ECDHE_RSA_AES_256_GCM_SHA384:256"
    ],
    "host.os.type.keyword": [
      "linux"
    ],
    "@timestamp": [
      "2023-01-10T16:45:55.104Z"
    ],
    "host.os.platform.keyword": [
      "x"
    ],
    "host.os.platform": [
      "x"
    ],
    "log.file.path": [
      "/var/log/exim4/mainlog"
    ],
    "agent.ephemeral_id": [
      "97890c-30c8-4aac-9bbe-86fc25askjdea2"
    ],
    "exim_router.keyword": [
      "dnslookup"
    ],
    "next_grok.keyword": [
      " foo@bar.net R=dnslookup T=remote_smtp H=mail.bar.net [198.27.92.8] X=TLS1.0:ECDHE_RSA_AES_256_GCM_SHA384:256 CV=no C=\"250 OK id=1pJdTd-0007FL-1Z\""
    ]
  }
}

# field example, where exim_flags: <=
{
  "_index": "logstash-exim-000135",
  "_id": "8fdf2oUBrnnsddllHTC",
  "_version": 1,
  "_score": 0,
  "_source": {
    "protocol": "esmtpa",
    "state": "mainlog",
    "real_exim_date": "2023-01-10 17:45:52",
    "exim_day": "10",
    "exim_flags": "<=",
    "agent": {
      "hostname": "SERVER1",
      "ephemeral_id": "929789asoid-40c8-4aac-9bbe-86gf3c7ccea2",
      "type": "filebeat",
      "id": "866e41e8-1fa9-445e-a44g-b2ff251bc346",
      "name": "SERVER1",
      "version": "7.17.6"
    },
    "exim_month": "01",
    "exim_msg_id": "1pJdTc-0001AR-Ma",
    "": {
      "type": "filestream"
    },
    "log": {
      "offset": 10459142,
      "file": {
        "path": "/var/log/exim4/mainlog"
      }
    },
    "exim_msg_state": "received",
    "tags": [
      "beats__codec_plain_applied"
    ],
    "next_grok": " noreply@domain.net H=local.domain.net (local.domain.net) [1.20.0.137] P=esmtpa A=login_virtual_exim:spool@domain.net S=15678,
    "exim_year": "2023",
    "remote_host": "10.10.0.137",
    "event": {
      "timezone": "+01:00"
    },
    "@version": "1",
    "exim_time": "17:45:52",
    "@timestamp": "2023-01-10T16:45:53.103Z",
    "remote_heloname": "local.domain.net",
    "host": {
      "mac": [
        "X"
      ],
      "os": {
        "version": "X",
        "codename": "X",
        "family": "X",
        "kernel": "X",
        "type": "linux",
        "name": "X",
        "platform": "X"
      },
      "containerized": false,
      "architecture": "x86_64",
      "id": "528e142b04567afd5ab7f1efd262385a85",
      "ip": [
        "10.10.0.137"
       ],
      "name": "SERVER1",
      "hostname": "SERVER1"
    },
    "exim_mtaspooler": "login_virtual_exim:spool@domain.net",
    "ecs": {
      "version": "1.12.0"
    },
    "exim_msg_size": "1147",
    "exim_sender": "noreply@domain.net",
    "remote_hostname": "local.domain.net",
    "message": "1pJdTc-0001AR-Ma <= noreply@domain.net H=local.domain.net (local.domain.net) [1.20.0.137] P=esmtpa A=login_virtual_exim:spooler@domain.net S=15678"
  },
  "fields": {
    "agent.version.keyword": [
      "7.17.6"
    ],
    "exim_mtaspooler.keyword": [
      "login_virtual_exim:spooler@domain.net"
    ],
    "exim_month.keyword": [
      "01"
    ],
    "exim_flags": [
      "<="
    ],
    "host.architecture.keyword": [
      "x86_64"
    ],
    "remote_hostname.keyword": [
      "local.domain.net""
    ],
    "host.name.keyword": [
      "SERVER1"
    ],
    "exim_msg_id.keyword": [
      "1pJdTc-0001AR-Ma"
    ],
    "exim_year": [
      "2023"
    ],
    "host.hostname": [
      "SERVER1"
    ],
    "host.mac": [
      "x"
    ],
    "agent.hostname.keyword": [
      "SERVER1"
    ],
    "exim_msg_state.keyword": [
      "received"
    ],
    "protocol": [
      "esmtpa"
    ],
    "real_exim_date": [
      "2023-01-10 17:45:52"
    ],
    "ecs.version.keyword": [
      "1.12.0"
    ],
    "host.ip.keyword": [
      "1.2.3.4"
    ],
    "host.os.version": [
      "x"
    ],
    "host.os.name": [
      "x"
    ],
    "exim_month": [
      "01"
    ],
    "exim_time": [
      "17:45:52"
    ],
    "host.id.keyword": [
      "528ashbk5917415ab7f1efd262385a85"
    ],
    "agent.name": [
      "SERVER1"
    ],
    "state": [
      "mainlog"
    ],
    "host.name": [
      "SERVER1"
    ],
    "real_exim_date.keyword": [
      "2023-01-10 17:45:52"
    ],
    "host.os.version.keyword": [
      "x"
    ],
    "exim_time.keyword": [
      "17:45:52"
    ],
    "host.os.type": [
      "linux"
    ],
    "agent.id.keyword": [
      "8684591e8-1fa9-4a7e-a33f-b2ee251bc205"
    ],
    "@version.keyword": [
      "1"
    ],
    ".type": [
      "filestream"
    ],
    "remote_hostname": [
      "local.domain.net"
    ],
    "log.offset": [
      10459142
    ],
    "agent.hostname": [
      "SERVER1"
    ],
    "remote_host": [
      "10.20.0.137"
    ],
    "tags": [
      "beats__codec_plain_applied"
    ],
    "remote_host.keyword": [
      "10.1.0.22"
    ],
    "host.architecture": [
      "x86_64"
    ],
    "agent.id": [
      "866641e9-1fa9-4a7e-a33f-b2ff2dfc205"
    ],
    "ecs.version": [
      "1.12.0"
    ],
    "host.containerized": [
      false
    ],
    "message.keyword": [
      "1pJdTc-0001AR-Ma <= noreply@domain.net H=local.domain.net (local.domain.net) [10.20.0.137] P=esmtpa A=login_virtual_exim:spooler@domain.net S=15678"
    ],
    "state.keyword": [
      "mainlog"
    ],
    "host.hostname.keyword": [
      "SERVER1"
    ],
    "agent.version": [
      "7.17.6"
    ],
    "protocol.keyword": [
      "esmtpa"
    ],
    "exim_year.keyword": [
      "2023"
    ],
    "host.os.family": [
      "x"
    ],
    "event.timezone.keyword": [
      "+01:00"
    ],
    ".type.keyword": [
      "filestream"
    ],
    "exim_msg_size": [
      "15678"
    ],
    "tags.keyword": [
      "beats__codec_plain_applied"
    ],
    "remote_heloname.keyword": [
      "local.domain.net"
    ],
    "exim_msg_id": [
      "1pJdTc-0001AR-Ma"
    ],
    "host.ip": [
      "1.2.1.1",
    ],
    "agent.type": [
      "filebeat"
    ],
    "remote_heloname": [
      "local.domain.net"
    ],
    "next_grok": [
      " noreply@domain.net H=local.domain.net (local.domain.net) [10.20.0.137] P=esmtpa A=login_virtual_exim:spooler@domain.net S=15678"
    ],
    "exim_sender": [
      "noreply@domain.net"
    ],
    "host.os.kernel.keyword": [
      "x"
    ],
    "host.os.kernel": [
      "x"
    ],
    "exim_msg_state": [
      "received"
    ],
    "@version": [
      "1"
    ],
    "host.os.name.keyword": [
      "x"
    ],
    "host.id": [
      "528e112340f13815ab7f1efd262385a85"
    ],
    "log.file.path.keyword": [
      "/var/log/exim4/mainlog"
    ],
    "event.timezone": [
      "+01:00"
    ],
    "agent.type.keyword": [
      "filebeat"
    ],
    "agent.ephemeral_id.keyword": [
      "92978d0c-40c8-4bbc-9bbe-86fc2c7ccea2"
    ],
    "host.os.codename.keyword": [
      "x"
    ],
    "host.mac.keyword": [
      "x"
    ],
    "agent.name.keyword": [
      "SERVER1"
    ],
    "exim_flags.keyword": [
      "<="
    ],
    "host.os.codename": [
      "x"
    ],
    "exim_day": [
      "10"
    ],
    "exim_msg_size.keyword": [
      ""
    ],
    "message": [
      "1pJdTc-0001AR-Ma <= noreply@domain.net H=local.domain.net (local.domain.net) [10.20.0.137] P=esmtpa A=login_virtual_exim:spooler@domain.net S=15678"
    ],
    "exim_day.keyword": [
      "22"
    ],
    "host.os.family.keyword": [
      "x"
    ],
    "exim_mtaspooler": [
      "login_virtual_exim:spooler@domian.net"
    ],
    "exim_sender.keyword": [
      "noreply@domain.net"
    ],
    "host.os.type.keyword": [
      "linux"
    ],
    "@timestamp": [
      "2023-01-10T16:45:53.103Z"
    ],
    "host.os.platform.keyword": [
      "x"
    ],
    "host.os.platform": [
      "x"
    ],
    "log.file.path": [
      "/var/log/exim4/mainlog"
    ],
    "agent.ephemeral_id": [
      "98978d0c-30c8-4aac-9bbe-86fc2c7ccea2"
    ],
    "next_grok.keyword": [
      " noreply@domain.net H=local.domain.net (local.domain.net) [10.20.0.137] P=esmtpa A=login_virtual_exim:spooler@domain.net S=15678"
    ]
  }
}

# field example, where exim_flags: **
{
  "_index": "logstash-exim-000245",
  "_id": "jA9JTYUBUskXasfdf5kzG1",
  "_version": 1,
  "_score": 0,
  "_ignored": [
    "message.keyword",
    "next_grok.keyword"
  ],
  "_source": {
    "event": {
      "timezone": "+01:00"
    },
    "": {
      "type": "filestream"
    },
    "tags": [
      "beats__codec_plain_applied",
      "_grokparsefailure"
    ],
    "exim_msg_state": "rejected",
    "@version": "1",
    "log": {
      "file": {
        "path": "/var/log/exim4/mainlog"
      },
      "offset": 7020686
    },
    "state": "mainlog",
    "exim_year": "2022",
    "host": {
      "mac": [
        "x"
      ],
      "os": {
        "kernel": "x",
        "family": "x",
        "codename": "x",
        "version": "x",
        "type": "linux",
        "name": "x",
        "platform": "x"
      },
      "containerized": false,
      "architecture": "x86_64",
      "ip": [
        "1.2.3.4",
      ],
      "id": "528e14skdflkdfb7f1efd262385a85",
      "name": "SERVER1",
      "hostname": "SERVER1"
    },
    "exim_month": "12",
    "exim_day": "26",
    "exim_time": "08:15:43",
    "@timestamp": "2022-12-10T07:15:44.076Z",
    "exim_flags": "**",
    "exim_sender": [
      "jane.doe@elasticsearch.com",
      "jane.doe@elasticsearch.com"
    ],
    "ecs": {
      "version": "1.12.0"
    },
    "agent": {
      "id": "866641e8-1fa9-4a7e-a33f-b2ee251bc205",
      "ephemeral_id": "92978d0c-30c8-4aac-9bbe-86fc2c7ccea2",
      "type": "filebeat",
      "version": "7.17.6",
      "name": "SERVER1",
      "hostname": "SERVER1"
    },
    "exim_msg_id": "1p9hi2-0005BO-EM",
    "next_grok": " jane.doe@elasticsearch.com <Jane.Doe@elasticsearch.com> R=dnslookup T=remote_smtp H=mail.elasticsearch.com [54.149.131.206] X=TLS1.2:DHE_RSA_AES_256_GCM_SHA384:256 CV=yes: SMTP error from remote mail server after RCPT TO:<Jane.Doe@elasticsearch.com>: 550 #5.1.0 Address rejected.",
    "real_exim_date": "2022-12-10 08:15:43",
    "message": "1p9hi2-0005BO-EM ** jane.doe@elasticsearch.com <Jane.Doe@elasticsearch.com> R=dnslookup T=remote_smtp H=mail.elasticsearch.com [54.149.131.206] X=TLS1.2:DHE_RSA_AES_256_GCM_SHA384:256 CV=yes: SMTP error from remote mail server after RCPT TO:<Jane.Doe@elasticsearch.com>: 550 #5.1.0 Address rejected."
  },
  "fields": {
    "agent.version.keyword": [
      "7.17.6"
    ],
    "exim_month.keyword": [
      "12"
    ],
    "exim_flags": [
      "**"
    ],
    "host.architecture.keyword": [
      "x86_64"
    ],
    "host.name.keyword": [
      "SERVER1"
    ],
    "exim_year": [
      "2022"
    ],
    "host.hostname": [
      "SERVER1"
    ],
    "exim_msg_id.keyword": [
      "1p9hi2-0005BO-EM"
    ],
    "host.mac": [
      "x"
    ],
    "agent.hostname.keyword": [
      "SERVER1"
    ],
    "exim_msg_state.keyword": [
      "rejected"
    ],
    "real_exim_date": [
      "2022-12-10 08:15:43"
    ],
    "host.ip.keyword": [
      "1.2.3.4"
    ],
    "ecs.version.keyword": [
      "1.12.0"
    ],
    "host.os.version": [
      "x"
    ],
    "exim_month": [
      "12"
    ],
    "host.os.name": [
      "x"
    ],
    "exim_time": [
      "08:15:43"
    ],
    "host.id.keyword": [
      "52567sjadfgb0f17415ab7f1efd262385a85"
    ],
    "agent.name": [
      "SERVER1"
    ],
    "host.name": [
      "SERVER1"
    ],
    "state": [
      "mainlog"
    ],
    "real_exim_date.keyword": [
      "2022-12-10 08:15:43"
    ],
    "exim_time.keyword": [
      "08:15:43"
    ],
    "host.os.version.keyword": [
      "x"
    ],
    "host.os.type": [
      "linux"
    ],
    "agent.id.keyword": [
      "869941e8-1fa9-4a7e-a33f-b2ee251bc205"
    ],
    "@version.keyword": [
      "1"
    ],
    ".type": [
      "filestream"
    ],
    "log.offset": [
      7020686
    ],
    "agent.hostname": [
      "SERVER1"
    ],
    "tags": [
      "beats__codec_plain_applied",
      "_grokparsefailure"
    ],
    "host.architecture": [
      "x86_64"
    ],
    "agent.id": [
      "86669238e8-1fa9-4a7e-a33f-b2ee251bc205"
    ],
    "ecs.version": [
      "1.12.0"
    ],
    "host.containerized": [
      false
    ],
    "state.keyword": [
      "mainlog"
    ],
    "host.hostname.keyword": [
      "SERVER1"
    ],
    "agent.version": [
      "7.17.6"
    ],
    "exim_year.keyword": [
      "2022"
    ],
    "host.os.family": [
      "x"
    ],
    "event.timezone.keyword": [
      "+01:00"
    ],
    ".type.keyword": [
      "filestream"
    ],
    "tags.keyword": [
      "beats__codec_plain_applied",
      "_grokparsefailure"
    ],
    "exim_msg_id": [
      "1p9hi2-0005BO-EM"
    ],
    "host.ip": [
      "1.2.3.4"
    ],
    "agent.type": [
      "filebeat"
    ],
    "next_grok": [
      "jane.doe@elasticsearch.com <Jane.Doe@elasticsearch,com> R=dnslookup T=remote_smtp H=mail.elesticsearch.com [54.149.131.206] X=TLS1.2:DHE_RSA_AES_256_GCM_SHA384:256 CV=yes: SMTP error from remote mail server after RCPT TO:<Jane.Doe@eleasticsearch.com>: 550 #5.1.0 Address rejected."
    ],
    "host.os.kernel.keyword": [
      "x"
    ],
    "exim_sender": [
      "jane.doe@elasticsearch.com",
      "jane.doe@elasticsearch.com"
    ],
    "host.os.kernel": [
      "x"
    ],
    "exim_msg_state": [
      "rejected"
    ],
    "@version": [
      "1"
    ],
    "host.os.name.keyword": [
      "x"
    ],
    "host.id": [
      "567hs42b0f17415ab7f1efd262385a85"
    ],
    "log.file.path.keyword": [
      "/var/log/exim4/mainlog"
    ],
    "event.timezone": [
      "+01:00"
    ],
    "agent.type.keyword": [
      "filebeat"
    ],
    "agent.ephemeral_id.keyword": [
      "92988d0c-30c8-4aac-9bbe-86fc2c7ccea2"
    ],
    "host.os.codename.keyword": [
      "x"
    ],
    "host.mac.keyword": [
      "x"
    ],
    "agent.name.keyword": [
      "SERVER1"
    ],
    "exim_flags.keyword": [
      "**"
    ],
    "host.os.codename": [
      "xr"
    ],
    "exim_day": [
      "26"
    ],
    "message": [
      "1p9hi2-0005BO-EM ** jane.doe@elasticsearch.com <Jane.Doe@elasticsearch.com> R=dnslookup T=remote_smtp H=mail.elasticsearch.com [54.149.131.206] X=TLS1.2:DHE_RSA_AES_256_GCM_SHA384:256 CV=yes: SMTP error from remote mail server after RCPT TO:<Jane.Doe@elasticsearch.com>: 550 #5.1.0 Address rejected."
    ],
    "exim_day.keyword": [
      "10"
    ],
    "host.os.family.keyword": [
      "x"
    ],
    "exim_sender.keyword": [
      "jane.doe@elasticsearch.com",
      "jane.doe@elasticsearch.com"
    ],
    "@timestamp": [
      "2022-12-10T07:15:44.076Z"
    ],
    "host.os.type.keyword": [
      "linux"
    ],
    "host.os.platform": [
      "x"
    ],
    "host.os.platform.keyword": [
      "x"
    ],
    "log.file.path": [
      "/var/log/exim4/mainlog"
    ],
    "agent.ephemeral_id": [
      "92978d0c-30c8-4aac-9bbe-86fc2c7ccea2"
    ]
  },
  "ignored_field_values": {
    "message.keyword": [
      "1p9hi2-0005BO-EM ** jane.doe@elasticsearch.com <Jane.Doe@elasticsearch.com> R=dnslookup T=remote_smtp H=mail.elasticsearch.com [54.149.131.206] X=TLS1.2:DHE_RSA_AES_256_GCM_SHA384:256 CV=yes: SMTP error from remote mail server after RCPT TO:<Jane.Doe@elasticsearch.com>: 550 #5.1.0 Address rejected."
    ],
    "next_grok.keyword": [
      " jane.doe@elasticsearch.com <Jane.Doe@elasticsearch.com> R=dnslookup T=remote_smtp H=mail.elasticsearch.com [54.149.131.206] X=TLS1.2:DHE_RSA_AES_256_GCM_SHA384:256 CV=yes: SMTP error from remote mail server after RCPT TO:<Jane.Doe@elasticsearch.com>: 550 #5.1.0 Address rejected."
    ]
  }
}

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.