Hmmm first pass working for me...
Below will show you how to test....
BTW interesting you are generating the _id from a fingerprint of a timestamp...... lots of potential pitfalls there ... not sure why you are doing that... it is a lot of overhead not sure for what... this looks like timeseries data... generally no need to do that... in short if you are hoping to update... probably not going to work with data streams... perhaps another thread on that.
Pipeline with target as @timestamp
PUT _ingest/pipeline/discuss-db2-logs
{
"processors": [
{
"grok": {
"field": "message",
"patterns": [
"""(?<dblogs.date>(\d\d\d\d-\d\d-\d\d))-%{DATA:dblogs.time}.(?<dblogs.timemsec>(\d)*)\+%{GREEDYDATA:temp.dblogs1}LEVEL:%{SPACE}%{DATA:dblogs.log.level}\nPID%{SPACE}:%{SPACE}%{DATA:dblogs.pid}%{SPACE}TID%{GREEDYDATA:temp.dblogs2}\nINSTANCE:%{SPACE}%{DATA:dblogs.instance}%{SPACE}NODE(?<temp.dblogs3>(.|\r|\n)*)HOSTNAME:%{SPACE}(?<dblogs.hostname>(\S)*)(?<temp.dblogs4>(.|\r|\n)*)FUNCTION:%{SPACE}%{GREEDYDATA:dblogs.function}\n(?<dblogs.log.message>(.|\r|\n)*)"""
]
}
},
{
"gsub": {
"field": "dblogs.time",
"pattern": """\.""",
"replacement": ":"
}
},
{
"set": {
"field": "dblogs.time",
"value": "{{dblogs.time}}.{{dblogs.timemsec}}"
}
},
{
"set": {
"field": "dblogs.datetime",
"value": "{{dblogs.date}}T{{dblogs.time}}"
}
},
{
"fingerprint": {
"fields": [
"dblogs.datetime"
],
"target_field": "fingerprint",
"method": "SHA-512",
"ignore_missing": true
}
},
{
"set": {
"field": "_id",
"copy_from": "fingerprint"
}
},
{
"date": {
"field": "dblogs.datetime",
"formats": [
"ISO8601"
],
"timezone": "GMT+8"
}
},
{
"set": {
"field": "test",
"value": "success"
}
}
]
}
Posting a document
POST filebeat-8.6.2/_doc?pipeline=discuss-db2-logs
{
"@timestamp": "2023-03-10T05:36:37.841Z",
"message": "2022-10-29-22.52.30.073009+480 I8038702E535 LEVEL: Error\nPID : 8173 TID : 46999353157376 PROC : db2sysc 0\nINSTANCE: db2inst1 NODE : 000 DB : ASSIST \nHOSTNAME: pksdc2pdb01\nEDUID : 60381 EDUNAME: db2hadrp.0.1 (ASSIST) 0\nFUNCTION: DB2 UDB, High Availability Disaster Recovery, hdrSendRedirectMsgToOneAddress, probe:31070\nMESSAGE : ZRC=0xFFFFFFFF=-1\nDATA #1 : <preformatted>\nThe HADR primary was not able to form a TCP connection with the standby: 10.20.7.11:51002.\n",
"tags": [
"db2logs"
],
"input": {
"type": "log"
}
}
# Results
{
"_index": ".ds-filebeat-8.6.2-2023.03.10-000002",
"_id": "PBTBOzjBaGXNsEMZbIxJknMjG5J47c0Btr30J0GpYfJQRhZLVcQOyyBkSz2Jp6e4LKQRFDHqcgpsOAoRUiNCWQ==",
"_version": 4,
"result": "created",
"_shards": {
"total": 2,
"successful": 1,
"failed": 0
},
"_seq_no": 3,
"_primary_term": 1
}
Search show correct timestamp in _source and fields
GET .ds-filebeat-8.6.2-2023.03.10-000002/_search
{
"fields": [
"*"
]
}
# Results note the _source @timestamp and the fields @timestamp dates are always stored in UTC
{
"took": 36,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 1,
"hits": [
{
"_index": ".ds-filebeat-8.6.2-2023.03.10-000002",
"_id": "PBTBOzjBaGXNsEMZbIxJknMjG5J47c0Btr30J0GpYfJQRhZLVcQOyyBkSz2Jp6e4LKQRFDHqcgpsOAoRUiNCWQ==",
"_score": 1,
"_source": {
"input": {
"type": "log"
},
"dblogs": {
"date": "2022-10-29",
"hostname": "pksdc2pdb01",
"datetime": "2022-10-29T22:52:30.073009",
"instance": "db2inst1",
"log": {
"level": "Error",
"message": """MESSAGE : ZRC=0xFFFFFFFF=-1
DATA #1 : <preformatted>
The HADR primary was not able to form a TCP connection with the standby: 10.20.7.11:51002.
"""
},
"function": "DB2 UDB, High Availability Disaster Recovery, hdrSendRedirectMsgToOneAddress, probe:31070",
"timemsec": "073009",
"pid": "8173",
"time": "22:52:30.073009"
},
"temp": {
"dblogs1": "480 I8038702E535 ",
"dblogs2": " : 46999353157376 PROC : db2sysc 0",
"dblogs3": """ : 000 DB : ASSIST
""",
"dblogs4": """
EDUID : 60381 EDUNAME: db2hadrp.0.1 (ASSIST) 0
"""
},
"@timestamp": "2022-10-29T22:52:30.073+08:00",
"test": "success",
"fingerprint": "PBTBOzjBaGXNsEMZbIxJknMjG5J47c0Btr30J0GpYfJQRhZLVcQOyyBkSz2Jp6e4LKQRFDHqcgpsOAoRUiNCWQ==",
"message": """2022-10-29-22.52.30.073009+480 I8038702E535 LEVEL: Error
PID : 8173 TID : 46999353157376 PROC : db2sysc 0
INSTANCE: db2inst1 NODE : 000 DB : ASSIST
HOSTNAME: pksdc2pdb01
EDUID : 60381 EDUNAME: db2hadrp.0.1 (ASSIST) 0
FUNCTION: DB2 UDB, High Availability Disaster Recovery, hdrSendRedirectMsgToOneAddress, probe:31070
MESSAGE : ZRC=0xFFFFFFFF=-1
DATA #1 : <preformatted>
The HADR primary was not able to form a TCP connection with the standby: 10.20.7.11:51002.
""",
"tags": [
"db2logs"
]
},
"fields": {
"dblogs.timemsec": [
"073009"
],
"test": [
"success"
],
"dblogs.date": [
"2022-10-29"
],
"dblogs.pid": [
"8173"
],
"input.type": [
"log"
],
"message": [
"""2022-10-29-22.52.30.073009+480 I8038702E535 LEVEL: Error
PID : 8173 TID : 46999353157376 PROC : db2sysc 0
INSTANCE: db2inst1 NODE : 000 DB : ASSIST
HOSTNAME: pksdc2pdb01
EDUID : 60381 EDUNAME: db2hadrp.0.1 (ASSIST) 0
FUNCTION: DB2 UDB, High Availability Disaster Recovery, hdrSendRedirectMsgToOneAddress, probe:31070
MESSAGE : ZRC=0xFFFFFFFF=-1
DATA #1 : <preformatted>
The HADR primary was not able to form a TCP connection with the standby: 10.20.7.11:51002.
"""
],
"dblogs.log.message": [
"""MESSAGE : ZRC=0xFFFFFFFF=-1
DATA #1 : <preformatted>
The HADR primary was not able to form a TCP connection with the standby: 10.20.7.11:51002.
"""
],
"dblogs.hostname": [
"pksdc2pdb01"
],
"tags": [
"db2logs"
],
"@timestamp": [
"2022-10-29T14:52:30.073Z"
],
"temp.dblogs4": [
"""
EDUID : 60381 EDUNAME: db2hadrp.0.1 (ASSIST) 0
"""
],
"fingerprint": [
"PBTBOzjBaGXNsEMZbIxJknMjG5J47c0Btr30J0GpYfJQRhZLVcQOyyBkSz2Jp6e4LKQRFDHqcgpsOAoRUiNCWQ=="
],
"dblogs.instance": [
"db2inst1"
],
"temp.dblogs3": [
""" : 000 DB : ASSIST
"""
],
"dblogs.time": [
"22:52:30.073009"
],
"dblogs.datetime": [
"2022-10-29T22:52:30.073009"
],
"temp.dblogs2": [
" : 46999353157376 PROC : db2sysc 0"
],
"dblogs.function": [
"DB2 UDB, High Availability Disaster Recovery, hdrSendRedirectMsgToOneAddress, probe:31070"
],
"temp.dblogs1": [
"480 I8038702E535 "
],
"dblogs.log.level": [
"Error"
]
}
}
]
}
}
