Hi there,
I have an index that looks like so:
{
"_index": "ec2_logs-index-2020.05.11",
"_type": "_doc",
"_id": "V3LAAnIBk2krYRHTuxCC",
"_version": 1,
"_score": null,
"_source": {
"owner": "104115521938",
"@version": "1",
"logStream": "mystream",
"subscriptionFilters": [
"To-Firehose"
],
"messageType": "DATA_MESSAGE",
"logGroup": "ec2-fluentd",
"@timestamp": "2020-05-11T08:01:24.400Z",
"logEvents": [
{
"timestamp": 1589167148000,
"id": "35439611645423604170418303021688828486859711444132757504",
"message": "{\"host\":\"ip-22-22-2-83\",\"ident\":\"dhclient\",\"pid\":\"2987\",\"message\":\"XMT: Solicit on eth0, interval 109630ms.\"}"
},
{
"timestamp": 1589167161000,
"id": "35439611935333291751316403861653166031288411021878165505",
"message": "{\"host\":\"ip-22-22-2-83\",\"ident\":\"sshd\",\"pid\":\"3864\",\"message\":\"Accepted publickey for ec2-user from 138.75.33.59 port 51868 ssh2: RSA SHA256:QY/2uJJiV5cYpErAb/KLg/505B6WQ4ZlvcBazh1Qfyo\"}"
},
{
"timestamp": 1589167161000,
"id": "35439611935333291751316403861653166031288411021878165506",
"message": "{\"host\":\"ip-22-22-2-83\",\"ident\":\"systemd\",\"message\":\"Created slice User Slice of ec2-user.\"}"
},
{
"timestamp": 1589167161000,
"id": "35439611935333291751316403861653166031288411021878165507",
"message": "{\"host\":\"ip-22-22-2-83\",\"ident\":\"systemd\",\"message\":\"Starting User Slice of ec2-user.\"}"
},
{
"timestamp": 1589167161000,
"id": "35439611935333291751316403861653166031288411021878165508",
"message": "{\"host\":\"ip-22-22-2-83\",\"ident\":\"systemd\",\"message\":\"Started Session 2 of user ec2-user.\"}"
},
{
"timestamp": 1589167161000,
"id": "35439611935333291751316403861653166031288411021878165509",
"message": "{\"host\":\"ip-22-22-2-83\",\"ident\":\"systemd-logind\",\"message\":\"New session 2 of user ec2-user.\"}"
}
],
"type": "ec2_logs"
},
"fields": {
"@timestamp": [
"2020-05-11T08:01:24.400Z"
]
},
"sort": [
1589184084400
]
}
The index mapping is:
{
"ec2_logs-index-2020.05.11" : {
"mappings" : {
"dynamic" : "true",
"_meta" : { },
"_source" : {
"includes" : [ ],
"excludes" : [ ]
},
"dynamic_date_formats" : [
"strict_date_optional_time",
"yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
],
"dynamic_templates" : [ ],
"date_detection" : true,
"numeric_detection" : false,
"properties" : {
"@timestamp" : {
"type" : "date",
"format" : "strict_date_optional_time"
},
"@version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"logEvents" : {
"type" : "nested",
"properties" : {
"id" : {
"type" : "text"
},
"message" : {
"type" : "text"
},
"timestamp" : {
"type" : "date_nanos"
}
}
},
"logGroup" : {
"type" : "text"
},
"logStream" : {
"type" : "text"
},
"messageType" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"owner" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"subscriptionFilters" : {
"type" : "text"
},
"type" : {
"type" : "text"
}
}
}
}
}
What I'm trying to do here is removing all forward slash and square brackets from logEvents.messages field and change logEvents.id to just a -
To that end, I have the following filter in logstash:
filter
{
mutate{
gsub => [
"[logEvents][message]", "[\[\]\\]", "",
"[logEvents][id]", ".*", "-"
]
}
}
But the resulting output is exactly the same. What did i do wrong?
Thanks in advance...
ck