とあるdate型のフィールドについて、date型のフィールドは保持しつつ、同じ値を文字列型のフィールドにも保持したいです。
■logstashのconfファイル
input {
～省略～
}
filter {
json {
source => "message"
}
date {
match => ["timestamp", "UNIX_MS"]
target => "timestamp_cloudwatch"
remove_field => ["timestamp"]
}
json {
source => "message"
}
date {
match => ["timestamp", "UNIX_MS"]
target => "timestamp_log"
remove_field => ["timestamp"]
}
ruby {
code => "
event.set('timestamp_cloudwatch_s',event.get('timestamp_cloudwatch').time.localtime.strftime('%Y-%m-%dT%T.%L'));
event.set('timestamp_log_s',event.get('timestamp_log').time.localtime.strftime('%Y-%m-%dT%T.%L'));
"
}
mutate {
convert => {
"timestamp_cloudwatch_s" => "string"
"timestamp_log_s" => "string"
}
}
}
output {
～省略～
}
■inputデータ
{"timestamp":1623986333029,"message":"{\"timestamp\": 1623986333029,\"formatVersion\": 1,\"webaclId\": \"arn:aws:wafv2:ap-southeast-2:EXAMPLE12345:regional/webacl/STMTest/1EXAMPLE-2ARN-3ARN-4ARN-123456EXAMPLE\",\"terminatingRuleId\": \"STMTest_SQLi_XSS\",\"terminatingRuleType\": \"REGULAR\",\"action\": \"BLOCK\",\"terminatingRuleMatchDetails\": [{\"conditionType\": \"SQL_INJECTION\",\"location\": \"HEADER\",\"matchedData\": [\"10\",\"AND\",\"1\"]}],\"httpSourceName\": \"-\",\"httpSourceId\": \"-\",\"ruleGroupList\": [],\"rateBasedRuleList\": [],\"nonTerminatingMatchingRules\": [],\"httpRequest\": {\"clientIp\": \"1.1.1.1\",\"country\": \"AU\",\"headers\": [{\"name\": \"Host\",\"value\": \"localhost:1989\"},{\"name\": \"User-Agent\",\"value\": \"curl/7.61.1\"},{\"name\": \"Accept\",\"value\": \"*/*\"},{\"name\": \"x-stm-test\",\"value\": \"10 AND 1=1\"}],\"uri\": \"/foo\",\"args\": \"\",\"httpVersion\": \"HTTP/1.1\",\"httpMethod\": \"GET\",\"requestId\": \"rid\"},\"labels\": [{\"name\": \"value\"}]}"}
■インデックステンプレート
PUT _index_template/index-s3-aws-waf-template
{
"index_patterns" : [
"index-s3-aws-waf*"
],
"template" : {
"settings" : {
"number_of_shards" : 1,
"number_of_replicas" : 1
},
"mappings" : {
"properties" : {
"timestamp_cloudwatch" : {
"type" : "date"
},
"timestamp_log" : {
"type" : "date"
},
"action" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword"
}
}
},
"sourceip" : {
"type" : "ip"
},
"statusnonTerminatingMatchingRules.ruleId" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword"
}
}
},
"uri" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword"
}
}
},
"terminatingRuleId" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword"
}
}
}
}
}
}
}
■登録結果
{
"_index": "index-s3-aws-waf-2021.10.21",
"_type": "_doc",
"_id": "OuR5oXwBrAiMnUbAmCP_",
"_version": 1,
"_score": null,
"_source": {
"terminatingRuleId": "STMTest_SQLi_XSS",
"httpSourceId": "-",
"@version": "1",
"timestamp_cloudwatch_s": "2021-06-18T12:18:53.029",
"action": "BLOCK",
"httpRequest": {
"args": "",
"country": "AU",
"headers": [
{
"name": "Host",
"value": "localhost:1989"
},
{
"name": "User-Agent",
"value": "curl/7.61.1"
},
{
"name": "Accept",
"value": "*/*"
},
{
"name": "x-stm-test",
"value": "10 AND 1=1"
}
],
"httpVersion": "HTTP/1.1",
"requestId": "rid",
"clientIp": "1.1.1.1",
"httpMethod": "GET",
"uri": "/foo"
},
"terminatingRuleMatchDetails": [
{
"matchedData": [
"10",
"AND",
"1"
],
"conditionType": "SQL_INJECTION",
"location": "HEADER"
}
],
"formatVersion": 1,
"timestamp_log_s": "2021-06-18T12:18:53.029",
"terminatingRuleType": "REGULAR",
"ruleGroupList": [],
"message": "{\"timestamp\": 1623986333029,\"formatVersion\": 1,\"webaclId\": \"arn:aws:wafv2:ap-southeast-2:EXAMPLE12345:regional/webacl/STMTest/1EXAMPLE-2ARN-3ARN-4ARN-123456EXAMPLE\",\"terminatingRuleId\": \"STMTest_SQLi_XSS\",\"terminatingRuleType\": \"REGULAR\",\"action\": \"BLOCK\",\"terminatingRuleMatchDetails\": [{\"conditionType\": \"SQL_INJECTION\",\"location\": \"HEADER\",\"matchedData\": [\"10\",\"AND\",\"1\"]}],\"httpSourceName\": \"-\",\"httpSourceId\": \"-\",\"ruleGroupList\": [],\"rateBasedRuleList\": [],\"nonTerminatingMatchingRules\": [],\"httpRequest\": {\"clientIp\": \"1.1.1.1\",\"country\": \"AU\",\"headers\": [{\"name\": \"Host\",\"value\": \"localhost:1989\"},{\"name\": \"User-Agent\",\"value\": \"curl/7.61.1\"},{\"name\": \"Accept\",\"value\": \"*/*\"},{\"name\": \"x-stm-test\",\"value\": \"10 AND 1=1\"}],\"uri\": \"/foo\",\"args\": \"\",\"httpVersion\": \"HTTP/1.1\",\"httpMethod\": \"GET\",\"requestId\": \"rid\"},\"labels\": [{\"name\": \"value\"}]}",
"labels": [
{
"name": "value"
}
],
"webaclId": "arn:aws:wafv2:ap-southeast-2:EXAMPLE12345:regional/webacl/STMTest/1EXAMPLE-2ARN-3ARN-4ARN-123456EXAMPLE",
"@timestamp": "2021-10-21T06:11:13.186198Z",
"timestamp_log": "2021-06-18T03:18:53.029Z",
"timestamp_cloudwatch": "2021-06-18T03:18:53.029Z",
"rateBasedRuleList": [],
"httpSourceName": "-",
"nonTerminatingMatchingRules": []
},
"fields": {
"@timestamp": [
"2021-10-21T06:11:13.186Z"
],
"timestamp_log": [
"2021-06-18T03:18:53.029Z"
],
"timestamp_cloudwatch": [
"2021-06-18T03:18:53.029Z"
],
"timestamp_cloudwatch_s": [
"2021-06-18T12:18:53.029Z"
],
"timestamp_log_s": [
"2021-06-18T12:18:53.029Z"
]
},
"sort": [
1623986333029
]
}
期待値としては、timestamp_log、およびtimestamp_cloudwatchはdate型、それぞれをJSTに変換した値を文字列型でtimestamp_cloudwatch_s、timestamp_log_sに登録したいです。
しかし、結果としてはJSTに変換されるのは期待通りなのですが、_sのフィールドにもdate型で格納されてしまいます。
実現方法についてご教示頂けないでしょうか。
以下のURLを参考にしています。mutateのconvertを使用すれば出実現できると思ったのですが、期待通りになりませんでした。
Mutate filter plugin | Logstash Reference [7.15] | ElasticProcessing: index_template.txt...