I tried to reindex today with the following:
POST _reindex
{
"source": {
"index": "var_log_nginx_access-2020.03.16"
},
"dest": {
"index": "var_log_nginx_access-2020.03.16-new"
}
}
After this I get the following message:
{
"index" : "var_log_nginx_access-2020.03.16-new",
"type" : "_doc",
"id" : "lv2x43AB4pUu6mpyqbN0",
"cause" : {
"type" : "mapper_parsing_exception",
"reason" : "failed to parse field [timestamp] of type [date] in document with id 'lv2x43AB4pUu6mpyqbN0'. Preview of field's value: '16/Mar/2020:15:13:45 +0100'",
"caused_by" : {
"type" : "illegal_argument_exception",
"reason" : "failed to parse date field [16/Mar/2020:15:13:45 +0100] with format [strict_date_optional_time||epoch_millis]",
"caused_by" : {
"type" : "date_time_parse_exception",
"reason" : "Failed to parse with all enclosed parsers"
}
}
},
"status" : 400
},
This is what the mapping of the old index looks like:
{
"mapping": {
"_doc": {
"properties": {
"@timestamp": {
"type": "date"
},
"@version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ClientIP": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Error_Message": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"agent": {
"properties": {
"ephemeral_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"hostname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"connection_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ecs": {
"properties": {
"version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"host": {
"properties": {
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"input": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"log": {
"properties": {
"file": {
"properties": {
"path": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"flags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"offset": {
"type": "long"
}
}
},
"loglevel": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"message": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"method": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"process_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"request": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"thread_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"timestamp_nginx": {
"type": "date",
"format": "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis"
}
}
}
}
}
This is the mapping of the newly created index:
{
"_doc": {
"_meta": {},
"_source": {},
"properties": {
"request": {
"type": "text"
},
"ClientIP_access": {
"type": "ip"
},
"http_referer": {
"type": "text"
},
"bytes_sent": {
"coerce": true,
"index": true,
"ignore_malformed": false,
"store": false,
"type": "byte",
"doc_values": true
},
"timestamp": {
"type": "date"
},
"status": {
"coerce": true,
"index": true,
"ignore_malformed": false,
"store": false,
"type": "integer",
"doc_values": true
},
"http_user_agent": {
"type": "text"
}
}
}
}
This is the grok filter pattern:
NGINX_ACCESS_DATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{ISO8601_TIMEZONE}
NGINX_ACCESS %{IP:ClientIP_access} - - \[%{NGINX_ACCESS_DATE:timestamp}\] %{QS:request} %{INT:status} %{INT:bytes_sent} %{QS:http_referer} %{QS:http_user_agent}
What can I do to debug this or is there something I have done wrong.
My motivation for this is because I wanted to create a visualisation using ipv4 range but I noticed that my field "ClientIP_access was not mapped to the "IP" type. With the new index that I checked this morning the mapping shows that the IP type and the field are mapped but actually it still complains there is not a valid field available for this.
As much help would be great, many thanks