Hi, I'm using Elasticsearch 6.3.2.
I set the Lambda function in AWS and put Elastic Load Balancer logs from s3 to Elasticsearch on EC2.
I'm using ingest node with this templating.
{
"elblog": {
"processors": [
{
"grok": {
"field": "message",
"patterns": [
"""%{NOTSPACE:type} %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:int} (?:(%{IP:targetip}:?:%{INT:targetport:int})|-) %{NUMBER:request_processing_time:float} %{NUMBER:target_processing_time:float} %{NUMBER:response_processing_time:float} %{INT:elb_status_code:int} (?:(%{INT:target_status_code:int})|-) %{INT:received_bytes:int} %{INT:sent_bytes:int} "(?:%{WORD:verb} %{URIPROTO:proto}://?(?:%{URIHOST:urihost})?(?:%{URIPATH:path}(?:%{URIPARAM:params})?)?(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" "%{DATA:agent}""""
],
"ignore_missing": true
}
},
{
"remove": {
"field": "message"
}
},
{
"user_agent": {
"field": "agent",
"target_field": "user_agent",
"ignore_failure": true
}
},
{
"remove": {
"field": "agent",
"ignore_failure": true
}
}
]
}
}
This is working well, but very few times in a day I found this error log at elasticsearch.log
My Lambda function set index name by "elb-log-name-yyyy.mm.dd".
I debugged and surely client put document with this index name above.
Also, the logs that the Lambda function is emitting did not generate any error.
I have no idea why error logs says "elb-log-name-2018.08.24/log/null" because I put document with "elb-log-name-yyyy.mm.dd".
It seems to me that client succeeded to put, but fails at only elasticsearch side.
Anyone assume why this is happening?
Thanks in advance.
[2018-08-24T06:30:33,881][DEBUG][o.e.a.b.TransportBulkAction] [xZIZVXR] failed to execute pipeline [elblog] for document [elb-log-name-2018.08.24/log/null]
org.elasticsearch.ElasticsearchParseException: Failed to parse content to map
at org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:144) ~[elasticsearch-6.3.2.jar:6.3.2]
at org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:112) ~[elasticsearch-6.3.2.jar:6.3.2]
at org.elasticsearch.action.index.IndexRequest.sourceAsMap(IndexRequest.java:293) ~[elasticsearch-6.3.2.jar:6.3.2]
at org.elasticsearch.ingest.PipelineExecutionService.innerExecute(PipelineExecutionService.java:153) ~[elasticsearch-6.3.2.jar:6.3.2]
at org.elasticsearch.ingest.PipelineExecutionService.access$100(PipelineExecutionService.java:43) ~[elasticsearch-6.3.2.jar:6.3.2]
at org.elasticsearch.ingest.PipelineExecutionService$1.doRun(PipelineExecutionService.java:78) [elasticsearch-6.3.2.jar:6.3.2]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:725) [elasticsearch-6.3.2.jar:6.3.2]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.3.2.jar:6.3.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_181]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_181]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181]
Caused by: com.fasterxml.jackson.core.JsonParseException: Unrecognized character escape 'x' (code 120)
at [Source: org.elasticsearch.transport.netty4.ByteBufStreamInput@6c095fc; line: 1, column: 252]
at com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1702) ~[jackson-core-2.8.10.jar:2.8.10]
at com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:558) ~[jackson-core-2.8.10.jar:2.8.10]
at com.fasterxml.jackson.core.base.ParserMinimalBase._handleUnrecognizedCharacterEscape(ParserMinimalBase.java:535) ~[jackson-core-2.8.10.jar:2.8.10]
at com.fasterxml.jackson.core.json.UTF8StreamJsonParser._decodeEscaped(UTF8StreamJsonParser.java:3247) ~[jackson-core-2.8.10.jar:2.8.10]
at com.fasterxml.jackson.core.json.UTF8StreamJsonParser._finishString2(UTF8StreamJsonParser.java:2514) ~[jackson-core-2.8.10.jar:2.8.10]
at com.fasterxml.jackson.core.json.UTF8StreamJsonParser._finishAndReturnString(UTF8StreamJsonParser.java:2469) ~[jackson-core-2.8.10.jar:2.8.10]
at com.fasterxml.jackson.core.json.UTF8StreamJsonParser.getText(UTF8StreamJsonParser.java:315) ~[jackson-core-2.8.10.jar:2.8.10]
at org.elasticsearch.common.xcontent.json.JsonXContentParser.text(JsonXContentParser.java:84) ~[elasticsearch-x-content-6.3.2.jar:6.3.2]
at org.elasticsearch.common.xcontent.support.AbstractXContentParser.readValue(AbstractXContentParser.java:416) ~[elasticsearch-x-content-6.3.2.jar:6.3.2]
at org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:364) ~[elasticsearch-x-content-6.3.2.jar:6.3.2]
at org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:327) ~[elasticsearch-x-content-6.3.2.jar:6.3.2]
at org.elasticsearch.common.xcontent.support.AbstractXContentParser.map(AbstractXContentParser.java:282) ~[elasticsearch-x-content-6.3.2.jar:6.3.2]
at org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:142) ~[elasticsearch-6.3.2.jar:6.3.2]
... 10 more