Hi everyone,
I created that topic, thinking I had a Logstash issue, but it seems like it's an Elasticsearch indexing issue.
Let me recap : I'm trying to parse and index k8s audit logs, which are valid JSONs, with Logstash.
Here's the config :
input{
http {
port => 8888
codec => "json"
type => "json"
}
}
filter{
json {
source => "message"
}
}
output{
file {
path => "/home/ubuntu/logstash-kubernetes/audit.log"
}
elasticsearch{
hosts => "localhost:9200"
index => "kubernetes"
}
}
Output to file is fine. However, indexing to Elasticsearch most of the time throws this error :
[2020-12-03T12:20:07,084][DEBUG][o.e.a.b.TransportShardBulkAction] [arc-concordia-controller1] [kubernetes][0] failed to execute bulk item (index) index {[kubernetes][_doc][bcuLKHYBxfUMFRPTnd0p], source[n/a, actual length: [2.9kb], max length: 2kb]}
org.elasticsearch.index.mapper.MapperParsingException: failed to parse
at org.elasticsearch.index.mapper.DocumentParser.wrapInMapperParsingException(DocumentParser.java:191) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:74) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:227) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:803) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:780) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:752) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:285) [elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:175) [elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:220) [elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:126) [elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:85) [elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:179) [elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:737) [elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.10.0.jar:7.10.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
at java.lang.Thread.run(Thread.java:832) [?:?]
Caused by: java.lang.ArrayIndexOutOfBoundsException: Index -1 out of bounds for length 0
at org.elasticsearch.index.mapper.DocumentParser.getMapper(DocumentParser.java:919) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObject(DocumentParser.java:499) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:415) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:395) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:482) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObject(DocumentParser.java:520) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:415) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:395) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:482) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObject(DocumentParser.java:520) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:415) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:395) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:482) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObject(DocumentParser.java:520) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:415) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:395) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:482) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObject(DocumentParser.java:520) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseNonDynamicArray(DocumentParser.java:592) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseArray(DocumentParser.java:557) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:417) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:395) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:482) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObject(DocumentParser.java:502) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:415) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:395) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:482) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObject(DocumentParser.java:502) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:415) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:395) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:482) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObject(DocumentParser.java:502) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseNonDynamicArray(DocumentParser.java:592) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseArray(DocumentParser.java:544) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:417) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:395) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:112) ~[elasticsearch-7.10.0.jar:7.10.0]
at org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:71) ~[elasticsearch-7.10.0.jar:7.10.0]
... 16 more
So it seems to fail here.
The bizarre fact is that, some entries will not throw that error, but most of them will.
And here's an example of one that'll throw the error :
{
"headers": {
"content_type": "application/json",
"request_method": "POST",
"accept_encoding": "gzip",
"request_path": "/",
"http_host": "****:8888",
"http_accept": "application/json, */*",
"content_length": "2668",
"http_version": "HTTP/1.1",
"http_user_agent": "Go-http-client/1.1"
},
"@version": "1",
"host": "****",
"apiVersion": "audit.k8s.io/v1",
"items": [
{
"requestObject": {
"metadata": {
"name": "kube-scheduler",
"namespace": "kube-system",
"creationTimestamp": "202...",
"annotations": {
"control-plane.alpha.kubernetes.io/leader": "{\"holderIdentity\":\"****38d6a87-bccb-4ea0-bd6b-31fc27e48b7a\",\"leaseDurationSeconds\":15,\"acquireTime\":\"2020-12-02T15:18:52Z\",\"renewTime\":\"2020-12-02T15:18:52Z\",\"leaderTransitions\":67}"
},
"managedFields": [
{
"operation": "Update",
"fieldsType": "FieldsV1",
"time": "2020-12-02T15:18:23Z",
"manager": "kube-scheduler",
"apiVersion": "v1",
"fieldsV1": {
"f:metadata": {
"f:annotations": {
".": {},
"f:control-plane.alpha.kubernetes.io/leader": {}
}
}
}
}
],
"resourceVersion": "2531",
"selfLink": "/api...",
"uid": "0e..."
},
"apiVersion": "v1",
"kind": "Endpoints"
},
"requestURI": "/api...",
"responseStatus": {
"code": 200,
"metadata": {}
},
"user": {
"groups": [
"system:authenticated"
],
"username": "system:kube-scheduler"
},
"annotations": {
"authorization.k8s.io/reason": "RBAC...",
"authorization.k8s.io/decision": "allow"
},
"stageTimestamp": "2020-...",
"userAgent": "kube-scheduler/v...",
"objectRef": {
"namespace": "kube-system",
"name": "kube-scheduler",
"apiVersion": "v1",
"resourceVersion": "2531",
"uid": "0ebf0...",
"resource": "endpoints"
},
"responseObject": {
"metadata": {
"name": "kube-scheduler",
"namespace": "kube-system",
"creationTimestamp": "2020-12-02T14:36:33Z",
"annotations": {
"control-plane.alpha.kubernetes.io/leader": "{\"holderIdentity\":\"****1_938d6a87-bccb-4ea0-bd6b-31fc27e48b7a\",\"leaseDurationSeconds\":15,\"acquireTime\":\"2020-12-02T15:18:52Z\",\"renewTime\":\"2020-12-02T15:18:52Z\",\"leaderTransitions\":67}"
},
"managedFields": [
{
"operation": "Update",
"fieldsType": "FieldsV1",
"time": "2020-12-02T15:18:52Z",
"manager": "kube-scheduler",
"apiVersion": "v1",
"fieldsV1": {
"f:metadata": {
"f:annotations": {
".": {},
"f:control-plane.alpha.kubernetes.io/leader": {}
}
}
}
}
],
"resourceVersion": "2542",
"selfLink": "/api...",
"uid": "0ebf0..."
},
"apiVersion": "v1",
"kind": "Endpoints"
},
"level": "RequestResponse",
"requestReceivedTimestamp": "2020-...",
"auditID": "a458...",
"sourceIPs": [
"****"
],
"stage": "ResponseComplete",
"verb": "update"
}
],
"@timestamp": "202...",
"type": "json",
"metadata": {},
"kind": "EventList"
}
(I reduced some fields)
This "index -1 out of bounds" terribly annoys me because, my entries are valid JSONs according to some online validator. Note that every correctly processed entries are about "responseStatus.code = 201" and the one that fail are "responseStatus.code = 200", this means they are of different types but I can't see why it throws that error.
Any help would be appreciated !
Best,
Hugo