Error in elasticsearch logs

Kaouther_Mechri · February 2, 2018, 1:29pm

Hello everyone,

I have this Error in my Elasticsearch logs . Do you have any idea please ?
ELK 6.1.1
thanks

[2018-02-02T14:20:03,624][DEBUG][o.e.a.b.TransportShardBulkAction] [logstash-2018.02.02][2] failed to execute bulk item (index) BulkShardRequest [[logstash-2018.02.02][2]] containing [index {[logstash-2018.02.02][doc][MCOsVmEB8wwdOBO9ikMm], source[{"user_id":85,"current_file#":8,"p1":8,"session_id":738,"event_id":2652584166,"blocking_session":null,"p3":1,"client_id":null,"p3text":"blocks","sql_opcode":3,"session_state":"WAITING","p2text":"block#","qc_instance_id":null,"session_type":"FOREGROUND","p1text":"file#","sql_child_number":0,"sql_id":"1yug1q2vk1kf0","capture_overhead":"N","port":46708,"plsql_object_id":null,"qc_session_id":null,"time_waited":10939,"plsql_entry_subprogram_id":null,"ecid":null,"plsql_entry_object_id":null,"p2":732732,"wait_class":"User I/O","force_matching_signature":17153010401499426907,"service_hash":3427055676,"module":"JDBC Thin Client","sample_id":82480688,"blocking_session_serial#":null,"blocking_session_status":"NO HOLDER","@version":"1","event":"db file sequential read","seq#":1129,"current_obj#":59388,"program":"JDBC Thin Client","current_block#":732732,"plsql_subprogram_id":null,"@timestamp":"2018-02-02T13:18:03.029Z","event#":117,"machine":"vm-app6-25.local","wait_class_id":1740759767,"flags":0,"session_serial#":3496,"sql_plan_hash_value":166146652,"action":null,"wait_time":0,"sample_time":"2018-02-02T13:18:03.029Z","xid":null,"is_captured":"N"}]}]
org.elasticsearch.index.mapper.MapperParsingException: failed to parse [force_matching_signature]
at org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:302) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:485) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:607) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:407) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:384) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:93) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:67) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:261) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:708) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:686) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:667) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.executeIndexRequestOnPrimary(TransportShardBulkAction.java:548) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.executeIndexRequest(TransportShardBulkAction.java:140) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:236) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:123) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:110) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:72) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:1033) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:1011) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:104) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:358) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:298) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:974) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:971) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:238) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2211) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryShardReference(TransportReplicationAction.java:983) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction.access$500(TransportReplicationAction.java:97) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:319) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:294) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:281) [elasticsearch-6.1.1.jar:6.1.1]

A_B · February 2, 2018, 1:34pm

What is the filed type for force_matching_signature in the index mapping?

org.elasticsearch.index.mapper.MapperParsingException: failed to parse [force_matching_signature]
at org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:302) ~[elasticsearch-6.1.1.jar:6.1.1]

Kaouther_Mechri · February 2, 2018, 1:49pm

It is number but I don't know how to change its type and to which type

A_B · February 2, 2018, 1:57pm

Does GET logstash-2018.02.02/_mapping also show it as number?

I'm not on ES6 yet so this is a bit of guess work... I have had fields that should be one type, be something else because the first document indexed in the daily index had a bad format. All subsequent documents failed... That was a fun day...

Anyway, double check what the type is in the specific index.

"force_matching_signature": 17153010401499427000

^^ sure looks like a number to me so can see anything wrong there.

I also ship all logs through Logstash so that I can enforce some rules and mutate fields is need be.

Hope that helps

Kaouther_Mechri · February 2, 2018, 2:02pm

thank you for your help . I 've this message in logs .

Caused by: com.fasterxml.jackson.core.JsonParseException: Numeric value (13491499919632766707) out of range of long (-9223372036854775808 - 9223372036854775807)

how ca I check the type . I've never touched to index that's why

A_B · February 2, 2018, 2:49pm

Well, looks like the type is ok. Just the number is too large.

Is that a field you really need?

Some possible ways to deal with this. None of them are great but this is what I have done before...

a. drop the force_matching_signature filed in Logstash.
b. truncate the force_matching_signature filed in Logstash to make it a valid number.
c. In ES5 you could change the type and that way get a new mapping in a new index which works ok for me using daily indices. Not sure how that works in ES6 as types have been removed.

Those are all workarounds. It would be good to hear from someone else if there are better longterm solutions

Kaouther_Mechri · February 2, 2018, 2:58pm

Oh thanks,

I've dropped the field and it is ok now .

Kaouther_Mechri · February 2, 2018, 3:01pm

I h've other errors in my logs :

[2018-02-02T16:00:07,827][DEBUG][o.e.a.s.TransportSearchAction] [elacticsearch.unealgroupe.fr] [packetbeat-6.1.2-2018.02.02][2], node[A3reGyDmStmJEmFIUy8nbQ], [P], s[STARTED], a[id=JGoQKyNI
SIagg_dSdGapEQ]: Failed to execute [SearchRequest{searchType=QUERY_THEN_FETCH, indices=[*], indicesOptions=IndicesOptions[id=39, ignore_unavailable=true, allow_no_indices=true, expand_wildc
ards_open=true, expand_wildcards_closed=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false], types=[], routing='null', preference='null', reques
tCache=null, scroll=null, maxConcurrentShardRequests=5, batchedReduceSize=512, preFilterShardSize=32, source={"size":0,"query":{"bool":{"must":[{"range":{"@timestamp":{"from":1517582707482,
"to":1517583607482,"include_lower":true,"include_upper":true,"format":"epoch_millis","boost":1.0}}},{"bool":{"must":[{"query_string":{"query":"beat.name:"vm-mule-25"","fields":[],"type":"
best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"
escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"agg
regations":{"3287e740-1b15-11e7-b09e-037021c4f8df":{"filter":{"match_all":{"boost":1.0}},"aggregations":{"timeseries":{"date_histogram":{"field":"@timestamp","time_zone":"Europe/Berlin","in
terval":"10s","offset":0,"order":{"_key":"asc"},"keyed":false,"min_doc_count":0,"extended_bounds":{"min":1517582707482,"max":1517583607482}},"aggregations":{"32880e50-1b15-11e7-b09e-037021c
4f8df":{"avg":{"field":"system.load.15"}}}}}}}}}] lastShard [true]
org.elasticsearch.transport.RemoteTransportException: [elacticsearch.unealgroupe.fr][192.168.9.160:9300][indices:data/read/search[phase/query]]
Caused by: org.elasticsearch.common.util.concurrent.EsRejectedExecutionException: rejected execution of org.elasticsearch.common.util.concurrent.TimedRunnable@320c90ed on QueueResizingEsThr
eadPoolExecutor[search, queue capacity = 1000, min queue capacity = 1000, max queue capacity = 1000, frame size = 2000, targeted response rate = 1s, task execution EWMA = 24.2ms, adjustment
amount = 50, QueueResizingEsThreadPoolExecutor[search, queue capacity = 1000, org.elasticsearch.common.util.concurrent.QueueResizingEsThreadPoolExecutor@2aab4d97[Running, pool size = 4, ac
tive threads = 4, queued tasks = 1271, completed tasks = 71276]]]
at org.elasticsearch.common.util.concurrent.EsAbortPolicy.rejectedExecution(EsAbortPolicy.java:48) ~[elasticsearch-6.1.1.jar:6.1.1]
at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823) ~[?:1.8.0_131]
at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369) ~[?:1.8.0_131]
at org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.doExecute(EsThreadPoolExecutor.java:94) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.common.util.concurrent.QueueResizingEsThreadPoolExecutor.doExecute(QueueResizingEsThreadPoolExecutor.java:93) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.execute(EsThreadPoolExecutor.java:89) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$0(SearchService.java:988) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:60) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:113) ~[elasticsearch-6.1.1.jar:6.1.1]

Kaouther_Mechri · February 7, 2018, 10:55am

Hello

It was an overload problem .

thanks to all

system · March 7, 2018, 10:56am

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.