Error in elasticsearch logs

Elastic Stack Elasticsearch
1

Hello everyone,

I have this Error in my Elasticsearch logs . Do you have any idea please ?
ELK 6.1.1
thanks

[2018-02-02T14:20:03,624][DEBUG][o.e.a.b.TransportShardBulkAction] [logstash-2018.02.02][2] failed to execute bulk item (index) BulkShardRequest [[logstash-2018.02.02][2]] containing [index {[logstash-2018.02.02][doc][MCOsVmEB8wwdOBO9ikMm], source[{"user_id":85,"current_file#":8,"p1":8,"session_id":738,"event_id":2652584166,"blocking_session":null,"p3":1,"client_id":null,"p3text":"blocks","sql_opcode":3,"session_state":"WAITING","p2text":"block#","qc_instance_id":null,"session_type":"FOREGROUND","p1text":"file#","sql_child_number":0,"sql_id":"1yug1q2vk1kf0","capture_overhead":"N","port":46708,"plsql_object_id":null,"qc_session_id":null,"time_waited":10939,"plsql_entry_subprogram_id":null,"ecid":null,"plsql_entry_object_id":null,"p2":732732,"wait_class":"User I/O","force_matching_signature":17153010401499426907,"service_hash":3427055676,"module":"JDBC Thin Client","sample_id":82480688,"blocking_session_serial#":null,"blocking_session_status":"NO HOLDER","@version":"1","event":"db file sequential read","seq#":1129,"current_obj#":59388,"program":"JDBC Thin Client","current_block#":732732,"plsql_subprogram_id":null,"@timestamp":"2018-02-02T13:18:03.029Z","event#":117,"machine":"vm-app6-25.local","wait_class_id":1740759767,"flags":0,"session_serial#":3496,"sql_plan_hash_value":166146652,"action":null,"wait_time":0,"sample_time":"2018-02-02T13:18:03.029Z","xid":null,"is_captured":"N"}]}]
org.elasticsearch.index.mapper.MapperParsingException: failed to parse [force_matching_signature]
at org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:302) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:485) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:607) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:407) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:384) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:93) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:67) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:261) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:708) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:686) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:667) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.executeIndexRequestOnPrimary(TransportShardBulkAction.java:548) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.executeIndexRequest(TransportShardBulkAction.java:140) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:236) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:123) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:110) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:72) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:1033) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:1011) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:104) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:358) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:298) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:974) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:971) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:238) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2211) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryShardReference(TransportReplicationAction.java:983) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction.access$500(TransportReplicationAction.java:97) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:319) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:294) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:281) [elasticsearch-6.1.1.jar:6.1.1]

2

What is the filed type for force_matching_signature in the index mapping?

org.elasticsearch.index.mapper.MapperParsingException: failed to parse [force_matching_signature]
at org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:302) ~[elasticsearch-6.1.1.jar:6.1.1]

3

It is number but I don't know how to change its type and to which type

4

Does GET logstash-2018.02.02/_mapping also show it as number?

I'm not on ES6 yet so this is a bit of guess work... I have had fields that should be one type, be something else because the first document indexed in the daily index had a bad format. All subsequent documents failed... That was a fun day...

Anyway, double check what the type is in the specific index.

"force_matching_signature": 17153010401499427000

^^ sure looks like a number to me so can see anything wrong there.

I also ship all logs through Logstash so that I can enforce some rules and mutate fields is need be.

Hope that helps :slight_smile:

5

thank you for your help . I 've this message in logs .

Caused by: com.fasterxml.jackson.core.JsonParseException: Numeric value (13491499919632766707) out of range of long (-9223372036854775808 - 9223372036854775807)

how ca I check the type . I've never touched to index that's why

6

Well, looks like the type is ok. Just the number is too large.

Is that a field you really need?

Some possible ways to deal with this. None of them are great but this is what I have done before...

a. drop the force_matching_signature filed in Logstash.
b. truncate the force_matching_signature filed in Logstash to make it a valid number.
c. In ES5 you could change the type and that way get a new mapping in a new index which works ok for me using daily indices. Not sure how that works in ES6 as types have been removed.

Those are all workarounds. It would be good to hear from someone else if there are better longterm solutions :slight_smile:

7

Oh thanks,

I've dropped the field and it is ok now .

8

I h've other errors in my logs :

[2018-02-02T16:00:07,827][DEBUG][o.e.a.s.TransportSearchAction] [elacticsearch.unealgroupe.fr] [packetbeat-6.1.2-2018.02.02][2], node[A3reGyDmStmJEmFIUy8nbQ], [P], s[STARTED], a[id=JGoQKyNI
SIagg_dSdGapEQ]: Failed to execute [SearchRequest{searchType=QUERY_THEN_FETCH, indices=[*], indicesOptions=IndicesOptions[id=39, ignore_unavailable=true, allow_no_indices=true, expand_wildc
ards_open=true, expand_wildcards_closed=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false], types=[], routing='null', preference='null', reques
tCache=null, scroll=null, maxConcurrentShardRequests=5, batchedReduceSize=512, preFilterShardSize=32, source={"size":0,"query":{"bool":{"must":[{"range":{"@timestamp":{"from":1517582707482,
"to":1517583607482,"include_lower":true,"include_upper":true,"format":"epoch_millis","boost":1.0}}},{"bool":{"must":[{"query_string":{"query":"beat.name:"vm-mule-25"","fields":[],"type":"
best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"
escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"agg
regations":{"3287e740-1b15-11e7-b09e-037021c4f8df":{"filter":{"match_all":{"boost":1.0}},"aggregations":{"timeseries":{"date_histogram":{"field":"@timestamp","time_zone":"Europe/Berlin","in
terval":"10s","offset":0,"order":{"_key":"asc"},"keyed":false,"min_doc_count":0,"extended_bounds":{"min":1517582707482,"max":1517583607482}},"aggregations":{"32880e50-1b15-11e7-b09e-037021c
4f8df":{"avg":{"field":"system.load.15"}}}}}}}}}] lastShard [true]
org.elasticsearch.transport.RemoteTransportException: [elacticsearch.unealgroupe.fr][192.168.9.160:9300][indices:data/read/search[phase/query]]
Caused by: org.elasticsearch.common.util.concurrent.EsRejectedExecutionException: rejected execution of org.elasticsearch.common.util.concurrent.TimedRunnable@320c90ed on QueueResizingEsThr
eadPoolExecutor[search, queue capacity = 1000, min queue capacity = 1000, max queue capacity = 1000, frame size = 2000, targeted response rate = 1s, task execution EWMA = 24.2ms, adjustment
amount = 50, QueueResizingEsThreadPoolExecutor[search, queue capacity = 1000, org.elasticsearch.common.util.concurrent.QueueResizingEsThreadPoolExecutor@2aab4d97[Running, pool size = 4, ac
tive threads = 4, queued tasks = 1271, completed tasks = 71276]]]
at org.elasticsearch.common.util.concurrent.EsAbortPolicy.rejectedExecution(EsAbortPolicy.java:48) ~[elasticsearch-6.1.1.jar:6.1.1]
at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823) ~[?:1.8.0_131]
at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369) ~[?:1.8.0_131]
at org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.doExecute(EsThreadPoolExecutor.java:94) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.common.util.concurrent.QueueResizingEsThreadPoolExecutor.doExecute(QueueResizingEsThreadPoolExecutor.java:93) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.execute(EsThreadPoolExecutor.java:89) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$0(SearchService.java:988) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:60) ~[elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:113) ~[elasticsearch-6.1.1.jar:6.1.1]

9

Hello

It was an overload problem .

thanks to all

10

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.