Hi,
My Logstash pipeline keeps failing during scroll because of an error. There are multiple documents causing that issue. Those are referral urls that sometimes look weird. The property itself is not indexed in elasticsearch ({"type": "object", "enabled": false}). Do you have any suggestions how to fix this?
Logstash Versions tested: 7.12.1, 7.10.1
Plugin: <LogStash::Inputs::Elasticsearch
Input Configuration:
input {
elasticsearch {
hosts => ["my-host:9200"]
index => "my-index"
docinfo => true
docinfo_fields => ["_type", "_id", "_index"]
query => '{"sort":["date"],"query": {"bool": {"must": [{"range": {"date": {"gte": "2020-01-01T00:00:00"}}}]}}}'
size => 10000
}
}
Example (shortened) document causing the error:
{
"date": "2020-01-11T08:00:00.000Z",
"count-by-referral-url": {
"https://bs.serving-sys.com/serving/adServer.bs?cn=trd&mc=click&pli=26859493&PluID=0&ord=[timestamp]&fbclid=IwAR3p4--K2EfVX0fzTv5FgX9sEJuIsWuhHVvr6W0zOTkAG166d6QB-yMGbwI": 1
}
}
Stacktrace:
Error: Invalid FieldReference: `https://bs.serving-sys.com/serving/adServer.bs?cn=trd&mc=click&pli=26859493&PluID=0&ord=[timestamp]&fbclid=IwAR3p4--K2EfVX0fzTv5FgX9sEJuIsWuhHVvr6W0zOTkAG166d6QB-yMGbwI`
Exception: Java::OrgLogstash::FieldReference::IllegalSyntaxException
Stack: org.logstash.FieldReference$StrictTokenizer.tokenize(FieldReference.java:312)
org.logstash.FieldReference.parse(FieldReference.java:213)
org.logstash.FieldReference.parseToCache(FieldReference.java:204)
org.logstash.FieldReference.from(FieldReference.java:127)
org.logstash.FieldReference.lambda$from$0(FieldReference.java:118)
java.base/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1737)
org.logstash.FieldReference.from(FieldReference.java:118)
org.logstash.ConvertedMap.convertKey(ConvertedMap.java:122)
org.logstash.ConvertedMap.access$000(ConvertedMap.java:44)
org.logstash.ConvertedMap$1.visit(ConvertedMap.java:55)
org.logstash.ConvertedMap$1.visit(ConvertedMap.java:49)
org.jruby.RubyHash.visitLimited(RubyHash.java:698)
org.jruby.RubyHash.visitAll(RubyHash.java:683)
org.logstash.ConvertedMap.newFromRubyHash(ConvertedMap.java:89)
org.logstash.ConvertedMap.newFromRubyHash(ConvertedMap.java:84)
org.logstash.Valuefier.lambda$initConverters$12(Valuefier.java:171)
org.logstash.Valuefier.convert(Valuefier.java:94)
org.logstash.ConvertedMap$1.visit(ConvertedMap.java:55)
org.logstash.ConvertedMap$1.visit(ConvertedMap.java:49)
org.jruby.RubyHash.visitLimited(RubyHash.java:698)
org.jruby.RubyHash.visitAll(RubyHash.java:683)
org.logstash.ConvertedMap.newFromRubyHash(ConvertedMap.java:89)
org.logstash.ext.JrubyEventExtLibrary$RubyEvent.ruby_initialize(JrubyEventExtLibrary.java:92)
usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_input_minus_elasticsearch_minus_4_dot_9_dot_1.lib.logstash.inputs.elasticsearch.RUBY$method$push_hit$0(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-4.9.1/lib/logstash/inputs/elasticsearch.rb:308)
usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_input_minus_elasticsearch_minus_4_dot_9_dot_1.lib.logstash.inputs.elasticsearch.RUBY$block$do_run_slice$1(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-4.9.1/lib/logstash/inputs/elasticsearch.rb:274)
org.jruby.runtime.CompiledIRBlockBody.yieldDirect(CompiledIRBlockBody.java:148)
org.jruby.runtime.BlockBody.yield(BlockBody.java:106)
org.jruby.runtime.Block.yield(Block.java:184)
org.jruby.RubyArray.each(RubyArray.java:1809)
usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_input_minus_elasticsearch_minus_4_dot_9_dot_1.lib.logstash.inputs.elasticsearch.RUBY$method$do_run_slice$0(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-4.9.1/lib/logstash/inputs/elasticsearch.rb:274)
usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_input_minus_elasticsearch_minus_4_dot_9_dot_1.lib.logstash.inputs.elasticsearch.RUBY$method$do_run$0(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-4.9.1/lib/logstash/inputs/elasticsearch.rb:250)
usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_input_minus_elasticsearch_minus_4_dot_9_dot_1.lib.logstash.inputs.elasticsearch.RUBY$method$run$0(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-4.9.1/lib/logstash/inputs/elasticsearch.rb:238)
usr.share.logstash.logstash_minus_core.lib.logstash.java_pipeline.RUBY$method$inputworker$0(/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405)
usr.share.logstash.logstash_minus_core.lib.logstash.java_pipeline.RUBY$method$inputworker$0$__VARARGS__(/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb)
org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:80)
org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:70)
org.jruby.ir.targets.InvokeSite.invoke(InvokeSite.java:207)
usr.share.logstash.logstash_minus_core.lib.logstash.java_pipeline.RUBY$block$start_input$1(/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396)
org.jruby.runtime.CompiledIRBlockBody.callDirect(CompiledIRBlockBody.java:138)
org.jruby.runtime.IRBlockBody.call(IRBlockBody.java:58)
org.jruby.runtime.IRBlockBody.call(IRBlockBody.java:52)
org.jruby.runtime.Block.call(Block.java:139)
org.jruby.RubyProc.call(RubyProc.java:318)
org.jruby.internal.runtime.RubyRunnable.run(RubyRunnable.java:105)
java.base/java.lang.Thread.run(Thread.java:834)