Hello Everyone, I hope you are good, I have this issue with logstash filter using Xpath,
I Don't really know if the xpath I Pass is wrong the error I'm getting from cmd is kinda syntax error....
What I'm seeing is like in the error log is /soapenv:Envelope[@ :but after the @ in the code follows a line that is xmlns and I think logstash are not understanding the xpath correctly if anyone could help me with this would be my hero, Im in the last step.
my conf:
input {
http_poller {
urls => {
testVersion => {
method => post
user => 'API_USER'
password => "API_PASS"
url => "http://testapm.osde.ar:8081/introscope-web-services/services/MetricsDataService"
headers => {
"Content-Type" => "text/xml; charset=UTF-8"
"SOAPAction" => "urn:mc_version"
}
body => '<soapenv:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:met="http://metricsdata.webservicesimpl.server.introscope.wily.com">
<soapenv:Header/>
<soapenv:Body>
<met:getLiveMetricData soapenv:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/">
<agentRegex xsi:type="xsd:string">.*\|Custom Metric Process \(Virtual\)\|Custom Metric Agent \(Virtual\)</agentRegex>
<metricPrefix xsi:type="xsd:string">Alerts\|[^|]+:.*Summary.*</metricPrefix>
</met:getLiveMetricData>
</soapenv:Body>
</soapenv:Envelope>'
}
}
request_timeout => 30
schedule => {'every' => "30s" }
codec => "plain"
# A hash of request metadata info (timing, response headers, etc.) will be sent here
metadata_target => "http_poller_metadata"
}
}
filter {
xml {
remove_namespaces => true
source => "message"
target => "doc"
xpath => ['/soapenv:Envelope[@xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"]/soapenv:Body/multiRef/agentName[@xsi:type="xsd:string"]/text()', 'new_field']
}
}
output {
stdout {
codec => rubydebug
}
}
Error log:
[2020-03-17T14:55:51,779][ERROR][org.logstash.execution.WorkerLoop][main] Exception in pipelineworker, the pipeline stopped processing new events, please check your filter configuration and restart Logstash.
org.jruby.exceptions.RaiseException: (SyntaxError) /soapenv:Envelope[@ :soapenv="http://schemas.xmlsoap.org/soap/envelope/"]/soapenv:Body/multiRef/agentName[@xsi:type="xsd:string"]/text()
at nokogiri.XmlXpathContext.evaluate(nokogiri/XmlXpathContext.java:128) ~[nokogiri.jar:?]
at C_3a_.logstash_minus_7_dot_6_dot_1.vendor.bundle.jruby.$2_dot_5_dot_0.gems.nokogiri_minus_1_dot_10_dot_8_minus_java.lib.nokogiri.xml.searchable.xpath_impl(C:/logstash-7.6.1/vendor/bundle/jruby/2.5.0/gems/nokogiri-1.10.8-java/lib/nokogiri/xml/searchable.rb:198) ~[?:?]
at C_3a_.logstash_minus_7_dot_6_dot_1.vendor.bundle.jruby.$2_dot_5_dot_0.gems.nokogiri_minus_1_dot_10_dot_8_minus_java.lib.nokogiri.xml.searchable.xpath_internal(C:/logstash-7.6.1/vendor/bundle/jruby/2.5.0/gems/nokogiri-1.10.8-java/lib/nokogiri/xml/searchable.rb:179) ~[?:?]
at C_3a_.logstash_minus_7_dot_6_dot_1.vendor.bundle.jruby.$2_dot_5_dot_0.gems.nokogiri_minus_1_dot_10_dot_8_minus_java.lib.nokogiri.xml.searchable.xpath(C:/logstash-7.6.1/vendor/bundle/jruby/2.5.0/gems/nokogiri-1.10.8-java/lib/nokogiri/xml/searchable.rb:154) ~[?:?]
at C_3a_.logstash_minus_7_dot_6_dot_1.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_filter_minus_xml_minus_4_dot_0_dot_7.lib.logstash.filters.xml.filter(C:/logstash-7.6.1/vendor/bundle/jruby/2.5.0/gems/logstash-filter-xml-4.0.7/lib/logstash/filters/xml.rb:153) ~[?:?]
at org.jruby.RubyHash.each(org/jruby/RubyHash.java:1428) ~[jruby-complete-9.2.9.0.jar:?]
at C_3a_.logstash_minus_7_dot_6_dot_1.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_filter_minus_xml_minus_4_dot_0_dot_7.lib.logstash.filters.xml.filter(C:/logstash-7.6.1/vendor/bundle/jruby/2.5.0/gems/logstash-filter-xml-4.0.7/lib/logstash/filters/xml.rb:152) ~[?:?]
at C_3a_.logstash_minus_7_dot_6_dot_1.logstash_minus_core.lib.logstash.filters.base.do_filter(C:/logstash-7.6.1/logstash-core/lib/logstash/filters/base.rb:143) ~[?:?]
at C_3a_.logstash_minus_7_dot_6_dot_1.logstash_minus_core.lib.logstash.filters.base.multi_filter(C:/logstash-7.6.1/logstash-core/lib/logstash/filters/base.rb:162) ~[?:?]
at org.jruby.RubyArray.each(org/jruby/RubyArray.java:1814) ~[jruby-complete-9.2.9.0.jar:?]
at C_3a_.logstash_minus_7_dot_6_dot_1.logstash_minus_core.lib.logstash.filters.base.multi_filter(C:/logstash-7.6.1/logstash-core/lib/logstash/filters/base.rb:159) ~[?:?]
at org.logstash.config.ir.compiler.AbstractFilterDelegatorExt.multi_filter(org/logstash/config/ir/compiler/AbstractFilterDelegatorExt.java:115) ~[logstash-core.jar:?]
at C_3a_.logstash_minus_7_dot_6_dot_1.logstash_minus_core.lib.logstash.java_pipeline.start_workers(C:/logstash-7.6.1/logstash-core/lib/logstash/java_pipeline.rb:262) ~[?:?]
warning: thread "[main]>worker2" terminated with exception (report_on_exception is true):
java.lang.IllegalStateException: org.jruby.exceptions.RaiseException: (SyntaxError) /soapenv:Envelope[@ :soapenv="http://schemas.xmlsoap.org/soap/envelope/"]/soapenv:Body/multiRef/agentName[@xsi:type="xsd:string"]/text()
at org.logstash.execution.WorkerLoop.run(org/logstash/execution/WorkerLoop.java:85)
at java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:498)
at org.jruby.javasupport.JavaMethod.invokeDirectWithExceptionHandling(org/jruby/javasupport/JavaMethod.java:441)
at org.jruby.javasupport.JavaMethod.invokeDirect(org/jruby/javasupport/JavaMethod.java:305)
at C_3a_.logstash_minus_7_dot_6_dot_1.logstash_minus_core.lib.logstash.java_pipeline.start_workers(C:/logstash-7.6.1/logstash-core/lib/logstash/java_pipeline.rb:262)
at org.jruby.RubyProc.call(org/jruby/RubyProc.java:274)
at java.lang.Thread.run(java/lang/Thread.java:748)
Caused by: org.jruby.exceptions.RaiseException: (SyntaxError) /soapenv:Envelope[@ :soapenv="http://schemas.xmlsoap.org/soap/envelope/"]/soapenv:Body/multiRef/agentName[@xsi:type="xsd:string"]/text()
at nokogiri.XmlXpathContext.evaluate(nokogiri/XmlXpathContext.java:128)
at C_3a_.logstash_minus_7_dot_6_dot_1.vendor.bundle.jruby.$2_dot_5_dot_0.gems.nokogiri_minus_1_dot_10_dot_8_minus_java.lib.nokogiri.xml.searchable.xpath_impl(C:/logstash-7.6.1/vendor/bundle/jruby/2.5.0/gems/nokogiri-1.10.8-java/lib/nokogiri/xml/searchable.rb:198)
at C_3a_.logstash_minus_7_dot_6_dot_1.vendor.bundle.jruby.$2_dot_5_dot_0.gems.nokogiri_minus_1_dot_10_dot_8_minus_java.lib.nokogiri.xml.searchable.xpath_internal(C:/logstash-7.6.1/vendor/bundle/jruby/2.5.0/gems/nokogiri-1.10.8-java/lib/nokogiri/xml/searchable.rb:179)
at C_3a_.logstash_minus_7_dot_6_dot_1.vendor.bundle.jruby.$2_dot_5_dot_0.gems.nokogiri_minus_1_dot_10_dot_8_minus_java.lib.nokogiri.xml.searchable.xpath(C:/logstash-7.6.1/vendor/bundle/jruby/2.5.0/gems/nokogiri-1.10.8-java/lib/nokogiri/xml/searchable.rb:154)
at C_3a_.logstash_minus_7_dot_6_dot_1.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_filter_minus_xml_minus_4_dot_0_dot_7.lib.logstash.filters.xml.filter(C:/logstash-7.6.1/vendor/bundle/jruby/2.5.0/gems/logstash-filter-xml-4.0.7/lib/logstash/filters/xml.rb:153)
at org.jruby.RubyHash.each(org/jruby/RubyHash.java:1428)
at C_3a_.logstash_minus_7_dot_6_dot_1.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_filter_minus_xml_minus_4_dot_0_dot_7.lib.logstash.filters.xml.filter(C:/logstash-7.6.1/vendor/bundle/jruby/2.5.0/gems/logstash-filter-xml-4.0.7/lib/logstash/filters/xml.rb:152)
at C_3a_.logstash_minus_7_dot_6_dot_1.logstash_minus_core.lib.logstash.filters.base.do_filter(C:/logstash-7.6.1/logstash-core/lib/logstash/filters/base.rb:143)
at C_3a_.logstash_minus_7_dot_6_dot_1.logstash_minus_core.lib.logstash.filters.base.multi_filter(C:/logstash-7.6.1/logstash-core/lib/logstash/filters/base.rb:162)
at org.jruby.RubyArray.each(org/jruby/RubyArray.java:1814)
at C_3a_.logstash_minus_7_dot_6_dot_1.logstash_minus_core.lib.logstash.filters.base.multi_filter(C:/logstash-7.6.1/logstash-core/lib/logstash/filters/base.rb:159)
at org.logstash.config.ir.compiler.AbstractFilterDelegatorExt.multi_filter(org/logstash/config/ir/compiler/AbstractFilterDelegatorExt.java:115)
at C_3a_.logstash_minus_7_dot_6_dot_1.logstash_minus_core.lib.logstash.java_pipeline.start_workers(C:/logstash-7.6.1/logstash-core/lib/logstash/java_pipeline.rb:262)
[2020-03-17T14:55:51,870][FATAL][logstash.runner ] An unexpected error occurred! {:error=>java.lang.IllegalStateException: org.jruby.exceptions.RaiseException: (SyntaxError) /soapenv:Envelope[@ :soapenv="http://schemas.xmlsoap.org/soap/envelope/"]/soapenv:Body/multiRef/agentName[@xsi:type="xsd:string"]/text(), :backtrace=>["org.logstash.execution.WorkerLoop.run(org/logstash/execution/WorkerLoop.java:85)", "java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:498)", "org.jruby.javasupport.JavaMethod.invokeDirectWithExceptionHandling(org/jruby/javasupport/JavaMethod.java:441)", "org.jruby.javasupport.JavaMethod.invokeDirect(org/jruby/javasupport/JavaMethod.java:305)", "C_3a_.logstash_minus_7_dot_6_dot_1.logstash_minus_core.lib.logstash.java_pipeline.start_workers(C:/logstash-7.6.1/logstash-core/lib/logstash/java_pipeline.rb:262)", "org.jruby.RubyProc.call(org/jruby/RubyProc.java:274)", "java.lang.Thread.run(java/lang/Thread.java:748)"]}
[2020-03-17T14:55:51,888][ERROR][org.logstash.Logstash ] java.lang.IllegalStateException: Logstash stopped processing because of an error: (SystemExit) exit