I have two ElasticStack setups running on 6.1.1, one of them has X-Pack installed with Centralized Pipeline Management configured, the other does not have x-pack. The following pipeline works on the non-xpack setup. On the Xpack setup, the error below the pipeline is thrown.
input {
file {
id => "C:\DMARC\*.xml"
path => "C:/DMARC/*.xml"
discover_interval => 5
codec => multiline {
auto_flush_interval => 5
negate => true
pattern => "<record>"
what => "previous"
}
}
}
filter {
xml {
id => "Field Extraction"
force_array => true
store_xml => false
source => "message"
xpath => [
"record/report_metadata/org_name/text()", "report.org",
"record/report_metadata/email/text()", "report.org_contact",
"record/report_metadata/extra_contact_info/text()", "report.additional_contact",
"record/report_metadata/report_id/text()", "report.id",
"record/report_metadata/date_range/begin/text()", "report.start",
"record/report_metadata/date_range/end/text()", "report.end",
"record/policy_published/domain/text()", "policy.domain",
"record/policy_published/aspf/text()", "policy.spf_mode",
"record/policy_published/adkim/text()", "policy.dkim_mode",
"record/policy_published/p/text()", "policy.dmarc.domain_action",
"record/policy_published/sp/text()", "policy.dmarc.subdomain_action",
"record/policy_published/pct/text()", "policy.percentage",
"record/row/source_ip/text()", "email.source_ip",
"record/row/count/text()", "email.count",
"record/row/policy_evaluated/disposition/text()", "email.dmarc_action",
"record/row/policy_evaluated/spf/text()", "email.spf_evaluation",
"record/row/policy_evaluated/dkim/text()", "email.dkim_evaluation",
"record/row/policy_evaluated/reason/type/text()", "dmarc.override_type",
"record/row/policy_evaluated/reason/comment/text()", "dmarc.override_comment",
"record/identifiers/envelope_to/text()", "email.envelope_to",
"record/identifiers/envelope_from/text()", "email.envelope_from",
"record/identifiers/header_from/text()", "email.header_from",
"record/auth_results/dkim/domain/text()", "authresult.dkim_domain",
"record/auth_results/dkim/result/text()", "authresult.dkim_result",
"record/auth_results/spf/domain/text()", "authresult.spf_domain",
"record/auth_results/spf/scope/text()", "authresult.spf_scope",
"record/auth_results/spf/result/text()", "authresult.spf_result"
]
}
geoip {
id => "IP Geo-Mapping"
source => "email.source_ip"
add_field => {
"[geoip][location][coordinates]" => "%{[geoip][location][lat]}, %{[geoip][location][lon]}"
}
}
}
output {
elasticsearch {
id => "Send to Elasticsearch"
hosts => ["ElasticStack:9200"]
http_compression => true
template => "C:/ELK/logstash/templates/dmarcxml.json"
template_name => "dmarcxml"
index => "dmarcxml-%{+YYYY.MM.DD}"
}
}
[2018-02-18T21:35:41,696][ERROR][logstash.agent ] Failed to execute action {:id=>:XML_DMARC, :action_type=>LogStash::ConvergeResult::FailedAction, :message=>"Something is wrong with your configuration.", :backtrace=>["C:/Logstash/logstash-core/lib/logstash/config/mixin.rb:89:inconfig_init'", "C:/Logstash/logstash-core/lib/logstash/outputs/base.rb:63:in initialize'", "C:/Logstash/logstash-core/lib/logstash/output_delegator_strategies/shared.rb:3:ininitialize'", "C:/Logstash/logstash-core/lib/logstash/output_delegator.rb:25:in initialize'", "C:/Logstash/logstash-core/lib/logstash/plugins/plugin_factory.rb:86:inplugin'", "C:/Logstash/logstash-core/lib/logstash/pipeline.rb:114:in plugin'", "(eval):60:in'", "org/jruby/RubyKernel.java:994:in eval'", "C:/Logstash/logstash-core/lib/logstash/pipeline.rb:86:ininitialize'", "C:/Logstash/logstash-core/lib/logstash/pipeline_action/reload.rb:34:in execute'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:335:inblock in converge_state'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:141:in with_pipelines'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:332:inblock in converge_state'", "org/jruby/RubyArray.java:1734:in each'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:319:inconverge_state'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:166:in block in converge_state_and_update'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:141:inwith_pipelines'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:164:in converge_state_and_update'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:105:inblock in execute'", "C:/Logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/interval.rb:18:in interval'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:94:inexecute'", "C:/Logstash/logstash-core/lib/logstash/runner.rb:343:in block in execute'", "C:/Logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:24:inblock in initialize'"]}`