Centralized Pipeline Management Issues


(Walker) #1

I have two ElasticStack setups running on 6.1.1, one of them has X-Pack installed with Centralized Pipeline Management configured, the other does not have x-pack. The following pipeline works on the non-xpack setup. On the Xpack setup, the error below the pipeline is thrown.

input {
  file {
    id => "C:\DMARC\*.xml"
    path => "C:/DMARC/*.xml"
    discover_interval => 5
    codec => multiline {
      auto_flush_interval => 5
      negate => true
      pattern => "<record>"
      what => "previous"
    }
  }
}
filter {
  xml {
    id => "Field Extraction"
    force_array => true
    store_xml => false
    source => "message"
    xpath => [
      "record/report_metadata/org_name/text()", "report.org",
      "record/report_metadata/email/text()", "report.org_contact",
      "record/report_metadata/extra_contact_info/text()", "report.additional_contact",
      "record/report_metadata/report_id/text()", "report.id",
      "record/report_metadata/date_range/begin/text()", "report.start",
      "record/report_metadata/date_range/end/text()", "report.end",
      "record/policy_published/domain/text()", "policy.domain",
      "record/policy_published/aspf/text()", "policy.spf_mode",
      "record/policy_published/adkim/text()", "policy.dkim_mode",
      "record/policy_published/p/text()", "policy.dmarc.domain_action",
      "record/policy_published/sp/text()", "policy.dmarc.subdomain_action",
      "record/policy_published/pct/text()", "policy.percentage",
      "record/row/source_ip/text()", "email.source_ip",
      "record/row/count/text()", "email.count",
      "record/row/policy_evaluated/disposition/text()", "email.dmarc_action",
      "record/row/policy_evaluated/spf/text()", "email.spf_evaluation",
      "record/row/policy_evaluated/dkim/text()", "email.dkim_evaluation",
      "record/row/policy_evaluated/reason/type/text()", "dmarc.override_type",
      "record/row/policy_evaluated/reason/comment/text()", "dmarc.override_comment",
      "record/identifiers/envelope_to/text()", "email.envelope_to",
      "record/identifiers/envelope_from/text()", "email.envelope_from",
      "record/identifiers/header_from/text()", "email.header_from",
      "record/auth_results/dkim/domain/text()", "authresult.dkim_domain",
      "record/auth_results/dkim/result/text()", "authresult.dkim_result",
      "record/auth_results/spf/domain/text()", "authresult.spf_domain",
      "record/auth_results/spf/scope/text()", "authresult.spf_scope",
      "record/auth_results/spf/result/text()", "authresult.spf_result"
    ]
  }
    geoip {
      id => "IP Geo-Mapping"
      source => "email.source_ip"
      add_field => {
        "[geoip][location][coordinates]" => "%{[geoip][location][lat]}, %{[geoip][location][lon]}"
      }
    }
}
output {
  elasticsearch {
    id => "Send to Elasticsearch"
    hosts => ["ElasticStack:9200"]
    http_compression => true
    template => "C:/ELK/logstash/templates/dmarcxml.json"
    template_name => "dmarcxml"
	index => "dmarcxml-%{+YYYY.MM.DD}"
  }
}

[2018-02-18T21:35:41,696][ERROR][logstash.agent ] Failed to execute action {:id=>:XML_DMARC, :action_type=>LogStash::ConvergeResult::FailedAction, :message=>"Something is wrong with your configuration.", :backtrace=>["C:/Logstash/logstash-core/lib/logstash/config/mixin.rb:89:inconfig_init'", "C:/Logstash/logstash-core/lib/logstash/outputs/base.rb:63:in initialize'", "C:/Logstash/logstash-core/lib/logstash/output_delegator_strategies/shared.rb:3:ininitialize'", "C:/Logstash/logstash-core/lib/logstash/output_delegator.rb:25:in initialize'", "C:/Logstash/logstash-core/lib/logstash/plugins/plugin_factory.rb:86:inplugin'", "C:/Logstash/logstash-core/lib/logstash/pipeline.rb:114:in plugin'", "(eval):60:in'", "org/jruby/RubyKernel.java:994:in eval'", "C:/Logstash/logstash-core/lib/logstash/pipeline.rb:86:ininitialize'", "C:/Logstash/logstash-core/lib/logstash/pipeline_action/reload.rb:34:in execute'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:335:inblock in converge_state'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:141:in with_pipelines'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:332:inblock in converge_state'", "org/jruby/RubyArray.java:1734:in each'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:319:inconverge_state'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:166:in block in converge_state_and_update'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:141:inwith_pipelines'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:164:in converge_state_and_update'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:105:inblock in execute'", "C:/Logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/interval.rb:18:in interval'", "C:/Logstash/logstash-core/lib/logstash/agent.rb:94:inexecute'", "C:/Logstash/logstash-core/lib/logstash/runner.rb:343:in block in execute'", "C:/Logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:24:inblock in initialize'"]}`


(Walker) #2

Disregard...debug log had additional information, bad path to elasticsearch template.


(system) #3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.