Hi! I'm trying to remove the "host" field but it seems that it's not working for me and every time I recreate the logstash instance, the second time that my queries are executed the following error appears over and over again:
_[2018-07-23T17:34:57,917][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"dev-infor-2018.07.23", :_type=>"doc", :_routing=>nil}, #LogStash::Event:0x588ca1c3], :response=>{"index"=>{"_index"=>"dev-infor-2018.07.23", "_type"=>"doc", "id"=>"New1yGQBFDLybt_JXRh1", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"object mapping for [host] tried to parse field [host] as object, but found a concrete value"}}}}
My pipeline is:
input {
jdbc {
jdbc_driver_library => "/opt/postgresql-{{ pg_jdbc_version }}.jar"
jdbc_driver_class => "org.postgresql.Driver"
jdbc_connection_string => "jdbc:postgresql://{{ postgres_host }}:5432/arts_v6"
jdbc_user => "postgres"
jdbc_password => "postgres"
schedule => "* * * * *"
statement_filepath => "/etc/logstash/pipelines/engage-register-count.sql"
clean_run => true
tags => [ "engage-register-count" ]
add_field => { "query_type" => "registercount" }
}
}
input {
jdbc {
jdbc_driver_library => "/opt/postgresql-{{ pg_jdbc_version }}.jar"
jdbc_driver_class => "org.postgresql.Driver"
jdbc_connection_string => "jdbc:postgresql://{{ postgres_host }}:5432/arts_v6"
jdbc_user => "postgres"
jdbc_password => "postgres"
schedule => "* 9,15,21 * * *"
statement_filepath => "/etc/logstash/pipelines/orphaned-item.sql"
clean_run => true
tags => [ "orphaned-items" ]
add_field => { "query_type" => "orphaned" }
}
}
input {
jdbc {
jdbc_driver_library => "/opt/postgresql-{{ pg_jdbc_version }}.jar"
jdbc_driver_class => "org.postgresql.Driver"
jdbc_connection_string => "jdbc:postgresql://{{ postgres_host }}:5432/arts_v6"
jdbc_user => "postgres"
jdbc_password => "postgres"
schedule => "* * * * *"
statement_filepath => "/etc/logstash/pipelines/engage-store-transaction-time.sql"
clean_run => true
tags => [ "engage-store-transaction-time" ]
add_field => { "query_type" => "store-transaction-time" }
}
}
input {
jdbc {
jdbc_driver_library => "/opt/postgresql-{{ pg_jdbc_version }}.jar"
jdbc_driver_class => "org.postgresql.Driver"
jdbc_connection_string => "jdbc:postgresql://{{ postgres_host }}:5432/arts_v6"
jdbc_user => "postgres"
jdbc_password => "postgres"
schedule => "* * * * *"
statement_filepath => "/etc/logstash/pipelines/engage-transaction-offline-amount.sql"
clean_run => true
tags => [ "engage-transaction-offline-amount" ]
add_field => { "query_type" => "register-saf-amount" }
}
}
input {
jdbc {
jdbc_driver_library => "/opt/postgresql-{{ pg_jdbc_version }}.jar"
jdbc_driver_class => "org.postgresql.Driver"
jdbc_connection_string => "jdbc:postgresql://{{ postgres_host }}:5432/arts_v6"
jdbc_user => "postgres"
jdbc_password => "postgres"
schedule => "* * * * *"
statement_filepath => "/etc/logstash/pipelines/engage-store-business-type.sql"
clean_run => true
tags => [ "engage-store-business-type" ]
add_field => { "query_type" => "store-type" }
}
}
input {
jdbc {
jdbc_driver_library => "/opt/postgresql-{{ pg_jdbc_version }}.jar"
jdbc_driver_class => "org.postgresql.Driver"
jdbc_connection_string => "jdbc:postgresql://{{ postgres_host }}:5432/arts_v6"
jdbc_user => "postgres"
jdbc_password => "postgres"
schedule => "* * * * *"
statement_filepath => "/etc/logstash/pipelines/engage-saf-validation.sql"
clean_run => true
tags => [ "engage-saf-validation" ]
add_field => { "query_type" => "saf-status-count" }
}
}
input {
jdbc {
jdbc_driver_library => "/opt/postgresql-{{ pg_jdbc_version }}.jar"
jdbc_driver_class => "org.postgresql.Driver"
jdbc_connection_string => "jdbc:postgresql://{{ postgres_host }}:5432/arts_v6"
jdbc_user => "postgres"
jdbc_password => "postgres"
schedule => "* * * * *"
statement_filepath => "/etc/logstash/pipelines/engage-store-transaction-time-histogram.sql"
clean_run => true
tags => [ "engage-store-transaction-time-histogram" ]
add_field => { "query_type" => "txn-time-history" }
}
}
filter {
mutate {
add_field => {"infor_stack_name" => "{{ env }}-{{ customer_code }}" }
add_field => {"infor_stack_version" => "{{ product_version }}" }
remove_field => [ "host" ]
}
if "engage-register-count" in [tags] {
mutate {
convert => {"registercount" => "integer" }
add_field => {"resultId" => "%{query_type}" }
}
}
if "engage-store-transaction-time" in [tags] {
mutate {
convert => {"percentile_50" => "float" }
convert => {"percentile_95" => "float" }
convert => {"percentile_99" => "float" }
add_field => {"resultId" => "%{query_type}-%{storeid}" }
}
}
if "engage-transaction-offline-amount" in [tags] {
mutate {
add_field => {"resultId" => "%{query_type}-%{storeid}-%{workstationId}" }
}
}
if "engage-store-business-type" in [tags] {
mutate {
add_field => {"resultId" => "%{query_type}-%{storeid}" }
}
}
if "engage-saf-validation" in [tags] {
mutate {
add_field => {"resultId" => "%{query_type}" }
}
}
if "engage-store-transaction-time-histogram" in [tags] {
mutate {
convert => {"averageTimeSeconds" => "float" }
add_field => {"resultId" => "%{query_type}-%{storeid}-%{+YYYY.MM.dd.HH.mm.ss.SSS}" }
}
}
}
output {
if "orphaned-items" not in [tags] {
elasticsearch {
hosts => "{{ external_elk_url }}"
user => "{{ external_elk_user }}"
password => "{{ external_elk_pass }}"
index => "engage-jdbc-{{ env}}-{{ customer_code }}-%{+YYYY.MM.dd}"
document_id => "engage-jdbc-%{resultId}"
}
}
{% if use_kinesis is defined and use_kinesis == "Enabled" %}
if "orphaned-items" in [tags] {
kinesis {
stream_name => "{{ env }}-{{ customer_code }}-base-RetailerLogStream"
region => "{{ region }}"
}
}
{% endif %}
}
Have no clue why is not working, since remove_field => [ "host" ] should be enough.
Any ideas?
Thanks.