input {
http {
host => "0.0.0.0" # default: 0.0.0.0
port => 0000 # default: 8080
}
}
filter {
urldecode {
all_fields => true
}
}
filter {
mutate { gsub => [ "message", "\]", "}", "message", "\[", "{" ] }
kv { field_split => "&" allow_empty_values => true }
}
filter {
mutate {
rename => { "campaign{name}" => 'campaign_name' }
rename => { "campaign{id}" => 'campaign_id' }
rename => { "contact{first_name}" => 'contact_first_name' }
rename => { "campaign{recipients}" => 'campaign_recipients' }
}
}
filter {
mutate {
split => { "campaign_name" => '+' }
split => { "campaign_id" => '+' }
split => { "message" => '+' }
split => { "contacti_ip" => '+' }
split => { "contact_first_name" => '+' }
split => { "campaign_recipients" => '+' }
split => { "date_time" => '+' }
}
}
filter {
mutate {
join => { "campaign_name" => " " }
join => { "campaign_id" => " " }
join => { "contacti_ip" => " " }
join => { "contact_first_name" => " " }
join => { "campaign_recipients" => " " }
join => { "date_time" => " " }
}
}
output {
Elasticsearch {
hosts => [ 'es:9200' ]
#ssl => "true"
#ssl_certificate_verification => "false"
#ecs_compatibility => disabled
#manage_template => false
index => "test"
}
stdout {
codec => "json"
}
}
output {
google_bigquery {
project_id => "id_09"
dataset => "database"
table_prefix => "table"
csv_schema => "campaign_name:STRING,timestamp:TIMESTAMP,date_time:DATETIME,campaign_id:STRING,account_id:STRING,type:STRING"
json_key_file => "/key/key.json"
error_directory => "/tmp/bigquery-errors"
#date_pattern => "activecapmain"
ignore_unknown_values => true
flush_interval_secs => 30
}
}
Hello Every one My Above Pipe line is working now I want to all input in to Elasticsearch and selective feilds output want to add in google bigquery any Idea How can I achive that
and second thing in my http input I am getting urlencoded data so I used decode option to remove &% kindof signs but now I want to to remove every filed {} replacing with _ it is acceptable in to bigquery but I have to right big code can any one suggest smart solution for this