Hi,
I've configured Logstash for send data to Elasticsearch but this not receive data and index not found (i dont have firewall/iptables/selinux.).
My config logstash:
Input
input {
file {
path => "/var/log/zimbra.log"
exclude => "*.gz"
}
}
Filter
filter {
# grok log lines by program name (listed alpabetically)
if [program] =~ /^postfix.*\/anvil$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_ANVIL}$" ]
tag_on_failure => [ "_grok_postfix_anvil_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/bounce$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_BOUNCE}$" ]
tag_on_failure => [ "_grok_postfix_bounce_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/cleanup$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_CLEANUP}$" ]
tag_on_failure => [ "_grok_postfix_cleanup_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/dnsblog$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_DNSBLOG}$" ]
tag_on_failure => [ "_grok_postfix_dnsblog_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/error$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_ERROR}$" ]
tag_on_failure => [ "_grok_postfix_error_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/local$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_LOCAL}$" ]
tag_on_failure => [ "_grok_postfix_local_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/master$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_MASTER}$" ]
tag_on_failure => [ "_grok_postfix_master_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/pickup$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_PICKUP}$" ]
tag_on_failure => [ "_grok_postfix_pickup_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/pipe$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_PIPE}$" ]
tag_on_failure => [ "_grok_postfix_pipe_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/postdrop$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_POSTDROP}$" ]
tag_on_failure => [ "_grok_postfix_postdrop_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/postscreen$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_POSTSCREEN}$" ]
tag_on_failure => [ "_grok_postfix_postscreen_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/qmgr$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_QMGR}$" ]
tag_on_failure => [ "_grok_postfix_qmgr_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/scache$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_SCACHE}$" ]
tag_on_failure => [ "_grok_postfix_scache_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/sendmail$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_SENDMAIL}$" ]
tag_on_failure => [ "_grok_postfix_sendmail_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/smtp$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_SMTP}$" ]
tag_on_failure => [ "_grok_postfix_smtp_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/lmtp$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_LMTP}$" ]
tag_on_failure => [ "_grok_postfix_lmtp_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/smtpd$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_SMTPD}$" ]
tag_on_failure => [ "_grok_postfix_smtpd_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/postsuper$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_POSTSUPER}$" ]
tag_on_failure => [ "_grok_postfix_postsuper_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/tlsmgr$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_TLSMGR}$" ]
tag_on_failure => [ "_grok_postfix_tlsmgr_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/tlsproxy$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_TLSPROXY}$" ]
tag_on_failure => [ "_grok_postfix_tlsproxy_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/trivial-rewrite$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_TRIVIAL_REWRITE}$" ]
tag_on_failure => [ "_grok_postfix_trivial_rewrite_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/discard$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_DISCARD}$" ]
tag_on_failure => [ "_grok_postfix_discard_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*\/virtual$/ {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "^%{POSTFIX_VIRTUAL}$" ]
tag_on_failure => [ "_grok_postfix_virtual_nomatch" ]
add_tag => [ "_grok_postfix_success" ]
}
} else if [program] =~ /^postfix.*/ {
mutate {
add_tag => [ "_grok_postfix_program_nomatch" ]
}
}
# process key-value data if it exists
if [postfix_keyvalue_data] {
kv {
source => "postfix_keyvalue_data"
trim_value => "<>,"
prefix => "postfix_"
remove_field => [ "postfix_keyvalue_data" ]
}
# some post processing of key-value data
if [postfix_client] {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => ["postfix_client", "^%{POSTFIX_CLIENT_INFO}$"]
tag_on_failure => [ "_grok_kv_postfix_client_nomatch" ]
remove_field => [ "postfix_client" ]
}
}
if [postfix_relay] {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => ["postfix_relay", "^%{POSTFIX_RELAY_INFO}$"]
tag_on_failure => [ "_grok_kv_postfix_relay_nomatch" ]
remove_field => [ "postfix_relay" ]
}
}
if [postfix_delays] {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => ["postfix_delays", "^%{POSTFIX_DELAYS}$"]
tag_on_failure => [ "_grok_kv_postfix_delays_nomatch" ]
remove_field => [ "postfix_delays" ]
}
}
}
# process command counter data if it exists
if [postfix_command_counter_data] {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => ["postfix_command_counter_data", "^%{POSTFIX_COMMAND_COUNTER_DATA}$"]
tag_on_failure => ["_grok_postfix_command_counter_data_nomatch"]
remove_field => ["postfix_command_counter_data"]
}
}
# Do some data type conversions
mutate {
convert => [
# list of integer fields
"postfix_anvil_cache_size", "integer",
"postfix_anvil_conn_count", "integer",
"postfix_anvil_conn_rate", "integer",
"postfix_client_port", "integer",
"postfix_cmd_auth", "integer",
"postfix_cmd_auth_accepted", "integer",
"postfix_cmd_count", "integer",
"postfix_cmd_count_accepted", "integer",
"postfix_cmd_data", "integer",
"postfix_cmd_data_accepted", "integer",
"postfix_cmd_ehlo", "integer",
"postfix_cmd_ehlo_accepted", "integer",
"postfix_cmd_helo", "integer",
"postfix_cmd_helo_accepted", "integer",
"postfix_cmd_mail", "integer",
"postfix_cmd_mail_accepted", "integer",
"postfix_cmd_quit", "integer",
"postfix_cmd_quit_accepted", "integer",
"postfix_cmd_rcpt", "integer",
"postfix_cmd_rcpt_accepted", "integer",
"postfix_cmd_rset", "integer",
"postfix_cmd_rset_accepted", "integer",
"postfix_cmd_starttls", "integer",
"postfix_cmd_starttls_accepted", "integer",
"postfix_cmd_unknown", "integer",
"postfix_cmd_unknown_accepted", "integer",
"postfix_nrcpt", "integer",
"postfix_postscreen_cache_dropped", "integer",
"postfix_postscreen_cache_retained", "integer",
"postfix_postscreen_dnsbl_rank", "integer",
"postfix_relay_port", "integer",
"postfix_server_port", "integer",
"postfix_size", "integer",
"postfix_status_code", "integer",
"postfix_termination_signal", "integer",
# list of float fields
"postfix_delay", "float",
"postfix_delay_before_qmgr", "float",
"postfix_delay_conn_setup", "float",
"postfix_delay_in_qmgr", "float",
"postfix_delay_transmission", "float",
"postfix_postscreen_violation_time", "float"
]
}
}
output {
elasticsearch {
hosts => ["http://X.X.X.X:9200"]
index => "postfix-%{+YYYY.MM.dd}"
}
#stdout { codec => rubydebug }
}
And the logstash-plain.log registre fine:
[2022-01-05T11:23:18,029][DEBUG][logstash.pipeline ] filter received {"event"=>{"@timestamp"=>2022-01-05T14:23:17.895Z, "host"=>"zproxy-01.domain.com", "path"=>"/var/log/zimbra.log", "message"=>"Jan 5 11:23:17 zproxy-01 postfix/submission/smtpd[15039]: 6C50A6D3DBA: filter: RCPT from unknown[X.X.X.X]: <dariquelme@domain.com>: Sender address triggers FILTER smtp-amavis:[127.0.0.1]:10026; from=<dariquelme@domain.com> to=<plermanda@domain.com> proto=ESMTP helo=<DARIQUELME>", "@version"=>"1"}}
[2022-01-05T11:23:18,029][DEBUG][logstash.pipeline ] filter received {"event"=>{"@timestamp"=>2022-01-05T14:23:17.897Z, "host"=>"zproxy-01.domain.com", "path"=>"/var/log/zimbra.log", "message"=>"Jan 5 11:23:17 zproxy-01 postfix/lmtp[20822]: 223326D4568: to=<galarcon-20200912@archive>, relay=archiving-05.domain.com[X.X.X.X]:7025, delay=4.7, delays=0.07/0.04/0.09/4.5, dsn=2.1.5, status=sent (250 2.1.5 Delivery OK)", "@version"=>"1"}}
In ths Elasticsearch server not list the index:
[root@elastic ~]# curl 'X.X.X.X:9200/_cat/indices?v'
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
green open .geoip_databases tUt-kQ83SQ2E98Ml4TyVQw 1 0 44 0 40.8mb 40.8mb
green open .apm-custom-link ogEJXaDmTEulpkUk3HAMjg 1 0 0 0 226b 226b
green open .kibana_task_manager_7.16.2_001 zPI6SDkvT5Sf1kwNmyv3hQ 1 0 17 54876 5.9mb 5.9mb
green open .kibana_7.16.2_001 NV-bF0POSeqffmBCWrXoYQ 1 0 46 5 2.3mb 2.3mb
green open .apm-agent-configuration 72sFdR__T1ajosIrh9Ngnw 1 0 0 0 226b 226b
green open .tasks lrrKWdfRRgu7zu_mrJsptA 1 0 2 0 7.8kb 7.8kb
[root@elastic ~]#
what's wrong?
Thanks.