Logstash not send data to Elasticsearch

Elastic Stack Logstash
1

Hi,
I've configured Logstash for send data to Elasticsearch but this not receive data and index not found (i dont have firewall/iptables/selinux.).

My config logstash:

Input

input {
  file {
    path => "/var/log/zimbra.log"
    exclude => "*.gz"
  }
}

Filter

filter {
    # grok log lines by program name (listed alpabetically)
    if [program] =~ /^postfix.*\/anvil$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_ANVIL}$" ]
            tag_on_failure => [ "_grok_postfix_anvil_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/bounce$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_BOUNCE}$" ]
            tag_on_failure => [ "_grok_postfix_bounce_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/cleanup$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_CLEANUP}$" ]
            tag_on_failure => [ "_grok_postfix_cleanup_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/dnsblog$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_DNSBLOG}$" ]
            tag_on_failure => [ "_grok_postfix_dnsblog_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/error$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_ERROR}$" ]
            tag_on_failure => [ "_grok_postfix_error_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/local$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_LOCAL}$" ]
            tag_on_failure => [ "_grok_postfix_local_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/master$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_MASTER}$" ]
            tag_on_failure => [ "_grok_postfix_master_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/pickup$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_PICKUP}$" ]
            tag_on_failure => [ "_grok_postfix_pickup_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/pipe$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_PIPE}$" ]
            tag_on_failure => [ "_grok_postfix_pipe_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/postdrop$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_POSTDROP}$" ]
            tag_on_failure => [ "_grok_postfix_postdrop_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/postscreen$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_POSTSCREEN}$" ]
            tag_on_failure => [ "_grok_postfix_postscreen_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/qmgr$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_QMGR}$" ]
            tag_on_failure => [ "_grok_postfix_qmgr_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/scache$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_SCACHE}$" ]
            tag_on_failure => [ "_grok_postfix_scache_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/sendmail$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_SENDMAIL}$" ]
            tag_on_failure => [ "_grok_postfix_sendmail_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/smtp$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_SMTP}$" ]
            tag_on_failure => [ "_grok_postfix_smtp_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/lmtp$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_LMTP}$" ]
            tag_on_failure => [ "_grok_postfix_lmtp_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/smtpd$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_SMTPD}$" ]
            tag_on_failure => [ "_grok_postfix_smtpd_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/postsuper$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_POSTSUPER}$" ]
            tag_on_failure => [ "_grok_postfix_postsuper_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/tlsmgr$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_TLSMGR}$" ]
            tag_on_failure => [ "_grok_postfix_tlsmgr_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/tlsproxy$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_TLSPROXY}$" ]
            tag_on_failure => [ "_grok_postfix_tlsproxy_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/trivial-rewrite$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_TRIVIAL_REWRITE}$" ]
            tag_on_failure => [ "_grok_postfix_trivial_rewrite_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/discard$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_DISCARD}$" ]
            tag_on_failure => [ "_grok_postfix_discard_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*\/virtual$/ {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => [ "message", "^%{POSTFIX_VIRTUAL}$" ]
            tag_on_failure => [ "_grok_postfix_virtual_nomatch" ]
            add_tag        => [ "_grok_postfix_success" ]
        }
    } else if [program] =~ /^postfix.*/ {
        mutate {
            add_tag => [ "_grok_postfix_program_nomatch" ]
        }
    }

    # process key-value data if it exists
    if [postfix_keyvalue_data] {
        kv {
            source       => "postfix_keyvalue_data"
            trim_value   => "<>,"
            prefix       => "postfix_"
            remove_field => [ "postfix_keyvalue_data" ]
        }

        # some post processing of key-value data
        if [postfix_client] {
            grok {
                patterns_dir   => "/etc/logstash/patterns.d"
                match          => ["postfix_client", "^%{POSTFIX_CLIENT_INFO}$"]
                tag_on_failure => [ "_grok_kv_postfix_client_nomatch" ]
                remove_field   => [ "postfix_client" ]
            }
        }
        if [postfix_relay] {
            grok {
                patterns_dir   => "/etc/logstash/patterns.d"
                match          => ["postfix_relay", "^%{POSTFIX_RELAY_INFO}$"]
                tag_on_failure => [ "_grok_kv_postfix_relay_nomatch" ]
                remove_field   => [ "postfix_relay" ]
            }
        }
        if [postfix_delays] {
            grok {
                patterns_dir   => "/etc/logstash/patterns.d"
                match          => ["postfix_delays", "^%{POSTFIX_DELAYS}$"]
                tag_on_failure => [ "_grok_kv_postfix_delays_nomatch" ]
                remove_field   => [ "postfix_delays" ]
            }
        }
    }

    # process command counter data if it exists
    if [postfix_command_counter_data] {
        grok {
            patterns_dir   => "/etc/logstash/patterns.d"
            match          => ["postfix_command_counter_data", "^%{POSTFIX_COMMAND_COUNTER_DATA}$"]
            tag_on_failure => ["_grok_postfix_command_counter_data_nomatch"]
            remove_field   => ["postfix_command_counter_data"]
        }
    }

    # Do some data type conversions
    mutate {
        convert => [
            # list of integer fields
            "postfix_anvil_cache_size", "integer",
            "postfix_anvil_conn_count", "integer",
            "postfix_anvil_conn_rate", "integer",
            "postfix_client_port", "integer",
            "postfix_cmd_auth", "integer",
            "postfix_cmd_auth_accepted", "integer",
            "postfix_cmd_count", "integer",
            "postfix_cmd_count_accepted", "integer",
            "postfix_cmd_data", "integer",
            "postfix_cmd_data_accepted", "integer",
            "postfix_cmd_ehlo", "integer",
            "postfix_cmd_ehlo_accepted", "integer",
            "postfix_cmd_helo", "integer",
            "postfix_cmd_helo_accepted", "integer",
            "postfix_cmd_mail", "integer",
            "postfix_cmd_mail_accepted", "integer",
            "postfix_cmd_quit", "integer",
            "postfix_cmd_quit_accepted", "integer",
            "postfix_cmd_rcpt", "integer",
            "postfix_cmd_rcpt_accepted", "integer",
            "postfix_cmd_rset", "integer",
            "postfix_cmd_rset_accepted", "integer",
            "postfix_cmd_starttls", "integer",
            "postfix_cmd_starttls_accepted", "integer",
            "postfix_cmd_unknown", "integer",
            "postfix_cmd_unknown_accepted", "integer",
            "postfix_nrcpt", "integer",
            "postfix_postscreen_cache_dropped", "integer",
            "postfix_postscreen_cache_retained", "integer",
            "postfix_postscreen_dnsbl_rank", "integer",
            "postfix_relay_port", "integer",
            "postfix_server_port", "integer",
            "postfix_size", "integer",
            "postfix_status_code", "integer",
            "postfix_termination_signal", "integer",

            # list of float fields
            "postfix_delay", "float",
            "postfix_delay_before_qmgr", "float",
            "postfix_delay_conn_setup", "float",
            "postfix_delay_in_qmgr", "float",
            "postfix_delay_transmission", "float",
            "postfix_postscreen_violation_time", "float"
        ]
    }
}

output {
  elasticsearch {
    hosts => ["http://X.X.X.X:9200"]
    index => "postfix-%{+YYYY.MM.dd}"
  }
  #stdout { codec => rubydebug }
}

And the logstash-plain.log registre fine:

[2022-01-05T11:23:18,029][DEBUG][logstash.pipeline        ] filter received {"event"=>{"@timestamp"=>2022-01-05T14:23:17.895Z, "host"=>"zproxy-01.domain.com", "path"=>"/var/log/zimbra.log", "message"=>"Jan  5 11:23:17 zproxy-01 postfix/submission/smtpd[15039]: 6C50A6D3DBA: filter: RCPT from unknown[X.X.X.X]: <dariquelme@domain.com>: Sender address triggers FILTER smtp-amavis:[127.0.0.1]:10026; from=<dariquelme@domain.com> to=<plermanda@domain.com> proto=ESMTP helo=<DARIQUELME>", "@version"=>"1"}}
[2022-01-05T11:23:18,029][DEBUG][logstash.pipeline        ] filter received {"event"=>{"@timestamp"=>2022-01-05T14:23:17.897Z, "host"=>"zproxy-01.domain.com", "path"=>"/var/log/zimbra.log", "message"=>"Jan  5 11:23:17 zproxy-01 postfix/lmtp[20822]: 223326D4568: to=<galarcon-20200912@archive>, relay=archiving-05.domain.com[X.X.X.X]:7025, delay=4.7, delays=0.07/0.04/0.09/4.5, dsn=2.1.5, status=sent (250 2.1.5 Delivery OK)", "@version"=>"1"}}

In ths Elasticsearch server not list the index:

[root@elastic ~]# curl 'X.X.X.X:9200/_cat/indices?v'
health status index                           uuid                   pri rep docs.count docs.deleted store.size pri.store.size
green  open   .geoip_databases                tUt-kQ83SQ2E98Ml4TyVQw   1   0         44            0     40.8mb         40.8mb
green  open   .apm-custom-link                ogEJXaDmTEulpkUk3HAMjg   1   0          0            0       226b           226b
green  open   .kibana_task_manager_7.16.2_001 zPI6SDkvT5Sf1kwNmyv3hQ   1   0         17        54876      5.9mb          5.9mb
green  open   .kibana_7.16.2_001              NV-bF0POSeqffmBCWrXoYQ   1   0         46            5      2.3mb          2.3mb
green  open   .apm-agent-configuration        72sFdR__T1ajosIrh9Ngnw   1   0          0            0       226b           226b
green  open   .tasks                          lrrKWdfRRgu7zu_mrJsptA   1   0          2            0      7.8kb          7.8kb
[root@elastic ~]#

what's wrong?

Thanks.

2

Are you seeing events when you write to stdout? If not, it could be that new data is not being read. If Lostash previously read an input file, it will not read it again. You may be interested in reading about sincedb in File input plugin | Logstash Reference [7.16] | Elastic

3

yes, i execute run manual logstash and it is visualized well:

{
          "host" => "zproxy-01.domain.com",
       "message" => "Jan  5 18:40:20 zproxy-01 postfix/submission/smtpd[24662]: warning: Illegal address syntax from unknown[X.X.X.X] in RCPT command: <'user@domain.com'>",
    "@timestamp" => 2022-01-05T21:40:20.375Z,
      "@version" => "1",
          "path" => "/var/log/zimbra.log"
}
{
          "host" => "zproxy-01.domain.com",
       "message" => "Jan  5 18:40:20 zproxy-01 postfix/submission/smtpd[25675]: warning: Illegal address syntax from unknown[X.X.X.X] in RCPT command: <'eaguerrero@domain.com'>",
    "@timestamp" => 2022-01-05T21:40:20.377Z,
      "@version" => "1",
          "path" => "/var/log/zimbra.log"
}
{
          "host" => "zproxy-01.domain.com",
       "message" => "Jan  5 18:40:20 zproxy-01 postfix/submission/smtpd[25675]: warning: Illegal address syntax from unknown[X.X.X.X] in RCPT command: <'kmatamala@domain.com'>",
    "@timestamp" => 2022-01-05T21:40:20.378Z,
      "@version" => "1",
          "path" => "/var/log/zimbra.log"
}
4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.