Connecting error publishing events (retrying)


(Ranjeet Ranjan) #1

Hi,
I am getting below error after installing X-pack.

Connecting error publishing events (retrying): dial tcp 192.168.154.170:5044: getsockopt: connection refused

Here is my filebeat.yml

############################# Filebeat ######################################
filebeat:
  # List of prospectors to fetch data.
  prospectors:
    # Each - is a prospector. Below are the prospector specific configurations
    -
     
      paths:
        - /var/log/*.log
        - /var/log/messages
        # foreman
        - /var/log/foreman/*.log
        - /var/log/foreman-proxy/*.log
        # openstack
        - /var/log/nova/*.log
        - /var/log/neutron/*.log
        - /var/log/cinder/*.log
        - /var/log/keystone/*.log
        - /var/log/horizon/*.log
        
       input_type: log

      document_type: syslog

############################# Output ##########################################

output:

  ### Elasticsearch as output
#  elasticsearch:
  logstash:
    hosts: ["192.168.154.170:5044"]
    bulk_max_size: 1024
    protocol: "https"
    username: "username"
    password: "password"

    ssl:
      # List of root certificates for HTTPS server verifications
      certificate_authorities: [/etc/filebeat/filebeat-forwarder.crt]

      # Certificate for TLS client authentication
      certificate: "/etc/filebeat/filebeat-forwarder.crt"

      # Client Certificate Key
      key: "/etc/filebeat/filebeat-forwarder.key"

      supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2, SSLv3]
      # Configure cipher suites to be used for TLS connections
      #cipher_suites: []

      # Configure curve types for ECDHE based cipher suites
      #curve_types: []

      # Configure minimum TLS version allowed for connection to logstash
      #min_version: 1.0
      min_version: 1.0
      # Configure maximum TLS version allowed for connection to logstash
      #max_version: 1.2
      max_version: 1.2

  ### Logstash as output
  #logstash:
    # The Logstash hosts
    #hosts: ["localhost:5044"]

    # Number of workers per Logstash host.
    #worker: 1

    # Optional load balance the events between the Logstash hosts
    #loadbalance: true

    #index: filebeat

    # Optional TLS. By default is off.
    #tls:
      # List of root certificates for HTTPS server verifications
      #certificate_authorities: ["/etc/pki/root/ca.pem"]

      # Certificate for TLS client authentication
      #certificate: "/etc/pki/client/cert.pem"

      # Client Certificate Key
      #certificate_key: "/etc/pki/client/cert.key"

      # Controls whether the client verifies server certificates and host name.
      # If insecure is set to true, all server host names and certificates will be
      # accepted. In this mode TLS based connections are susceptible to
      # man-in-the-middle attacks. Use only for testing.
      #insecure: true

      # Configure cipher suites to be used for TLS connections
      #cipher_suites: []

      # Configure curve types for ECDHE based cipher suites
      #curve_types: []


  ### File as output
  #file:
    # Path to the directory where to save the generated files. The option is mandatory.
    #path: "/tmp/filebeat"

    # Name of the generated files. The default is `filebeat` and it generates files: `filebeat`, `filebeat.1`, `filebeat.2`, etc.
    #filename: filebeat

    # Maximum size in kilobytes of each file. When this size is reached, the files are
    # rotated. The default value is 10 MB.
    #rotate_every_kb: 10000

    # Maximum number of files under path. When this number of files is reached, the
    # oldest file is deleted and the rest are shifted from last to first. The default
    # is 7 files.
    #number_of_files: 7


  ### Console output
  # console:
    # Pretty print json event
    #pretty: false



############################# Logging #########################################

# There are three options for the log ouput: syslog, file, stderr.
# Under Windos systems, the log files are per default sent to the file output,
# under all other system per default to syslog.
logging:

  # Send all logging output to syslog. On Windows default is false, otherwise
  # default is true.
  #to_syslog: true

  # Write all logging output to files. Beats automatically rotate files if rotateeverybytes
  # limit is reached.
  #to_files: false

  # To enable logging to files, to_files option has to be set to true
  files:
    # The directory where the log files will written to.
    #path: /var/log/mybeat

    # The name of the files where the logs are written to.
    #name: mybeat

    # Configure log file size limit. If limit is reached, log file will be
    # automatically rotated
    rotateeverybytes: 10485760 # = 10MB

    # Number of rotated log files to keep. Oldest files will be deleted first.
    #keepfiles: 7

  # Enable debug output for selected components. To enable all selectors use ["*"]
  # Other available selectors are beat, publish, service
  # Multiple selectors can be chained.
  #selectors: [ ]

  # Sets log level. The default log level is error.
  # Available log levels are: critical, error, warning, info, debug
  level: debug

Here is Logstash.conf

[root@elk ~]# cat /etc/logstash/conf.d/logstash.conf
input {
  tcp {
    port => 5000
    type => syslog
  }
  udp {
    port => 5000
    type => syslog
  }
}

filter {
  if [type] == "syslog" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{host}" ]
    }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }
  }
}

output {
  elasticsearch { 
    hosts => ["localhost:9200"] 
    user => username
    password => password 

# uncomment this for debug messages
#  stdout { codec => rubydebug 
 }
}

input {
  beats {
    port => 5044
    ssl => true
    ssl_certificate => ["/usr/share/logstash/filebeat-forwarder.crt"]
    ssl_key => ["/usr/share/logstash/filebeat-forwarder.key"]
    ssl_verify_mode => none
  }
}

(Tudor Golubenco) #2

The error indicates that port 5044 of Logstash is not accessible from Filebeat. Can you check with telnet from the Filebeat machine? I.e. telnet 192.168.154.170 5044. Perhaps Logstash didn't start or there's some connectivity issue.


(Ranjeet Ranjan) #3

I can see logstash is not started. Getting below error during start.

[2017-05-31T15:20:21,114][ERROR][logstash.inputs.tcp      ] Could not start TCP server: Address in use {:host=>"0.0.0.0", :port=>5000}
[2017-05-31T15:20:21,225][ERROR][logstash.pipeline        ] Error registering plugin {:plugin=>"<LogStash::Inputs::Tcp port=>5000, type=>\"syslog\", id=>\"c4f07f53729a9716fdabd05f491f407187b9cf36-7\", enable_metric=>true, codec=><LogStash::Codecs::Line id=>\"line_3a57e7a6-8ef3-4dde-b173-a4335682a11d\", enable_metric=>true, charset=>\"UTF-8\", delimiter=>\"\\n\">, host=>\"0.0.0.0\", data_timeout=>-1, mode=>\"server\", proxy_protocol=>false, ssl_enable=>false, ssl_verify=>true, ssl_key_passphrase=><password>>", :error=>"Address already in use - bind - Address already in use"}
[2017-05-31T15:20:22,777][ERROR][logstash.agent           ] Pipeline aborted due to error {:exception=>#<Errno::EADDRINUSE: Address already in use - bind - Address already in use>, :backtrace=>["org/jruby/ext/socket/RubyTCPServer.java:118:in `initialize'", "org/jruby/RubyIO.java:871:in `new'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-input-tcp-4.1.0/lib/logstash/inputs/tcp.rb:279:in `new_server_socket'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-input-tcp-4.1.0/lib/logstash/inputs/tcp.rb:89:in `register'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:268:in `register_plugin'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:279:in `register_plugins'", "org/jruby/RubyArray.java:1613:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:279:in `register_plugins'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:429:in `start_inputs'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:323:in `start_workers'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:214:in `run'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:398:in `start_pipeline'"]}
[2017-05-31T15:20:24,083][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}
[2017-05-31T15:20:26,539][WARN ][logstash.agent           ] stopping pipeline {:id=>"main"}

(Tudor Golubenco) #4

It's an address in use error, you have something listening on the same port. Perhaps another instance of LS?


(Ranjeet Ranjan) #5

I dont see any other instance of LS. Please help to find if other instance is running. Also see this error after we install x-pack


(system) #6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.