Pipeline definition in filebeat.yml doesn't work

[Burga]

Hi ,

trying to configure filebeat to use pipeline , but it doesnt work

my filebeat.yml pipeline configuration looks like this

filebeat.inputs:

- type: log

  paths:

    - /var/log/messages
#     - /var/log/*.log
  document_type: syslog
  fields:
        log_type: "syslog"
  enable: true
  fields_under_root: true
  pipeline: "filebeat-7.6.1-system-auth-pipeline"
  multiline.pattern: '^[0-9][0-9]:[0-9][0-9]:'
  multiline.negate: true
  multiline.match: after

- type: log

  paths:

   - /var/log/secure
 #   - /var/log/*.log

  document_type: syslog
  fields:
        log_type: "secure"
  enable: true
  pipeline: "filebeat-7.6.1-system-auth-pipeline"
  fields_under_root: true
  multiline.pattern: '^[0-9][0-9]:[0-9][0-9]:'
  multiline.negate: true
  multiline.match: after

filebeat is up and working but it doesn't use pipeline

need your assistance.

Thanks in advance.

[shaunak (Shaunak Kashyap)]

Please always post your complete filebeat.yml configuration (with sensitive information redacted), otherwise we can't tell if some other configuration setting is interfering and we also can't easily try to reproduce your issue.

Shaunak

[Burga]

sorry

filebeat.modules:


filebeat.inputs:

- type: log

  paths:

    - /var/log/messages
#     - /var/log/*.log
  document_type: syslog
  fields:
        log_type: "syslog"
  enable: true
  fields_under_root: true
  **pipeline: "filebeat-7.6.1-system-auth-pipeline"**
  multiline.pattern: '^[0-9][0-9]:[0-9][0-9]:'
  multiline.negate: true
  multiline.match: after

- type: log

  paths:

   - /var/log/secure
 #   - /var/log/*.log

  document_type: syslog
  fields:
        log_type: "secure"
  enable: true
   **pipeline: "filebeat-7.6.1-system-auth-pipeline"**
  fields_under_root: true
  multiline.pattern: '^[0-9][0-9]:[0-9][0-9]:'
  multiline.negate: true
  multiline.match: after


- type: log

  paths:

  - /var/log/yum.log

  document_type: syslog
  fields:
        log_type: "yum"
  enable: true
  pipeline: "filebeat-7.6.1-system-auth-pipeline"
  fields_under_root: true
  multiline.pattern: '^[0-9][0-9]:[0-9][0-9]:'
  multiline.negate: true
  multiline.match: after



#----------------------------- Logstash output --------------------------------
output.logstash:
  enabled: true
  # The Logstash hosts
  #hosts: ["localhost:5044"]
  hosts: ["il-infra-logs2:5044","il-infra-logs1:5044"]
  loadbalance: true
  #logging.level: debug
  #logging.to_files: true
  #logging.files:
  #path: /var/log/filebeat
  # The name of the files where the logs are written to.
  #name: filebeat
  name: filebeat


  # Optional SSL. By default is off.
  # List of root certificates for HTTPS server verifications
  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]

  # Certificate for SSL client authentication
  #ssl.certificate: "/etc/pki/client/cert.pem"

  # Client Certificate Key
  #ssl.key: "/etc/pki/client/cert.key"



#============================= Filebeat modules ===============================


filebeat.config.modules:
  # Glob pattern for configuration loading
  path: ${path.config}/modules.d/*.yml

  # Set to true to enable config reloading
  reload.enabled: false

  # Period on which files under path should be checked for changes
  #reload.period: 10s




#================================ Processors =====================================

# Configure processors to enhance or manipulate events generated by the beat.

processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~
  - add_docker_metadata: ~
  - add_kubernetes_metadata: ~








#================================ Logging =====================================

# Sets log level. The default log level is info.
# Available log levels are: error, warning, info, debug
#logging.level: debug

# At debug level, you can selectively enable logging only for some components.
# To enable all selectors use ["*"]. Examples of other selectors are "beat",
# "publish", "service".
#logging.selectors: ["*"]


logging.level: info
logging.to_files: true
logging.to_syslog: false
logging.json: true
logging.files:
    path: '/var/log/filebeat'
    name: 'filebeat'
    keepfiles: '3'
    permissions: '0644'