Filebeat > Nginx Module

Part 4: Final

    "panw.panos.destination.zone",
    "panw.panos.destination.interface",
    "panw.panos.network.pcap_id",
    "panw.panos.network.nat.community_id",
    "panw.panos.file.hash",
    "panw.panos.url.category",
    "panw.panos.flow_id",
    "panw.panos.threat.resource",
    "panw.panos.threat.id",
    "panw.panos.threat.name",
    "postgresql.log.timestamp",
    "postgresql.log.database",
    "postgresql.log.query",
    "rabbitmq.log.pid",
    "redis.log.role",
    "redis.slowlog.cmd",
    "redis.slowlog.key",
    "redis.slowlog.args",
    "santa.action",
    "santa.decision",
    "santa.reason",
    "santa.mode",
    "santa.disk.volume",
    "santa.disk.bus",
    "santa.disk.serial",
    "santa.disk.bsdname",
    "santa.disk.model",
    "santa.disk.fs",
    "santa.disk.mount",
    "certificate.common_name",
    "certificate.sha256",
    "hash.sha256",
    "suricata.eve.event_type",
    "suricata.eve.app_proto_orig",
    "suricata.eve.tcp.tcp_flags",
    "suricata.eve.tcp.tcp_flags_tc",
    "suricata.eve.tcp.state",
    "suricata.eve.tcp.tcp_flags_ts",
    "suricata.eve.fileinfo.sha1",
    "suricata.eve.fileinfo.state",
    "suricata.eve.fileinfo.sha256",
    "suricata.eve.fileinfo.md5",
    "suricata.eve.dns.type",
    "suricata.eve.dns.rrtype",
    "suricata.eve.dns.rrname",
    "suricata.eve.dns.rdata",
    "suricata.eve.dns.rcode",
    "suricata.eve.flow_id",
    "suricata.eve.email.status",
    "suricata.eve.http.redirect",
    "suricata.eve.http.protocol",
    "suricata.eve.http.http_content_type",
    "suricata.eve.in_iface",
    "suricata.eve.alert.category",
    "suricata.eve.alert.signature",
    "suricata.eve.ssh.client.proto_version",
    "suricata.eve.ssh.client.software_version",
    "suricata.eve.ssh.server.proto_version",
    "suricata.eve.ssh.server.software_version",
    "suricata.eve.tls.issuerdn",
    "suricata.eve.tls.sni",
    "suricata.eve.tls.version",
    "suricata.eve.tls.fingerprint",
    "suricata.eve.tls.serial",
    "suricata.eve.tls.subject",
    "suricata.eve.app_proto_ts",
    "suricata.eve.flow.state",
    "suricata.eve.flow.reason",
    "suricata.eve.app_proto_tc",
    "suricata.eve.smtp.rcpt_to",
    "suricata.eve.smtp.mail_from",
    "suricata.eve.smtp.helo",
    "suricata.eve.app_proto_expected",
    "system.auth.ssh.method",
    "system.auth.ssh.signature",
    "system.auth.ssh.event",
    "system.auth.sudo.error",
    "system.auth.sudo.tty",
    "system.auth.sudo.pwd",
    "system.auth.sudo.user",
    "system.auth.sudo.command",
    "system.auth.useradd.home",
    "system.auth.useradd.shell",
    "traefik.access.user_identifier",
    "traefik.access.frontend_name",
    "traefik.access.backend_url",
    "zeek.session_id",
    "zeek.connection.state",
    "zeek.connection.history",
    "zeek.connection.orig_l2_addr",
    "zeek.connection.resp_l2_addr",
    "zeek.dns.trans_id",
    "zeek.dns.query",
    "zeek.dns.qclass_name",
    "zeek.dns.qtype_name",
    "zeek.dns.rcode_name",
    "zeek.dns.answers",
    "zeek.http.status_msg",
    "zeek.http.info_msg",
    "zeek.http.tags",
    "zeek.http.password",
    "zeek.http.proxied",
    "zeek.http.client_header_names",
    "zeek.http.server_header_names",
    "zeek.http.orig_fuids",
    "zeek.http.orig_mime_types",
    "zeek.http.orig_filenames",
    "zeek.http.resp_fuids",
    "zeek.http.resp_mime_types",
    "zeek.http.resp_filenames",
    "zeek.files.fuid",
    "zeek.files.session_ids",
    "zeek.files.source",
    "zeek.files.analyzers",
    "zeek.files.mime_type",
    "zeek.files.filename",
    "zeek.files.parent_fuid",
    "zeek.files.md5",
    "zeek.files.sha1",
    "zeek.files.sha256",
    "zeek.files.extracted",
    "zeek.ssl.version",
    "zeek.ssl.cipher",
    "zeek.ssl.curve",
    "zeek.ssl.server_name",
    "zeek.ssl.next_protocol",
    "zeek.ssl.cert_chain",
    "zeek.ssl.cert_chain_fuids",
    "zeek.ssl.client_cert_chain",
    "zeek.ssl.client_cert_chain_fuids",
    "zeek.ssl.issuer",
    "zeek.ssl.client_issuer",
    "zeek.ssl.validation_status",
    "zeek.ssl.validation_code",
    "zeek.ssl.subject",
    "zeek.ssl.client_subject",
    "zeek.ssl.last_alert",
    "zeek.notice.connection_id",
    "zeek.notice.icmp_id",
    "zeek.notice.file.id",
    "zeek.notice.file.parent_id",
    "zeek.notice.file.source",
    "zeek.notice.file.mime_type",
    "zeek.notice.fuid",
    "zeek.notice.note",
    "zeek.notice.msg",
    "zeek.notice.sub",
    "zeek.notice.peer_name",
    "zeek.notice.peer_descr",
    "zeek.notice.actions",
    "zeek.notice.email_body_sections",
    "zeek.notice.email_delay_tokens",
    "zeek.notice.identifier",
    "fields.*"

Did that come from?

GET /_template/filebeat-7.1.1

If so further down you will see this.......which is correct. The out put is very long.

   "source" : {
      "properties" : {
        "geo" : {
          "properties" : {
            "region_iso_code" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "continent_name" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "city_name" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "country_iso_code" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "country_name" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "name" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "region_name" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "location" : {
              "type" : "geo_point"
            }
          }
        },

In the index pattern you should see these fields like this shown below...

if not delete that index pattern from the Kibana GUI
Mangement -> Index Patterns
and run ./filebeat setup again.

Apologies but I don't think I am able to help much more ... to do a clean install ... uninstall and re-install elasticsearch and make sure the the data directory under the elasticsearch install is removed before you reinstall.

Hi may I check with you. My client side is installing filebeat 7.3.0 while the ES server's Filebeat is on 7.1.1. Will there be an issue in terms of compatibility and the data format?

Hi @stephenb, I manage to get this fields, but the dashboard still cannot capture.

What I did is I removed the client Filebeat and installed the 7.1.1 version.

image

After that I notice I am facing some difficulties installing the logstash on the client server, its Centos 6.10 and with Java 1.8.0. Will there be any problem connecting these 2 servers ?

Hi is there anyone can help to troubleshoot this issue? thx

@stephenb , may I know what is your index name for this Nginx elastic ?

I came across some forum suggest that the template only recognize nginx-* indexes in order to load those templates.

If you use the filebeat nginx module with all the default settings the nginx logs will be indexed into indexes name with pattern filebeat-*

Hi bro, so this is expected? Are you able to help me on this issue? I have no solution to it

I found this online, wanted to load this template, but it fails. https://github.com/elastic/beats/blob/master/filebeat/module/nginx/_meta/kibana/7/dashboard/Filebeat-nginx-logs.json

Can anyone share why? is it incompatible ?

How can I check the Filebeat's dashboard compatibility and which version did I installed ?

Dear all, is there anyone can help on my question? It's been sometimes.

Almost 99% of my dashboards cannot display the data collected, while there are data collected from the other servers stored and received into our Elasticsearch DB. Wonder why the default dashboard cannot display?

I have tried various method to resolve this, but all of it just doesn't work.

Next, the indexes ./filebeat has been deleted and reindex previously, but yet there is no data is reflecting on the dashboards.

Hi, is there any command I can used to check what went wrong?

Currently I found most solution was to use "logstash" to compile, but I am facing some challenges to get it work. May I know is it possible to use "filebeat" to do the data collection instead?

Hi all, how come after I added these lines, I cant get the filebeat to start properly?

filebeat.config.modules:
  # Glob pattern for configuration loading
  path: ${path.config}/modules.d/*.yml

  # Set to true to enable config reloading
  reload.enabled: false

  # Period on which files under path should be checked for changes
  #reload.period: 10s

Once I add the code above, the module listing is able to work, but my filebeat will have issue running and keep restarting. What is the correct way to do it?