Unable to fetch mappings - BRO IDS + ELK Stack


(Adam) #1

Hello Everyone, i'll start by saying...newbe here and i'm lost. I'm trying to setup BRO and ELK stack. I'm following https://github.com/BrashEndeavours/bro-elk-IDS. I have my BRO ids and ELK stack on the same server for testing. I'm running Ubuntu Desktop 16.04.2 I'm running into an issue where when I login to Kibana its saying it is unable to fetch mappings. I've confirmed that BRO is generating logs and running. I'm thinking its an issue with my logstash_bro.conf but honestly.... i'm not sure. Here is my current configuration:

Logstash_bro.conf

16:42:53 ✔ root:/nsm/logstash/config/pipeline$ cat logstash_bro.conf 
input {
  stdin {
  }

# Production Logs#############################
  bro {
    type => "app_stats"
    path => "/nsm/bro/logs/current/app_stats.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "conn"
    path => "/nsm/bro/logs/current/conn.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "dns_entropy"
    path => "/nsm/bro/logs/current/dns_entropy.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "dhcp"
    path => "/nsm/bro/logs/current/dhcp.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "dns"
    path => "/nsm/bro/logs/current/dns.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "dpd"
    path => "/nsm/bro/logs/current/dpd.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "files"
    path => "/nsm/bro/logs/current/files.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "http"
    path => "/nsm/bro/logs/current/http.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "http_entropy"
    path => "/nsm/bro/logs/current/http_entropy.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "kerberos"
    path => "/nsm/bro/logs/current/kerberos.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "known_certs"
    path => "/nsm/bro/logs/current/known_certs.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "known_hosts"
    path => "/nsm/bro/logs/current/known_hosts.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "known_services"
    path => "/nsm/bro/logs/current/known_services.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "intel"
    path => "/nsm/bro/logs/current/intel.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "notice"
    path => "/nsm/bro/logs/current/notice.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "pe"
    path => "/nsm/bro/logs/current/pe.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "rdp"
    path => "/nsm/bro/logs/current/rdp.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "sip"
    path => "/nsm/bro/logs/current/sip.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "smtp"
    path => "/nsm/bro/logs/current/smtp.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "socks"
    path => "/nsm/bro/logs/current/socks.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "software"
    path => "/nsm/bro/logs/current/software.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "ssh"
    path => "/nsm/bro/logs/current/ssh.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "ssl"
    path => "/nsm/bro/logs/current/ssl.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "syslog"
    path => "/nsm/bro/logs/current/syslog.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "traceroute"
    path => "/nsm/bro/logs/current/traceroute.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "weird"
    path => "/nsm/bro/logs/current/weird.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  bro {
    type => "x509"
    path => "/nsm/bro/logs/current/x509.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
}

# Benchmark!
filter {
    metrics {
        meter => "benchmark"
        add_tag => "metric"
    }
}

output {
    if "metric" in [tags] {
        stdout {
            codec => line {
                format => "%{[benchmark][rate_1m]} requests/sec, ( %{[benchmark][count]} total)"
            }
        }
    }

#  stdout { codec => rubydebug }
  elasticsearch {
    hosts => ["127.0.0.1:9200"]
  }
}

Any help would be greatly appreciated!


(Adam) #2

Also here is my netstat and bro log directory.

Netstat

tcp        0      0 0.0.0.0:47760           0.0.0.0:*               LISTEN      2041/bro        
tcp        0      0 127.0.1.1:53            0.0.0.0:*               LISTEN      938/dnsmasq     
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      1390/sshd       
tcp        0      0 0.0.0.0:5601            0.0.0.0:*               LISTEN      776/node        
tcp6       0      0 127.0.0.1:9200          :::*                    LISTEN      1388/java       
tcp6       0      0 :::47760                :::*                    LISTEN      2041/bro        
tcp6       0      0 127.0.0.1:9300          :::*                    LISTEN      1388/java       
tcp6       0      0 :::22                   :::*                    LISTEN      1390/sshd       

--
BRO logs Directory

16:46:06 ✔ root:/nsm/bro/logs/current$ ls -la
total 308
drwxr-xr-x 4 bro nsm  4096 Apr 20 16:32 .
drwxr-xr-x 5 bro nsm  4096 Apr 20 16:19 ..
-rw-r--r-- 1 bro nsm   259 Apr 20 16:32 capture_loss.log
-rw-r--r-- 1 bro nsm   122 Apr 20 16:17 .cmdline
-rw-r--r-- 1 bro nsm   591 Apr 20 16:17 communication.log
-rw-r--r-- 1 bro nsm 22741 Apr 20 16:46 conn.log
-rw-r--r-- 1 bro nsm   749 Apr 20 16:46 dhcp.log
-rw-r--r-- 1 bro nsm 82177 Apr 20 16:46 dns.log
-rw-r--r-- 1 bro nsm   303 Apr 20 16:17 .env_vars
drwxr-xr-x 2 bro nsm 12288 Apr 20 16:42 extract_files
-rw-r--r-- 1 bro nsm 31652 Apr 20 16:42 files.log
-rw-r--r-- 1 bro nsm 10749 Apr 20 16:42 http.log
-rw-r--r-- 1 bro nsm   202 Apr 20 16:17 known_hosts.log
-rw-r--r-- 1 bro nsm   284 Apr 20 16:31 known_services.log
-rw-r--r-- 1 bro nsm 23534 Apr 20 16:17 loaded_scripts.log
-rw-r--r-- 1 bro nsm   226 Apr 20 16:17 packet_filter.log
-rw-r--r-- 1 bro nsm     5 Apr 20 16:17 .pid
-rw-r--r-- 1 bro nsm   619 Apr 20 16:18 software.log
-rw-r--r-- 1 bro nsm   848 Apr 20 16:18 ssh.log
-rw-r--r-- 1 bro nsm 11579 Apr 20 16:25 ssl.log
-rw-r--r-- 1 bro nsm    58 Apr 20 16:17 .startup
drwx------ 3 bro nsm  4096 Apr 20 16:17 .state
-rw-r--r-- 1 bro nsm  1182 Apr 20 16:42 stats.log
-rwx------ 1 bro nsm    18 Apr 20 16:17 .status
-rw-r--r-- 1 bro nsm    20 Apr 20 16:17 stderr.log
-rw-r--r-- 1 bro nsm   188 Apr 20 16:17 stdout.log
-rw-r--r-- 1 bro nsm  1641 Apr 20 16:40 weird.log
-rw-r--r-- 1 bro nsm 14720 Apr 20 16:25 x509.log
16:46:07 ✔ root:/nsm/bro/logs/current$ 

(Magnus Bäck) #3

I'm running into an issue where when I login to Kibana its saying it is unable to fetch mappings.

Have you verified that Elasticsearch indexes have been created? Use e.g. ES's /_cat/indices API.


(system) #4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.