Compare 2 fields and drop the matching

[Dr.Dark92 (Majid Aljerdani)]

Hi,
I am trying to make a Dashbaord for Bind9 engine,
in brief i have 3 indexes,

first index for Blacklist logs
second index for whitelist logs
third index for resolver logs.

the goal is : to drop any blacklisted domains or whitelisted domains from resolver logs.
my current progress, if i compare the field with a value it works fine E.g

if [URL] in ["website.com", "website2.com"] {
  	drop{}

however if i try to compare this way, it doesn't work

if [URL] in [Website] {
  	drop{}

this is my full logstash config file

########################## 1.0 - Stream-1-Resolver-logs-Stream1-Resolver ############################

input {
  beats {
  type => "resolver"
    port => 5044
  }
}


####################################################################################################

################################### timezone filter ########################################
filter {
  ruby {
    code => "
      require 'time'
      tz = Time.now.getlocal('+04:00')
      event.set('timezone', tz.strftime('%Y.%m.%d.%H'))
    "
 }
 }
#####################################################################################################

filter {
	if "domains" in [tags] {
    	grok { match => {"message" => "%{GREEDYDATA:BlacklistWebsite}"
  }

    }
  }
  }



########################## 1.1 - Stream-1-Resolver Resolver Grok ####################################
filter {

 if "resolver" in [tags] {
grok { match => {"message" => "%{MONTHDAY:Day}-%{MONTH:Month}-%{YEAR:Year} (?<time>\d{2}):%{GREEDYDATA}%{SPACE}%{WORD:Client}%{SPACE}@%{WORD:word}%{SPACE}%{IPV4:IP}#%{SPACE}%{WORD:request_Number}%{SPACE}%{DATA:word}:%{SPACE}%{DATA:ACTION_Type}:%{SPACE}%{GREEDYDATA:URL} IN A %{SPACE}%{GREEDYDATA:Method} +%{GREEDYDATA:Resolver}"
}
}


mutate {
	add_field => {"logtype" => "resolver"}
}
}
  if [URL] in [Website] {
  	drop{}
}
}
#####################################################################################################



################### 1.2 -  Stream-1-Resolver  Index #########################################
output {
if "resolver" in [tags] {
  stdout { codec => rubydebug }
  elasticsearch {
    hosts => ["https://localhost:9200"]
    index => "resolver-%{timezone}"
    user => "elastic"
    password => "user@user.com"
    ssl => true
ssl_certificate_verification => false
cacert => "/etc/logstash/http_ca.crt"
}
}
}
################################################################################################

########################## 2.0 - Stream-2-RPZ-logs-Stream2-RPZ ################################

####################################################################################################

########################## 2.2 - Stream-2-RPZ Grok #################################################

filter {
if "blacklist" in [tags] {
 grok { match=> {"message" => "%{MONTHDAY:Day}-%{MONTH:Month}-%{YEAR:Year} (?<time>\d{2}):%{GREEDYDATA}%{SPACE}%{WORD:Type}:%{SPACE}%{LOGLEVEL:LogLevel}:%{SPACE}%{WORD:Client}%{SPACE}@%{WORD:Data}%{SPACE}%{IPV4:IP}#%{WORD:Number}%{SPACE}\(%{GREEDYDATA:Website}\):%{SPACE}%{WORD:Type}%{SPACE}%{WORD:DNSRESPONSE}%{DATA:local-data}%{SPACE}%{WORD:Word}%{SPACE}%{WORD:Redirect}%{SPACE}%{DATA:URL}.*(?<RPZ>whitelist|blacklist)\.local\.com.*"
 }
 }
if [RPZ] == "whitelist" {
	mutate {
	replace => { "tags" => "whitelist" }
	add_field => {"logtype" => "whitelist"}
 }
 }
 }
 }
####################################################################################################

########################## 2.3 - Stream-2-RPZ Index #################################################

output {

 if [RPZ] == "blacklist" {
  stdout { codec => rubydebug }
  elasticsearch {
    hosts => ["https://localhost:9200"]
    index => "blacklist-%{timezone}"
    user => "elastic"
    password => "elastic@elastic.com"
    ssl => true
ssl_certificate_verification => false
cacert => "/etc/logstash/http_ca.crt"
}
}
if [RPZ] == "whitelist" {
  stdout { codec => rubydebug }
  elasticsearch {
    hosts => ["https://localhost:9200"]
    index => "whitelist-%{timezone}"
    user => "elastic"
    password => "elastic@elastic.com"
    ssl => true
ssl_certificate_verification => false
cacert => "/etc/logstash/http_ca.crt"
}
}
}

output {
if "domains" in [tags] {
stdout { codec => rubydebug }
  elasticsearch {
    hosts => ["https://localhost:9200"]
    index => "domains-%{timezone}"
    user => "elastic"
    password => "user@user.com"
    ssl => true
ssl_certificate_verification => false
cacert => "/etc/logstash/http_ca.crt"
}
}
}


#####################################################################################################

i have been working on it for 3 weeks and cant get any progress for, planning to fix it using bash script but bash only works on single thread it wont work on large amount of data.

so thanks in advance

Majid Aljerdani

[Badger]

@Dr.Dark92

if [URL] in [Website] {
  	drop{}

You are doing that comparison in section 1.1 of your configuration, but [Website] does not exist until the grok in section 2.2 executes, so that comparison will never match.