Access data event and field in logstash condition error

tatdat · December 17, 2017, 5:06am

Im ussing LS 6.1

When i use condition in logstash, it not work. i dont know that why? I should be run into condition (True) instead of False.

It not run in " if [nginx][access][remote_ip] { ... } "

Sample log

{
  "_index": "fb-fim-websites-2017.50",
  "_type": "doc",
  "_id": "vyPVYmABWfAqnld66ayn",
  "_version": 1,
  "_score": null,
  "_source": {
    "type": "nginx-access",
    "nginx.access.user_name": "-",
    "@timestamp": "2017-12-17T04:58:03.767Z",
    "nginx.access.ident": "-",
    "nginx.access.url": "/2017/12/07/co-nang-lam-nhung-viec-nho-voi-mot-tinh-yeu-lon-1512629884_660x0.jpg",
    "beat": {
      "version": "6.0.1",
      "hostname": "Nginx-LB-Outside-01",
      "name": "Nginx-LB-Outside-01"
    },
    "offset": 26800425,
    "@version": "1",
    "host": "Nginx-LB-Outside-01",
    "nginx.access.body_sent.bytes": "0",
    "message": "66.160.201.42 - - [17/Dec/2017:11:58:03 +0700] \"my-website.com\" \"GET /2017/12/07/mot-tinh-yeu-lon-1512629884_660x0.jpg HTTP/1.1\" 304 0 \"http://my-website.com/\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36\" \"-\"",
    "nginx.access.referrer": "http://my-website.com/",
    "nginx.access.http_forwarder": "-",
    "beatname": "fb-fim-websites",
    "source": "/var/log/nginx/log/chungta-access.log",
    "tags": [
      "beats_input_codec_plain_applied"
    ],
    "beattype": "doc",
    "nginx.access.remote_ip": "66.160.201.42",
    "prospector": {
      "type": "log"
    },
    "nginx.access.time": "17/Dec/2017:11:58:03 +0700",
    "nginx.access.method": "GET",
    "nginx.access.response_code": "304",
    "nginx.access.agent": "\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36\"",
    "nginx.access.http_version": "1.1",
    "nginx.access.domain": "my-website.com"
  },
  "fields": {
    "@timestamp": [
      "2017-12-17T04:58:03.767Z"
    ]
  },
  "highlight": {
    "type": [
      "@kibana-highlighted-field@nginx-access@/kibana-highlighted-field@"
    ]
  },
  "sort": [
    1513486683767
  ]
}

LS Config

filter {
  if [type] == "nginx-access" {
    grok {
      patterns_dir => ["/etc/logstash/patterns"]
      match => { "message" => "%{NGINXACCESS}" }
      match => { "message" => "%{NGINXACCESSCUSTOME}" }
    }

    if [nginx][access][remote_ip] {
      # Check if destination IP address is private.
      cidr {
        id => "nginx-postproc-cidr-remote_ip"
        address => [ "%{[nginx][access][remote_ip]}" ]
        network => [ "0.0.0.0/32", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fc00::/7", "127.0.0.0/8", "::1/128","169.254.0.0/16", "fe80::/10","224.0.0.0/4", "ff00::/8","255.255.255.255/32" ]
        add_field => { "[nginx][access][remote_ip_locality]" => "private" }
      }
      # Check to see if dst_locality exists. If it doesn't the remote_ip didn't match a private address space and locality must be public.
      if ![nginx][access][remote_ip_locality] {
        mutate {
        id => "nginx-postproc-remote_ip-public"
        add_field => { "[nginx][access][remote_ip_locality]" => "public" }
        replace => { "[nginx][access][remote_ip_locality]" => "public" }
        }
        geoip {
          id => "nginx-postproc-geoip_remote_ip-city"
          source => "[nginx][access][remote_ip]"
          default_database_type => "City"
          target => "[nginx][access][geoip]"
          database => "/etc/logstash/GeoLite2-City.mmdb"
        }
        geoip {
          id => "nginx-postproc-geoip_remote_ip-asn"
          source => "[nginx][access][remote_ip]"
          default_database_type => "ASN"
          target => "[nginx][access][geoip]"
          database => "/etc/logstash/GeoLite2-ASN.mmdb"
        }
        # Populate geoip_dst.autonomous_system.
        if [nginx][access][geoip][as_org] {
          if [nginx][access][geoip][asn] {
            mutate {
            id => "nginx-postproc-remote_ip-as-from-as_org-asn"
            add_field => { "[nginx][access][geoip][autonomous_system]" => "%{[nginx][access][geoip][as_org]} (%{[nginx][access][geoip][asn]})" }
          }
          } else {
            mutate {
            id => "nginx-postproc-remote_ip-as-from-as_org"
            add_field => { "[nginx][access][geoip][autonomous_system]" => "%{[nginx][access][geoip][as_org]}" }
            }
          }
        } else if [geoip][asn] {
          mutate {
            id => "nginx-postproc-remote_ip-as-from-asn"
            add_field => { "[nginx][access][geoip][autonomous_system]" => "%{[nginx][access][geoip][asn]}" }
          }
        }
      } else {
        mutate {
          id => "nginx-postproc-remote_ip-as-private"
          add_field => { "[nginx][access][geoip][autonomous_system]" => "PRIVATE" }
        }
        translate {
          regex => true
          dictionary_path => "/etc/logstash/translates/internal-ip.yaml"
          field => "nginx.access.remote_ip"
        }
        json {
          source => "translation"
          remove_field => ["translation"]   
        }
      }
    } 
        
    date {
        match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
    }
  }
 }

Pipelines overview

And another question.
WHY Filter worked with nginx.access.remote_ip

cidr {
        id => "nginx-postproc-cidr-remote_ip_2"
        address => [ "%{nginx.access.remote_ip}" ]
        network => [ "0.0.0.0/32", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fc00::/7", "127.0.0.0/8", "::1/128","169.254.0.0/16", "fe80::/10","224.0.0.0/4", "ff00::/8","255.255.255.255/32" ]
        add_field => { "nginx.access.remote_ip_locality" => "private" }
      }

And not work with [nginx][access][remote_ip]. It return invalid ip address in log of logstash

cidr {
        id => "nginx-postproc-cidr-remote_ip_2"
        address => [ "%{[nginx][access][remote_ip]}" ]
        network => [ "0.0.0.0/32", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fc00::/7", "127.0.0.0/8", "::1/128","169.254.0.0/16", "fe80::/10","224.0.0.0/4", "ff00::/8","255.255.255.255/32" ]
        add_field => { "[nginx][access][remote_ip_locality]" => "private" }
      }

jjensen · December 17, 2017, 8:56am

[nginx][access][remote_ip] refers to a nested field, which you don't have. This would look like the following in your log:

"nginx" : {
    "access" : {
        "remote_ip" : some_ip_address
    }
}

The field name is instead literally "nginx.access.remote_ip" and is what you should be matching on.

tatdat · December 17, 2017, 8:58am

Thank @jjensen

You can see in my sample log above, i have this field

"nginx.access.remote_ip": "66.160.201.42"

or in kibana view :

jjensen · December 17, 2017, 9:00am

Right. Your CIDR filter needs to look like first one you posted, and not the second. You're getting the error because you're trying to reference a nested field ([nginx][access][remote_ip]) that doesn't exist.

tatdat · December 17, 2017, 9:06am

im not clearly ...
Why nested field ([nginx][access][remote_ip]) is not exist ? I still saw this field data event.

if [nginx][access][remote_ip] { ... } " and cidr filter is placed after grok filter

So [nginx][access][remote_ip] field must have been. Right?

jjensen · December 17, 2017, 9:12am

Are you interested in creating nested fields or will single level fields be ok?

tatdat · December 17, 2017, 9:15am

I think nested fields is good, It's easy to manage and see clearly

jjensen · December 17, 2017, 9:21am

Try this

filter {
  if [type] == "nginx-access" {
    grok {
      patterns_dir => ["/etc/logstash/patterns"]
      match => { "message" => "%{NGINXACCESS}" }
      match => { "message" => "%{NGINXACCESSCUSTOME}" }
    }    

    if [nginx.access.remote_ip] {
      # Check if destination IP address is private.
      cidr {
        id => "nginx-postproc-cidr-remote_ip"
        address => [ "%{[nginx.access.remote_ip]}" ]
        network => [ "0.0.0.0/32", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fc00::/7", "127.0.0.0/8", "::1/128","169.254.0.0/16", "fe80::/10","224.0.0.0/4", "ff00::/8","255.255.255.255/32" ]
        add_field => { "[nginx][access][remote_ip_locality]" => "private" }
      }
      # Check to see if dst_locality exists. If it doesn't the remote_ip didn't match a private address space and locality must be public.
      if ![nginx][access][remote_ip_locality] {
        mutate {
        id => "nginx-postproc-remote_ip-public"
        add_field => { "[nginx][access][remote_ip_locality]" => "public" }
        }
        geoip {
          id => "nginx-postproc-geoip_remote_ip-city"
          source => "[nginx.access.remote_ip]"
          default_database_type => "City"
          target => "[nginx][access][geoip]"
          database => "/etc/logstash/GeoLite2-City.mmdb"
        }
        geoip {
          id => "nginx-postproc-geoip_remote_ip-asn"
          source => "[nginx.access.remote_ip]"
          default_database_type => "ASN"
          target => "[nginx][access][geoip]"
          database => "/etc/logstash/GeoLite2-ASN.mmdb"
        }
        # Populate geoip_dst.autonomous_system.
        if [nginx][access][geoip][as_org] {
          if [nginx][access][geoip][asn] {
            mutate {
            id => "nginx-postproc-remote_ip-as-from-as_org-asn"
            add_field => { "[nginx][access][geoip][autonomous_system]" => "%{[nginx][access][geoip][as_org]} (%{[nginx][access][geoip][asn]})" }
          }
          } else {
            mutate {
            id => "nginx-postproc-remote_ip-as-from-as_org"
            add_field => { "[nginx][access][geoip][autonomous_system]" => "%{[nginx][access][geoip][as_org]}" }
            }
          }
        } else if [geoip][asn] {
          mutate {
            id => "nginx-postproc-remote_ip-as-from-asn"
            add_field => { "[nginx][access][geoip][autonomous_system]" => "%{[nginx][access][geoip][asn]}" }
          }
        }
      } else {
        mutate {
          id => "nginx-postproc-remote_ip-as-private"
          add_field => { "[nginx][access][geoip][autonomous_system]" => "PRIVATE" }
        }
        translate {
          regex => true
          dictionary_path => "/etc/logstash/translates/internal-ip.yaml"
          field => "nginx.access.remote_ip"
        }
        json {
          source => "translation"
          remove_field => ["translation"]   
        }
      }
    } 
        
    date {
        match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
    }
  }
 }

jjensen · December 17, 2017, 9:39am

an example of a nested field that you have in your log is beat. example:

"beat": {
  "version": "6.0.1",
  "hostname": "Nginx-LB-Outside-01",
  "name": "Nginx-LB-Outside-01"
}

if you wanted to match on the hostname in the beat field you would use [beat][hostname]. there is no such nested field for the remote IP in the access log, just a single-level field identified by [nginx.access.remote_ip]. I get that it's confusing because you would expect the dots to signify the object mapping, but in this case they don't. The dots are part of the literal field name.

tatdat · December 17, 2017, 9:42am

Thank @jjensen, it worked, im changing all field to [field.nestedfield] .

jjensen · December 17, 2017, 9:46am

system · January 14, 2018, 9:46am

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.