Mapping Netflow Data (NSEL)


#1

Am new to the Elastic Stack and my 1st project with it is to analyze Netflow data from a Cisco ASA which actually pushes out NSEL as opposed to regular Netflow [v5,9] packets.

The problem that I am running into is that the index pulled into Kibana doesn't seem classify the 'type' within each property correctly.

For example, flow_seq_number, I set the type as a 'long', yet Kibana is showing the type as a 'number'
Each IP address property is loading into Kibana as type 'string'

I included the mappings below and would appreciate any thoughts on how to fix this.

Regards.

curl -XPUT localhost:9200/_template/logstash_netflow-9995 -d '{
"template" : "logstash_netflow-9995*",
"settings": {
"index.refresh_interval": "5s"
},
"mappings" : {
"default" : {
"_all" : {"enabled" : false},

    "properties" : {
      "@version": { "index": "analyzed", "type": "integer" },
      "@timestamp": { "index": "analyzed", "type": "date" },
      "host": {"index": "analyzed", "type": "ip"},
      
      "netflow": {
        "dynamic": true,
        "type": "object",

        "properties": {
        
          "version": { 
            "index": "not_analyzed", 
            "type": "integer" 
          },

          "flow_seq_num": {
            "index": "not_analyzed",
            "type": "long"
          },

          "flowset_id": {
            "index": "not_analyzed",
            "type": "long"
          },

          "nf_f_conn_id": {
            "index": "not_analyzed",
            "type": "long"
          },

          "nf_f_src_addr_ipv4": {
            "index": "analyzed",
            "type": "ip"
          },

          "nf_f_src_port": {
            "index": "analyzed",
            "type": "long"
          },

          "nf_f_src_intd_id": {
            "index": "not_analyzed",
            "type": "long"
          },

          "nf_f_dst_addr_ipv4": {
            "index": "analyzed",
            "type": "ip"
          },

          "nf_f_dst_port": {
            "index": "analyzed",
            "type": "long"
          },

          "nf_f_dst_intf_id": {
            "index": "not_analyzed",
            "type": "long"
          },              

          "nf_f_protocol": {
            "index": "not_analyzed",
            "type": "integer"
          },

          "nf_f_ingress_acl_id": {
            "index": "not_analyzed",
            "type": "string"
          },

          "nf_f_egress_acl_id": {
            "type": "string",
            "index": "not_analyzed"
          },

          "nf_f_fwd_flow_delta_bytes": {
            "type": "long",
            "index": "analyzed"
          },

          "nf_f_rev_flow_delta_bytes": {
            "type": "long",
            "index": "analyzed"
          },

          "nf_f_icmp_code": {
            "type": "long",
            "index": "analyzed"
          },

          "nf_f_icmp_type": {
            "type": "long",
            "index": "analyzed"
          },

          "nf_f_event_time_msec": {
            "type": "string",
            "index": "analyzed"
          },

          "nf_f_flow_create_time_msec": {
            "type": "date",
            "index": "analyzed"
          },

          "nf_f_xlate_dst_addr_ipv4": {
            "type": "ip",
            "index": "analyzed"
          },

          "nf_f_xlate_dst_port": {
            "type": "long",
            "index": "analyzed"
          },

          "nf_f_xlate_src_addr_ipv4": {
            "type": "ip",
            "index": "analyzed"
          },

          "nf_f_xlate_src_port": {
            "type": "long",
            "index": "analyzed"
          }              
        }
      }
    }
  }
}

}'


(Mark Walkom) #2

What version are you on, what does the mapping that is applied look like?


#3

Running 5.5.1

And here is what the mapping looks like once it is applied.

curl -XGET 'localhost:9200/logstash_netflow-2017.08.17/_mapping/?pretty'
{
"logstash_netflow-2017.08.17" : {
"mappings" : {
"logs" : {
"properties" : {
"@timestamp" : {
"type" : "date"
},
"@version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"host" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"netflow" : {
"properties" : {
"flow_seq_num" : {
"type" : "long"
},
"flowset_id" : {
"type" : "long"
},
"fw_event" : {
"type" : "long"
},
"in_permanent_bytes" : {
"type" : "long"
},
"nf_f_conn_id" : {
"type" : "long"
},
"nf_f_csrc_intf_id" : {
"type" : "long"
},
"nf_f_dst_addr_ipv4" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"nf_f_dst_intf_id" : {
"type" : "long"
},
"nf_f_dst_port" : {
"type" : "long"
},
"nf_f_egress_acl_id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"nf_f_event_time_msec" : {
"type" : "long"
},
"nf_f_flow_create_time_msec" : {
"type" : "long"
},
"nf_f_fw_ext_event" : {
"type" : "long"
},
"nf_f_icmp_code" : {
"type" : "long"
},
"nf_f_icmp_type" : {
"type" : "long"
},
"nf_f_ingress_acl_id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"nf_f_protocol" : {
"type" : "long"
},
"nf_f_src_addr_ipv4" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"nf_f_src_port" : {
"type" : "long"
},
"nf_f_username" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"nf_f_xlate_dst_addr_ipv4" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"nf_f_xlate_dst_port" : {
"type" : "long"
},
"nf_f_xlate_src_addr_ipv4" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"nf_f_xlate_src_port" : {
"type" : "long"
},
"version" : {
"type" : "long"
}
}
}
}
}
}
}
}


(Mark Walkom) #4

I can't see flow_seq_number in either mapping?


#5

Sorry Mark, I do not understand what it was that you were asking.


(Mark Walkom) #6

The flow_seq_number you mention in the first post is not visible in any of the mappings you have posted.


#7

Unless I am looking at it wrong, after retrieving the mapping, 'flow_seq_num' shows to be one of the 1st properties of listed under netflow.

“netflow” : {
“properties” : {
“flow_seq_num” : {
“type” : “long”
},


(Mark Walkom) #8

Oh, see I was searching for flow_seq_number, when it's flow_seq_num. Looks like we had wires crossed! :slight_smile:

Kibana does a bit of hand waving here and depicts any long/float/numeric as a number, so as long as the mapping is correct then you have nothing to be worried about. The docs should call this out, so if you really want to create an issue then please do, otherwise I will :slight_smile:

The IP should show as one though, but looking at (eg) the nf_f_src_addr_ipv4 field, it's showing as a text field. Are you able to edit your posts and wrap the json on code tags, makes it easier to read and debug :slight_smile:


#9

Hoping this turns out and it is a little easier to read.

"logstash_netflow-2017.08.29" : {
 "mappings" : {
  "netflow" : {
    "properties" : {
      "@timestamp" : {
        "type" : "date"
      },
      "@version" : {
        "type" : "text",
        "fields" : {
          "keyword" : {
            "type" : "keyword",
            "ignore_above" : 256
          }
        }
      },
      "host" : {
        "type" : "text",
        "fields" : {
          "keyword" : {
            "type" : "keyword",
            "ignore_above" : 256
          }
        }
      },
      "netflow" : {
        "properties" : {
          "flow_seq_num" : {
            "type" : "long"
          },
          "flowset_id" : {
            "type" : "long"
          },
          "fw_event" : {
            "type" : "long"
          },
          "in_permanent_bytes" : {
            "type" : "long"
          },
          "nf_f_conn_id" : {
            "type" : "long"
          },
          "nf_f_csrc_intf_id" : {
            "type" : "long"
          },
          "nf_f_dst_addr_ipv4" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "nf_f_dst_intf_id" : {
            "type" : "long"
          },
          "nf_f_dst_port" : {
            "type" : "long"
          },
          "nf_f_egress_acl_id" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "nf_f_event_time_msec" : {
            "type" : "long"
          },
          "nf_f_flow_create_time_msec" : {
            "type" : "long"
          },
          "nf_f_fw_ext_event" : {
            "type" : "long"
          },
          "nf_f_icmp_code" : {
            "type" : "long"
          },
          "nf_f_icmp_type" : {
            "type" : "long"
          },
          "nf_f_ingress_acl_id" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "nf_f_protocol" : {
            "type" : "long"
          },
          "nf_f_src_addr_ipv4" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "nf_f_src_port" : {
            "type" : "long"
          },
          "nf_f_username" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "nf_f_xlate_dst_addr_ipv4" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "nf_f_xlate_dst_port" : {
            "type" : "long"
          },
          "nf_f_xlate_src_addr_ipv4" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "nf_f_xlate_src_port" : {
            "type" : "long"
          },
          "version" : {
            "type" : "long"
          }
        }
      },
      "type" : {
        "type" : "text",
        "fields" : {
          "keyword" : {
            "type" : "keyword",
            "ignore_above" : 256
          }
        }
      }
    }
  }
}

(Mark Walkom) #10

It is, thanks!

Looking at that one then you definitely have the IP fields mapped as text+keyword. Can you show the template for it, also formatted?


#11

Here is the template that was used

curl -XPUT localhost:9200/_template/logstash_netflow- -d

'{
  "template" : "logstash_netflow-",
  "settings": {
  "index.refresh_interval": "5s",
  "number_of_shards": "1",
  "number_of_replicas": "0"
},
"mappings" : {
  "_default_" : {
    "_all" : {"enabled" : true},

    "properties" : {
      "@version": { "index": "analyzed", "type": "integer" },
      "@timestamp": { "index": "analyzed", "type": "date" },
      "host": {"index": "analyzed", "type": "ip"},
      
      "netflow": {
        "dynamic": false,
        "type": "object",

        "properties": {
        
          "version": { 
            "index": "not_analyzed", 
            "type": "integer" 
          },

          "flow_seq_num": {
            "index": "not_analyzed",
            "type": "long"
          },

          "flowset_id": {
            "index": "not_analyzed",
            "type": "long"
          },

          "nf_f_conn_id": {
            "index": "not_analyzed",
            "type": "long"
          },

          "nf_f_src_addr_ipv4": {
            "index": "analyzed",
            "type": "ip"
          },

          "nf_f_src_port": {
            "index": "analyzed",
            "type": "long"
          },

          "nf_f_src_intd_id": {
            "index": "not_analyzed",
            "type": "long"
          },

          "nf_f_dst_addr_ipv4": {
            "index": "analyzed",
            "type": "ip"
          },

          "nf_f_dst_port": {
            "index": "analyzed",
            "type": "long"
          },

          "nf_f_dst_intf_id": {
            "index": "not_analyzed",
            "type": "long"
          },              

          "nf_f_protocol": {
            "index": "not_analyzed",
            "type": "integer"
          },

          "nf_f_ingress_acl_id": {
            "index": "not_analyzed",
            "type": "string"
          },

          "nf_f_egress_acl_id": {
            "type": "string",
            "index": "not_analyzed"
          },

          "nf_f_fwd_flow_delta_bytes": {
            "type": "long",
            "index": "analyzed"
          },

          "nf_f_rev_flow_delta_bytes": {
            "type": "long",
            "index": "analyzed"
          },

          "nf_f_icmp_code": {
            "type": "long",
            "index": "analyzed"
          },

          "nf_f_icmp_type": {
            "type": "long",
            "index": "analyzed"
          },

          "nf_f_event_time_msec": {
            "type": "string",
            "index": "analyzed"
          },

          "nf_f_flow_create_time_msec": {
            "type": "date",
            "index": "analyzed"
          },

          "nf_f_xlate_dst_addr_ipv4": {
            "type": "ip",
            "index": "analyzed"
          },

          "nf_f_xlate_dst_port": {
            "type": "long",
            "index": "analyzed"
          },

          "nf_f_xlate_src_addr_ipv4": {
            "type": "ip",
            "index": "analyzed"
          },

          "nf_f_xlate_src_port": {
            "type": "long",
            "index": "analyzed"
          }              
        }
      }
    }
  }
}
}'

(system) #12

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.