GeoIP challenges - Custom-City.mmdb

Elastic Stack Elasticsearch
13

I figured out how you can make the mmdb file yourself.
From all my postings I believe that many people can benefit from all this work.
GeoIP Mapping of my internal IP space is very helpful in debug activities.

See Below:

  1. new GO code that reads a CSV file,
  2. the CSV file,
  3. the Simulate Syntax.

Go Source Code

(main.go)

package main

import (
  "encoding/csv"
  "io"
  "log"
  "net"
  "os"
  "fmt"

  "github.com/maxmind/mmdbwriter"
  "github.com/maxmind/mmdbwriter/mmdbtype"
)

func main() {
  writer, err := mmdbwriter.New(
    mmdbwriter.Options{
      DatabaseType:            "GeoLite2-City",
      RecordSize:              28,
      IPVersion:               6,
      IncludeReservedNetworks: true,
    },
  )
  if err != nil {
    log.Fatal(err)
  }

  // Test for the file
  file, err := os.Open("Custom-City.csv")
  defer file.Close()
  if err != nil {
    log.Fatal("Error while reading the file", err)
  }

  r := csv.NewReader(file)
  // first line header info
  r.Read()

  // start looping thru the csv
  for {
    // check that there are csv records or are we at the end
    row, err := r.Read()
    if err == io.EOF {
      break
    }
    if err != nil {
      log.Fatal(err)
    }
    // verify that there are 11 columns
    if len(row) != 11 {
      log.Fatalf("unexpected CSV rows: %v", row)
    }

    //  0 geoipip
    //  1 geoipcontinent_code
    //  2 geoipcontinent_name
    //  3 geoipcountry_code3
    //  4 geoipcountry_name
    //  5 geoipregion_code
    //  6 geoipregion_name
    //  7 geoipcity_name
    //  8 geoiptime_zone
    //  9 geoiplatitude
    // 10 geoiplongitude

    // MMDB Section
    // ==== range
    _, network, err := net.ParseCIDR(row[0])
    if err != nil {
      log.Fatal(err)
    }

    // all the records are in this map
    record := mmdbtype.Map{

      // ==== continent
      "continent": mmdbtype.Map {
        "code": mmdbtype.String(row[1]),
        "names": mmdbtype.Map{
          "en": mmdbtype.String(row[2]),
        },
      },

      // ==== country
      "country": mmdbtype.Map{
        "iso_code": mmdbtype.String(row[3]),
        "names": mmdbtype.Map{
          "en": mmdbtype.String(row[4]),
        },
      },

      // ==== city
      "city": mmdbtype.Map {
        "names": mmdbtype.Map{
          "en": mmdbtype.String(row[7]),
        },
      },

      // ==== location
      "location": mmdbtype.Map{
        "latitude":  mmdbtype.String(row[9]),
        "longitude": mmdbtype.String(row[10]),
        "time_zone": mmdbtype.String(row[8]),
      },
    }

    // Write the record and loop back
    err = writer.Insert(network, record)
    if err != nil {
      fmt.Println(err)
      return
    }
  }

  // data is looped so now write all the records to the file
  fh, err := os.Create("Custom-City.mmdb")
  defer fh.Close()
  if err != nil {
    fmt.Println(err)
    return
    // log.Fatal(err)
  }

  _, err = writer.WriteTo(fh)
  if err != nil {
    log.Fatal(err)
  }
  // State the obvious - we are done
  fmt.Println("The End")
}

CSV File

(Custom-City.csv)
no blank lines please and yes it has a header

geoipip,geoipcontinent_code,geoipcontinent_name,geoipcountry_code3,geoipcountry_name,geoipregion_code,geoipregion_name,geoipcity_name,geoiptime_zone,geoiplatitude,geoiplongitude
10.103.212.0/24,NA,North America,USA,United States,0,0,Jacksonville,America/New_York,30.33218400,-81.65564700
10.103.23.0/24,EU,Europe,DEU,Germany,0,0,Hamburg,Europe/Berlin,53.55108600,9.99368200
10.103.61.0/24,NA,North America,USA,United States,0,0,Gloucester,America/New_York,35.10807800,-80.78914600
10.104.128.0/24,EU,Europe,GBR,United Kingdon,ENG,England,Marston Green,Europe/London,50.99800000,-2.64920000

Elastic Command line Simulate Syntax:

curl --cacert /etc/elasticsearch/certs/http_ca.crt -X POST -u user:pass "https://localhost:9200/_ingest/pipeline/_simulate?pretty" -H 'Content-Type: application/json' -d'
{
  "pipeline" :
  {
    "description": "_GEO",
    "processors": [
      {
        "geoip": {
          "field": "ip",
          "target_field": "geo",
          "database_file": "Custom-City.mmdb"
        }
      }
    ]
  },
  "docs": [
    {
      "_index": "index",
      "_id": "id",
      "_source": {
        "foo": "bar",
        "ip": "10.103.212.44"
      }
    }
  ]
}
'

All code advice is welcome. Please see if you can also see the failure.

Thank You !!

Steve B