Template index creation based on time


(Andrew) #1

Hello All,

I am having some issues with attempting to create a dynamic template for an index. I keep getting a 404 index not found error. Below is my template and logstash config. If someone could possibly take a look and tell me what I am doing wrong that would be great.

   PUT _template/blocklist {
  "template": "blocklist-*",
  "mappings": {
    "tor_node": {
      "_all": {
        "enabled": true
      }
    },
    "ip": {
      "_all": {
        "enabled": false
      },
      "properties": {
        "IP": {
          "type": "ip"
        },
        "PullTime": {
          "type": "date",
          "format": "yyyy'-'MM'-'dd' 'HH:mm:ss"
        },
        "Source": {
          "type": "text"
        }
      }
    },
    "url": {
      "_all": {
        "enabled": false
      },
      "properties": {
        "URL": {
          "type": "text"
        },
        "Date": {
          "type": "date",
          "format": "dd'/'MMM'/'yyyy','HH:mm:ss"
        },
        "Source": {
          "type": "text"
        }
      }
    },
    "domain": {
      "_all": {
        "enabled": false
      },
      "properties": {
        "Domain": {
          "type": "text"
        },
        "Date": {
          "type": "date",
          "format": "dd'/'MMM'/'yyyy','HH:mm:ss"
        },
        "Source": {
          "type": "text"
        }
      }
    }
  }
}


input {

        file {
                path => "/home/ubuntu/enviroments/Blocklists/IP/*.csv"
                start_position => "beginning"
                sincedb_path => "/dev/null"
        }
}
filter {
    csv {
       separator => ","
       columns => ["IP","Source","PullTime"]
        add_tag => ["csvOK"]
    }
    if ([PullTime] == "PullTime") {
        drop { }
    }

    geoip {
        source => "IP"
        target => "geoip"
        database => "/etc/logstash/GeoLite2-City.mmdb"
        add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
        add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
    }

    date {
        match => ["PullTime", "yyy-MM-dd' 'HH:mm:ss"]
    }
}
output {
    stdout{ codec => rubydebug }
    elasticsearch
    {
        hosts=>"http://XX.XX.XX.XX:9200"
        user=>"elastic"
        password=>"changeme"
        template_name => "blocklist"
        template_overwrite => "true"
        index=>"blocklist-%{+YYYY-MM-dd}"
        document_type=>"ip"
    }
}

(Mark Walkom) #2

Please format your code using the </> button, or markdown style back ticks, it's really hard to read as is :slight_smile:


(system) #3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.