Logstash.outputs.elasticsearch : Failed to install template

Elastic Stack Logstash
1

this is logstash conf file

input {
  file {
    path            => "/home/skills180/Es/templetes/test2.txt"
    start_position  => "beginning"
    sincedb_path    => "/dev/null"
    codec         => "json" 
  }
}
filter {
  json {
    source  => "message"
  }
  date {
    match => [ "timestamp", "UNIX_MS" ]
    target => "public_time"
  }
  ruby {

    code => "event.set('index_day', event.get('[public_time]').time.localtime.strftime('%Y-%m-%d'))"
  }
  ruby {
    code => "event.set('timestamp', event.get('timestamp').to_f / 1000)"
  }
  mutate {
    add_field => {  "[location][lat]" => "%{[bidRequest][device][geo][lat]}"
      "[location][lon]" => "%{[bidRequest][device][geo][lon]}"
      "ideditor"        => "6"
      "idapp"           => "81"
      "geo_type"=>"%{[bidRequest][device][geo][type]}"
    }
    rename => {     "[bidRequest][device][ifa]"             => "idfa"
      "timestamp"                             => "happenon"
      "[bidRequest][device][geo][country]"    => "country"
      #"[bidRequest][app][publisher][id]"     => "idapp"
    }
    convert => {
      "happenon" => "integer"
    }
    remove_field    => [ "@version","@timestamp","path", "host","public_time",  "[user]", "[app]", "[device]", "[imp]", "[http]", "[bidRequest]", "[enrichment]", "[geo]", "[sspId]", "[organizationId]", "[domain]"]
  }
}
output {
  if [geo_type] == "GPS_LOCATION" {
    if [idfa] != "" {
      elasticsearch {
hosts => ["http://localhost:9200/"]
template =>"/home/skills180/Es/templetes/template_test.json"
template_name=>["template_test"]
template_overwrite => true
index => "datamobi_in_%{index_day}"
document_type => "ping"
}
}
}
# stdout { codec => rubydebug }
}

this is text2.txt

{"timestamp":"1524635328682","bidRequest":{"id":"550D288074CBCB09A01C59D3473DDB6D","imp":[{"id":"1","banner":{"w":320,"h":50,"id":"1","pos":"UNKNOWN","btype":["IFRAME"],"battr":["POP","ANNOYING"],"topframe":false,"wmax":320,"hmax":50},"displaymanager":"third_party_sdk","displaymanagerver":"3.0","instl":false,"bidfloor":0.637338,"bidfloorcur":"USD","secure":false}],"app":{"id":"246228_4-4793","name":"Women Saree Photo Making","domain":"silvermob.com","cat":["IAB1"],"bundle":"com.formationapps.womensaree","publisher":{"id":"48757"},"storeurl":"https://play.google.com/store/apps/details?id=com.formationapps.womensaree"},"device":{"dnt":false,"ua":"Mozilla/5.0 (Linux; Android 6.0.1; SM-T580 Build/MMB29K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.106 Safari/537.36","ip":"88.179.216.6","geo":{"lat":48.68,"lon":1.7557,"country":"FRA","type":"GPS_LOCATION"},"dpidsha1":"DA38637008C08832341EC8DED77CA2AAE9715276","dpidmd5":"577DBDDED8C396BE3E0E130C477621C6","carrier":"WIFI","make":"Samsung","model":"SM-T580","os":"Android","osv":"6.0","js":true,"connectiontype":"WIFI","devicetype":"TABLET","ifa":"CA42A60D-558A-4443-8B88-57D1C3C3BD0D"},"user":{"id":"CA42A60D-558A-4443-8B88-57D1C3C3BD0D"},"at":"SECOND_PRICE","tmax":300,"cur":["USD"],"bcat":["IAB24","IAB25","IAB26"]},"http":{"requestUrl":"http://mobfox-display.rtb.adx1.com/display","headers":{"host":"mobfox-display.rtb.adx1.com","xRealIp":"54.209.102.201","contentType":"application/json","xOpenrtbVersion":"2.3","accept":"*/*","acceptEncoding":"gzip"}},"organizationId":3,"user":{"platormId":"device.ifa:ca42a60d-558a-4443-8b88-57d1c3c3bd0d","externalId":"ca42a60d-558a-4443-8b88-57d1c3c3bd0d"},"sspId":320,"enrichment":{"geo":{"country":"FRA","region":"78","city":"Poigny-la-Foret","cityID":0,"timeZoneOffset":-6.72549154E8,"locationCombinations":["FRA","FRA|78"]},"os":"Android 6","browser":"Chrome 66","carrier":"","device":"Tablet Samsung SM-T580","cellular":false},"domain":"com.formationapps.womensaree"}

this template_test.json

{
  "template_test": {
    "order": 0,
    "index_patterns": [
      "datatest_in*"
    ],
    "settings": {
      "index": {
        "number_of_shards": "2",
        "number_of_replicas": "1"
      }
    },
    "mappings": {
      "ping": {
        "dynamic": "false",
        "properties": {
          "idapp": {
            "type": "integer"
          },
          "idfa": {
            "type": "text",
            "fields": {
              "raw": {
                "type": "keyword"
              }
            }
          },
          "happenon": {
            "type": "date",
            "format": "epoch_second"
          },
          "location": {
            "type": "geo_point"
          }
        }
      }
    },
    "aliases": {}
  }
}

after running logstash , i am getting this error

[2018-10-22T19:01:52,136][ERROR][logstash.outputs.elasticsearch] Failed to install template. {:message=>"Got response code '400' contacting Elasticsearch at URL 'http://localhost:9200/_template/datamobi'", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError", :backtrace=>["/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.1-java/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb:80:inperform_request'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:291:in perform_request_to_url'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:278:inblock in perform_request'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:373:in with_connection'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:277:inperform_request'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:285:in block in Pool'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:348:intemplate_put'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:86:in `template_install'", "/usr/share/logs........

2

Your log shows:

[2018-10-22T19:01:52,136][ERROR][logstash.outputs.elasticsearch] Failed to install template. {:message=>"Got response code '400' contacting Elasticsearch at URL 'http://localhost:9200/_template/datamobi'"

It gets a 400 error code.

  • What do you get in the ES logs?
  • What do you get when you call GET /_template?
  • Why do you specify inside output->elasticsearch ->index the index name datamobi_in_%{index_day}, but you create a template for the index template_test?
  • Why do you have an array inside output->elasticsearch->template_name? It should be a string according to documentation: doc
1 Like
3

this was the main error, after reading documentation i got that. thank u for pointing that. i am new in ELK but loving it.

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.