Hi Im trying to use logstash to use elasticsearch output plugin to send logs to elasticsearch.
The version of elastic that Im using is 7.4.2 as well as logstash.
The issue that Im having is when I try to load the template, I get the follwing error
[INFO ] 2020-01-24 15:44:03.776 [Ruby-0-Thread-5: :1] elasticsearch - Attempting to install template {:manage_template=>{"template"=>["test-*"], "order"=>0, "settings"=>{"index.refresh_interval"=>"5s", "index.routing.allocation.require.node_type"=>"hot", "number_of_shards"=>45, "analysis"=>{"analyzer"=>{"default"=>{"type"=>"standard", "stopwords"=>"_none_"}}}}, "mappings"=>{"_default_"=>{"_all"=>{"enabled"=>true}, "dynamic_templates"=>[{"message"=>{"match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"string", "index"=>"analyzed", "omit_norms"=>true, "index_options"=>"offsets"}}}, {"stacktrace"=>{"match"=>"stacktrace", "match_mapping_type"=>"string", "mapping"=>{"type"=>"string", "index"=>"analyzed", "omit_norms"=>true, "index_options"=>"offsets"}}}, {"javathread"=>{"match"=>"javathread", "match_mapping_type"=>"string", "mapping"=>{"type"=>"string", "index"=>"analyzed", "omit_norms"=>true, "index_options"=>"offsets"}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"multi_field", "fields"=>{"{name}"=>{"type"=>"string", "index"=>"not_analyzed", "omit_norms"=>true, "index_options"=>"offsets"}}}}}], "properties"=>{"@version"=>{"type"=>"string", "index"=>"not_analyzed"}, "geoip"=>{"type"=>"object", "dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}}}, "tags"=>{"type"=>"string", "index"=>"not_analyzed"}, "pid"=>{"type"=>"long", "index"=>"not_analyzed"}, "priority"=>{"type"=>"integer", "index"=>"not_analyzed"}, "severity"=>{"type"=>"integer", "index"=>"not_analyzed"}, "facility"=>{"type"=>"integer", "index"=>"not_analyzed"}, "syslog_severity_code"=>{"type"=>"integer", "index"=>"not_analyzed"}, "syslog_facility_code"=>{"type"=>"integer", "index"=>"not_analyzed"}}}}}}
[INFO ] 2020-01-24 15:44:03.810 [Ruby-0-Thread-5: :1] elasticsearch - Installing elasticsearch template to _template/test-*
[ERROR] 2020-01-24 15:44:03.904 [Ruby-0-Thread-5: :1] elasticsearch - Failed to install template. {:message=>"Got response code '400' contacting Elasticsearch at URL 'http://app1431.com:9200/_template/test-*'", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError", :backtrace=>["/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.1.0-java/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb:80:in `perform_request'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.1.0-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:291:in `perform_request_to_url'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.1.0-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:278:in `block in perform_request'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.1.0-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:373:in `with_connection'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.1.0-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:277:in `perform_request'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.1.0-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:285:in `block in Pool'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.1.0-java/lib/logstash/outputs/elasticsearch/http_client.rb:352:in `template_put'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.1.0-java/lib/logstash/outputs/elasticsearch/http_client.rb:86:in `template_install'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.1.0-java/lib/logstash/outputs/elasticsearch/template_manager.rb:28:in `install'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.1.0-java/lib/logstash/outputs/elasticsearch/template_manager.rb:16:in `install_template'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.1.0-java/lib/logstash/outputs/elasticsearch/common.rb:130:in `install_template'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.1.0-java/lib/logstash/outputs/elasticsearch/common.rb:51:in `block in setup_after_successful_connection'"]}quote`
Listed below is the config that I have under /etc/logstash/conf.d:
input { file { path => "/data_test" start_position => "beginning" } } output { elasticsearch { manage_template => true template_name => "test" template => "/etc/logstash/templates/test.json" template_overwrite => true index => ["test-%{+YYYY.MM.dd}"] hosts => ["http://app1417.com:9200","http://app1431.com:9200"] workers => 1 } stdout { codec => rubydebug { } } }
and the template looks like this, this template was build for version 2.3, so Im trying to figure out if this applies or I theres some key changes:
{ "template" : ["test-*"], "order" : 0, "settings" : { "index.refresh_interval" : "5s", "index.routing.allocation.require.node_type": "hot", "number_of_shards": 45, "analysis" : { "analyzer" : { "default" : { "type" : "standard", "stopwords" : "_none_" } } } }, "mappings" : { "_default_" : { "_all" : {"enabled" : true}, "dynamic_templates" : [ { "message" : { "match" : "message", "match_mapping_type" : "string", "mapping" : { "type" : "string", "index" : "analyzed", "omit_norms" : true, "index_options" : "offsets" } } }, { "stacktrace" : { "match" : "stacktrace", "match_mapping_type" : "string", "mapping" : { "type" : "string", "index" : "analyzed", "omit_norms" : true, "index_options" : "offsets" } } }, { "javathread" : { "match" : "javathread", "match_mapping_type" : "string", "mapping" : { "type" : "string", "index" : "analyzed", "omit_norms" : true, "index_options" : "offsets" } } }, { "string_fields" : { "match" : "*", "match_mapping_type" : "string", "mapping" : { "type" : "multi_field", "fields" : { "{name}" : {"type": "string", "index" : "not_analyzed", "omit_norms" : true, "index_options" : "offsets"} } } } } ], "properties" : { "@version": { "type": "string", "index": "not_analyzed" }, "geoip" : { "type" : "object", "dynamic": true, "properties" : { "ip" : { "type": "ip" }, "location" : { "type" : "geo_point" } } }, "tags": { "type": "string", "index": "not_analyzed" }, "pid": { "type": "long", "index": "not_analyzed" }, "priority": { "type": "integer", "index": "not_analyzed" }, "severity": { "type": "integer", "index": "not_analyzed" }, "facility": { "type": "integer", "index": "not_analyzed" }, "syslog_severity_code": { "type": "integer", "index": "not_analyzed" }, "syslog_facility_code": { "type": "integer", "index": "not_analyzed" } } } } }
can someone please point me to the right direction.
