using Ubuntu 16.04
rev1.conf file
input {
file {
path => "/home/skills34/es/templetes/test1.txt"
start_position => "beginning"
sincedb_path => "/dev/null"
codec => "json"
}
}
filter { }
output {
elasticsearch {
hosts => ["http://localhost:9200/"]
template =>"/home/skills34/es/templetes/datamobi.json"
template_name=>"datamobi"
template_overwrite => true
index => "datamobi_in_11_2"
}
}
datamobi.json file
{
"index_patterns": [
"datamobi_in_11*"
],
"settings": {
"index": {
"number_of_shards": "2",
"number_of_replicas": "1"
}
},
"mappings": {
"doc" : {
"dynamic": "false",
"properties" : {
"reviewId": { "type": "text" },
"displayName": { "type": "text" },
"profilePhotoUrl": { "type": "text" },
"comment": { "type": "text" },
"name": { "type": "text" },
"createTime": { "type": "date" },
"updateTime": { "type": "date" }
}
}
},
"aliases": {}
}
test1.txt file
{"reviewId":"test","displayName":"Afrid Khan","profilePhotoUrl":"dfdf","comment":"Good working environment","createTime":"2018-11-21T11:38:54.366207Z","updateTime":"2018-11-21T11:39:11.527644Z","name" : "dnfd"}
on Terminal ran this command :
sudo /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/rev1.conf
response:
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
[WARN ] 2018-12-06 12:15:29.138 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified
[INFO ] 2018-12-06 12:15:29.153 [LogStash::Runner] runner - Starting Logstash {"logstash.version"=>"6.5.1"}
[INFO ] 2018-12-06 12:15:31.528 [Converge PipelineAction::Create] pipeline - Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>8, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50}
[INFO ] 2018-12-06 12:15:31.991 [[main]-pipeline-manager] elasticsearch - Elasticsearch pool URLs updated {:changes=>{:removed=>, :added=>[http://localhost:9200/]}}
[INFO ] 2018-12-06 12:15:31.999 [[main]-pipeline-manager] elasticsearch - Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://localhost:9200/, :path=>"/"}
[WARN ] 2018-12-06 12:15:32.194 [[main]-pipeline-manager] elasticsearch - Restored connection to ES instance {:url=>"http://localhost:9200/"}
[INFO ] 2018-12-06 12:15:32.371 [[main]-pipeline-manager] elasticsearch - ES Output version determined {:es_version=>6}
[WARN ] 2018-12-06 12:15:32.375 [[main]-pipeline-manager] elasticsearch - Detected a 6.x and above cluster: thetypeevent field won't be used to determine the document _type {:es_version=>6}
[INFO ] 2018-12-06 12:15:32.398 [[main]-pipeline-manager] elasticsearch - New Elasticsearch output {:class=>"LogStash::Outputs::Elasticsearch", :hosts=>["http://localhost:9200/"]}
[INFO ] 2018-12-06 12:15:32.419 [Ruby-0-Thread-5: :1] elasticsearch - Using mapping template from {:path=>"/home/skills34/es/templetes/datamobi.json"}
[INFO ] 2018-12-06 12:15:32.428 [Ruby-0-Thread-5: :1] elasticsearch - Attempting to install template {:manage_template=>{"index_patterns"=>["datamobi_in_11*"], "settings"=>{"index"=>{"number_of_shards"=>"2", "number_of_replicas"=>"1"}}, "mappings"=>{"doc"=>{"dynamic"=>"false", "properties"=>{"reviewId"=>{"type"=>"text"}, "displayName"=>{"type"=>"text"}, "profilePhotoUrl"=>{"type"=>"text"}, "comment"=>{"type"=>"text"}, "name"=>{"type"=>"text"}, "createTime"=>{"type"=>"date"}, "updateTime"=>{"type"=>"date"}}}}, "aliases"=>{}}}
[INFO ] 2018-12-06 12:15:32.588 [Ruby-0-Thread-5: :1] elasticsearch - Installing elasticsearch template to _template/datamobi
[INFO ] 2018-12-06 12:15:32.759 [Converge PipelineAction::Create] pipeline - Pipeline started successfully {:pipeline_id=>"main", :thread=>"#<Thread:0x605bc3d4 sleep>"}
[INFO ] 2018-12-06 12:15:32.814 [Ruby-0-Thread-1: /usr/share/logstash/lib/bootstrap/environment.rb:6] agent - Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>}
[INFO ] 2018-12-06 12:15:32.820 [[main]<file] observingtail - START, creating Discoverer, Watch with file and sincedb collections
[INFO ] 2018-12-06 12:15:33.176 [Api Webserver] agent - Successfully started Logstash API endpoint {:port=>9601}
Kibana response
GET datamobi_in_11_2/_search
{
"error" : {
"root_cause" : [
{
"type" : "index_not_found_exception",
"reason" : "no such index",
"resource.type" : "index_or_alias",
"resource.id" : "datamobi_in_11_2",
"index_uuid" : "_na_",
"index" : "datamobi_in_11_2"
}
],
"type" : "index_not_found_exception",
"reason" : "no such index",
"resource.type" : "index_or_alias",
"resource.id" : "datamobi_in_11_2",
"index_uuid" : "_na_",
"index" : "datamobi_in_11_2"
},
"status" : 404
}
thats all, please help. It was creating index intially then sddenll what happened i dnt know .
