Lgstash not loading data

Elastic Stack Logstash
1

hi
i am trying to load data to ES from logstash using template json
my data -

"s1","salesman01","2017-06-22 00:00:00.000","2017-06-22 16:10:23.000","2017-06-22 16:11:44.000","5","25.252130,55.349060"
"s2","salesman01","2017-06-22 00:00:00.000","2017-06-22 17:10:23.000","2017-06-22 17:21:44.000","8","25.246060,55.359770"
"s3","salesman01","2017-06-22 00:00:00.000","2017-06-22 13:10:23.000","2017-06-22 14:11:44.000","7","25.266920,55.341000"
"s4","salesman01","2017-06-22 00:00:00.000","2017-06-22 12:10:23.000","2017-06-22 13:11:44.000","4","25.242660,55.370600"
"s5","salesman01","2017-06-22 00:00:00.000","2017-06-22 15:10:23.000","2017-06-22 16:10:44.000","2","25.258430,55.341720"
"s1","salesman01","2017-06-22 00:00:00.000","2017-06-22 20:10:23.000","2017-06-22 20:12:44.000","1","25.246260,55.366790"
"s2","salesman01","2017-06-22 00:00:00.000","2017-06-22 20:10:23.000","2017-06-22 21:21:44.000","3","25.266920,55.341030"
"s3","salesman01","2017-06-22 00:00:00.000","2017-06-22 19:10:23.000","2017-06-22 20:11:44.000","5","25.258310,55.341660"
"s4","salesman01","2017-06-22 00:00:00.000","2017-06-22 19:10:23.000","2017-06-22 20:11:44.000","8","25.294370,55.389790"
"s5","salesman01","2017-06-22 00:00:00.000","2017-06-22 21:10:23.000","2017-06-22 22:11:44.000","5","25.248610,55.355260"
"s1","salesman01","2017-06-23 00:00:00.000","2017-06-23 11:10:23.000","2017-06-23 12:11:44.000","5","25.258930,55.342600"
"s2","salesman01","2017-06-23 00:00:00.000","2017-06-23 12:10:23.000","2017-06-23 13:21:44.000","8","25.242500,55.372130"
"s3","salesman01","2017-06-23 00:00:00.000","2017-06-23 13:10:23.000","2017-06-23 14:11:44.000","7","25.252490,55.347850"
"s4","salesman01","2017-06-23 00:00:00.000","2017-06-23 14:10:23.000","2017-06-23 15:11:44.000","4","25.267010,55.341250"
"s5","salesman01","2017-06-23 00:00:00.000","2017-06-23 15:10:23.000","2017-06-23 16:10:44.000","2","25.252300,55.348590"
"s1","salesman01","2017-06-23 00:00:00.000","2017-06-23 16:10:23.000","2017-06-23 17:12:44.000","1","25.251540,55.348840"
"s2","salesman01","2017-06-23 00:00:00.000","2017-06-23 17:10:23.000","2017-06-23 18:21:44.000","3","25.266240,55.340200"
"s3","salesman01","2017-06-23 00:00:00.000","2017-06-23 18:10:23.000","2017-06-23 19:11:44.000","5","25.294190,55.389820"
"s4","salesman01","2017-06-23 00:00:00.000","2017-06-23 19:10:23.000","2017-06-23 20:11:44.000","8","25.246630,55.358600"
"s5","salesman01","2017-06-23 00:00:00.000","2017-06-23 20:10:23.000","2017-06-23 21:11:44.000","5","25.267150,55.341280"

template file -

{ 
    "template" : "testtemp*",
    "settings": {
        "number_of_shards": 3,
        "number_of_replicas": 0
    },
    "mappings": {
        "testindexdoc": {
            "properties": {

                "SalesmanCode": {
                    "type": "keyword"
                },
                "SName": {
                    "type": "keyword",
					 "doc_values": false
                },
                "VisitDate": {
                    "type": "date",
                    "format" : "YYYY-MM-dd HH:mm:ss:SSS || YYYY-MM-dd HH:mm:ss || YYYY-MM-dd HH:mm"
                },
				"VisitStartDateTime": {
                    "type": "date",
                    "format" : "YYYY-MM-dd HH:mm:ss:SSS || YYYY-MM-dd HH:mm:ss || YYYY-MM-dd HH:mm"
                },
				"VisitEndDateTime": {
                    "type": "date",
                    "format" : "YYYY-MM-dd HH:mm:ss:SSS || YYYY-MM-dd HH:mm:ss || YYYY-MM-dd HH:mm"
                },
                "TotalVisit": {
                    "type": "integer"
                },
				"CustomerGeocode": {
				"type": "geo_point"
				}

            }
        }
    },
	"aliases" : {"{index}-alias": {}}
}

log stash config file -

input {
	file {
	path => "/home/admin/logstashV5/templatetesting/testtempldata.csv"
	start_position => "beginning"
	sincedb_path => "/dev/null"
	}
}


filter {
	csv {
		separator => ","
		columns  => [ "SalesmanCode" , "SName", "VisitDate" ,  "VisitStartDateTime" , "VisitEndDateTime" , "TotalVisit" , "CustomerGeocode"]

	}

	
}

	

output {
	elasticsearch {
		hosts => "localhost"
		index => "testtempindex%{+YYYY.MM.dd}"
		template => "/home/admin/logstashV5/templatetesting/testtempl.json"
		template_overwrite => true
		template_name => "testtemp"
		document_type => "testindexdoc"
		
	}
	stdout{}

}

its run without any error and creating index with all mapping , but its doesn't load any data.

Note - If i am using normal logstash file without template i able to load data correctly .

any suggestion or help .

thanks

2

Have you looked in the Logstash log for clues?

3

yes , no error , and its reading data from .csv also

4

please find log for ur reference

[2017-12-06T02:46:29,523][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://localhost:9200/]}}
[2017-12-06T02:46:29,526][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://localhost:9200/, :path=>"/"}
[2017-12-06T02:46:29,615][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>#<Java::JavaNet::URI:0x28743552>}
[2017-12-06T02:46:29,619][INFO ][logstash.outputs.elasticsearch] Using mapping template from {:path=>"/home/admin/logstashV5/templatetesting/testtempl.json"}
[2017-12-06T02:46:29,652][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{"template"=>"testtemp*", "settings"=>{"number_of_shards"=>3, "number_of_replicas"=>0}, "mappings"=>{"testindexdoc"=>{"properties"=>{"SalesmanCode"=>{"type"=>"keyword"}, "SName"=>{"type"=>"keyword", "doc_values"=>false}, "VisitDate"=>{"type"=>"date", "format"=>"YYYY-MM-dd HH:mm:ss:SSS || YYYY-MM-dd HH:mm:ss || YYYY-MM-dd HH:mm"}, "VisitStartDateTime"=>{"type"=>"date", "format"=>"YYYY-MM-dd HH:mm:ss:SSS || YYYY-MM-dd HH:mm:ss || YYYY-MM-dd HH:mm"}, "VisitEndDateTime"=>{"type"=>"date", "format"=>"YYYY-MM-dd HH:mm:ss:SSS || YYYY-MM-dd HH:mm:ss || YYYY-MM-dd HH:mm"}, "TotalVisit"=>{"type"=>"integer"}, "CustomerGeocode"=>{"type"=>"geo_point"}}}}, "aliases"=>{"{index}-alias"=>{}}}}
[2017-12-06T02:46:29,655][INFO ][logstash.outputs.elasticsearch] Installing elasticsearch template to _template/testtemp
[2017-12-06T02:46:29,699][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>[#<Java::JavaNet::URI:0x2578a014>]}
[2017-12-06T02:46:29,701][INFO ][logstash.pipeline        ] Starting pipeline {"id"=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>250}
[2017-12-06T02:46:29,892][INFO ][logstash.pipeline        ] Pipeline main started
[2017-12-06T02:46:29,984][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}
5

How do you know Logstash is reading the file at all? If you comment the elasticsearch output and use a stdout { codec => rubydebug } output, are you getting the events you expect?

1 Like
6

hi now its working
from the template i removed ,
"format" : "YYYY-MM-dd HH:mm:ss:SSS || YYYY-MM-dd HH:mm:ss || YYYY-MM-dd HH:mm"

and in logstash config i added

	  date {
  match => [ "VisitEndDateTime", "ISO8601","YYYY-MM-dd HH:mm:ss:SSS", "YYYY-MM-dd HH:mm:ss","YYYY-MM-dd HH:mm" ]
  target => "VisitEndDateTime"
  locale => "en"
}

thanks ur suggestion for add debug helped me to find issue .

7

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.