[^/]+)/" }
}
if ("wpdirect_order.log" in [path]) {
mutate {
convert => { "thread" => "string" }
convert => { "integration" => "string" }
convert => { "class" => "string" }
convert => { "orderid" => "integer" }
}
grok {
patterns_dir => ["/usr/share/logstash/patterns"]
match => { "message" => "^%{TIMESTAMP_ISO8601:timestamp};%{LOGLEVEL:loglevel}(.*);%{OPTIONALUSERNAME:user};%{TNR:tnr};%{INTEGRATION:integration};%{THREAD:thread};%{CLASS:class};%{CLASS:irgendwas};%{NUMBER:orderid:int}$" }
add_field => ["logtype", "wpdirect_order"]
}
}
date {
match => ["timestamp", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-d HH:mm:ss,SSS"]
}
}
output {
redis {
id => "output_redis"
data_type => "list"
host => ["REDIS-SERVER:6379"]
key => "logstash"
}
stdout { codec => json }
}
```
---
### Author:  [@magnusbaeck](https://discuss.elastic.co/u/magnusbaeck)
#### Post date: [November 21, 2017, 1:32pm UTC](https://discuss.elastic.co/t/logstash-index-problem-after-logrotate/108463/4 "2017-11-21T13:32:59Z")
Yes, try using `start_position => "beginning"` instead.
---
### Author:  [@Vafa\_Ronaghi](https://discuss.elastic.co/u/Vafa_Ronaghi)
#### Post date: [November 21, 2017, 3:08pm UTC](https://discuss.elastic.co/t/logstash-index-problem-after-logrotate/108463/5 "2017-11-21T15:08:34Z")
Thanks Magnus, It works now.
---
### Author:  [@Vafa\_Ronaghi](https://discuss.elastic.co/u/Vafa_Ronaghi)
#### Post date: [November 22, 2017, 8:38am UTC](https://discuss.elastic.co/t/logstash-index-problem-after-logrotate/108463/6 "2017-11-22T08:38:31Z")
Hi Magnus
I found another problem after changing start\_position to beginning.
It seems that Logstash loses position of last read offset when ever we have to restart the instance.
Is this a normal behavior or it is a bug ?
Thanks
Vafa
---
### Author:  [@magnusbaeck](https://discuss.elastic.co/u/magnusbaeck)
#### Post date: [November 22, 2017, 8:57am UTC](https://discuss.elastic.co/t/logstash-index-problem-after-logrotate/108463/7 "2017-11-22T08:57:50Z")
That's not normal behavior. Increasing the log level will make Logstash emit more messages related to sincedb and you can e.g. verify whether Logstash saves the correct offset upon shutdown.
---
### Author:  [@Vafa\_Ronaghi](https://discuss.elastic.co/u/Vafa_Ronaghi)
#### Post date: [November 22, 2017, 9:13am UTC](https://discuss.elastic.co/t/logstash-index-problem-after-logrotate/108463/8 "2017-11-22T09:13:40Z")
I am not using sincedb in this configuration, because due to this help page , sincedb will be created in home dir of the user rungs logstash.
[https://www.elastic.co/guide/en/logstash/current/plugins-inputs-file.html#\_tracking\_of\_current\_position\_in\_watched\_files](https://www.elastic.co/guide/en/logstash/current/plugins-inputs-file.html#_tracking_of_current_position_in_watched_files)
I can't find any sincedb file.
What shall i do, shall i create a sincedb\_path for each file inside inplut plugin ?
Thanks in advance for your support.
Regards
Vafa
---
### Author:  [@magnusbaeck](https://discuss.elastic.co/u/magnusbaeck)
#### Post date: [November 22, 2017, 9:15am UTC](https://discuss.elastic.co/t/logstash-index-problem-after-logrotate/108463/9 "2017-11-22T09:15:48Z")
Well, if you disable sincedb then Logstash will of course start from the beginning each time. If the default sincedb location isn't suitable (and overriding the HOME variable for Logstash isn't an option) you'll have to set `sincedb_path` for each file input.
---
### Author:  [@Vafa\_Ronaghi](https://discuss.elastic.co/u/Vafa_Ronaghi)
#### Post date: [November 22, 2017, 11:13am UTC](https://discuss.elastic.co/t/logstash-index-problem-after-logrotate/108463/10 "2017-11-22T11:13:23Z")
Ok, this means i should use a sincedb for every file
path=\> "file1"
sincedb\_path =\> "sincedb\_file1"
Shall i still use start\_position =\> "beginning" ?
* * *
I ask beause now everytime make a config change and restart logstash, it start to read files from beginning
---
### Author:  [@magnusbaeck](https://discuss.elastic.co/u/magnusbaeck)
#### Post date: [November 22, 2017, 11:19am UTC](https://discuss.elastic.co/t/logstash-index-problem-after-logrotate/108463/11 "2017-11-22T11:19:07Z")
> Ok, this means i should use a sincedb for every file
> path=\> "file1"
> sincedb\_path =\> "sincedb\_file1"
Yes.
> Shall i still use start\_position =\> "beginning" ?
Yes.
---
### Author:  [@Vafa\_Ronaghi](https://discuss.elastic.co/u/Vafa_Ronaghi)
#### Post date: [November 25, 2017, 10:19pm UTC](https://discuss.elastic.co/t/logstash-index-problem-after-logrotate/108463/12 "2017-11-25T22:19:37Z")
Thanks Magnus,
I did this but I still have the problem after logrotate , when the file is only having one line of log.
This application most of the time writes only one sigle lineto the file after every logrotate.
Logs are looking like this:
`2017-11-25 02:00:01,022;INFO ;NA;NA;;[Timer-0];AdminFacade;Logging started`
After log rotate this single line of log is not read by Logstash and also not indexed.
**But if we write a second line of log , than it will be read by Logstash and indexed by ES.**
Any Idea ?
Thanks in advance
---
### Author:  [@magnusbaeck](https://discuss.elastic.co/u/magnusbaeck)
#### Post date: [November 26, 2017, 7:58pm UTC](https://discuss.elastic.co/t/logstash-index-problem-after-logrotate/108463/13 "2017-11-26T19:58:26Z")
How are the files being rotated? What's the logrotate configuration?
---
### Author:  [@system](https://discuss.elastic.co/u/system)
#### Post date: [December 24, 2017, 7:58pm UTC](https://discuss.elastic.co/t/logstash-index-problem-after-logrotate/108463/14 "2017-12-24T19:58:31Z")
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.