I am having trouble with the logstash pipeline
. I've had the pipeline working with the exact same settings on a different server.
The config works when I run it manually using the command line command.
The discoverer does not find any files at the specified location.
Is there anything that I'm missing? I can't figure out why it is not discovering any files when all the paths are correct.
pipeline log
[2020-07-22T14:00:06,380][DEBUG][org.logstash.execution.PeriodicFlush] Pushing flush onto pipeline.
[2020-07-22T14:00:11,380][DEBUG][org.logstash.execution.PeriodicFlush] Pushing flush onto pipeline.
[2020-07-22T14:00:16,380][DEBUG][org.logstash.execution.PeriodicFlush] Pushing flush onto pipeline.
[2020-07-22T14:00:17,469][TRACE][filewatch.discoverer ] discover_files {"count"=>0}
[2020-07-22T14:00:21,380][DEBUG][org.logstash.execution.PeriodicFlush] Pushing flush onto pipeline.
[2020-07-22T14:00:26,380][DEBUG][org.logstash.execution.PeriodicFlush] Pushing flush onto pipeline.
[2020-07-22T14:00:31,380][DEBUG][org.logstash.execution.PeriodicFlush] Pushing flush onto pipeline.
[2020-07-22T14:00:32,474][TRACE][filewatch.discoverer ] discover_files {"count"=>0}
[root@dev-01 /path/version7-0/integrations/test/data]# ls -la
drwxr-xr-x. 2 root root 4096 Jul 21 15:17 .
drw-r--r--. 3 root root 80 Jul 21 11:03 ..
-rw-r--r--. 1 root root 5997 Jul 21 15:16 167772682.json
-rw-r--r--. 1 root root 5996 Jul 21 15:16 167772684.json
-rw-r--r--. 1 root root 5331 Jul 21 15:17 2887132417.json
/etc/logstash/conf.d/test.conf
input {
file {
path => "/path/version7-0/integrations/test/data/*.json"
start_position => "beginning"
sincedb_path => "/dev/null"
type => "json"
mode => "read"
file_completed_action => "delete"
file_chunk_size => 64000
}
}
filter {
json {
source => "message"
}
mutate {
remove_field => [ "message" ]
}
}
output {
elasticsearch {
hosts => "127.0.0.1:9200"
index => "test-%{+YYYY.MM.dd.HH}"
manage_template => true
template => '/path/version7-0/integrations/templates/test_template.json'
template_name => 'test_template' # The default is logstash
template_overwrite => true
}
stdout {
codec => rubydebug
}
}
pipeline.yml
- pipeline.id: test-pipeline
path.config: "/etc/logstash/conf.d/test.conf"