Hello
I'm trying to create an export using logstash (5.6.12) with the pipeline as written below. But unfortunately it exports everything continously in stead of everything just once.
I start it with the following command in a pod in a kubernetes cluster
logstash --path.data /tmp/ --http.port 9601 -f ./test-erik-acc.conf
What do i do wrong?
test-erik-acc.conf:
input {
elasticsearch {
hosts => "es-client.cts-prd:9200"
index => "cts-portaal*"
scroll => "30s"
query => '
{
"query": {
"bool": {
"must": [
{
"query_string": {
"query": "batchRequestInfo.receiverID: (8716871000002)",
"analyze_wildcard": true,
"all_fields": true
}
},
{
"match_phrase": {
"topic": {
"query": "pushed-meteringrequests"
}
}
},
{
"range": {
"batchRequestInfo.intakeDateTime": {
"gte": "now-9d/d",
"lt": "now-8d/d",
"format": "epoch_millis"
}
}
}
],
"must_not":
}
}
}'
}}
filter {
mutate {
rename => {
"pushDateTime" => "[headEndPushInfo][pushDateTime]"
}
}
ruby {
init => "
begin
@@csv_file = '/tmp/pushed.csv'
@@csv_headers = ['headEndPushInfo', 'correlationId', 'facilitatingRNB' ,]
if File.zero?(@@csv_file) || !File.exist?(@@csv_file)
CSV.open(@@csv_file, 'w') do |csv|
csv << @@csv_headers
end
end
end
"
code => "
begin
@@csv_file = event.get('[@metadata][csv_file]')
@@csv_headers = event.get('[@metadata][csv_headers]')
end
"
}
csv {
columns => ["pushDateTime", "correlationId", "facilitatingRNB"]
}
}
output {
csv {
fields => ["[headEndPushInfo][pushDateTime]", "correlationId", "facilitatingRNB"]
path => "/tmp/pushed.csv"
}
stdout {
codec => rubydebug {
metadata => true
}
}
}
after a few hours (export should be 900000 lines) the export looks like(and is 4660000 lines):
headEndPushInfo,correlationId,facilitatingRNB
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696003509,b8497f80-6696-49a7-a1f0-52b24f562c29,8716871000002
1545696022448,786cc485-4efd-4387-8a43-a12e56c1605a,8716871000002
1545696022448,786cc485-4efd-4387-8a43-a12e56c1605a,8716871000002
1545696022448,786cc485-4efd-4387-8a43-a12e56c1605a,8716871000002
1545696022448,786cc485-4efd-4387-8a43-a12e56c1605a,8716871000002
1545696022448,786cc485-4efd-4387-8a43-a12e56c1605a,8716871000002
1545696022448,786cc485-4efd-4387-8a43-a12e56c1605a,8716871000002
1545696022448,786cc485-4efd-4387-8a43-a12e56c1605a,8716871000002
1545696022448,786cc485-4efd-4387-8a43-a12e56c1605a,8716871000002
1545696022448,786cc485-4efd-4387-8a43-a12e56c1605a,8716871000002
1545696022448,786cc485-4efd-4387-8a43-a12e56c1605a,8716871000002
1545696022448,786cc485-4efd-4387-8a43-a12e56c1605a,8716871000002