CSV as input and CSV as output

Dear All,

I am new to logstash and I am trying to input a .csv file to logstash and output it also as a .csv format for my testing.
However, I find that the output .csv is not ordered row by row as the input .csv file. The order is somehow randomized even I specified start_position is beginning.
Moreover, since the input .csv come with the field name in 1st row, logstash regarded it as a document rather than field name, this frustrated me.
It would be glad if someone could solve my problem. Thank you in advance.

input {
    file {
        path => "/home/elk/logstash-6.5.4/input_data/M10d-splunk-wk.csv"
        start_position => "beginning"
		sincedb_path => "/dev/null"
    }
}
filter {
	csv {
		separator => ","
		columns => ["TimeStamp","username","User Agent","Region","Source IP","altemail","countrycode","phone","acctype","kuemail"]
		autodetect_column_names => true
		autogenerate_column_names => true
	}

        fingerprint {
            method => "SHA256"
            source => ["username"]
        }

        mutate { add_field => { 'username_key' => "%{fingerprint}"  'username_value' => "%{username}" }  }
        mutate { replace => { "username" => "%{fingerprint}" } }

        fingerprint {
            method => "SHA256"
            source => ["phone"]
        }

        mutate { add_field => { 'phone_key' => "%{fingerprint}"  'phone_value' => "%{phone}" } }
        mutate { replace => { "phone" => "%{fingerprint}" } }
}
output {
	csv {
		fields => ["TimeStamp","username","User Agent","Region","Source IP","altemail","countrycode","phone","acctype","kuemail"]
		path => "/home/elk/logstash-6.5.4/input_data/a.csv"
	}
	csv {
		fields => ["username_key","username_value","phone_key","phone_value"]
		path => "/home/elk/logstash-6.5.4/input_data/b.csv"
	}
}

That is working as expected. If you add "--pipeline.workers 1" to the command line to make logstash single-threaded that may help, but logstash does not guarantee ordering.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.