Read a gzip file with gzip_lines codec

Elastic Stack Logstash
12

Clone Source
git clone https://github.com/logstash-plugins/logstash-codec-gzip_lines

Change gzip_lines.rb
logstash-codec-gzip_lines/lib/logstash/codecs/gzip_lines.rb
line 32 @decoder = Zlib::GzipReader.open(data)

Build Gem File
cd logstash-codec-gzip_lines
gem build logstash-code-gzip_lines.gemspec

Reinstall logstash-codec-gzip_lines
cd logstash
bin/plugin install /opt/logstash-codec-gzip_lines/logstash-codec-gzip_lines-2.0.2.gem

Create list.txt File
Write by yourself gzip_lines.rb and run it. It will create list.txt file.

Sample code:
class GzipLines
attr_accessor :list_path

  public
  def create_list(data_path)
    #unless File.exist?(self.list_path)
    open(self.list_path,"w")
    #end

    folders = entries_list(data_path)

    folders.each do |folder|
      files = file_list(data_path+folder)
      files.each do |file|
        open(self.list_path, "a") { |f| f << data_path+folder+'/'+file.to_s+"\n" }
      end
    end
  end

  def entries_list(basedir)
    Dir.entries(basedir).select {|entry| File.directory? File.join(basedir, entry) and !(entry =='.' || entry =='..')}
  end

  def file_list(folder_path)
    Dir.chdir(folder_path)
    files = Dir.glob('*.gz')
  end
end

    gzip_lines = GzipLines.new
    gzip_lines.list_path = '/opt/logstash/list.txt'
    gzip_lines.create_list('/home/tsanchin/rawdata/')

Config logstash.conf to read list.txt file
file {
type => "gzip"
path => "/opt/logstash/list.txt"
start_position => "beginning"
sincedb_path => "gzip"
codec => gzip_lines
}

2 Likes