I am trying to setup tests for Logstash filters. We use Chef, and are having trouble installing plugins for our filters. Here's the spec file that sets up and run the tests. Tests are described in a YAML file, which I don't think are relevant to this question... because we are not even reaching that step.
# frozen_string_literal: true
require_relative '../spec_helper'
describe 'logstash::events' do
FILTER_VERIFIER_VERSION = '1.6.0'
LOGSTASH_VERSION = '6.6.2'
LOGSTASH_RUBY_VERSION = '2.3.0'
cached(:chef_run) do
ChefSpec::SoloRunner.new(platform: 'ubuntu', version: '16.04') do |runner|
runner.node.set['consul']['server'] = 'true'
runner.node.set['base_environment'] = 'chefspec'
runner.node.set['logstash']['events']['type_mapping'] = {
'default-es-cluster' => 'mock-elastic-search-cluster',
}
runner.node.set['logstash-testing']['es_cluster_map'] = {
'mock-elastic-search-cluster' => '"127.0.0.19:9200"',
}
runner.node.set['logstash']['events']['default_es_cluster'] = 'mock-elastic-search-cluster'
runner.node.set['consul']['config_dir'] = '/mnt/consul/config'
runner.node.set['envoy']['listeners_log_path'] = '/mnt/envoy/listeners.log'
# We're using a relative path, because we can't declare a property that lazily evaluates the @temp_dir.
runner.node.set['maxmind']['db']['city_file'] = "logstash-#{LOGSTASH_VERSION}/vendor/bundle/jruby/#{LOGSTASH_RUBY_VERSION}/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-City.mmdb"
end.converge(described_recipe)
end
before(:all) do
@temp_dir = Dir.mktmpdir
# DEBUG
# @temp_dir = '/tmp/d20200403-29347-rjfl7n'
# Install the test software.
unless system("wget 'https://github.com/magnusbaeck/logstash-filter-verifier/releases/download/#{FILTER_VERIFIER_VERSION}/logstash-filter-verifier_#{FILTER_VERIFIER_VERSION}_linux_amd64.tar.gz' -O - | tar zx", chdir: @temp_dir)
raise 'unable to download logstash-filter-verifier code'
end
# Download and extract logstash.
puts "Downloading and extracting Logstash version #{LOGSTASH_VERSION}..."
unless system("curl -s https://artifacts.elastic.co/downloads/logstash/logstash-#{LOGSTASH_VERSION}.tar.gz | tar zx", chdir: @temp_dir)
raise 'unable to download logstash'
end
# Download and install logstash plugins
logstash_plugins = {
'logstash-codec-protobuf' => '1.1.0',
'logstash-filter-kv' => '4.2.1',
'logstash-input-cloudwatch_logs' => '1.0.3',
'logstash-filter-age' => '1.0.2'
}
logstash_directory = "#{@temp_dir}/logstash-#{LOGSTASH_VERSION}"
ENV['DEBUG'] = '1'
ENV['LOGSTASH_HOME'] = logstash_directory
ENV['JRUBY_HOME'] = "#{logstash_directory}/vendor/jruby"
ENV['GEM_HOME'] = "#{logstash_directory}/vendor/bundle/jruby/#{LOGSTASH_RUBY_VERSION}"
ENV['GEM_PATH'] = "#{logstash_directory}/vendor/bundle/jruby/#{LOGSTASH_RUBY_VERSION}"
ENV['LS_GEM_PATH'] = ENV['GEM_PATH']
ENV['PATH'] = "#{logstash_directory}/bin:#{logstash_directory}/vendor/jruby/bin:#{ENV['GEM_HOME']}/bin:#{ENV['PATH']}"
system(ENV, "for f in $(grep --recursive --files-with-matches '/home/vagrant/projects/logstash/vendor/jruby/bin/jruby' #{logstash_directory}/*); do sed -i'' 's|/home/vagrant/projects/logstash|#{logstash_directory}|g' $f; done")
logstash_plugins.each do |plugin_name, plugin_version|
puts "Installing Logstash plugin #{plugin_name} (v#{plugin_version})..."
next if system(ENV,
"#{logstash_directory}/bin/logstash-plugin install --version #{plugin_version} #{plugin_name}",
chdir: @temp_dir)
raise "Unable to install plugin #{plugin_name} (v#{plugin_version})..."
end
# Install slack patterns, because the filters need them.
Dir.mkdir "#{@temp_dir}/patterns"
FileUtils.cp(File.join(__dir__, '../../files/default/patterns'), "#{@temp_dir}/patterns/patterns")
FileUtils.cp(File.join(__dir__, '../../files/default/patterns'), "#{logstash_directory}/vendor/bundle/jruby/#{LOGSTASH_RUBY_VERSION}/gems/logstash-patterns-core-4.1.2/patterns/patterns")
end
after(:all) do
# DEBUG
# FileUtils.remove_entry_secure @temp_dir unless @temp_dir.nil?
end
before do
# Pretend everything is installed (because we're not testing installation).
stub_command('/usr/bin/dpkg-query --status logstash | /bin/grep -P "Status.+unpacked"').and_return(true)
stub_command("curl -s --max-time 5 -XGET 'localhost:9600/_node/plugins' | jq '.plugins[] | select(.name == \"logstash-codec-protobuf\" and .version == \"1.1.0\")' | grep -q logstash-codec-protobuf").and_return(true)
stub_command("curl -s --max-time 5 -XGET 'localhost:9600/_node/plugins' | jq '.plugins[] | select(.name == \"logstash-filter-kv\" and .version == \"4.2.1\")' | grep -q logstash-filter-kv").and_return(true)
stub_command("curl -s --max-time 5 -XGET 'localhost:9600/_node/plugins' | jq '.plugins[] | select(.name == \"logstash-input-cloudwatch_logs\" and .version == \"1.0.3\")' | grep -q logstash-input-cloudwatch_logs").and_return(true)
stub_command("curl -s --max-time 5 -XGET 'localhost:9600/_node/plugins' | jq '.plugins[] | select(.name == \"logstash-filter-age\" and .version == \"1.0.2\")' | grep -q logstash-filter-age").and_return(true)
stub_command("curl -s --max-time 5 -XGET 'localhost:9600/_node/plugins' | jq .plugins[].name | grep -q logstash-codec-protobuf").and_return(true)
stub_command("curl -s --max-time 5 -XGET 'localhost:9600/_node/plugins' | jq .plugins[].name | grep -q logstash-filter-kv").and_return(true)
stub_command("curl -s --max-time 5 -XGET 'localhost:9600/_node/plugins' | jq .plugins[].name | grep -q logstash-input-cloudwatch_logs").and_return(true)
stub_command("curl -s --max-time 5 -XGET 'localhost:9600/_node/plugins' | jq .plugins[].name | grep -q logstash-filter-age").and_return(true)
end
let(:tests) { Dir[File.join(__dir__, '*.yaml')] }
it 'passes the filter tests' do
expect(chef_run).to render_file('/etc/logstash/conf.d/events.conf').with_content { |content|
# Create the config in our temporary directory.
File.write("#{@temp_dir}/events.conf", content)
# Use this file to invoke the tests.
unless system("#{@temp_dir}/logstash-filter-verifier", "--logstash-path=#{@temp_dir}/logstash-#{LOGSTASH_VERSION}/bin/logstash", __dir__, "#{@temp_dir}/events.conf", chdir: @temp_dir)
raise 'filter tests do not pass'
end
}
end
end
I get the following exception:
Installing Logstash plugin logstash-filter-age (v1.0.2)...
Using GEM_HOME=/tmp/d20200403-29347-rjfl7n/logstash-6.6.2/vendor/bundle/jruby/2.3.0
Using GEM_PATH=/tmp/d20200403-29347-rjfl7n/logstash-6.6.2/vendor/bundle/jruby/2.3.0
DEBUG: exec /tmp/d20200403-29347-rjfl7n/logstash-6.6.2/vendor/jruby/bin/jruby /tmp/d20200403-29347-rjfl7n/logstash-6.6.2/lib/pluginmanager/main.rb install --version 1.0.2 logstash-filter-age
Could not find rake-13.0.1 in any of the sources
/home/meowtochondria/.rbenv/versions/2.4.1/lib/ruby/gems/2.4.0/gems/bundler-2.1.4/lib/bundler/spec_set.rb:86:in `block in materialize'
org/jruby/RubyArray.java:2518:in `map!'
/home/meowtochondria/.rbenv/versions/2.4.1/lib/ruby/gems/2.4.0/gems/bundler-2.1.4/lib/bundler/spec_set.rb:80:in `materialize'
/home/meowtochondria/.rbenv/versions/2.4.1/lib/ruby/gems/2.4.0/gems/bundler-2.1.4/lib/bundler/definition.rb:170:in `specs'
/home/meowtochondria/.rbenv/versions/2.4.1/lib/ruby/gems/2.4.0/gems/bundler-2.1.4/lib/bundler/definition.rb:237:in `specs_for'
/home/meowtochondria/.rbenv/versions/2.4.1/lib/ruby/gems/2.4.0/gems/bundler-2.1.4/lib/bundler/definition.rb:226:in `requested_specs'
/home/meowtochondria/.rbenv/versions/2.4.1/lib/ruby/gems/2.4.0/gems/bundler-2.1.4/lib/bundler/runtime.rb:101:in `block in requested_specs'
/home/meowtochondria/.rbenv/versions/2.4.1/lib/ruby/gems/2.4.0/gems/bundler-2.1.4/lib/bundler/runtime.rb:20:in `setup'
/home/meowtochondria/.rbenv/versions/2.4.1/lib/ruby/gems/2.4.0/gems/bundler-2.1.4/lib/bundler.rb:149:in `setup'
/home/meowtochondria/.rbenv/versions/2.4.1/lib/ruby/gems/2.4.0/gems/bundler-2.1.4/lib/bundler/setup.rb:10:in `block in (root)'
/home/meowtochondria/.rbenv/versions/2.4.1/lib/ruby/gems/2.4.0/gems/bundler-2.1.4/lib/bundler/ui/shell.rb:136:in `with_level'
/home/meowtochondria/.rbenv/versions/2.4.1/lib/ruby/gems/2.4.0/gems/bundler-2.1.4/lib/bundler/ui/shell.rb:88:in `silence'
/home/meowtochondria/.rbenv/versions/2.4.1/lib/ruby/gems/2.4.0/gems/bundler-2.1.4/lib/bundler/setup.rb:10:in `<main>'
org/jruby/RubyKernel.java:955:in `require'
/tmp/d20200403-29347-rjfl7n/logstash-6.6.2/vendor/jruby/lib/ruby/stdlib/rubygems/core_ext/kernel_require.rb:1:in `(root)'
Run `bundle install` to install missing gems.
I verified that rake is available in JRuby's installation of Logstash. I have rbenv installed, which is providing ruby 2.4.1. Rake is installed there as well, but bundler is not able to find even that. I tried going through source code, but have not able to find any useful clues. Is there an environment variable or something that i can set to have installation proceed?