Summary
I am collecting asset data for Jira using automations to generate a HTTP POST request to Logstash, the following code then takes the input from the request and filters it and sends it to OpenSearch. I have one yaml file that is working using webhooks to collect issue data however the asset yaml doesn't. The error thrown refers to the logstash.javapipeline. The problem however seems to be with the Ruby filter as hashing out those lines seems to work however doesn't provide any data.
[ERROR][logstash.javapipeline][main] Pipeline error {:pipeline_id="main", :exception=>#JSON::ParserError: unexpected token at ''>, :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:238:in `register_plugins`", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:599:in `maybe_setup_out_plugins`", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:245:in `start_workers`", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:190:in `run`", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:142:in `block in start`"], "pipeline.sources"=>["/usr/share/logstash/pipeline/logstash.conf"], :thread=>"#<Thread:0x36246de8 run>"}
Code
input: |-
http {
port => 8080
codec => "json"
}
filter: |-
json {
source => "message"
target => "assets"
}
ruby {
init => '
require "net/http"
require "uri"
jira_url = ENV["JIRA_DEV_URL"]
uri = URI(jira_url)
token = ENV["JIRA_BEARER"]
request = Net::HTTP::Get.new(uri)
request["Authorization"] = "Bearer " + token
response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
http.request(request)
end
@@json_list = JSON.parse(response.body)
'
code => '
begin
event.set("[assets][environment]", "dev")
event.get("[assets][object][data]").each do |key, value|
should_remove = false
if value.nil?
should_remove = true
elsif (value.is_a?(String) || value.is_a?(Array) || value.is_a?(Hash)) && value.empty?
should_remove = true
end
if should_remove
event.remove("[assets][object][data][#{key}]")
else
matching_item = @@json_list.find { |item| item["id"] == key }
if matching_item
new_key = matching_item["name"].downcase.gsub(" ", "_")
event.remove("[assets][object][data][#{key}]")
event.set("[assets][object][data][#{new_key}]", value)
end
end
end
rescue Exception => e
logger.error("{e.message}")
end
'
}
output: |-
opensearch {
hosts => ["url:port"]
index => "index_string"
ssl => true
ssl_certificate_verification => true
keystore => "keystore.p12"
keystore_password => "${KEYSTORE_PASSWORD}"
truststore => "truststore.jks"
truststore_password => "${TRUSTSTORE_PASSWORD}"
manage_template => false
}
stdout {}
)