Failed to query elasticsearch for previous event {:query=>"_index:s1index AND type:a AND A:1 ",....:error=>#<NoMethodError: undefined method `start_with?' for nil:NilClass>, :level=>:warn}

Elastic Stack Logstash
1

Hi
I have two csv files , s1.csv with following data,
1,2,3
1,2,3
2,3,4

And s2.csv with following data,
1,4
2,5

Consider column name as specified in configuration files like for s1 - A,B,C and for s2 Aa,Bb
Now I want to add new field called "Bbb" and copy "Bb" column from s2 file to s1 event by matching A = Aa. so I want to get following result

1,2,3,4
1,2,3,4
2,3,4,5

logstash-sample1.conf with following configuration

input {
file {
path => "/home/workload/s1.csv"
start_position => "beginning"
stat_interval => 30
type => "a"
}
}
filter {
if ([type] == "a") {
csv {
columns => ["A","B","C"]
separator => ","
convert => {"A"=>"integer"}
convert => {"B"=>"integer"}
convert => {"C" => "integer"}
}
}
}
output {
elasticsearch {
action => "index"
index => "s1index"
hosts => "10.0.1.73"
}
stdout {
codec=>"rubydebug"
}
}

And logstash-sample2.conf with following
input {
file {
path => "/home/workload/s2.csv"
start_position => "beginning"
stat_interval => 30
type => "b"
}
}
filter {
if ([type] == "b") {
csv {
columns => ["Aa","Bb"]
separator => ","
convert => {"Aa"=>"integer"}
convert => {"Bb"=>"integer"}
}
}
elasticsearch {
hosts => ["centrifyfw2"]
query => "_index:s1index AND type:a AND A:%{[Aa]} "
fields => ["B", "Bbb"]
}
}
output {
stdout {
codec=>"rubydebug"
}
}

I am getting following error
Starting pipeline {:id=>"base", :pipeline_workers=>4, :batch_size=>125, :batch_delay=>5, :max_inflight=>500, :level=>:info}
Pipeline started {:level=>:info}
Logstash startup completed
Failed to query elasticsearch for previous event {:query=>"_index:s1index AND type:a AND A:1 ", :event=>#<LogStash::Event:0x193eed2b @metadata_accessors=#<LogStash::Util::Accessors:0x1de75774 @store={"path"=>"/home/workload/s2.csv"}, @lut={"[path]"=>[{"path"=>"/home/workload/s2.csv"}, "path"]}>, @cancelled=false, @data={"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, @metadata={"path"=>"/home/workload/s2.csv"}, @accessors=#<LogStash::Util::Accessors:0x184abbb @store={"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, @lut={"path"=>[{"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, "path"], "host"=>[{"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, "host"], "type"=>[{"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, "type"], "[type]"=>[{"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, "type"], "message"=>[{"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, "message"], "[Aa]"=>[{"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, "Aa"], "[Bb]"=>[{"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, "Bb"]}>>, :error=>#<NoMethodError: undefined method `start_with?' for nil:NilClass>, :level=>:warn}
{
"message" => "1,4\r",
"@version" => "1",
"@timestamp" => "2016-04-13T13:17:38.436Z",
"path" => "/home/workload/s2.csv",
"host" => "centrifyfw2",
"type" => "b",
"Aa" => 1,
"Bb" => 4
}
Failed to query ..
{
"message" => "2,5\r",
"@version" => "1",
"@timestamp" => "2016-04-13T13:17:38.603Z",
"path" => "/home/workload/s2.csv",
"host" => "centrifyfw2",
"type" => "b",
"Aa" => 2,
"Bb" => 5
}

How to resolve it?

2

In the subject field you mention a NoMethodError error. Please show the complete logs surrounding that.

3

As there is limit to subject text I put "Failed to query elasticsearch for previous event {:query=>”_index:s1index AND type:a AND A:1 “,….:error=>#<NoMethodError: undefined method `start_with?’ for nil:NilClass>, :level=>:warn}" in subject

4

Well, is there a NoMethodError error message in your logs? Then post the full context of it. What you've included in the body of your post never mentions NoMethodError.

5

Yes. There is NoMethodError error message in logs. Here is log.

Registering file input {:path=>["/home/workload/s2.csv"], :level=>:info}
No sincedb_path set, generating one based on the file path {:sincedb_path=>"/root/.sincedb_5cdf64f518cc8621047416ef3996332c", :path=>["/home/workload/s2.csv"], :level=>:info}
New ElasticSearch filter {:hosts=>["centrifyfw2"], :level=>:info}
Starting pipeline {:id=>"base", :pipeline_workers=>4, :batch_size=>125, :batch_delay=>5, :max_inflight=>500, :level=>:info}
Pipeline started {:level=>:info}
Logstash startup completed
Failed to query elasticsearch for previous event {:query=>"_index:s1index AND type:a AND A:1 ", :event=>#<LogStash::Event:0x193eed2b @metadata_accessors=#<LogStash::Util::Accessors:0x1de75774 @store={"path"=>"/home/workload/s2.csv"}, @lut={"[path]"=>[{"path"=>"/home/workload/s2.csv"}, "path"]}>, @cancelled=false, @data={"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, @metadata={"path"=>"/home/workload/s2.csv"}, @accessors=#<LogStash::Util::Accessors:0x184abbb @store={"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, @lut={"path"=>[{"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, "path"], "host"=>[{"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, "host"], "type"=>[{"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, "type"], "[type]"=>[{"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, "type"], "message"=>[{"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, "message"], "[Aa]"=>[{"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, "Aa"], "[Bb]"=>[{"message"=>"1,4\r", "@version"=>"1", "@timestamp"=>"2016-04-13T13:17:38.436Z", "path"=>"/home/workload/s2.csv", "host"=>"centrifyfw2", "type"=>"b", "Aa"=>1, "Bb"=>4}, "Bb"]}>>, :error=>#NoMethodError: undefined method `start_with?' for nil:NilClass, :level=>:warn}
{
"message" => "1,4\r",
"@version" => "1",
"@timestamp" => "2016-04-13T13:17:38.436Z",
"path" => "/home/workload/s2.csv",
"host" => "centrifyfw2",
"type" => "b",
"Aa" => 1,
"Bb" => 4
}

6

But the log snippet you just posted doesn't contain the word NoMethodError.

7

Please check it again.

8

Hi
I have the same logstash.log error with NoMethodError .
here the conf files:

input {
beats {
port => 5044
ssl => false

ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"

ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"

}
}
filter {
if [type] == "conn" {
grok {
tag_on_failure => ["connlog_long_parse_fail"]
match => {"message" => [ ".?\s+.?\s+(?<src_ip>\d+.\d+.\d+.\d+)\s\S+\s(?<dst_ip>\d+.\d+.\d+.\d+)"] }
}
elasticsearch {
hosts => ["localhost"]
query => "type:maltrail AND dst_ip:%{dst_ip}"
fields => [ "reason" ]
}
geoip {
source => "src_ip"
target => "geoip"
database => "/etc/logstash/GeoLiteCity.dat"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
geoip {
source => "dst_ip"
target => "geoip"
database => "/etc/logstash/GeoLiteCity.dat"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
}
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
sniffing => true
manage_template => false
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
sniffing => true
manage_template => false
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}

there's the log-

{:timestamp=>"2016-07-18T09:10:18.083000+0300", :message=>"Failed to query elasticsearch for previous event", :index=>"", :query=>"type:maltrail AND dst_ip:172.16.1.10", :event=>#<LogStash::Event:0x51b7e5c7 @metadata_accessors=#<LogStash::Util::Accessors:0x6e95d2e7 @store={"type"=>"conn", "beat"=>"filebeat"}, @lut={}>, @cancelled=false, @data={"message"=>"1468822199.963471\tC3gEHk1b9GOQXYX5Z3\t172.17.1.2\t45124\t172.16.1.10\t53\tudp\tdns\t0.026759\t0\t506\tSHR\tT\tT\t0\tCd\t0\t0\t1\t534\t(empty)\t-\t-\t#siftworkstation-eth0", "@version"=>"1", "@timestamp"=>"2016-07-18T06:10:17.432Z", "type"=>"conn", "source"=>"/nsm/bro/logs/current/conn.log", "count"=>1, "fields"=>nil, "beat"=>{"hostname"=>"siftworkstation", "name"=>"siftworkstation"}, "offset"=>504473, "input_type"=>"log", "host"=>"siftworkstation", "tags"=>["beats_input_codec_plain_applied"], "syslog_severity_code"=>5, "syslog_facility_code"=>1, "syslog_facility"=>"user-level", "syslog_severity"=>"notice", "src_ip"=>"172.17.1.2", "dst_ip"=>"172.16.1.10"}, @metadata={"type"=>"conn", "beat"=>"filebeat"}, @accessors=#<LogStash::Util::Accessors:0x2669ac33 @store={"message"=>"1468822199.963471\tC3gEHk1b9GOQXYX5Z3\t172.17.1.2\t45124\t172.16.1.10\t53\tudp\tdns\t0.026759\t0\t506\tSHR\tT\tT\t0\tCd\t0\t0\t1\t534\t(empty)\t-\t-\t#siftworkstation-eth0", "@version"=>"1", "@timestamp"=>"2016-07-18T06:10:17.432Z", "type"=>"conn", "source"=>"/nsm/bro/logs/current/conn.log", "count"=>1, "fields"=>nil, "beat"=>{"hostname"=>"siftworkstation", "name"=>"siftworkstation"}, "offset"=>504473, "input_type"=>"log", "host"=>"siftworkstation", "tags"=>["beats_input_codec_plain_applied"], "syslog_severity_code"=>5, "syslog_facility_code"=>1, "syslog_facility"=>"user-level", "syslog_severity"=>"notice", [{"message"=>"1468822199.963471\tC3gEHk1b9GOQXYX5Z3\t172.17.1.2\t45124\t172.16.1.10\t53\tudp\tdns\t0.026759\t0\t506\tSHR\tT\tT\t0\tCd\t0\t0\t1\t534\t(empty)\t-\t-\t#siftworkstation-eth0", "@version"=>"1", "@timestamp"=>"2016-07-18T06:10:17.432Z", "type"=>"conn", "source"=>"/nsm/bro/logs/current/conn.log", "count"=>1, "fields"=>nil, "beat"=>{"hostname"=>"siftworkstation", "name"=>"siftworkstation"}, "offset"=>504473, "input_type"=>"log", "host"=>"siftworkstation", "tags"=>["beats_input_codec_plain_applied"], "syslog_severity_code"=>5, "syslog_facility_code"=>1, "syslog_facility"=>"user-level", "syslog_severity"=>"notice", "src_ip"=>"172.17.1.2", "dst_ip"=>"172.16.1.10"}, "src_ip"]}>>, :error=>#<NoMethodError: undefined method `start_with?' for nil:NilClass>, :level=>:warn}

9

[{"message"=>"1468822626.133305\tCv3pcx65PJ07Jdv25\t172.17.1.2\t37239\t172.16.1.10\t53\tudp\tdns\t0.027748\t0\t506\tSHR\tT\tT\t0\tCd\t0\t0\t1\t534\t(empty)\t-\t-\t#siftworkstation-eth0", "@version"=>"1", "@timestamp"=>"2016-07-18T06:17:17.440Z", "source"=>"/nsm/bro/logs/current/conn.log", "offset"=>505790, "input_type"=>"log", "count"=>1, "beat"=>{"hostname"=>"siftworkstation", "name"=>"siftworkstation"}, "type"=>"conn", "fields"=>nil, "host"=>"siftworkstation", "tags"=>["beats_input_codec_plain_applied"], "syslog_severity_code"=>5, "syslog_facility_code"=>1, "syslog_facility"=>"user-level", "syslog_severity"=>"notice", "src_ip"=>"172.17.1.2", "dst_ip"=>"172.16.1.10"}, "src_ip"]}>>, :error=>#<NoMethodError: undefined method start_with?' for nil:NilClass>, :level=>:warn} {:timestamp=>"2016-07-18T09:17:20.595000+0300", :message=>"Failed to query elasticsearch for previous event", :index=>"", :query=>"type:maltrail AND dst_ip:172.16.1.10", :event=>#<LogStash::Event:0x4b653d18 @metadata_accessors=#<LogStash::Util::Accessors:0x614d158e @store={"type"=>"conn", "beat"=>"filebeat"}, @lut={}>, @cancelled=false, @data={"message"=>"1468822626.137517\tCJ06wGBTIOaRavwR2\t172.17.1.2\t50456\t172.16.1.10\t53\tudp\tdns\t0.035619\t0\t1008\tSHR\tT\tT\t0\tCd\t0\t0\t2\t1064\t(empty)\t-\t-\t#siftworkstation-eth0", "@version"=>"1", "@timestamp"=>"2016-07-18T06:17:17.440Z", "source"=>"/nsm/bro/logs/current/conn.log", "type"=>"conn", "input_type"=>"log", "count"=>1, "fields"=>nil, "beat"=>{"hostname"=>"siftworkstation", "name"=>"siftworkstation"}, "offset"=>505938, "host"=>"siftworkstation", "tags"=>["beats_input_codec_plain_applied"], "syslog_severity_code"=>5, "syslog_facility_code"=>1, "syslog_facility"=>"user-level", "syslog_severity"=>"notice", "src_ip"=>"172.17.1.2", "dst_ip"=>"172.16.1.10"}, @metadata={"type"=>"conn", "beat"=>"filebeat"}, @accessors=#<LogStash::Util::Accessors:0x5f71b8e1 @store={"message"=>"1468822626.137517\tCJ06wGBTIOaRavwR2\t172.17.1.2\t50456\t172.16.1.10\t53\tudp\tdns\t0.035619\t0\t1008\tSHR\tT\tT\t0\tCd\t0\t0\t2\t1064\t(empty)\t-\t-\t#siftworkstation-eth0", "@version"=>"1", "@timestamp"=>"2016-07-18T06:17:17.440Z", "source"=>"/nsm/bro/logs/current/conn.log", "type"=>"conn", "input_type"=>"log", "count"=>1, "fields"=>nil, "beat"=>{"hostname"=>"siftworkstation", "name"=>"siftworkstation"}, "offset"=>505938, "host"=>"siftworkstation", "tags"=>["beats_input_codec_plain_applied"], "syslog_severity_code"=>5, "syslog_facility_code"=>1, "syslog_facility"=>"user-level", "syslog_severity"=>"notice", "src_ip"=>"172.17.1.2", "dst_ip"=>"172.16.1.10"}, @lut={"@timestamp"=>[{"message"=>"1468822626.137517\tCJ06wGBTIOaRavwR2\t172.17.1.2\t50456\t172.16.1.10\t53\tudp\tdns\t0.035619\t0\t1008\tSHR\tT\tT\t0\tCd\t0\t0\t2\t1064\t(empty)\t-\t-\t#siftworkstation-eth0", "@version"=>"1", "@timestamp"=>"2016-07-18T06:17:17.440Z", "source"=>"/nsm/bro/logs/current/conn.log", "type"=>"conn", "input_type"=>"log", "count"=>1, "fields"=>nil, "beat"=>{"hostname"=>"siftworkstation", "name"=>"siftworkstation"}, "offset"=>505938, "host"=>"siftworkstation", "tags"=>["beats_input_codec_plain_applied"], "syslog_severity_code"=>5, "syslog_facility_code"=>1, "syslog_facility"=>"user-level", "syslog_severity"=>"notice", "src_ip"=>"172.17.1.2", "dst_ip"=>"172.16.1.10"}, "@timestamp"], "source"=>[{"message"=>"1468822626.137517\tCJ06wGBTIOaRavwR2\t172.17.1.2\t50456\t172.16.1.10\t53\tudp\tdns\t0.035619\t0\t1008\tSHR\tT\tT\t0\tCd\t0\t0\t2\t1064\t(empty)\t-\t-\t#siftworkstation-eth0", "@version"=>"1", "@timestamp"=>"2016-07-18T06:17:17.440Z", "source"=>"/nsm/bro/logs/current/conn.log", "type"=>"conn", "input_type"=>"log", "count"=>1, "fields"=>nil, "beat"=>{"hostname"=>"siftworkstation", "name"=>"siftworkstation"}, "offset"=>505938, "host"=>"siftworkstation", "tags"=>["beats_input_codec_plain_applied"], "syslog_severity_code"=>5, "syslog_facility_code"=>1, "syslog_facility"=>"user-level", "syslog_severity"=>"notice", "src_ip"=>"172.17.1.2", "dst_ip"=>"172.16.1.10"}, "source"], "type"=>[{"message"=>"1468822631.454374\tCfj4LIvtsNP39EI4\t172.17.1.2\t33237\t172.16.1.10\t53\tudp\tdns\t0.012831\t0\t1008\tSHR\tT\tT\t0\tCd\t0\t0\t2\t1064\t(empty)\t-\t-\t#siftworkstation-eth0", "@version"=>"1", "@timestamp"=>"2016-07-18T06:17:24.440Z", "fields"=>nil, "beat"=>{"hostname"=>"siftworkstation", "name"=>"siftworkstation"}, "offset"=>506088, "type"=>"conn", "input_type"=>"log", "source"=>"/nsm/bro/logs/current/conn.log", "count"=>1, "host"=>"siftworkstation", "tags"=>["beats_input_codec_plain_applied"], "syslog_severity_code"=>5, "syslog_facility_code"=>1, "syslog_facility"=>"user-level", "syslog_severity"=>"notice", "src_ip"=>"172.17.1.2", "dst_ip"=>"172.16.1.10"}, "src_ip"]}>>, :error=>#<NoMethodError: undefined methodstart_with?' for nil:NilClass>, :level=>:warn}