Logstash writing into wrong index

chandukreddi · May 9, 2019, 5:44pm

Hello Team,

I have 2 conf files you can see below 1st conf file using output index alias "stats_write_swift_proxy_alias" to process proxy-nodes-monitor-stats-.csv & proxy-nodes-latency-stats-.csv files but I see this alias not processing mentioned files instead its processing /opt/swiftstats/aco/new/.csv* files, anything I am doing wrong but it was working as expected in 6.1.3 version I am seeing this issues right after upgrade 6.7.1.

proxy-nodes-monitor-stats-.csv & proxy-nodes-latency-stats-.csv files supposed to go to index (stats_write_swift_proxy_alias:alias) but you see its processing csv file which is given in 2nd confile

)

Here are my indexes and alias details:
green open swift-proxy-stats-2019.05.09-1 qSZopH_LTziMbNXfUL5Szg 5 0 200750 0 17.5mb 17.5mb
green open swift-aco-stats-os-2019.05.09-1 o36VEV07Rd-WdHkaor7VYg 5 0 1669229 0 197.3mb 197.3mb

alias index filter routing.index routing.search
stats_write_swift_proxy_alias swift-proxy-stats-2019.05.09-1 - - -
stats_write_swift_aco_alias swift-aco-stats-os-2019.05.09-1 - - -

1. 1st conf file:

input {
file {
path => "/opt/swiftstats/proxy/proxy-nodes-monitor-stats-*.csv"
start_position => "beginning"
sincedb_path => "/opt/data/logstash/plugins/inputs/file/proxy-stats.log"
tags => ["E1-proxy-stats"]
}

file {
path => "/opt/swiftstats/proxy-latency/proxy-nodes-latency-stats-*.csv"
start_position => "beginning"
sincedb_path => "/opt/data/logstash/plugins/inputs/file/proxy-latency-stats.log"
tags => ["E1-proxy-latency"]
}
}

--filters

output {

elasticsearch {
hosts => [ "http://10.1.28.176:9200" ]
index => "stats_write_swift_proxy_alias"
template => "/opt/logstash/config/templates/swift_proxy_stats.json"
template_name => "swift_proxy_stats_template"
template_overwrite => true

}

Template:
[elk@elk2 572019]$ cat /opt/logstash/config/templates/swift_proxy_stats.json
{
"template": "swift_proxy_stats_template",
"index_patterns": ["swift-proxy-stats-*"],
"settings": {
"index.refresh_interval": "5s",
"index.codec": "best_compression",
"number_of_shards": 5,
"number_of_replicas": 0
},
"aliases": {
"stats_write_swift_proxy_alias": {}
}
}

1. 2nd conf file:

input {
file {
path => "/opt/swiftstats/aco/new/*.csv"
start_position => "beginning"
#sincedb_path => "/dev/null"
sincedb_path => "/opt/data/logstash/plugins/inputs/file/swift_aco_os_handoff.log"
tags => ["E1-aco"]
}
}
-- filters
output {
if "E1-aco" in [tags] {
elasticsearch {
hosts => [ "http://10.1.28.176:9200" ]
index => "stats_write_swift_aco_alias"
template => "/opt/logstash/config/templates/swift_aco_stats.json"
template_name => "swift_aco_stats_template"
template_overwrite => true

}
}
}

2nd conf file Template:

[elk@elk2 572019]$ cat /opt/logstash/config/templates/swift_aco_stats.json
{
"template": "swift_aco_stats_template",
"index_patterns": ["swift-aco-stats-os*"],
"settings": {
"index.refresh_interval": "5s",
"index.codec": "best_compression",
"number_of_shards": 5,
"number_of_replicas": 0
},
"aliases": {
"stats_write_swift_aco_alias": {}
}
}

Thanks
Chandra

yaauie · May 9, 2019, 8:52pm

Are you explicitly using multiple pipelines (e.g., have you configured a pipelines.yml)?

If not, Logstash concatenates the files found in its config directory into a single pipeline, in which all inputs generate events that go through all filters and to all outputs, regardless of the source file.

See: https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html

chandukreddi · May 9, 2019, 9:21pm

Hi @yaauie, I am not using pipelines but the same config was working earlier and it was going into write indexes.

And if you notice I am redirecting specific csv file specific index.

Thanks
Chandra

Badger · May 9, 2019, 10:10pm

Well, writing to the stats_write_swift_aco_alias index is conditional upon the E1-aco tag, but writing to the stats_write_swift_proxy_alias index is unconditional.

chandukreddi · May 9, 2019, 10:23pm

@Badger I made one conf file for all files adding output tag condition, now I see at least its reading correct csv file but seems like parsing failing.

Conf file:

[elk@elk2 all_config]$ cat swift_proxy_monitor.conf
input {
file {
path => "/opt/swiftstats/proxy/proxy-nodes-monitor-stats-*.csv"
start_position => "beginning"
#sincedb_path => "/dev/null"
sincedb_path => "/opt/data/logstash/plugins/inputs/file/proxy-stats.log"
tags => ["E1-proxy-stats"]
}

file {
path => "/opt/swiftstats/proxy-latency/proxy-nodes-latency-stats-.csv"
start_position => "beginning"
#sincedb_path => "/dev/null"
sincedb_path => "/opt/data/logstash/plugins/inputs/file/proxy-latency-stats.log"
tags => ["E1-proxy-latency"]
}
file {
path => "/opt/swiftstats/aco/new/.csv"
start_position => "beginning"
#sincedb_path => "/dev/null"
sincedb_path => "/opt/data/logstash/plugins/inputs/file/swift_aco_os_handoff.log"
tags => ["E1-aco"]
}

}

filter {
if "E1-proxy-stats" in [tags] {
csv {
separator => ","
columns => ["fename","si-card","logdate","status-type","total-count","404_count","413_count","4xx_count","2xx_count","5xx_count"]
}

date {
match => ["logdate","MMM dd HH:mm:ss"]
target => "logdate"
timezone => "PST8PDT"
}

mutate {
convert => {
#"fename" => "integer"
#"si-card" => "integer"
#"status-type" => "integer"
"total-count" => "integer"
"404_count" => "integer"
"413_count" => "integer"
"4xx_count" => "integer"
"2xx_count" => "integer"
"5xx_count" => "integer"
}
remove_field => [ "message" ]
}
}

if "E1-proxy-latency" in [tags] {
csv {
separator => ","
columns => ["script","si-card","fename","logdate","request_type","response_min","response_max","response_avg"]
}

date {
match => ["logdate","MMM dd HH:mm:ss"]
target => "logdate"
timezone => "PST8PDT"
}

mutate {
convert => {
"response_min" => "float"
"response_max" => "float"
"response_avg" => "float"
}
remove_field => [ "message","script" ]
}
}

Aco node stats

if "E1-aco" in [tags] {
csv {
separator => ","
columns => ["logdate","hostname","handoff","primarypart","cpuused","load","iowait","memused","memfree","membuff"]
}

date {
match => ["logdate","MMM dd HH:mm:ss"]
target => "logdate"
timezone => "PST8PDT"
}

mutate {
convert => {
"handoff" => "integer"
"primarypart" => "integer"
"cpuused" => "float"
"load" => "float"
"iowait" => "float"
"memused" => "float"
"memfree" => "float"
"membuff" => "float"
}
remove_field => [ "message" ]
}
}
}

output {
if "E1-proxy-stats" in [tags] {
elasticsearch {
hosts => [ "http://10.1.1.1:9200" ]
index => "stats_write_swift_proxy_alias"
template => "/opt/logstash/config/templates/swift_proxy_stats.json"
template_name => "swift_proxy_stats_template"
template_overwrite => true
#user => 'admin'
#password => 'mavenir'
#ssl => true
#cacert => "/opt/logstash-6.1.3/config/ssl/logstash.pem"
#ssl_certificate_verification => false
}
}

if "E1-proxy-latency" in [tags] {
elasticsearch {
hosts => [ "http://10.1.1.2:9200" ]
index => "stats_write_swift_proxy_alias"
template => "/opt/logstash/config/templates/swift_proxy_stats.json"
template_name => "swift_proxy_stats_template"
template_overwrite => true
#user => 'admin'
#password => 'mavenir'
#ssl => true
#cacert => "/opt/logstash-6.1.3/config/ssl/logstash.pem"
#ssl_certificate_verification => false
}
}
if "E1-aco" in [tags] {
elasticsearch {
hosts => [ "http://10.1.1.2:9200" ]
index => "stats_write_swift_aco_alias"
template => "/opt/logstash/config/templates/swift_aco_stats.json"
template_name => "swift_aco_stats_template"
template_overwrite => true
#user => 'admin'
#password => 'mavenir'
#ssl => true
#cacert => "/opt/logstash-6.1.3/config/ssl/logstash.pem"
#ssl_certificate_verification => false
}
}

#stdout {}
}

[elk@elk2 all_config]$

Thanks
Chandra

Badger · May 9, 2019, 10:37pm

The configuration you show does not include a grok filter, so it is hard to tell why you are getting _grokparsefailure tags.

chandukreddi · May 9, 2019, 10:55pm

your correct, I am not using any grok patterns its just csv filter I am using, Any thoughts on further troubleshooting?

Thanks
Chandra

Christian_Dahlqvist · May 10, 2019, 5:21am

You probably have some other file in that directory that contains a grok filter and possibly also some output config. Logstash will concatenation ALL files in that directory.

chandukreddi · May 10, 2019, 1:23pm

Yes @Christian_Dahlqvist, I do have one more config file which uses grok but I am still confused I am giving exact path and file name(each csv file in different paths) why another conf files are touching which doesn't belong to.

and I am not sure why csv files are not parsed using csv filter which used to parse earlier, if I create different config files each file then at least getting parsed but rest of them are going into same index with parse failure.
I am missing some basic thing here, not sure what it could be.

for your better understanding I am giving sample rows in those files(it was working before upgrade with out any issues, and having 2 csv conf files under same config folder).

path => "/opt/swiftstats/proxy/proxy-nodes-monitor-stats-*.csv"
/opt/swiftstats/proxy
[root@elk2 proxy]# ll | tail -n 1
-rw-rw-r--. 1 elk elk 144060 Mar 31 23:08 proxy-nodes-monitor-stats-Mar-31-19-23.csv
[root@elk2 proxy]# tail -4 proxy-nodes-monitor-stats-Mar-31-19-23.csv
hostname,0-15,Mar 31 23:02:02,DELETE,489,36,0,206,246,0
hostname,0-15,Mar 31 23:02:02,HEAD,489,36,0,206,246,0
hostname,0-15,Mar 31 23:02:02,ERROR,0
hostname,0-15,Mar 31 23:02:02,TIMEOUT,8

path => "/opt/swiftstats/proxy-latency/proxy-nodes-latency-stats-*.csv"
[root@elk2 proxy-latency]# ll | tail -n 1
-rw-rw-r--. 1 elk elk 2905666 Apr 1 12:54 proxy-nodes-latency-stats-Mar-31-19.csv
[root@elk2 proxy-latency]# tail -4 proxy-nodes-latency-stats-Mar-31-19.csv
latency_monitor.sh,0-16,hostname,Mar 31 23:00:1,GET,0.0099,9.1529,0.15058
latency_monitor.sh,0-16,hostname,Mar 31 23:00:1,PUT,0.0034,8.9893,0.470697
latency_monitor.sh,0-16,hostname,Mar 31 23:00:1,POST,0.1053,8.1968,0.323621
latency_monitor.sh,0-16,hostname,Mar 31 23:00:1,DELETE,0.1156,9.9212,1.43149

path => "/opt/swiftstats/aco/new/*.csv"
[root@elk2 new]# pwd
/opt/swiftstats/aco/new
[root@elk2 new]# ll | tail -n 1
-rw-rw-r--. 1 elk elk 147178 Apr 10 09:08 proxy-nodes-monitor-stats-Apr-10-19-09.csv
[root@elk2 new]# tail -4 proxy-nodes-monitor-stats-Apr-10-19-09.csv
hostname,0-15,Apr 10 09:02:01,DELETE,2394,382,0,0,2006,0
hostname,0-15,Apr 10 09:02:01,HEAD,2394,382,0,0,2006,0
hostname,0-15,Apr 10 09:02:01,ERROR,35
hostname,0-15,Apr 10 09:02:01,TIMEOUT,59

Do you think I can resolve this problem with pipeline?

elasticforme · May 10, 2019, 2:24pm

It will make huge difference. as I mention in other post I was having some weird output. but as soon as I introduce proper pipeline, all worked out.

Logstash loads all config in to it's pipeline. if you do not have pipeline define everything goes in to default one and then who knows what goes out.

chandukreddi · May 10, 2019, 2:44pm

Thanks @elasticforme.. will try that and update here.

chandukreddi · May 13, 2019, 5:55pm

working perfectly fine with pipeline.

Thanks @elasticforme

system · June 10, 2019, 5:55pm

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.