I have two logstash configuration files - when I run logstash it is creating a bizarre situation where one of the indexes from config1 has fields from the other index in config2
See below for steps :-
- Create an index for CSV files (java class/method signiatures)
elk>cat create_sig_profiler_index.json
{
"settings": {
"number_of_shards": 1,
"number_of_replicas": 1
},
"mappings": {
"doc": {
"properties": {
"testsuite_sp": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"java_version_sp": {
"type": "integer"
},
"OS_sp": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"platform_sp": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"class_method_sp": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"checksum_sp": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"count_sp": {
"type": "integer"
}
}
}
}
}
- Create index the index in ES
elk>curl -s -XPUT http://localhost:9200/sig_profiler -H 'Content-Type: application/json' -d @create_sig_profiler_index.json {"acknowledged":true,"shards_acknowledged":true,"index":"sig_profiler"}
- Verify logstash config files in place (2 files - 2 separate indexes, one for CSV files, the other for jenkins console logs)
twhelan@qa-elk>ls -l /etc/logstash/conf.d/*.conf -rw-r--r--. 1 root root 4948 Oct 29 11:55 /etc/logstash/conf.d/10-jenkins-qa.conf -rw-r--r--. 1 root root 448 Oct 29 16:50 /etc/logstash/conf.d/sig_profiler.conf
-
Verify CSV logstash configuration (file input for CSV files)
elk>cat sig_profiler.conf
input {
file {
path => "/PROFILER/*.csv"
start_position => "beginning"
}
}filter {
csv {
separator => ","
columns => [ "sp_testsuite", "sp_java_version", "sp_java_vendor", "sp_OS", "sp_platform", "sp_class_method", "sp_checksum", "sp_count" ]
}
}output {
elasticsearch {
hosts => ["localhost:9200"]
index => "sig_profiler"
}
}
-
Start logstash
twhelan@qa-elk>sudo systemctl start logstash -
Now check sig_profiler index - is corrupted, it has fields from other logstash configuration (??)
twhelan@qa-elk>curl -XGET "http://localhost:9200/sig_profiler?pretty" { "sig_profiler" : { "aliases" : { }, "mappings" : { "doc" : { "properties" : { "@timestamp" : { "type" : "date" }, "@version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "OS_sp" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "branch_name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "build" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "checksum_sp" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "class_method_sp" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "console_data" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "count_sp" : { "type" : "integer" }, "finish" : { "type" : "date" }, "guest_vm_version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "host_vm_version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "java_version_sp" : { "type" : "integer" }, "jenkins_job_logs_id" : { "type" : "long" }, "jenkins_job_number" : { "type" : "long" }, "os_name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "owner" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "platform_sp" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "start" : { "type" : "date" }, "state" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "task_id" : { "type" : "long" }, "testsuite" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "testsuite_sp" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } } }, "settings" : { "index" : { "creation_date" : "1603989435423", "number_of_shards" : "1", "number_of_replicas" : "1", "uuid" : "7W2bXWWWRoi92TB01zlZvA", "version" : { "created" : "6050399" }, "provided_name" : "sig_profiler" } } } }