Hi,
This is a fresh installation of ELK 7.10.0. I am trying to configure Logstash to process .csv files and index into ES. I created a config file and a custom index template. It looks like the .csv are being processed, but with default auto-mapping or other mapping template but the one I need to use. All fields are being indexed as text fields, including date and IP fields, and that's not the way I defined those in my custom template. No errors were found in the logs and apparently the config file and the custom template are being processed correctly, then what is failing? I have spent a lot of time trying to find what I could be missing or doing wrong without luck so far.
Can somebody please help me to fix this?
ts_reports.conf
input {
file {
id => "TS_Reports"
path => "/opt/ts_reports/*.csv"
mode => "read"
start_position => "beginning"
file_completed_action => "delete"
type => "TS"
}
}
filter {
csv {
columns => [
"Time",
"Device",
"Source IP",
"Source Port",
"Destination IP",
"Destination Port",
"Action",
"Direction",
"Targets",
"ID"
]
separator => ","
}
}
output {
elasticsearch {
# action => "index"
hosts => ["https://127.0.0.1:9200"]
index => "ts_reports-%{+YYYY.MM}"
manage_template => true
template => "/etc/logstash/ts_reports-template.json"
user => "[username]"
password => "[password]"
ssl => true
ssl_certificate_verification => true
cacert => "[cert]"
}
}
ts_reports-template.json
{
"index_patterns": "ts_reports-*",
"settings": {
"index.number_of_replicas": 0,
"index.refresh_interval" : "5s"
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"Action": {
"type": "keyword"
},
"Destination IP": {
"type": "ip"
},
"Destination Port": {
"type": "long"
},
"Device": {
"type": "keyword"
},
"Direction": {
"type": "keyword"
},
"ID": {
"type": "keyword"
},
"Source IP": {
"type": "ip"
},
"Source Port": {
"type": "long"
},
"Targets": {
"type": "text"
},
"Time": {
"type": "date",
"format": "MM/dd/yyyy HH:mm"
}
}
}
}
GET ts_reports-2020.12/_mapping
{
"ts_reports-2020.12" : {
"mappings" : {
"properties" : {
"@timestamp" : {
"type" : "date"
},
"@version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"Action" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"Destination IP" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"Destination Port" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"Device" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"Direction" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"ID" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"Source IP" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"Source Port" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"Targets" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"Time" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"host" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"message" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"path" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
}
