Hello.
I have ELK stack on the single server.
Logstash have a two config files, first listen nginx logs over udp,
Second config read logs from files.
Nginx input:
input {
syslog {
host => "0.0.0.0"
port => 5150
timezone => "UTC"
tags => "aws-nginx"
type => "log"
}
}filter {...}
output {
elasticsearch {
hosts => ["http://localhost:9200"]
index => "nginx-%{+YYYY.MM.dd}"
}
}
Logstash second config is:
input {
file {
path => "/path/to/logs/*"
start_position => "beginning"
mode => "read"
}
}filter {
grok {
patterns_dir => "/etc/logstash/patterns"
match => { "message" => "%{POSTGRESQL}" }
}
}
output {
elasticsearch {
hosts => ["http://localhost:9200"]
index => "rds-%{+YYYY.MM.dd}"
}
}
And from kibana dev tools, i create two templates for each index.
PUT _template/nginx1
{
"index_patterns": "nginx*",
"settings": {
"refresh_interval": "5s",
"number_of_replicas": 1,
"number_of_shards": 2
},
"mappings": {
"doc": {
"dynamic_templates": [
{
"message_field": {
"path_match": "message",
"match_mapping_type": "string",
"mapping": {
"type": "text",
"norms": false
}
}
},
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
],
"properties" : {
"@timestamp" : {
"type" : "date"
},
"remote_addr" : {
"type" : "ip"
},
"method" : {
"type" : "text"
},
"refferer" : {
"type" : "keyword"
},
"request_time" : {
"type" : "float"
},
"response_bytes" : {
"type" : "integer"
},
"user_agent" : {
"type" : "text"
},
"args_body" : {
"type" : "text"
},
"forwarded_for" : {
"type" : "text"
},
"@version" : {
"type" : "keyword"
},
"geoip" : {
"dynamic" : true,
"properties" : {
"ip" : {
"type" : "ip"
},
"location" : {
"type" : "geo_point"
},
"latitude" : {
"type" : "half_float"
},
"longitude" : {
"type" : "half_float"
}
}
}
}
}
}
}
Second:
PUT _template/rds
{
"index_patterns": "rds*",
"settings": {
"refresh_interval": "5s",
"number_of_replicas": 1,
"number_of_shards": 2
},
"mappings": {
"doc": {
"dynamic_templates": [
{
"message_field": {
"path_match": "message",
"match_mapping_type": "string",
"mapping": {
"type": "text",
"norms": false
}
}
},
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
],
"properties" : {
"@timestamp" : {
"type" : "date"
},
"private_ip" : {
"type" : "ip"
},
"request_size" : {
"type" : "integer"
},
"username" : {
"type" : "text"
},
"database" : {
"type" : "text"
},
"connection" : {
"type" : "integer"
},
"duration" : {
"type" : "float"
},
"query" : {
"type" : "text"
}
}
}
}
}
The problem appears, when elasticsearch create indexes. In the first(nginx) index i see mapping fields, which created for the second(rds) index.
rds-*** index starts receiving messages from nginx(over UDP), whilst I've configured elasticsearch and logstash to read messages from static files for RDS index.
If it's needed I can provide additional information.
What im doing wrong ?