je veux consulter des logs avec ELK (ElasticSearch – Logstash – Kibana)
j'ai des problème avec logstash j'ai crée un fichier config (logstash.conf)
input {
file {
path => 'C:\Users\younes\Desktop\projet\Apache24\logs\logtest.log'
}
}
filter {
grok {
match => {
"message" => "%{COMBINEDAPACHELOG} %{IPORHOST:serverip} %{NUMBER:serverport} %{NUMBER:elapsed_millis} %{NOTSPACE:sessionid} %{QS:proxiedip} %{QS:loginame}"
}
overwrite => [ "message" ]
remove_field => [ "ident", "auth" ]
}
useragent {
source => "agent"
target => "ua"
remove_field => [ "agent" ]
}
mutate {
gsub => [
"request", "?.+", "",
"proxiedip", "(^"|"$)", "",
"loginame", "(^"|"$)" , "",
"referrer", "(^"|"$)" , ""
]
}
if [proxiedip] != "-" {
mutate {
replace => {
"clientip" => "%{proxiedip}"
}
}
}
if ![bytes] {
mutate {
add_field => {
"bytes" => "0"
}
}
}
mutate {
remove_field => ["proxiedip"]
}
mutate {
convert => {
"bytes" => "integer"
"elapsed_millis" => "integer"
"serverport" => "integer"
}
}
date {
match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
}
}
output {
stdout {
codec => plain {
charset => "ISO-8859-1"
}
}
elasticsearch {
template => "apache_template.json"
template_name => "apache_log"
template_overwrite => true
}
}
et un autre JSON
{
"template": "apache_log",
"settings": {
"index.refresh_interval": "5s"
},
"mappings": {
"default": {
"dynamic_templates": [
{
"message_field": {
"mapping": {
"index": "analyzed",
"omit_norms": true,
"type": "string"
},
"match_mapping_type": "string",
"match": "message"
}
},
{
"string_fields": {
"mapping": {
"index": "analyzed",
"omit_norms": true,
"type": "string",
"fields": {
"raw": {
"index": "not_analyzed",
"ignore_above": 256,
"type": "string"
}
}
},
"match_mapping_type": "string",
"match": "*"
}
}
],
"properties": {
"geoip": {
"dynamic": true,
"properties": {
"location": {
"type": "geo_point"
}
},
"type": "object"
},
"@version": {
"index": "not_analyzed",
"type": "string"
}
},
"all": {
"enabled": true
}
}
}
}
voila mon fichier log (c'est just un exemple)
45.217.184.114 - - [05/Sep/2016:21:58:26 +0000] "GET /rs-web/rechercherContratAutoPagineted.do?sEcho=1&iColumns=13&sColumns=%2C%2C%2C%2C%2C%2C%2C%2C%2C%2C%2C%2C&iDisplayStart=0&iDisplayLength=5&mDataProp_0=0&sSearch_0=&bRegex_0=false&bSearchable_0=true&mDataProp_1=1&sSearch_1=&bRegex_1=false&bSearchable_1=true&mDataProp_2=2&sSearch_2=&bRegex_2=false&bSearchable_2=true&mDataProp_3=3&sSearch_3=&bRegex_3=false&bSearchable_3=true&mDataProp_4=4&sSearch_4=&bRegex_4=false&bSearchable_4=true&mDataProp_5=5&sSearch_5=&bRegex_5=false&bSearchable_5=true&mDataProp_6=6&sSearch_6=&bRegex_6=false&bSearchable_6=true&mDataProp_7=7&sSearch_7=&bRegex_7=false&bSearchable_7=true&mDataProp_8=8&sSearch_8=&bRegex_8=false&bSearchable_8=true&mDataProp_9=9&sSearch_9=&bRegex_9=false&bSearchable_9=true&mDataProp_10=10&sSearch_10=&bRegex_10=false&bSearchable_10=true&mDataProp_11=11&sSearch_11=&bRegex_11=false&bSearchable_11=true&mDataProp_12=12&sSearch_12=&bRegex_12=false&bSearchable_12=true&sSearch=&bRegex=false&=1473108677432 HTTP/1.1" 200 330 "https://196.12.229.28:9448/rs-web/initRechercheContratAuto.do" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36" 10.10.11.168 9448 104 7706FC223C392F1E1F8DEB2AE74A0EE1.jvm1 "-" "-"
quand j’exécute la commande logstash -f logstash.conf
elasticsearch me donne
{
"took": 4,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 1,
"hits": [
{
"_index": ".kibana",
"_type": "config",
"_id": "4.4.2",
"_score": 1,
"_source": {
"buildNum": 9732
}
}
]
}
}
je comprend pas ! de l'aide svp