Logstash filter seems not working

Elastic Stack Logstash
1

Hi,

I hv below configuration with my logststash :

input {
beats {
port => 5044
}
}
filter{
if [fields][sourcelog]=="cdrlog" {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:logdate:date} %{LOGLEVEL:debugtype} %{DATA:source} %{TIMESTAMP_ISO8601:smsdate:date},%{WORD:sourceaddr},%{NUMBER:addrton},%{NUMBER:addrnpi},%{WOR
D:destaddr},%{NUMBER:sourceton},%{NUMBER:sourcenpi},%{WORD:status},%{WORD:transport},%{WORD:trtype},%{WORD:trsender}" }}

    grok { match => [ "sourceaddr", "^(?<operator>.....)" ] }

    translate {

field => "operator"
destination => "operator_name"
dictionary => [
"62811", "Telkomsel",
"62812", "Telkomsel",
"62813", "Telkomsel",
"62821", "Telkomsel",
"62822", "Telkomsel",
"62823", "Telkomsel",
"62851", "Telkomsel",
"62852", "Telkomsel",
"62853", "Telkomsel",
"62814", "Indosat",
"62815", "Indosat",
"62816", "Indosat",
"62855", "Indosat",
"62856", "Indosat",
"62857", "Indosat",
"62858", "Indosat",
"62817", "XL",
"62818", "XL",
"62819", "XL",
"62859", "XL",
"62877", "XL",
"62878", "XL",
"62831", "XL",
"62832", "XL",
"62833", "XL",
"62838", "XL",
"62895", "Tri",
"62896", "Tri",
"62897", "Tri",
"62898", "Tri",
"62899", "Tri",
"62881", "Smartfren",
"62882", "Smartfren",
"62883", "Smartfren",
"62884", "Smartfren",
"62885", "Smartfren",
"62886", "Smartfren",
"62887", "Smartfren",
"62888", "Smartfren",
"62889", "Smartfren",
"62828", "Net1"
]

     }


    translate {

field => "operator"
destination => "smstype"
dictionary => [
"62811", "Incoming",
"62812", "Incoming",
"62813", "Incoming",
"62821", "Incoming",
"62822", "Incoming",
"62823", "Incoming",
"62851", "Incoming",
"62852", "Incoming",
"62853", "Incoming",
"62814", "Incoming",
"62815", "Incoming",
"62816", "Incoming",
"62855", "Incoming",
"62856", "Incoming",
"62857", "Incoming",
"62858", "Incoming",
"62817", "Incoming",
"62818", "Incoming",
"62819", "Incoming",
"62859", "Incoming",
"62877", "Incoming",
"62878", "Incoming",
"62831", "Incoming",
"62832", "Incoming",
"62833", "Incoming",
"62838", "Incoming",
"62895", "Incoming",
"62896", "Incoming",
"62897", "Incoming",
"62898", "Incoming",
"62899", "Incoming",
"62881", "Incoming",
"62882", "Incoming",
"62883", "Incoming",
"62884", "Incoming",
"62885", "Incoming",
"62886", "Incoming",
"62887", "Incoming",
"62888", "Incoming",
"62889", "Incoming",
"62828", "Outgoing"
]

}


date {
  match => [ "smsdate", "ISO8601" ]
  target => ["@timestamp"]
}

date {
  match => [ "smsdate", "ISO8601" ]
  target => ["smsdate"]
}






    }

}
output {
elasticsearch {
hosts => ["localhost:9200"]
sniffing => true
manage_template => false
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
}
}

The input is coming from filebeat with the following :

{
  "_index": "filebeat-2019.12.14",
  "_type": "doc",
  "_id": "nlAUA28BDii-OZlyZvGn",
  "_version": 1,
  "_score": null,
  "_source": {
    "log": {
      "file": {
        "path": "/opt/eolos/TelScale-smsc-jboss-7.5.1-95/jboss-5.1.0.GA/server/default/log/cdr.log"
      }
    },
    "source": "/opt/eolos/TelScale-smsc-jboss-7.5.1-95/jboss-5.1.0.GA/server/default/log/cdr.log",
    "offset": 66451087,
    "message": "2019-12-14 06:24:24,440 DEBUG [org.mobicents.smsc.library.CdrGenerator] 2019-12-14 06:24:24.424,6282800082800,1,1,6285769178147 ,1,1,failed,SMPP,message,serdadu,11257533,null,null,null,null,null,null,null,25,0,null,1,6,,,,,\"438482\",\"MAPException when sending SRI from sendSRI(): org.mobicents.protocols.ss7.map.api.MAPException: char should be between 0 - 9  *  #  a  b  c for Telephony Binary Coded Decimal String. Received  \",,,",
    "input": {
      "type": "log"
    },
    "fields": {
      "sourcelog": "cdrlog"
    },
    "@timestamp": "2019-12-14T06:24:24.676Z",
    "tags": [
      "beats_input_codec_plain_applied",
      "_grokparsefailure"
    ],
    "beat": {
      "hostname": "TelScale-3",
      "name": "TelScale-3",
      "version": "6.8.4"
    },
    "@version": "1",
    "prospector": {
      "type": "log"
    },
    "host": {
      "name": "TelScale-3",
      "id": "05cb8c7b39fe0f70e3ce97e5beab809d",
      "architecture": "x86_64",
      "os": {
        "name": "CentOS Linux",
        "version": "7 (Core)",
        "codename": "Core",
        "platform": "centos",
        "family": "redhat"
      },
      "containerized": false
    }
  },
  "fields": {
    "@timestamp": [
      "2019-12-14T06:24:24.676Z"
    ]
  },
  "sort": [
    1576304664676
  ]
}

But it seems above data is not being process by the filter.
I would like to understand where is the issue. Anyone has this kind of problem before ?

Regards

Pradana

2

I would start by looking at logstash logs and elasticsearch logs to see where the problem is. You are likely to be able to find where the problem is with logstash logs

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.