After mutate split and add_field,my message is repetition

Elastic Stack Logstash
1

logstash,kibana and elastic version: 6.5.1

select same index。
some document， kibana show some field value is repetition，
some document didn`t.
error kibana screenshot：

image
image.png1162×804 151 KB

source message:

	2019-03-27 17:06:45.344	{"ws_log_level":"INFO","ws_prev_page":"Home","ws_current_page":"PickOrderDetail","ws_device_type":"MI 6X","ws_user_id":"keke.yan","entry_detail":{"data":[{"orderId":"2310034487650070","printCode":"#184","ffStatus":"FINISH","packageId":"10083250157515_2"}]},"entry_key":"dms_shop_app.View_Home.confirm_sure_ffstatus","package_info":{"project_id":"dms-shop-app","version":8056},"client_time":"2019-03-27 17:06:36.298","boot_time":26636757,"app_version":"8010019","pid":"80134","coordinate_system":"BD09","session":"0defc27444f4bbdc3730cb5639f2f9f4","bssid":"0a:74:9c:a9:bc:96","device_id":"4b798afb-4f25-3129-92fe-fcbe3bb8be0a","gid":"d52a9b81-6442-4a10-9c06-d7a5500368b4","server_time":"2019-03-27 17:06:36.424","gps":"116.434652,39.850578","client_ip":"222.131.163.19"}

another no problem screenshot:

image
image.png1107×785 124 KB

source message:

2019-03-27 16:54:46.282	{"ws_log_level":"INFO","ws_prev_page":"Home","ws_current_page":"PickOrderDetail","ws_device_type":"JKM-AL00b","ws_user_id":"hongwei.yi","entry_detail":{"data":[{"orderId":"2310034461897909","printCode":"#470","ffStatus":"FINISH","packageId":"10083252823760_1"}]},"entry_key":"dms_shop_app.View_Home.confirm_sure_ffstatus","package_info":{"project_id":"dms-shop-app","version":8056},"client_time":"2019-03-27 16:54:38.818","boot_time":265460916,"app_version":"8010019","pid":"80134","coordinate_system":"BD09","session":"fa7f81f8098a444c3e842c9807e5498b","bssid":"0a:05:88:0f:bc:13","device_id":"cd28e5c1-2f4e-310c-b8eb-89335407a45f","gid":"6dcae258-3c87-43d6-ac45-8c492a72757b","server_time":"2019-03-27 16:54:39.535","gps":"116.114013,39.662855","client_ip":"223.104.254.36"}

logstash config:

input {
        kafka{
                bootstrap_servers => "kafka1-dh.vip.blibee.com:9092"
                group_id => "logstash10"
                topics => ["fe_pid_log"]
                consumer_threads => 1
                decorate_events => true
                max_partition_fetch_bytes => "104857600"
        }
}

filter{
    json {
        source => "message"
    }
    grok{
        match=>{ "message" => "%{TIMESTAMP_ISO8601:time_iso8601}\s%{GREEDYDATA:log_json}"  }
    }
    json {
        source => "log_json"
    }
    mutate {
        replace => { "type" => "fe_app_log" }
        split => ["entry_detail",","]
        add_field => { "new_entry_detail" => "%{entry_detail}" }
        split => ["native_value",","]
        add_field => { "new_native_value" => "%{native_value}" }
        split => ["data",',']
        add_field => { "new_data" => "%{data}" }
    }
    mutate {
        remove_field => ["entry_detail","log_json","wmonitor","data","native_value"]
    }

}

output {
  elasticsearch {
    hosts => ["felog1.logsystem.ops.bj1.wormpex.com:9200"]
    manage_template => false
    #index => "fe_log_pid-%{pid}-%{+xxxx.ww}"
    index => "fe_log_pid-%{pid}-%{+YYYY.MM.dd}"
  }
}

kibana show:

{
  "_index": "fe_log_pid-80134-2019.03.27",
  "_type": "doc",
  "_id": "VR1mvmkBkM87AQir4ZvD",
  "_version": 1,
  "_score": null,
  "_source": {
    "offset": 612772223,
    "input": {
      "type": "log"
    },
    "client_ip": "222.131.163.19",
    "package_info": {
      "project_id": "dms-shop-app",
      "version": 8056
    },
    "tags": [
      "_jsonparsefailure"
    ],
    "ws_prev_page": "Home",
    "new_entry_detail": [
      "{\"data\":[{\"packageId\":\"10083250157515_2\",\"ffStatus\":\"FINISH\",\"orderId\":\"2310034487650070\",\"printCode\":\"#184\"}]}",
      "{\"data\":[{\"packageId\":\"10083250157515_2\",\"ffStatus\":\"FINISH\",\"orderId\":\"2310034487650070\",\"printCode\":\"#184\"}]}"
    ],
    "client_time": "2019-03-27 17:06:36.298",
    "gid": "d52a9b81-6442-4a10-9c06-d7a5500368b4",
    "ws_log_level": "INFO",
    "gps": "116.434652,39.850578",
    "app_version": "8010019",
    "@version": "1",
    "bssid": "0a:74:9c:a9:bc:96",
    "beat": {
      "hostname": "dataserver1.dp.data.bj1.wormpex.com",
      "version": "6.4.0",
      "name": "dataserver1.dp.data.bj1.wormpex.com"
    },
    "ws_device_type": "MI 6X",
    "coordinate_system": "BD09",
    "entry_key": "dms_shop_app.View_Home.confirm_sure_ffstatus",
    "host": {
      "name": "dataserver1.dp.data.bj1.wormpex.com"
    },
    "time_iso8601": [
      "2019-03-27 17:06:45.344",
      "2019-03-27 17:06:45.344"
    ],
    "prospector": {
      "type": "log"
    },
    "boot_time": 26636757,
    "source": "/data/w/www/data-log-terminator.wormpex.com/logs/data-collector-pid.2019-03-27_17.log",
    "session": "0defc27444f4bbdc3730cb5639f2f9f4",
    "device_id": "4b798afb-4f25-3129-92fe-fcbe3bb8be0a",
    "server_time": "2019-03-27 17:06:36.424",
    "ws_current_page": "PickOrderDetail",
    "type": "fe_app_log",
    "new_data": [
      "%{data}",
      "%{data}"
    ],
    "@timestamp": "2019-03-27T09:06:45.575Z",
    "ws_user_id": "keke.yan",
    "message": "2019-03-27 17:06:45.344\t{\"ws_log_level\":\"INFO\",\"ws_prev_page\":\"Home\",\"ws_current_page\":\"PickOrderDetail\",\"ws_device_type\":\"MI 6X\",\"ws_user_id\":\"keke.yan\",\"entry_detail\":{\"data\":[{\"orderId\":\"2310034487650070\",\"printCode\":\"#184\",\"ffStatus\":\"FINISH\",\"packageId\":\"10083250157515_2\"}]},\"entry_key\":\"dms_shop_app.View_Home.confirm_sure_ffstatus\",\"package_info\":{\"project_id\":\"dms-shop-app\",\"version\":8056},\"client_time\":\"2019-03-27 17:06:36.298\",\"boot_time\":26636757,\"app_version\":\"8010019\",\"pid\":\"80134\",\"coordinate_system\":\"BD09\",\"session\":\"0defc27444f4bbdc3730cb5639f2f9f4\",\"bssid\":\"0a:74:9c:a9:bc:96\",\"device_id\":\"4b798afb-4f25-3129-92fe-fcbe3bb8be0a\",\"gid\":\"d52a9b81-6442-4a10-9c06-d7a5500368b4\",\"server_time\":\"2019-03-27 17:06:36.424\",\"gps\":\"116.434652,39.850578\",\"client_ip\":\"222.131.163.19\"}",
    "topic": "fe_pid_log",
    "pid": "80134",
    "new_native_value": [
      "%{native_value}",
      "%{native_value}"
    ]
  },
  "fields": {
    "@timestamp": [
      "2019-03-27T09:06:45.575Z"
    ]
  },
  "highlight": {
    "entry_key": [
      "@kibana-highlighted-field@dms_shop_app.View_Home.confirm_sure_ffstatus@/kibana-highlighted-field@"
    ]
  },
  "sort": [
    1553677605575
  ]
}

why new_entry_detail, time_iso8601 or new_native_value was repetition。

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.