Remove / using logstash filter

[Giridharan_C]

I used multiple mutate filters to remove the / in the message field, which is not working. Also, I want to parse these message fields into valid JSON.

Please suggest and help.

{
"host" => "mylocalhost.mydomain.com",
"@version" => "1",
"message" => "2022-10-07T16:58:22.209Z 10.30.94.20 {"ap_count": 0, "cpu_utilization": 0, "firmware_backup_version": "8.7.0.0-2.3.0.4_82267", "firmware_version": "8.7.0.0-2.3.0.4_82267", "group_name": "my-group", "ip_address": "10.10.10.10", "is_firmware_version_supported": true, "labels": ["Corp", "LAB"], "labels_info": [{"id": 888, "name": "Corp"}, {"id": 777, "name": "LAB"}], "location": {"gps": {"latitude": "99.9215415", "longitude": "-00.935501"}, "override": false, "serial_number": "", "street_address": {"address_1": "6 city", "city": "lux", "country": "UK", "postal_code": "08054", "state": "New Jersey"}}, "mac_range": "20:4c:03:04:af:40 - 20:4c:03:04:af:5f", "macaddr": "20:4c:03:04:af:40", "mem_free": 0, "mem_total": 0, "mm_hostname": null, "mobility_master": null, "mode": "GATEWAY", "model": "A555", "devi_info": null, "name": "L-8x", "ntp_server_info": {"server": "host01-mydomain.net", "sync": 1}, "peers": , "poe_available": "-", "poe_budget": "-", "poe_consumption": "-", "poe_supported": true, "public_ip": "77.77.55.91", "reason": "I took", "recommended_version": "8.7.0.0-2.3.0.7_83952", "redundancy_peer_hostname": null, "redundancy_peer_sn": null, "role": "", "serial": "5555", "site": "5555", "site_info": {"id": 890, "name": "555"}, "status": "Down", "uplinks": [{"default_gw": "192.168.1.1", "description": "-0/0/15", "is_backup": true, "is_virtual_uplink": false, "link_index": 101, "link_status": "Down", "link_tag": "inet", "name": "uplink101", "priority": 200, "private_ip": "192.168.1.0", "public_ip": "77.55.77.98", "status": "Down", "vlllanan": 4094, "vlakaanfd_description": "inet", "LAN_status": "Down", "Bus_type": "Internet"}, {"default_gw": "0.0.0.0", "description": "metro-channel/0/14", "is_backup": false, "is_virtual_uplink": false, "link_index": 102, "link_status": "Down", "link_tag": "ethernet", "name": "uplink102", "priority": 200, "private_ip": "0.0.0.0", "public_ip": "0.0.0.0", "status": "Down", "vlan": 4093, "vlan_description": "ethernet", "wan_status": "Down", "wan_type": "Metro-E"}], "uptime": 0, "usage": 0}",
"@timestamp" => 2022-10-07T19:03:01.786Z,
"path" => "/tmp/sample/58.pat3.txt"
}

[Rios]

How do you try to remove and which field?

According to this sample, the message field hasn't been parsed at all.
You can use grok to parse:
%{TIMESTAMP_ISO8601:time}\s*%{IP:ip}\s*%{GREEDYDATA:jsonmsg}

Then extract fields from jsonmsg. After that you can remove field(s).

[Giridharan_C]

@Rios - I have tried the above and still the \ is there. Please find the conf and output below.

input {
    file {
        path => "/tmp/sample/*.txt"
        start_position => "beginning"
            sincedb_path => "/dev/null"
    }
}

filter {
        grok {
                match => { "message" => "%{TIMESTAMP_ISO8601:time}\s*%{IP:ip}\s*%{GREEDYDATA:jsonmsg}"  }
        }
        mutate { remove_field => [ "message" ] }
}

output {
    stdout {}
    file {
        #codec => json
        path => "/tmp/output/output.txt"
    }
}

output:

{
"@version" => "1",
"ip" => "10.10.10.11",
"path" => "/tmp/sample/part3.txt",
"@timestamp" => 2022-10-08T04:39:31.702Z,
"time" => "2022-10-07T16:58:22.230Z",
"jsonmsg" => "{"ap_count": 0, "cpu_utilization": 11, "firmware_backup_version": "82660", "firmware_version": "82660", "group_name": "7010", "ip_address": "10.10.10.11", "is_firmware_version_supported": true, "labels": ["West", "West"], "labels_info": [{"id": 1021, "name": "West"}, {"id": 10237878, "name": "West"}], "location": {"gps": {"latitude": "100.5989497", "longitude": "-100.4229842"}, "override": false, "serial_number": "", "street_address": {"address_1": "1755 Way", "city": "gold", "country": "moracco", "postal_code": "55555", "state": "goa"}}, "mac_range": ":9c - 20:4c:03:", "macaddr": "20:4c", "mem_free": 1678475264, "mem_total": 3142696960, "mm_hostname": null, "mobility_master": null, "mode": "GATEWAY", "model": "A7012310", "modem_info": null, "name": "mydevice.myhost.net", "ntp_server_info": {"server": "abce.myhost-super.mydomain.net", "sync": 1}, "peers": , "poe_available": 119.9, "poe_budget": 150.0, "poe_consumption": 30.1, "poe_supported": true, "public_ip": "75.75.75.99", "reason": "heat temp", "recommended_version": "83952", "redundancy_peer_hostname": null, "redundancy_peer_sn": null, "role": "", "serial": "abc671283", "site": "super_market", "site_info": {"id": 11, "name": "supermarket"}, "status": "Up", "uplinks": [{"default_gw": "192.0.0.1", "description": "inet-0/0/15", "is_backup": true, "is_virtual_uplink": false, "link_index": 101, "link_status": "Down", "link_tag": "inet", "name": "uplink101", "priority": 200, "private_ip": "0.0.0.0", "public_ip": "0.0.0.0", "status": "Down", "vlan": 4094, "vlan_description": "inet", "wan_status": "Down", "wan_type": "Internet"}, {"default_gw": "75.75.75.1", "description": "metro-ethernet-0/0/14", "is_backup": false, "is_virtual_uplink": false, "link_index": 102, "link_status": "Up", "link_tag": "metro-ethernet", "name": "uplink102", "priority": 200, "private_ip": "71.71.71.17", "public_ip": "71.71.71.17", "status": "Up", "vlan": 4093, "vlan_description": "metro-ethernet", "wan_status": "Up", "wan_type": "Metro-E"}], "uptime": 559149, "usage": 0}",
"host" => myhost.mydomain.com"
}

[Rios]

filter {

  grok {
    match => { "message" => "%{TIMESTAMP_ISO8601:time}\s*%{IP:ip}\s*%{GREEDYDATA:jsonmsg}" }
  }
  
  date {
    match => [ "time", "ISO8601" ]
    remove_field => [ "time", "log", "host", "event" ]
  }
# remove / 
  mutate {
    gsub => ["jsonmsg","[\/]", ""]
  }
# replace missing values with null
 mutate {
   gsub => ["jsonmsg",'\": ,', '": null,']
 }
    json {
      source => "jsonmsg"
    }

}

"description" => "metro-channel014" <- removed "/"