Logstash Remove help

Elastic Stack Logstash
1

I pull in the twitter json and want to remove certain fields from the user. object as well as some other fields. I cannot get it to work.

here is my filter:

filter {
if [field] =~ /in_reply_to*/ {
json {
source => "message"
remove_field => [field]
}
}
if [field] =~ /user.profile*/ {
json {
source => "message"
target => "user"
remove_field => [field]
}
}
}

2

Double-quote the field name:

json {
  ...
  remove_field => ["field"]
}
3

What I am trying to do is to remove any field that starts with "in_reply_to". even after double quoting, it does not work.

4

Here is a sample json. I want to remove anything with in_reply_to* as well as user.timezone field. how can I do it?

{
"_index": "twitter",
"_type": "tweet",
"_id": "AU3i9z7NNs7nuNHjMvVu",
"_score": null,
"_source": {
"created_at": "Thu Jun 11 14:12:43 +0000 2015",
"id": 609000309688574000,
"id_str": "609000309688573952",
"text": "RT @Nesrin_ulema: ",
"source": "<a href="http://twitter.com/download/android" rel="nofollow">Twitter for Android",
"truncated": false,
"in_reply_to_status_id": null,
"in_reply_to_status_id_str": null,
"in_reply_to_user_id": null,
"in_reply_to_user_id_str": null,
"in_reply_to_screen_name": null,
"user": {
"id": 2187249188,
"id_str": "2187249188",
"name": "Ayşe Arabacı",
"screen_name": "AyseArabacii",
"location": "",
"url": null,
"description": "",
"protected": false,
"verified": false,
"followers_count": 452,
"friends_count": 609,
"listed_count": 3,
"favourites_count": 1154,
"statuses_count": 6615,
"created_at": "Sun Nov 10 22:12:53 +0000 2013",
"utc_offset": null,
"time_zone": null,
"geo_enabled": true,
"lang": "tr",
"contributors_enabled": false,
"is_translator": false,
"profile_background_color": "C0DEED",
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png",
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png",
"profile_background_tile": false,
"profile_link_color": "0084B4",
"profile_sidebar_border_color": "C0DEED",
"profile_sidebar_fill_color": "DDEEF6",
"profile_text_color": "333333",
"profile_use_background_image": true,
"profile_image_url": "http://pbs.twimg.com/profile_images/594269698004484097/rOEI46FR_normal.jpg",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/594269698004484097/rOEI46FR_normal.jpg",
"default_profile": true,
"default_profile_image": false,
"following": null,
"follow_request_sent": null,
"notifications": null
},
"geo": null,
"coordinates": null,
"place": null,
"contributors": null,
"retweeted_status": {
"created_at": "Thu Jun 11 08:18:28 +0000 2015",
"id": 608911160432296000,
"id_str": "608911160432295936",
"text": "Sayın Başbakanımız ",
"source": "<a href="http://twitter.com/download/android" rel="nofollow">Twitter for Android",
"truncated": false,
"in_reply_to_status_id": null,
"in_reply_to_status_id_str": null,
"in_reply_to_user_id": null,
"in_reply_to_user_id_str": null,
"in_reply_to_screen_name": null,
"user": {
"id": 387303168,
"id_str": "387303168",
"name": "Nesrin ULEMA",
"screen_name": "Nesrin_ulema",
"location": "İzmir / Ankara",
"url": null,
"description": "",
"protected": false,
"verified": false,
"followers_count": 7025,
"friends_count": 337,
"listed_count": 60,
"favourites_count": 186,
"statuses_count": 5495,
"created_at": "Sat Oct 08 20:25:51 +0000 2011",
"utc_offset": 10800,
"time_zone": "Istanbul",
"geo_enabled": true,
"lang": "tr",
"contributors_enabled": false,
"is_translator": false,
"profile_background_color": "C0DEED",
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png",
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png",
"profile_background_tile": false,
"profile_link_color": "0084B4",
"profile_sidebar_border_color": "C0DEED",
"profile_sidebar_fill_color": "DDEEF6",
"profile_text_color": "333333",
"profile_use_background_image": true,
"profile_image_url": "http://pbs.twimg.com/profile_images/569595885521477633/Fi18kf4J_normal.jpeg",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/569595885521477633/Fi18kf4J_normal.jpeg",
"profile_banner_url": "https://pbs.twimg.com/profile_banners/387303168/1433104916",
"default_profile": true,
"default_profile_image": false,
"following": null,
"follow_request_sent": null,
"notifications": null
},
"geo": null,
"coordinates": null,
"place": null,
"contributors": null,
"retweet_count": 27,
"favorite_count": 18,
"entities": {
"hashtags": [],
"trends": [],
"urls": [],
"user_mentions": [],
"symbols": []
},
"favorited": false,
"retweeted": false,
"possibly_sensitive": true,
"filter_level": "low",
"lang": "tr"
},
"retweet_count": 0,
"favorite_count": 0,
"entities": {
"hashtags": [],
"trends": [],
"urls": [],
"user_mentions": [
{
"screen_name": "Nesrin_ulema",
"name": "Nesrin ULEMA",
"id": 387303168,
"id_str": "387303168",
"indices": [
3,
16
]
}
],
"symbols": []
},
"favorited": false,
"retweeted": false,
"possibly_sensitive": false,
"filter_level": "low",
"lang": "tr",
"timestamp_ms": "1434031963165",
"@version": "1",
"@timestamp": "2015-06-11T14:12:43.000Z"
},
"fields": {
"text": [
"RT @Nesrin_ulema: "
],
"@timestamp": [
1434031963000
]
},
"sort": [
1434031963000
]
}

5

Okay. remove_field doesn't support wildcards so you'll have to use a ruby filter. Untested:

filter {
  ruby {
    code => "
      event.to_hash.keys.each { |k|
        event.remove(k) if k.start_with?('in_reply_to_')
      }
    "
  }
}
3 Likes
6

Awesome. Thanks.