Error processing Twitter

Elastic Stack Logstash
1

Good morning,

  1. I've deployed the input plugin filter and it works but during the parsing occurs the following error

[2017-07-06T11:00:15,593][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"twitter-traffic", :_type=>"twitter", :_routing=>nil}, 2017-07-06T09:00:14.000Z %{host} %{message}], :response=>{"index"=>{"_index"=>"twitter-traffic", "_type"=>"twitter", "_id"=>"AV0XINs0X2aWZtZ_FrYC", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse [user]", "caused_by"=>{"type"=>"illegal_state_exception", "reason"=>"Can't get text on a START_OBJECT at 1:1202"}}}}}
[2017-07-06T11:00:15,797][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"pan-traffic", :_type=>"twitter", :_routing=>nil}, 2017-07-06T09:00:14.000Z %{host} %{message}], :response=>{"index"=>{"_index"=>"pan-traffic", "_type"=>"twitter", "_id"=>"AV0XINt7X2aWZtZ_FrYD", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse [user]", "caused_by"=>{"type"=>"illegal_state_exception", "reason"=>"Can't get text on a START_OBJECT at 1:1202"}}}}}
[2017-07-06T11:00:15,978][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"pan-traffic", :_type=>"twitter", :_routing=>nil}, 2017-07-06T09:00:14.000Z %{host} %{message}], :response=>{"index"=>{"_index"=>"pan-traffic", "_type"=>"twitter", "_id"=>"AV0XINxZX2aWZtZ_Frd6", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse [user]", "caused_by"=>{"type"=>"illegal_state_exception", "reason"=>"Can't get text on a START_OBJECT at 1:1202"}}}}}

An example of this kind of tweets is the following:

{"in_reply_to_status_id_str":null,"in_reply_to_status_id":null,"created_at":"Thu Jul 06 09:00:14 +0000 2017","in_reply_to_user_id_str":null,"source":"<a href=\"http://www.egasolineras.com/egasolineras_bot\" rel=\"nofollow\">egasolineras_bot</a>","type":"twitter","retweet_count":0,"retweeted":false,"geo":null,"filter_level":"low","in_reply_to_screen_name":null,"is_quote_status":false,"id_str":"882886899555340288","in_reply_to_user_id":null,"@version":"1","favorite_count":0,"id":882886899555340288,"text":"#Gasolina 95 +barata hasta 5km #Alcorcón #España 1.047€ ALCAMPO AVENIDA EUROPA, S/N L-D: 24H ALCORCON https://t.co/sP4145GWFd","place":null,"lang":"es","favorited":false,"possibly_sensitive":false,"coordinates":null,"truncated":false,"timestamp_ms":"1499331614756","@timestamp":"2017-07-06T09:00:14.000Z","entities":{"urls":[{"display_url":"egasolineras.com/gasolinera/mTz…","indices":[102,125],"expanded_url":"http://www.egasolineras.com/gasolinera/mTztSEiZqcDbrxH","url":"https://t.co/sP4145GWFd"}],"hashtags":[{"indices":[0,9],"text":"Gasolina"},{"indices":[31,40],"text":"Alcorcón"},{"indices":[41,48],"text":"España"}],"user_mentions":[],"symbols":[]},"contributors":null,"user":{"utc_offset":7200,"friends_count":1991,"profile_image_url_https":"https://pbs.twimg.com/profile_images/3004255282/ef6d7a800ca9bfe3ea669b7f9f487f97_normal.png","listed_count":78,"profile_background_image_url":"http://abs.twimg.com/images/themes/theme1/bg.png","default_profile_image":false,"favourites_count":1,"description":"Buscador de gasolineras + baratas de España y Chile. Búsquedas por geolocalización, dirección, localidad/provincia, código postal y ruta.","created_at":"Fri Dec 21 22:58:04 +0000 2012","is_translator":false,"profile_background_image_url_https":"https://abs.twimg.com/images/themes/theme1/bg.png","protected":false,"screen_name":"egasolineras","id_str":"1027319756","profile_link_color":"1DA1F2","id":1027319756,"geo_enabled":true,"profile_background_color":"C0DEED","lang":"es","profile_sidebar_border_color":"C0DEED","profile_text_color":"333333","verified":false,"profile_image_url":"http://pbs.twimg.com/profile_images/3004255282/ef6d7a800ca9bfe3ea669b7f9f487f97_normal.png","time_zone":"Madrid","url":"http://www.egasolineras.com","contributors_enabled":false,"profile_background_tile":false,"profile_banner_url":"https://pbs.twimg.com/profile_banners/1027319756/1419617181","statuses_count":169713,"follow_request_sent":null,"followers_count":2590,"profile_use_background_image":true,"default_profile":true,"following":null,"name":"egasolineras.com","location":"Valencia, Spain","profile_sidebar_fill_color":"DDEEF6","notifications":null}}

the logstash.conf file related to twitter is the following:

twitter {
        consumer_key => ""
        consumer_secret => ""
        oauth_token =>  ""
        oauth_token_secret => ""
        keywords=>["1","2","23","4","5","6","7"]
        type=>"twitter"
        full_tweet=> true
    }

    output{
     elasticsearch{
                index => "twitter-traffic"
                hosts => ["localhost:9200"]
            }

    }

any ideas?

2

It looks like the user field has been mapped as a string yet the event you're attempting to index is an object.

3

then I've to do the following

mutate {
convert => [ "User", "object" ]

o generate the different elements included inside user?

4

Why is user mapped as a string? Was it by mistake while you were experimenting or do you have other logs using that field name to store strings?

You can't change existing mappings without reindexing, so depending on the answers to the questions above there are a couple of different things to do.

5

Good morning Magnus,

I've regenerate the template and its works.

many thanks

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.