Hello,
I'm accessing tweets with Tweepy and generating complete .JSON files for each Tweet.
I set up my logstash to read each of these files and index them in Elasticsearch
See below my logstash configuration file:
input{
file{
codec => json
path => ["C:/----my directory-----/twitter_logs*.json*"]
start_position => "beginning"
sincedb_path => "nul"
}
}
filter {
json {
source => "message"
}
}
output{
elasticsearch {
hosts => "localhost:9200"
index => "twitter"
}
}
See the contents of each .JSON file.
{
"created_at":"Tue May 21 22:18:17 +0000 2019",
"id":1130961032392990700,
"id_str":"1130961032392990720",
"text":"Ainda não superou o final de Game of Thrones? Nós também não! Confira o que preparamos para os fãs que já não sab… https://t.co/G93RryP9OS",
"truncated":true,
"entities":{
"hashtags":{},
"symbols":{},
"user_mentions":{},
"urls":[{
"url":"https://t.co/G93RryP9OS",
"expanded_url":"https://twitter.com/i/web/status/1130961032392990720",
"display_url":"twitter.com/i/web/status/1…",
"indices":[
117,
140
]
}]
},
"source":"Twitter Web Client",
"in_reply_to_status_id":null,
"in_reply_to_status_id_str":null,
"in_reply_to_user_id":null,
"in_reply_to_user_id_str":null,
"in_reply_to_screen_name":null,
"user":{
"id":183639847,
"id_str":"183639847",
"name":"Conheça a BRQ",
"screen_name":"brqdigital",
"location":"São Paulo",
"description":"Paixão por transformar negócios com tecnologia, esse é o propósito que move a BRQ em 26 anos de história.",
"url":"https://t.co/IXZCQUM9nb",
"entities":{
"url":{
"urls":[{
"url":"https://t.co/IXZCQUM9nb",
"expanded_url":"http://www.brq.com",
"display_url":"brq.com",
"indices":[
0,
23
]
}]
},
"description":{
"urls":{}
}
},
"protected":false,
"followers_count":551,
"friends_count":84,
"listed_count":9,
"created_at":"Fri Aug 27 14:25:19 +0000 2010",
"favourites_count":8,
"utc_offset":null,
"time_zone":null,
"geo_enabled":true,
"verified":false,
"statuses_count":606,
"lang":null,
"contributors_enabled":false,
"is_translator":false,
"is_translation_enabled":false,
"profile_background_color":"4D8BB7",
"profile_background_image_url":"http://abs.twimg.com/images/themes/theme4/bg.gif",
"profile_background_image_url_https":"https://abs.twimg.com/images/themes/theme4/bg.gif",
"profile_background_tile":false,
"profile_image_url":"http://pbs.twimg.com/profile_images/1026460187195043840/LDTi5Im9_normal.jpg",
"profile_image_url_https":"https://pbs.twimg.com/profile_images/1026460187195043840/LDTi5Im9_normal.jpg",
"profile_banner_url":"https://pbs.twimg.com/profile_banners/183639847/1523283084",
"profile_link_color":"4D8BB7",
"profile_sidebar_border_color":"FFFFFF",
"profile_sidebar_fill_color":"FBECD5",
"profile_text_color":"685157",
"profile_use_background_image":false,
"has_extended_profile":false,
"default_profile":false,
"default_profile_image":false,
"following":false,
"follow_request_sent":false,
"notifications":false,
"translator_type":"none"
},
"geo":null,
"coordinates":null,
"place":null,
"contributors":null,
"is_quote_status":false,
"retweet_count":0,
"favorite_count":0,
"favorited":false,
"retweeted":false,
"possibly_sensitive":false,
"lang":"pt"
}
When I run logstash it does not recognize any changes in the directory
I think I'm setting something up wrong.
Can someone help me?
