Hi. Trying to get Twitter feeds into my ES System but its not working like the way it should?
If I remove the 'full_tweet = true' from the input file I get some of the message (see attached) but not any Geo or other goodness...
Below is my Logstash config and attached is an example tweet that I currently get in ES.
Also, when I put full_tweet back in my /var/log/logstash/logstash.log file fills up with lots of 'unauthorized' messages and just scrolls with errors.
root@log01:/etc/logstash/conf.d# cat *
input {
twitter {
consumer_key => "A"
consumer_secret => "B"
oauth_token => "C"
oauth_token_secret => "D"
keywords => ["I","am","testing","this"]
type => "twitter-log01"
}
beats {
port => 5044
type => "beats-log01"
}
beats {
port => 5045
type => "winfilebeat-log01"
}
tcp {
port => 5514
type => "syslog5514-log01"
}
tcp {
port => 5515
type => "syslog5515-log01"
}
}
filter {
if [type] == "syslog5514-log01" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:[%{POSINT:syslog_pid}])?: %{GREEDYDATA:syslog_message}" }
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
}
output {
if [type] == "twitter-log01" {
elasticsearch {
hosts => ["els03","els04"]
sniffing => true
manage_template => false
index => "twitter-%{+YYYY.MM.dd}"
document_type => "twitter-log01"
}
} else if [type] == "beats-log01" {
elasticsearch {
hosts => ["els03","els04"]
sniffing => true
manage_template => false
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
} else if [type] == "winfilebeat-log01" {
elasticsearch {
hosts => ["els03","els04"]
sniffing => true
manage_template => false
index => "winfilebeat-%{+YYYY.MM.dd}"
document_type => "winfilebeat-log01"
}
} else if [type] == "syslog5514-log01" {
elasticsearch {
hosts => ["els03","els04"]
sniffing => true
manage_template => false
index => "syslog5514-%{+YYYY.MM.dd}"
document_type => "syslog5514-log01"
}
} else if [type] == "syslog5515-log01" {
elasticsearch {
hosts => ["els03","els04"]
sniffing => true
manage_template => false
index => "syslog5515-%{+YYYY.MM.dd}"
document_type => "syslog5515-log01"
}
} else {
elasticsearch {
hosts => ["els03","els04"]
sniffing => true
manage_template => false
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
}
Thanks!