hi experts,
i am using logstash to read data from kafka cluster and output to elasticsearch. but i encountered this error - in the message it contains hex characters, so failed to parse the json.
{:timestamp=>"2016-06-01T21:36:03.222000+0000", :message=>"JSON parse failure. Falling back to plain-text", :error=>#<LogStash::Json::ParserError: Unexpected character ('' (code 92)): was expecting double-quote to start field name
at [Source: [B@4e292087; line: 1, column: 3]>, :data=>"{\"Operator\":\"tls\",\"Environment\":\"proda\",\"Site\":\"A\",\"Slot\":\"S58\",\"SessionId\":\"Null Client InstrumentationContext\",\"Association\":null,\"ServerOrigin\":\"InstrumentationContextExtensions\",\"ListFields\":\"{\\\"titleId\\\":\\\"1257_907395_Live\\\",\\\"matchStatus\\\":\\\"PartialMatchFound\\\",\\\"title\\\":\\\"Il \xE9tait une fois... les Explorateurs\\\",\\\"episodeTitle\\\":null,\\\"releaseYear\\\":null,\\\"originalAirDate\\\":\\\"1996-12-10T00:00:00Z\\\",\\\"matchedProvider\\\":\\\"tms\\\",\\\"matchedProgramId\\\":null,\\\"matchedSeriesId\\\":\\\"SH002351360000\\\",\\\"matchedCandidates\\\":\\\"SH001587420000,SH018724030000,SH017853210000,SH023917590000,SH000742910000\\\",\\\"frmt\\\":\\\"Matching response for title {titleId}: {matchStatus}.\\\\nRequest:\\\\n\\\\tTitle: {title}\\\\n\\\\tEpisodeTitle: {episodeTitle}\\\\n\\\\tReleaseYear: {releaseYear}\\\\n\\\\tOriginalAirDate: {originalAirDate}\\\\nResponse:\\\\n\\\\tProvider: {matchedProvider}\\\\n\\\\tProgramId: {matchedProgramId}\\\\n\\\\tSeriesId: {matchedSeriesId}\\\\n\\\\tCandidates: {matchedCandidates}\\\"}\",\"RoleName\":\"catalogworker\",\"InstanceNumber\":\"2\",\"DeploymentId\":\"catalogworker-s58-a-proda-tls_1.0.1591.1\",\"LogTime\":\"2016-05-29T18:28:57.2095678Z\",\"Group\":8,\"Name\":\"UniversalMatchingManager\",\"esIndex\":\"logs-2016.05.29\",\"esType\":\"tls_proda_A\"}", :level=>:info}
{:timestamp=>"2016-06-01T21:36:03.224000+0000", :message=>"Received an event that has a different character encoding than you configured.", :text=>"{\"Operator\":\"tls\",\"Environment\":\"proda\",\"Site\":\"A\",\"Slot\":\"S58\",\"SessionId\":\"Null Client InstrumentationContext\",\"Association\":null,\"ServerOrigin\":\"InstrumentationContextExtensions\",\"ListFields\":\"{\\\"titleId\\\":\\\"1257_626316_Live\\\",\\\"matchStatus\\\":\\\"MatchFound\\\",\\\"title\\\":\\\"Hors s\xE9rie\\\",\\\"episodeTitle\\\":null,\\\"releaseYear\\\":null,\\\"originalAirDate\\\":\\\"2012-11-09T00:00:00Z\\\",\\\"matchedProvider\\\":\\\"tms\\\",\\\"matchedProgramId\\\":\\\"EP016354030001\\\",\\\"matchedSeriesId\\\":\\\"SH016354030000\\\",\\\"matchedCandidates\\\":\\\"EP016354030002,EP016354030003,EP016354030004,EP016354030005,EP016354030006\\\",\\\"frmt\\\":\\\"Matching response for title {titleId}: {matchStatus}.\\\\nRequest:\\\\n\\\\tTitle: {title}\\\\n\\\\tEpisodeTitle: {episodeTitle}\\\\n\\\\tReleaseYear: {releaseYear}\\\\n\\\\tOriginalAirDate: {originalAirDate}\\\\nResponse:\\\\n\\\\tProvider: {matchedProvider}\\\\n\\\\tProgramId: {matchedProgramId}\\\\n\\\\tSeriesId: {matchedSeriesId}\\\\n\\\\tCandidates: {matchedCandidates}\\\"}\",\"RoleName\":\"catalogworker\",\"InstanceNumber\":\"3\",\"DeploymentId\":\"catalogworker-s58-a-proda-tls_1.0.1591.1\",\"LogTime\":\"2016-05-29T18:28:46.1512684Z\",\"Group\":8,\"Name\":\"UniversalMatchingManager\",\"esIndex\":\"logs-2016.05.29\",\"esType\":\"tls_proda_A\"}", :expected_charset=>"UTF-8", :level=>:warn}
here is my config
input {
kafka {
topic_id => "elasticsearchtopiclogs"
group_id => "elasticsearchtopiclogs-tls-a-debug"
zk_connect => "xxx"
auto_offset_reset => "smallest"
reset_beginning => "true"
consumer_threads => 3
codec => json {
charset => "UTF-8"
}
}
}
filter {}
output {
if "_jsonparsefailure" in [tags] {
stdout { codec => rubydebug { metadata => true }}
}
elasticsearch {
hosts => ["https://xxx:9200"]
ssl => true
index => "debug-logs-%{+YYYY.MM.dd}"
user => "xxx"
password => "xxx"
document_type => "%{estype}"
template_name => "logs-logstash-template"
workers => 5
}
}
can someone help me to correctly handle this ? how to ignore the error without block the whole pipeline ?
Thanks
Hongwei