Help Indexing Twitter created_at field


(Sparrowdev) #1

I'm trying to index objects coming in from the Twitter Stream api.
I've defined the index mapping as following, but the date-picker and histogram panels in Kibana are not accepting the created_at field properly, so I am unable to filter by time.
Could my index mapping format be incorrect, or is there another issue at large?

This is how the Twitter created_at field looks like:

"created_at": "Wed Nov 04 03:59:55 +0000 2015"

And here are my index settings:

 {
  "settings": {
    "analysis": {},
    "mapping.ignore_malformed": false
  },
  "mappings": {
    "tweet": {
      "properties": {
        "created_at": {
          "type": "date",
          "format": "EEE MMM dd HH:mm:ss Z yyyy"
        },
        "truncated": {
          "type": "boolean"
        },
        "timestamp_ms": {
          "type": "string"
        },
        "source": {
          "type": "string"
        },
        "retweet_count": {
          "type": "long"
        },
        "retweeted": {
          "type": "boolean"
        },
        "filter_level": {
          "type": "string"
        },
        "is_quote_status": {
          "type": "boolean"
        },
        "entities": {
          "type": "object"
        },
        "id_str": {
          "type": "string"
        },
        "favorite_count": {
          "type": "long"
        },
        "id": {
          "type": "long"
        },
        "text": {
          "type": "string"
        },
        "lang": {
          "type": "string"
        },
        "user": {
          "properties": {
            "utc_offset": {
              "type": "long"
            },
            "friends_count": {
              "type": "long"
            },
            "profile_image_url_https": {
              "type": "string"
            },
            "listed_count": {
              "type": "long"
            },
            "profile_background_image_url": {
              "type": "string"
            },
            "default_profile_image": {
              "type": "boolean"
            },
            "favourites_count": {
              "type": "long"
            },
            "created_at": {
              "type": "string"
            },
            "description": {
              "type": "string"
            },
            "is_translator": {
              "type": "boolean"
            },
            "profile_background_image_url_https": {
              "type": "string"
            },
            "protected": {
              "type": "boolean"
            },
            "screen_name": {
              "type": "string"
            },
            "id_str": {
              "type": "string"
            },
            "profile_link_color": {
              "type": "string"
            },
            "geo_enabled": {
              "type": "boolean"
            },
            "id": {
              "type": "long"
            },
            "profile_background_color": {
              "type": "string"
            },
            "lang": {
              "type": "string"
            },
            "profile_sidebar_border_color": {
              "type": "string"
            },
            "profile_text_color": {
              "type": "string"
            },
            "verified": {
              "type": "boolean"
            },
            "profile_image_url": {
              "type": "string"
            },
            "time_zone": {
              "type": "string"
            },
            "contributors_enabled": {
              "type": "boolean"
            },
            "profile_background_tile": {
              "type": "boolean"
            },
            "statuses_count": {
              "type": "long"
            },
            "default_profile": {
              "type": "boolean"
            },
            "followers_count": {
              "type": "long"
            },
            "profile_use_background_image": {
              "type": "boolean"
            },
            "name": {
              "type": "string"
            },
            "location": {
              "type": "string"
            },
            "profile_sidebar_fill_color": {
              "type": "string"
            }
          }
        },
        "favorited": {
          "type": "boolean"
        }
      }
    }
  }
}

(Andresmmujica) #2

bump.. having the same issue..


#3

created_at field is string. Use the timestamp_ms field instead


(system) #4