How to debug Logstash output for custom logstash (solved)

Elastic Stack Logstash
1

(solved) Never mind I found the problem. One of my main varibles was initialized outside the main for loop, which meant information was being over written.

        process = Hash.new
        Dir.foreach("/proc"){ |file|
        ...     

        Dir.foreach("/proc"){ |file|
               process = Hash.new
               ...

Original Message

I think I am seeing a problem with my logstash custom plugin, some events don't get indexed and I can't tell why, even though they will print to stdout.

I thought it was some mapping mismatch but when I manually issue a curl -XPUT of the data it inserts with no problem.

I don't see any errors or warnings even in --debug mode.

How do I go about troubleshooting it?

here is the JSON of the one that fails

{
    "file": "/proc",
    "host": "coperdragon",
    "type": "pidstats",
    "process": {
        "cmdline": "",
        "environ": {},
        "cwd": null,
        "fd": {},
        "exe": {},
        "nlwp": 1,
        "root": 1,
        "io": {},
        "pid": 25,
        "comm": "migration/2",
        "state": "S",
        "ppid": 2,
        "pgrp": 0,
        "session": 0,
        "tty_nr": 0,
        "tpgid": -1,
        "flags": 69247040,
        "minflt": 0,
        "cminflt": 0,
        "majflt": 0,
        "cmajflt": 0,
        "utime": 0,
        "stime": 23,
        "cutime": 0,
        "cstime": 0,
        "priority": -100,
        "nice": 0,
        "itrealvalue": 0,
        "starttime": 9,
        "vsize": 0,
        "rss": 0,
        "rlim": "18446744073709551615",
        "startcode": 0,
        "endcode": 0,
        "startstack": 0,
        "kstkesp": 0,
        "kstkeip": 0,
        "signal": 0,
        "blocked": 0,
        "sigignore": 2147483647,
        "sigcatch": 0,
        "wchan": "18446744073709551615",
        "nswap": 0,
        "cnswap": 0,
        "exit_signal": 17,
        "processor": 2,
        "rt_priority": 99,
        "policy": 1,
        "name": "watchdog",
        "uid": 0,
        "euid": 0,
        "gid": 0,
        "egid": 0
    },
    "@version": "1",
    "@timestamp": "2015-06-21T03:32:43.697Z"
}

here is a json of one that succeeds

{
    "file": "/proc",
    "host": "coperdragon",
    "type": "pidstats",
    "process": {
        "cmdline": "",
        "environ": {},
        "cwd": null,
        "fd": {},
        "exe": {},
        "nlwp": 1,
        "root": 1,
        "io": {},
        "pid": 47,
        "comm": "md",
        "state": "S",
        "ppid": 2,
        "pgrp": 0,
        "session": 0,
        "tty_nr": 0,
        "tpgid": -1,
        "flags": 69247072,
        "minflt": 0,
        "cminflt": 0,
        "majflt": 0,
        "cmajflt": 0,
        "utime": 0,
        "stime": 0,
        "cutime": 0,
        "cstime": 0,
        "priority": 0,
        "nice": -20,
        "itrealvalue": 0,
        "starttime": 21,
        "vsize": 0,
        "rss": 0,
        "rlim": "18446744073709551615",
        "startcode": 0,
        "endcode": 0,
        "startstack": 0,
        "kstkesp": 0,
        "kstkeip": 0,
        "signal": 0,
        "blocked": 0,
        "sigignore": 2147483647,
        "sigcatch": 0,
        "wchan": "18446744073709551615",
        "nswap": 0,
        "cnswap": 0,
        "exit_signal": 17,
        "processor": 2,
        "rt_priority": 0,
        "policy": 0,
        "name": "md",
        "uid": 0,
        "euid": 0,
        "gid": 0,
        "egid": 0
    },
    "@version": "1",
    "@timestamp": "2015-06-21T03:32:49.406Z"
}

Here is the mapping

{
"state": "open",
"settings": {
"index": {
"creation_date": "1434852236454",
"uuid": "uLQngz9UQhio9zcznaywew",
"number_of_replicas": "0",
"number_of_shards": "1",
"version": {
"created": "1060099"
}
}
},
"mappings": {
"pidstats": {
"properties": {
"process": {
"properties": {
"uid": {
"type": "long"
},
"nlwp": {
"type": "long"
},
"tty_nr": {
"type": "long"
},
"utime": {
"type": "long"
},
"rt_priority": {
"type": "long"
},
"egid": {
"type": "long"
},
"stime": {
"type": "long"
},
"root": {
"type": "long"
},
"kstkeip": {
"type": "long"
},
"nice": {
"type": "long"
},
"sigcatch": {
"type": "long"
},
"vsize": {
"type": "long"
},
"priority": {
"type": "long"
},
"rss": {
"type": "long"
},
"cmajflt": {
"type": "long"
},
"processor": {
"type": "long"
},
"rlim": {
"type": "string"
},
"euid": {
"type": "long"
},
"starttime": {
"type": "long"
},
"environ": {
"type": "object"
},
"minflt": {
"type": "long"
},
"exit_signal": {
"type": "long"
},
"itrealvalue": {
"type": "long"
},
"session": {
"type": "long"
},
"majflt": {
"type": "long"
},
"gid": {
"type": "long"
},
"cnswap": {
"type": "long"
},
"cutime": {
"type": "long"
},
"policy": {
"type": "long"
},
"exe": {
"type": "object"
},
"cminflt": {
"type": "long"
},
"startcode": {
"type": "long"
},
"state": {
"type": "string"
},
"signal": {
"type": "long"
},
"nswap": {
"type": "long"
},
"sigignore": {
"type": "long"
},
"pgrp": {
"type": "long"
},
"kstkesp": {
"type": "long"
},
"endcode": {
"type": "long"
},
"name": {
"type": "string"
},
"blocked": {
"type": "long"
},
"tpgid": {
"type": "long"
},
"comm": {
"type": "string"
},
"flags": {
"type": "long"
},
"ppid": {
"type": "long"
},
"cmdline": {
"type": "string"
},
"pid": {
"type": "long"
},
"fd": {
"type": "object"
},
"startstack": {
"type": "long"
},
"io": {
"type": "object"
},
"cstime": {
"type": "long"
},
"wchan": {
"type": "string"
}
}
},
"@timestamp": {
"format": "dateOptionalTime",
"type": "date"
},
"host": {
"type": "string"
},
"file": {
"type": "string"
},
"type": {
"type": "string"
},
"@version": {
"type": "string"
}
}
}
},
"aliases": [
"pidstats"
]
}