This is json document that I get from one of the kafka topic.
Steps are different and more/less in quantity.
{
"jobid" : 300872305,
"dataflow" : {
"phases" : {
"SETUP" : {
"start" : "1583425476",
"end" : "1583425499"
},
"PREPARE" : {
"start" : "1583425499",
"end" : "1583425513"
},
"GROUP" : {
"start" : "1583425513",
"end" : "1583427707"
},
"CLEANUP" : {
"start" : "1583427707",
"end" : "1583427709"
},
"Overall job time" : {
"start" : "1583425476",
"end" : "1583427707"
}
},
"steps" : {
"stack" : {
"version" : "v207.18",
"instances" : 1858,
"setup.cpu" : " 0.01",
"setup.elapsed" : " 0.01",
"prepare.cpu" : " 0.00",
"prepare.elapsed" : " 0.00",
"group.cpu" : " 1.89",
"group.elapsed" : " 1.80",
"cleanup.cpu" : " 0.00",
"cleanup.elased" : " 0.00"
},
"hfnmo" : {
"version" : "v218.02",
"instances" : 163533,
"setup.cpu" : " 0.08",
"setup.elapsed" : " 0.09",
"prepare.cpu" : " 4.17",
"prepare.elapsed" : " 12.31",
"group.cpu" : " 48.47",
"group.elapsed" : " 48.25",
"cleanup.cpu" : " 0.00",
"cleanup.elased" : " 0.00"
}
}
},
"elmname" : "2020.02.21",
"io" : {
"esp" : {
"readebytes" : 16845431672,
"readlatency" : 233434552,
"readrate" : 72.1634030938145,
"writebytes" : 74286795,
"writelatency" : 1879879,
"writerate" : 39.5167960278295
},
"kernel" : {
"blockreadbytes" : 6096404480,
"blockwritebytes" : 7859302400,
"rawreadbytes" : 24650385306,
"rawwritebytes" : 7857807527
}
},
"runtime" : {
"cputime" : 1987.27,
"memory" : {
"peakvirtual" : 20743581696.0,
"virtual" : 10412249088.0
},
"ncores" : 16,
"overallstatus" : "true",
"pagefaults" : 3,
"processes" : {
"peak_run" : 10,
"peak_scheduled" : 369,
"swaps" : 0
},
"processingfactor" : 44.5984614156633,
"protected" : {
"count" : 48,
"elapsed" : 21.1101427078247
},
"sysload" : {
"avg" : 6.79199981689453,
"peak" : 8.36999988555908
},
"totalelapsed" : 2232.7202129364,
"start" : 1583425475
},
"status" : "Success",
"scheduling" : {
"submission" : 1583328372
},
"user" : "sachin",
"version" : 1,
"cleanup" : {
"end" : 1583427712,
"postprocess" : 1583427709
},
"setup" : {
"link" : 1583425454,
"start" : 1583425451
}
}
When I load this document in ELK it comes up like this
||@timestamp|Apr 2, 2020 @ 08:52:37.056|
||@version|1|
||_id|300872305|
||_index|sachin_quick_test|
||_score|0|
||_type|_doc|
||cleanup.end|1,583,427,712|
||cleanup.postprocess|1,583,427,709|
||dataflow.phases.CLEANUP.end|1583427709|
||dataflow.phases.CLEANUP.start|1583427707|
||dataflow.phases.GROUP.end|1583427707|
||dataflow.phases.GROUP.start|1583425513|
||dataflow.phases.Overall job time.end|1583427707|
||dataflow.phases.Overall job time.start|1583425476|
||dataflow.phases.PREPARE.end|1583425513|
||dataflow.phases.PREPARE.start|1583425499|
||dataflow.phases.SETUP.end|1583425499|
||dataflow.phases.SETUP.start|1583425476|
||dataflow.steps.hfnmo.cleanup.cpu|0.00|
||dataflow.steps.hfnmo.cleanup.elased|0.00|
||dataflow.steps.hfnmo.group.cpu|48.47|
||dataflow.steps.hfnmo.group.elapsed|48.25|
||dataflow.steps.hfnmo.instances|163,533|
||dataflow.steps.hfnmo.prepare.cpu|4.17|
||dataflow.steps.hfnmo.prepare.elapsed|12.31|
||dataflow.steps.hfnmo.setup.cpu|0.08|
||dataflow.steps.hfnmo.setup.elapsed|0.09|
||dataflow.steps.hfnmo.version|v218.02|
||dataflow.steps.stack.cleanup.cpu|0.00|
||dataflow.steps.stack.cleanup.elased|0.00|
||dataflow.steps.stack.group.cpu|1.89|
||dataflow.steps.stack.group.elapsed|1.80|
||dataflow.steps.stack.instances|1,858|
||dataflow.steps.stack.prepare.cpu|0.00|
||dataflow.steps.stack.prepare.elapsed|0.00|
||dataflow.steps.stack.setup.cpu|0.01|
||dataflow.steps.stack.setup.elapsed|0.01|
||dataflow.steps.stack.version|v207.18|
||elmname|2020.02.21|
||io.esp.readebytes|16,845,431,672|
||io.esp.readlatency|233,434,552|
||io.esp.readrate|72.163|
||io.esp.writebytes|74,286,795|
||io.esp.writelatency|1,879,879|
||io.esp.writerate|39.517|
||io.kernel.blockreadbytes|6,096,404,480|
||io.kernel.blockwritebytes|7,859,302,400|
||io.kernel.rawreadbytes|24,650,385,306|
||io.kernel.rawwritebytes|7,857,807,527|
||jobid|300,872,305|
||runtime.cputime|1,987.27|
||runtime.memory.peakvirtual|20,743,581,696|
||runtime.memory.virtual|10,412,249,088|
||runtime.ncores|16|
||runtime.overallstatus|true|
||runtime.pagefaults|3|
||runtime.processes.peak_run|10|
||runtime.processes.peak_scheduled|369|
||runtime.processes.swaps|0|
||runtime.processingfactor|44.598|
||runtime.protected.count|48|
||runtime.protected.elapsed|21.11|
||runtime.start|1,583,425,475|
||runtime.sysload.avg|6.792|
||runtime.sysload.peak|8.37|
||runtime.totalelapsed|2,232.72|
||scheduling.submission|1,583,328,372|
||setup.link|1,583,425,454|
||setup.start|1,583,425,451|
||status|Success|
||user|sachin|
||version|1|
Which does not serve the purpose.
i.e using this I can't tell how many time steps hfnmo was executed. ( because everything is key-value and value is nail down to some digit.
I can't find how many time user sachin used xyz steps in last month. because "hfnmo" is no where in value field.
how do solve this?
main thing is steps. each topic will have multiple steps, and sometime same steps twice in same json file.