i have below record which i want to process.
[
{
"_id": "GFk-X5cBY6REVzo7i86y",
"_index": "processor_test",
"_source": {
"event.original":"172.16.102.98 - - [11/Jun/2025:19:05:43 +0530] \"POST /api/elasticsearchData HTTP/1.1\" 200 397 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 Edg/137.0.0.0\" \"{\\x22query\\x22:\\x22/home/overview/trafficOverview.json\\x22,\\x22querySource\\x22:\\x22file\\x22,\\x22startDate\\x22:\\x222025-06-11T12:41:30.760Z\\x22,\\x22endDate\\x22:\\x222025-06-11T12:56:30.760Z\\x22,\\x22customUserFilter\\x22:false,\\x22tabify\\x22:false,\\x22debug\\x22:false}"
}
}
]
And below is my ingress pipeline
PUT _ingest/pipeline/logs-nginx.access@custom
{
"processors": [
{
"grok": {
"field": "event.original",
"tag": "grok_extract_raw_body",
"patterns": [
"(?:%{IPORHOST} %{HTTPDUSER} %{HTTPDUSER} \\[%{HTTPDATE}\\] \"(?:%{WORD} %{NOTSPACE}(?: HTTP/%{NUMBER})?|%{DATA})\" %{NUMBER} (?:%{NUMBER}|-) %{QS} %{QS}) %{GREEDYDATA:request_body_raw}"
],
"on_failure": [
{
"set": {
"field": "pipeline_failure.processor_type",
"value": "{{ _ingest.on_failure_processor_type }}"
}
},
{
"set": {
"field": "pipeline_failure.processor_tag",
"value": "{{ _ingest.on_failure_processor_tag }}"
}
},
{
"set": {
"field": "pipeline_failure.message",
"value": "Grok failed: {{ _ingest.on_failure_message }}"
}
}
]
}
},
{
"script": {
"lang": "painless",
"tag": "data_sanitization",
"source": """
if (ctx.containsKey('request_body_raw') && ctx.request_body_raw != null) {
String tempBody = ctx.request_body_raw;
// Remove outermost double quotes (if present) captured by Grok
if (tempBody.length() >= 2 && tempBody.startsWith('"') && tempBody.endsWith('"')) {
tempBody = tempBody.substring(1, tempBody.length() - 1);
}
// Correctly replace \\x22 with a single double quote "
ctx.request_body = tempBody.replace("\\\\x22", "\\\""); // Correct Painless escape for literal "
}
"""
}
},
{
"json": {
"field": "request_body",
"tag": "json_parsing",
"target_field": "parsed_request_body",
"if": "ctx.request_body != null && (ctx.request_body.startsWith('{') || ctx.request_body.startsWith('['))",
"ignore_failure": true,
"on_failure": [
{
"set": {
"field": "json_parsing_error",
"value": "JSON parsing failed for 'request_body': {{ _ingest.on_failure_message }}"
}
}
]
}
},
{
"rename": {
"tag": "renaming_parsed_request_body.query",
"field": "parsed_request_body.query",
"target_field": "request.query",
"ignore_failure": true
}
},
{
"rename": {
"tag": "renaming_parsed_request_body.startDate",
"field": "parsed_request_body.startDate",
"target_field": "request.startDate",
"ignore_failure": true
}
},
{
"rename": {
"tag": "renaming_parsed_request_body.endDate",
"field": "parsed_request_body.endDate",
"target_field": "request.endDate",
"ignore_failure": true
}
},
{
"remove": {
"field": "event.original",
"ignore_failure": true
}
},
{
"script": {
"tag": "timerange_calculation",
"lang": "painless",
"source": "\n if (ctx.containsKey('request') && ctx.request.startDate != null && ctx.request.endDate != null) {\n Instant start = Instant.parse(ctx.request.startDate);\n Instant end = Instant.parse(ctx.request.endDate);\n long diffMillis = Duration.between(start, end).toMillis();\n ctx.request.durationMinutes = diffMillis / 60000.0;\n }\n ",
"ignore_failure": true,
"on_failure": [
{
"set": {
"field": "timerange_calculation_error",
"value": "Timerange calculation failed"
}
}
]
}
},
{
"append": {
"field": "test",
"value": "kuldeep"
}
}
],
"on_failure": [
{
"set": {
"field": "pipeline_failure.processor_type",
"value": "{{ _ingest.on_failure_processor_type }}"
}
},
{
"set": {
"field": "pipeline_failure.processor_tag",
"value": "{{ _ingest.on_failure_processor_tag }}"
}
},
{
"set": {
"field": "pipeline_failure.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
]
}
My goal is to parse time range of query in minutes. and in my nginx record(provided above) i have startTime and endTime field.
I tried testing for the given document and got this output:
{
"docs": [
{
"processor_results": [
{
"processor_type": "grok",
"status": "error",
"tag": "grok_extract_raw_body",
"error": {
"root_cause": [
{
"type": "illegal_argument_exception",
"reason": "field [event] not present as part of path [event.original]"
}
],
"type": "illegal_argument_exception",
"reason": "field [event] not present as part of path [event.original]"
}
},
{
"processor_type": "set",
"status": "success",
"doc": {
"_index": "processor_test",
"_id": "HVlqX5cBY6REVzo7CdMj",
"_version": "-3",
"_source": {
"event.original": "172.16.102.98 - - [11/Jun/2025:19:05:43 +0530] \"POST /api/elasticsearchData HTTP/1.1\" 200 397 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 Edg/137.0.0.0\" \"{\\x22query\\x22:\\x22/home/overview/trafficOverview.json\\x22,\\x22querySource\\x22:\\x22file\\x22,\\x22startDate\\x22:\\x222025-06-11T12:41:30.760Z\\x22,\\x22endDate\\x22:\\x222025-06-11T12:56:30.760Z\\x22,\\x22customUserFilter\\x22:false,\\x22tabify\\x22:false,\\x22debug\\x22:false}",
"pipeline_failure": {
"processor_type": "grok"
}
},
"_ingest": {
"pipeline": "_simulate_pipeline",
"on_failure_message": "field [event] not present as part of path [event.original]",
"on_failure_processor_tag": "grok_extract_raw_body",
"on_failure_pipeline": "_simulate_pipeline",
"timestamp": "2025-06-11T14:39:41.801Z",
"on_failure_processor_type": "grok"
}
}
},
{
"processor_type": "set",
"status": "success",
"doc": {
"_index": "processor_test",
"_id": "HVlqX5cBY6REVzo7CdMj",
"_version": "-3",
"_source": {
"event.original": "172.16.102.98 - - [11/Jun/2025:19:05:43 +0530] \"POST /api/elasticsearchData HTTP/1.1\" 200 397 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 Edg/137.0.0.0\" \"{\\x22query\\x22:\\x22/home/overview/trafficOverview.json\\x22,\\x22querySource\\x22:\\x22file\\x22,\\x22startDate\\x22:\\x222025-06-11T12:41:30.760Z\\x22,\\x22endDate\\x22:\\x222025-06-11T12:56:30.760Z\\x22,\\x22customUserFilter\\x22:false,\\x22tabify\\x22:false,\\x22debug\\x22:false}",
"pipeline_failure": {
"processor_type": "grok",
"processor_tag": "grok_extract_raw_body"
}
},
"_ingest": {
"pipeline": "_simulate_pipeline",
"on_failure_message": "field [event] not present as part of path [event.original]",
"on_failure_processor_tag": "grok_extract_raw_body",
"on_failure_pipeline": "_simulate_pipeline",
"timestamp": "2025-06-11T14:39:41.801Z",
"on_failure_processor_type": "grok"
}
}
},
{
"processor_type": "set",
"status": "success",
"doc": {
"_index": "processor_test",
"_id": "HVlqX5cBY6REVzo7CdMj",
"_version": "-3",
"_source": {
"event.original": "172.16.102.98 - - [11/Jun/2025:19:05:43 +0530] \"POST /api/elasticsearchData HTTP/1.1\" 200 397 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 Edg/137.0.0.0\" \"{\\x22query\\x22:\\x22/home/overview/trafficOverview.json\\x22,\\x22querySource\\x22:\\x22file\\x22,\\x22startDate\\x22:\\x222025-06-11T12:41:30.760Z\\x22,\\x22endDate\\x22:\\x222025-06-11T12:56:30.760Z\\x22,\\x22customUserFilter\\x22:false,\\x22tabify\\x22:false,\\x22debug\\x22:false}",
"pipeline_failure": {
"processor_type": "grok",
"processor_tag": "grok_extract_raw_body",
"message": "Grok failed: field [event] not present as part of path [event.original]"
}
},
"_ingest": {
"pipeline": "_simulate_pipeline",
"on_failure_message": "field [event] not present as part of path [event.original]",
"on_failure_processor_tag": "grok_extract_raw_body",
"on_failure_pipeline": "_simulate_pipeline",
"timestamp": "2025-06-11T14:39:41.801Z",
"on_failure_processor_type": "grok"
}
}
},
{
"processor_type": "script",
"status": "success",
"tag": "data_sanitization",
"doc": {
"_index": "processor_test",
"_id": "HVlqX5cBY6REVzo7CdMj",
"_version": "-3",
"_source": {
"event.original": "172.16.102.98 - - [11/Jun/2025:19:05:43 +0530] \"POST /api/elasticsearchData HTTP/1.1\" 200 397 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 Edg/137.0.0.0\" \"{\\x22query\\x22:\\x22/home/overview/trafficOverview.json\\x22,\\x22querySource\\x22:\\x22file\\x22,\\x22startDate\\x22:\\x222025-06-11T12:41:30.760Z\\x22,\\x22endDate\\x22:\\x222025-06-11T12:56:30.760Z\\x22,\\x22customUserFilter\\x22:false,\\x22tabify\\x22:false,\\x22debug\\x22:false}",
"pipeline_failure": {
"processor_type": "grok",
"processor_tag": "grok_extract_raw_body",
"message": "Grok failed: field [event] not present as part of path [event.original]"
}
},
"_ingest": {
"pipeline": "_simulate_pipeline",
"timestamp": "2025-06-11T14:39:41.801Z"
}
}
},
{
"processor_type": "json",
"status": "skipped",
"tag": "json_parsing",
"if": {
"condition": "ctx.request_body != null && (ctx.request_body.startsWith('{') || ctx.request_body.startsWith('['))",
"result": false
}
},
{
"processor_type": "rename",
"status": "error_ignored",
"tag": "renaming_parsed_request_body.query",
"ignored_error": {
"error": {
"root_cause": [
{
"type": "illegal_argument_exception",
"reason": "field [parsed_request_body.query] doesn't exist"
}
],
"type": "illegal_argument_exception",
"reason": "field [parsed_request_body.query] doesn't exist"
}
},
"doc": {
"_index": "processor_test",
"_id": "HVlqX5cBY6REVzo7CdMj",
"_version": "-3",
"_source": {
"event.original": "172.16.102.98 - - [11/Jun/2025:19:05:43 +0530] \"POST /api/elasticsearchData HTTP/1.1\" 200 397 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 Edg/137.0.0.0\" \"{\\x22query\\x22:\\x22/home/overview/trafficOverview.json\\x22,\\x22querySource\\x22:\\x22file\\x22,\\x22startDate\\x22:\\x222025-06-11T12:41:30.760Z\\x22,\\x22endDate\\x22:\\x222025-06-11T12:56:30.760Z\\x22,\\x22customUserFilter\\x22:false,\\x22tabify\\x22:false,\\x22debug\\x22:false}",
"pipeline_failure": {
"processor_type": "grok",
"processor_tag": "grok_extract_raw_body",
"message": "Grok failed: field [event] not present as part of path [event.original]"
}
},
"_ingest": {
"pipeline": "_simulate_pipeline",
"timestamp": "2025-06-11T14:39:41.801Z"
}
}
},
{
"processor_type": "rename",
"status": "error_ignored",
"tag": "renaming_parsed_request_body.startDate",
"ignored_error": {
"error": {
"root_cause": [
{
"type": "illegal_argument_exception",
"reason": "field [parsed_request_body.startDate] doesn't exist"
}
],
"type": "illegal_argument_exception",
"reason": "field [parsed_request_body.startDate] doesn't exist"
}
},
"doc": {
"_index": "processor_test",
"_id": "HVlqX5cBY6REVzo7CdMj",
"_version": "-3",
"_source": {
"event.original": "172.16.102.98 - - [11/Jun/2025:19:05:43 +0530] \"POST /api/elasticsearchData HTTP/1.1\" 200 397 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 Edg/137.0.0.0\" \"{\\x22query\\x22:\\x22/home/overview/trafficOverview.json\\x22,\\x22querySource\\x22:\\x22file\\x22,\\x22startDate\\x22:\\x222025-06-11T12:41:30.760Z\\x22,\\x22endDate\\x22:\\x222025-06-11T12:56:30.760Z\\x22,\\x22customUserFilter\\x22:false,\\x22tabify\\x22:false,\\x22debug\\x22:false}",
"pipeline_failure": {
"processor_type": "grok",
"processor_tag": "grok_extract_raw_body",
"message": "Grok failed: field [event] not present as part of path [event.original]"
}
},
"_ingest": {
"pipeline": "_simulate_pipeline",
"timestamp": "2025-06-11T14:39:41.801Z"
}
}
},
{
"processor_type": "rename",
"status": "error_ignored",
"tag": "renaming_parsed_request_body.endDate",
"ignored_error": {
"error": {
"root_cause": [
{
"type": "illegal_argument_exception",
"reason": "field [parsed_request_body.endDate] doesn't exist"
}
],
"type": "illegal_argument_exception",
"reason": "field [parsed_request_body.endDate] doesn't exist"
}
},
"doc": {
"_index": "processor_test",
"_id": "HVlqX5cBY6REVzo7CdMj",
"_version": "-3",
"_source": {
"event.original": "172.16.102.98 - - [11/Jun/2025:19:05:43 +0530] \"POST /api/elasticsearchData HTTP/1.1\" 200 397 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 Edg/137.0.0.0\" \"{\\x22query\\x22:\\x22/home/overview/trafficOverview.json\\x22,\\x22querySource\\x22:\\x22file\\x22,\\x22startDate\\x22:\\x222025-06-11T12:41:30.760Z\\x22,\\x22endDate\\x22:\\x222025-06-11T12:56:30.760Z\\x22,\\x22customUserFilter\\x22:false,\\x22tabify\\x22:false,\\x22debug\\x22:false}",
"pipeline_failure": {
"processor_type": "grok",
"processor_tag": "grok_extract_raw_body",
"message": "Grok failed: field [event] not present as part of path [event.original]"
}
},
"_ingest": {
"pipeline": "_simulate_pipeline",
"timestamp": "2025-06-11T14:39:41.801Z"
}
}
},
{
"processor_type": "remove",
"status": "error_ignored",
"ignored_error": {
"error": {
"root_cause": [
{
"type": "illegal_argument_exception",
"reason": "field [event] not present as part of path [event.original]"
}
],
"type": "illegal_argument_exception",
"reason": "field [event] not present as part of path [event.original]"
}
},
"doc": {
"_index": "processor_test",
"_id": "HVlqX5cBY6REVzo7CdMj",
"_version": "-3",
"_source": {
"event.original": "172.16.102.98 - - [11/Jun/2025:19:05:43 +0530] \"POST /api/elasticsearchData HTTP/1.1\" 200 397 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 Edg/137.0.0.0\" \"{\\x22query\\x22:\\x22/home/overview/trafficOverview.json\\x22,\\x22querySource\\x22:\\x22file\\x22,\\x22startDate\\x22:\\x222025-06-11T12:41:30.760Z\\x22,\\x22endDate\\x22:\\x222025-06-11T12:56:30.760Z\\x22,\\x22customUserFilter\\x22:false,\\x22tabify\\x22:false,\\x22debug\\x22:false}",
"pipeline_failure": {
"processor_type": "grok",
"processor_tag": "grok_extract_raw_body",
"message": "Grok failed: field [event] not present as part of path [event.original]"
}
},
"_ingest": {
"pipeline": "_simulate_pipeline",
"timestamp": "2025-06-11T14:39:41.801Z"
}
}
},
{
"processor_type": "script",
"status": "success",
"tag": "timerange_calculation",
"doc": {
"_index": "processor_test",
"_id": "HVlqX5cBY6REVzo7CdMj",
"_version": "-3",
"_source": {
"event.original": "172.16.102.98 - - [11/Jun/2025:19:05:43 +0530] \"POST /api/elasticsearchData HTTP/1.1\" 200 397 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 Edg/137.0.0.0\" \"{\\x22query\\x22:\\x22/home/overview/trafficOverview.json\\x22,\\x22querySource\\x22:\\x22file\\x22,\\x22startDate\\x22:\\x222025-06-11T12:41:30.760Z\\x22,\\x22endDate\\x22:\\x222025-06-11T12:56:30.760Z\\x22,\\x22customUserFilter\\x22:false,\\x22tabify\\x22:false,\\x22debug\\x22:false}",
"pipeline_failure": {
"processor_type": "grok",
"processor_tag": "grok_extract_raw_body",
"message": "Grok failed: field [event] not present as part of path [event.original]"
}
},
"_ingest": {
"pipeline": "_simulate_pipeline",
"timestamp": "2025-06-11T14:39:41.801Z"
}
}
},
{
"processor_type": "append",
"status": "success",
"doc": {
"_index": "processor_test",
"_id": "HVlqX5cBY6REVzo7CdMj",
"_version": "-3",
"_source": {
"event.original": "172.16.102.98 - - [11/Jun/2025:19:05:43 +0530] \"POST /api/elasticsearchData HTTP/1.1\" 200 397 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 Edg/137.0.0.0\" \"{\\x22query\\x22:\\x22/home/overview/trafficOverview.json\\x22,\\x22querySource\\x22:\\x22file\\x22,\\x22startDate\\x22:\\x222025-06-11T12:41:30.760Z\\x22,\\x22endDate\\x22:\\x222025-06-11T12:56:30.760Z\\x22,\\x22customUserFilter\\x22:false,\\x22tabify\\x22:false,\\x22debug\\x22:false}",
"test": [
"kuldeep"
],
"pipeline_failure": {
"processor_type": "grok",
"processor_tag": "grok_extract_raw_body",
"message": "Grok failed: field [event] not present as part of path [event.original]"
}
},
"_ingest": {
"pipeline": "_simulate_pipeline",
"timestamp": "2025-06-11T14:39:41.801Z"
}
}
}
]
}
]
}
