we have fields like hour of day, week of month,quarter of year.. derived from @timestamp field using scripted field in kibana.
We want to create a transform index to summarize our operational data, we want to store those fields hour of day, week of month,.. into transformed index instead of creating scripted fields in kibana.
we ran the following query in elasticsearch 7.5.1 and 7.8.0. both versions are giving back the same error. Kindly help us to solve the problem.
DSL:
POST /_transform/_preview?pretty
{
"source": {
"index": "kibana_sample_data_ecommerce"
},
"pivot": {
"group_by": {
"customer_first_name.keyword": {
"terms": {
"field": "customer_first_name.keyword"
}
},
"day_of_week": {
"terms": {
"field": "day_of_week"
}
}
},
"aggregations": {
"order_date": {
"max": {
"field": "order_date"
}
},
"script_fields": {
"hour_of_day": {
"script": {
"lang": "painless",
"source": """
ZonedDateTime date = doc['order_date'].value;
if(!doc['order_date'].empty)
{
return date.getHour();
}
"""
}
}
}
}
},
"dest": {
"index": "pivot_logs"
}
}
Mapping for todel002
{
"pivot_logs" : {
"mappings" : {
"properties" : {
"customer_first_name" : {
"properties" : {
"keyword" : {
"type" : "keyword"
}
}
},
"day_of_week" : {
"type" : "keyword"
},
"order_date" : {
"properties" : {
"max" : {
"type" : "date"
}
}
},
"script_fields" : {
"properties" : {
"hour_of_day" : {
"type" : "integer"
}
}
}
}
}
}
}
after adding script_field to aggregation throwing the following error.
{
"error" : {
"root_cause" : [
{
"type" : "named_object_not_found_exception",
"reason" : "[1:77] unknown field [hour_of_day]"
}
],
"type" : "x_content_parse_exception",
"reason" : "[1:466] [data_frame_transform_config] failed to parse field [pivot]",
"caused_by" : {
"type" : "x_content_parse_exception",
"reason" : "[1:466] [data_frame_transform_pivot] failed to parse field [aggregations]",
"caused_by" : {
"type" : "named_object_not_found_exception",
"reason" : "[1:77] unknown field [hour_of_day]"
}
}
},
"status" : 400
}