Transforms failed status with runtime fields

[Ann_Abbott]

Hi,
I am trying to create a transforms index with runtime-fields called adoption_time_stamp that's in a date format.

Every time I try to create a transforms index with this adoption_time_stamp runtime_field, I get this error saying it has failed to parse the date field of [-9223372036854775808]. This epoch does not exist in my adoption_time_stamp and am confused how to fix this error.

[przemekwitek (Przemysław Witek)]

Hi,
Welcome to the Elastic discuss channel!

The number -9223372036854775808 is just the minimum number an integer type can hold. So it may be that there are some missing values of the adoption_time_stamp field that get translated to the minimum integer value.

A few questions:

1. How do you configure the adoption_time_stamp runtime field? Is it in the index mapping for all of the indices matching events-* pattern?

You can find out by calling:

GET events-*/_mapping

2. How are you using this adoption_time_stamp field in transform?
   Is it configured as a sync field?
   Is the transform continuous?
   Does the transform itself specify runtime fields?

Would be good if you posted the transform config here.
You can obtain it by calling:

GET _transform/max_adoption

[Ann_Abbott]

I am trying to use adoption_time_stamp as a main @timestamp in transform.
Not sure if it's configured as a sync field but the transform is continuous.
This is the result when I call

GET _transform/max_adoption

{
  "count" : 1,
  "transforms" : [
    {
      "id" : "max_adoption",
      "version" : "8.1.1",
      "create_time" : 1650863362505,
      "source" : {
        "index" : [
          "events-*"
        ],
        "query" : {
          "match_all" : { }
        },
        "runtime_mappings" : {
          "order_item_fill_count" : {
            "type" : "double",
            "script" : {
              "source" : """if (doc['event_ctx.event_type'].value ==~ /ORDER_CREATED/ && doc['order_item.type'].value ==~ /FILL/)
{
emit(1);
}
else if (doc['event_ctx.event_type'].value ==~ /ORDER_ITEM_STATUS_CHANGED/ && doc['order_item.type'].value ==~ /FILL/)
{
emit(-1)
}
else
{
emit(0)
}"""
            }
          },
          "ordered_fill_count_with_cancellation" : {
            "type" : "double",
            "script" : {
              "source" : """if (doc['event_ctx.event_type'].value ==~ /ORDER_CREATED/ && doc['order_item.type'].value ==~ /FILL/)
{
emit(1);
}
else
{
emit(0)
}"""
            }
          },
          "pharmacy_sold_fill_count" : {
            "type" : "double",
            "script" : {
              "source" : """if (doc['event_ctx.event_type'].value ==~ /RX_FILL_STATE_CHANGED/) 
{ 
    if (doc.containsKey('data_tags.FILL_EVENT_TYPE.keyword') && 
    doc['data_tags.FILL_EVENT_TYPE.keyword'].size() > 0)
    {
        if (doc['data_tags.FILL_EVENT_TYPE.keyword'].value ==~ /FILL_SOLD_BY_PHARMACY/)
        {
        emit(1)
        }
        else 
        {
        emit(0)   
        }
    }
    else 
    {
    emit(0)
    }
}
else
{
emit(0)
}"""
            }
          },
          "adoption_time_stamp" : {
            "type" : "date",
            "script" : {
              "source" : """if (doc['event_ctx.event_type'].value ==~ /RX_FILL_STATE_CHANGED/) 
{ 
    if (doc.containsKey('data_tags.FILL_EVENT_TYPE.keyword') && 
    doc['data_tags.FILL_EVENT_TYPE.keyword'].size() > 0)
    {
        if (doc['data_tags.FILL_EVENT_TYPE.keyword'].value ==~ /FILL_SOLD_BY_PHARMACY/)
        {
        emit(doc['event_ctx.timestamp'].value.millis)
        }
    }
}
else if (doc['event_ctx.event_type'].value ==~ /ORDER_CREATED/ && doc['order_item.type'].value ==~ /FILL/)
{
emit(doc['event_ctx.timestamp'].value.millis);
}"""
            }
          },
          "order_counting" : {
            "type" : "double",
            "script" : {
              "source" : """if (doc['event_ctx.event_type'].value ==~ /ORDER_CREATED/)
{
emit(1);
}
else if (doc['event_ctx.event_type'].value ==~ /ORDER_ITEM_STATUS_CHANGED/)
{
emit(-1)
}
else
{
emit(0)
}"""
            }
          },
          "count_package" : {
            "type" : "double",
            "script" : {
              "source" : """if (doc['event_ctx.event_type'].value ==~ /PACKAGE_CREATED_FOR_ORDER/)
{
emit(1)
}
else if (doc['event_ctx.event_type'].value ==~ /PACKAGE_STATUS_CHANGED/) 
{ 
    if (doc.containsKey('package.delivery_status.keyword') && 
    doc['package.delivery_status.keyword'].size() > 0)
    {
        if (doc['package.delivery_status.keyword'].value ==~ /C/)
        {
        emit(-1)
        }
        else 
        {
        emit(0)   
        }
    }
}
else
{
emit(0)
}"""
            }
          }
        }
      },
      "dest" : {
        "index" : "max_adoption"
      },
      "frequency" : "1m",
      "sync" : {
        "time" : {
          "field" : "@timestamp",
          "delay" : "60s"
        }
      },
      "pivot" : {
        "group_by" : {
          "pharmacy.name" : {
            "terms" : {
              "field" : "pharmacy.name"
            }
          },
          "adoption_time_stamp" : {
            "date_histogram" : {
              "field" : "adoption_time_stamp",
              "calendar_interval" : "1d"
            }
          },
          "pharmacy.launch_date" : {
            "date_histogram" : {
              "field" : "pharmacy.launch_date",
              "calendar_interval" : "1d"
            }
          }
        },
        "aggregations" : {
          "ordered_fill_count_with_cancellation.sum" : {
            "sum" : {
              "field" : "ordered_fill_count_with_cancellation"
            }
          },
          "pharmacy_sold_fill_count.sum" : {
            "sum" : {
              "field" : "pharmacy_sold_fill_count"
            }
          }
        }
      },
      "settings" : {
        "max_page_search_size" : 500
      }
    }
  ]
}

[przemekwitek (Przemysław Witek)]

Thanks for the config.
adoption_time_stamp is used in the group-by section but it is not the sync.time field. Maybe that's ok for your use-case, but please double-check.

What is more worrying, the function (script) calculating adoption_time_stamp is not complete, i.e. it doesn't emit a value in certain conditions:

    if (doc.containsKey('data_tags.FILL_EVENT_TYPE.keyword') && 
    doc['data_tags.FILL_EVENT_TYPE.keyword'].size() > 0)
    {
        if (doc['data_tags.FILL_EVENT_TYPE.keyword'].value ==~ /FILL_SOLD_BY_PHARMACY/)
        {
        emit(doc['event_ctx.timestamp'].value.millis)
        }
    }

What I mean is, when if (doc['data_tags.FILL_EVENT_TYPE.keyword'].value ==~ /FILL_SOLD_BY_PHARMACY/) evaluates to true, a value is emitted, but if, on the other hand, it evaluates to false, nothing is emitted and this can manifest itself with the error message you are getting. Please add some else branch and re-check the transform.