Index is empty using Transform

Elastic Stack Elasticsearch
1

Hi all,

I'm trying to create a transform on ElasticSearch.

I started by creating this using console and got the expected results.

In the console I use:

POST _transform/_preview
{
  "source": {
    "index": "test",
    "query": {
      "bool":{
        "filter":[
          { "term": { "os_message.request_eventname.keyword": "WebScreenServerExecuted"}}
        ]
      }
    }
  }, 
  "dest" : { 
    "index" : "testdest"
  },
  "sync" : { 
    "time": {
      "field": "timestamp",
      "delay": "60s"
    }
  },
  "pivot": {
     "group_by": { 
      "carrier": { "terms": { "field": "os_message.application_name.keyword" }}
  },
    "aggregations": {
       "totalToday": {
          "filter": { 
            "bool": { 
              "must": [ 
                  {"range": { "@timestamp": { "from" : "now-5d", "to": "now"}}}
              ]
            }
          }
        },
        
        "totalOld": {
          "filter": { 
            "bool": { 
              "must": [ 
                  {"range": { "@timestamp": { "from" : "now-10d", "to": "now-5d"}}}
              ]
            }
          }
        },
      
        "today": {
          "filter": { 
              "bool": { 
                "must": [ 
                      {"range": { "@timestamp": { "from" : "now-5d", "to": "now"}}},
                      {"range": {"os_message.eventdetails.D": {"gte" : 1000}}}
                ]
              }
            }
        },
        
        "old": {
          "filter":  { 
            "bool": { 
              "must": [ 
                  {"range": { "@timestamp": { "from" : "now-10d", "to": "now-5d"}}},
                  {"range": {"os_message.eventdetails.D": {"gte" : 1000}}}
              ]
            }
          }
        },
        
      "diffRequests": {
        "bucket_script": {
          "buckets_path": {
            "today": "today>_count",
            "old": "old>_count",
            "totalToday": "totalToday>_count",
            "totalOld": "totalOld>_count"
          },
          "script": "params.today - params.old"
        }
      },
      
       "PercentageToday": {
        "bucket_script": {
          "buckets_path": {
            "today": "today>_count",
            "old": "old>_count",
            "totalToday": "totalToday>_count",
            "totalOld": "totalOld>_count"
          },
          "script": "params.today/params.totalToday*100"
        }
      }
    }
  }
}

And the results are:

{
  "preview": [
    {
      "carrier": "Administration",
      "totalOld": 263,
      "old": 41,
      "today": 4,
      "totalToday": 13,
      "diffRequests": -37,
      "PercentageToday": 30.76923076923077
    },
    {
      "carrier": "Auth0 Web Connector Extended",
      "totalOld": 57,
      "old": 2,
      "today": 0,
      "totalToday": 94,
      "diffRequests": -2,
      "PercentageToday": 0
    },
    {
      "carrier": "Azure Login Backoffice",
      "totalOld": 8,
      "old": 6,
      "today": 0,
      "totalToday": 0,
      "diffRequests": -6,
      "PercentageToday": null
    },
    {
      "carrier": "DB Cleaner on Steroids",
      "totalOld": 0,
      "old": 0,
      "today": 0,
      "totalToday": 0,
      "diffRequests": 0,
      "PercentageToday": null
    },
    {
      "carrier": "Discovery",
      "totalOld": 54,
      "old": 9,
      "today": 11,
      "totalToday": 48,
      "diffRequests": 2,
      "PercentageToday": 22.916666666666664
    }
  ],
  "generated_dest_index": {
    "mappings": {
      "_meta": {
        "_transform": {
          "transform": "transform-preview",
          "version": {
            "created": "8.3.2"
          },
          "creation_date_in_millis": 1658333989664
        },
        "created_by": "transform"
      },
      "properties": {
        "carrier": {
          "type": "keyword"
        },
        "totalOld": {
          "type": "long"
        },
        "old": {
          "type": "long"
        },
        "today": {
          "type": "long"
        },
        "totalToday": {
          "type": "long"
        }
      }
    },
    "settings": {
      "index": {
        "number_of_shards": "1",
        "auto_expand_replicas": "0-1"
      }
    },
    "aliases": {}
  }

But If I create a transform, the index is empty (no docs) but in the preview, I get some results:

image
image962×251 19.3 KB

Any idea why my index has not been populated?

Best regards,
Ruben Marinho

2

Can you post the output (executed in dev console) of GET _transform/<transform_id>/_stats?

3

Hi Hendrik,

Thanks for your reply.

The output is:

{
  "count": 1,
  "transforms": [
    {
      "id": "plus_overall2",
      "state": "started",
      "node": {
        "id": "lcR4ooDTSYe6Rou2AOZYCA",
        "name": "instance-0000000001",
        "ephemeral_id": "vYed6BWWSNKJakkwln8L-g",
        "transport_address": "I removed the IP address",
        "attributes": {}
      },
      "stats": {
        "pages_processed": 1,
        "documents_processed": 0,
        "documents_indexed": 0,
        "documents_deleted": 0,
        "trigger_count": 1006,
        "index_time_in_ms": 0,
        "index_total": 0,
        "index_failures": 0,
        "search_time_in_ms": 36,
        "search_total": 1,
        "search_failures": 0,
        "processing_time_in_ms": 0,
        "processing_total": 1,
        "delete_time_in_ms": 0,
        "exponential_avg_checkpoint_duration_ms": 63,
        "exponential_avg_documents_indexed": 0,
        "exponential_avg_documents_processed": 0
      },
      "checkpointing": {
        "last": {
          "checkpoint": 1,
          "timestamp_millis": 1658331275013,
          "time_upper_bound_millis": 1658331215013
        },
        "changes_last_detected_at": 1658331275011,
        "last_search_time": 1658391575087
      }
    }
  ]
}
4

Thanks!

This is indeed strange, the stats show a running transform, but it did not find any documents.

I suggest to try it in batch mode, meaning without specifying sync. I actually wonder: you configured timestamp but in aggregations you use @timestamp.

5

Hi Hendrik,

The problem was with timestamp name. After change it to @timestamp , it worked.

Thanks for your help.

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.