Elastic-Index Join

Elastic Stack Elasticsearch
1

Hi Team,

My requirement is to join two index as mentioned below.Is it possible? if yes how to achieve it.

Index 1:
{
"A": 1,
"B": "Sam",
"C": 1234
}
Index 2:
{
"C": 1234,
"D": "elastic",
"E": "Support"
}
Final:
{
"A": 1,
"B": "Sam",
"C": 1234,
"D": "elastic",
"E": "Support"
}

2

Welcome to our community! :smiley:

You could do that with a reindex and an ingest pipeline with an enrich processor.

3

Thanks Warkolm!!!Can you elaborate more or with samples?

4

My mistake here, a transform works best when you already have the data in Elasticsearch. Here's an example;

PUT index1/_doc/1
{
"A": 1,
"B": "Sam",
"C": 1234
}

PUT index2/_doc/1
{
"C": 1234,
"D": "elastic",
"E": "Support"
}

PUT _transform/my-join
{
  "source": {
    "index": [
      "index1",
      "index2"
    ]
  },
  "dest": {
    "index": "joined-index"
  }, 
  "pivot": {
    "group_by": {
      "id": {
        "terms": {
          "field": "C"
        }
      }
    },
    "aggregations": {
      "combineFields": {
      "scripted_metric": {
        "init_script": "state.join = new HashMap()",
        "map_script": """
          String[] fields = new String[] {'A', 'B', 'C', 'D', 'E'};
          for (field in fields) {
            if (doc.containsKey(field + ".keyword")) {
              state.join.put(field, doc[field + ".keyword"]);
            } else if (doc.containsKey(field)) {
              state.join.put(field, doc[field]);
            }
          }
        """,
        "combine_script": "return state.join",
        "reduce_script": """
          String[] fields = new String[] {'A', 'B', 'C', 'D', 'E'};
          Map combined=new HashMap();
          for (state in states) {
            for (field in fields) {
              if (state.containsKey(field)) {
                combined.put(field, state[field].get(0));
              }
            }
          }
          return combined;
        """
      }
    }
    }
  }
}

POST _transform/my-join/_start

GET joined-index/_search
{
  "took": 0,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 1,
      "relation": "eq"
    },
    "max_score": 1,
    "hits": [
      {
        "_index": "joined-index",
        "_id": "ALnOexvkEM03Dun-taspfc0AAAAAAAAA",
        "_score": 1,
        "_source": {
          "combineFields": {
            "A": 1,
            "B": "Sam",
            "C": 1234,
            "D": "elastic",
            "E": "Support"
          },
          "id": 1234
        }
      }
    ]
  }
}
```
5

You can also do this in a search, but it won't be stored into another index like a transform;

GET index1,index2/_search
{
  "aggregations": {
    "combineFields": {
      "scripted_metric": {
        "init_script": "state.join = new HashMap()",
        "map_script": """
          String[] fields = new String[] {'A', 'B', 'C', 'D', 'E'};
          for (field in fields) {
            if (doc.containsKey(field + ".keyword")) {
              state.join.put(field, doc[field + ".keyword"]);
            } else if (doc.containsKey(field)) {
              state.join.put(field, doc[field]);
            }
          }
        """,
        "combine_script": "return state.join",
        "reduce_script": """
          String[] fields = new String[] {'A', 'B', 'C', 'D', 'E'};
          Map combined=new HashMap();
          for (state in states) {
            for (field in fields) {
              if (state.containsKey(field)) {
                combined.put(field, state[field].get(0));
              }
            }
          }
          return combined;
        """
      }
    }
  }
}
6

Awesome brother...exactly this is what looking..you made my day

1 Like
7

Thanks @xeraa too.

1 Like
8

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.