Mapping for dynamic JSON payload

Elastic Stack Elasticsearch
1

Good day to everyone!
I'm trying to send JSON formated data directly to elasticsearch by using elasticsearch python library but I'm having a fight here with the index mapping.
How can one define dynamic mapping to JSON data where JSON key changes, but the child keys under this main key are static?

From the data source, I receive following data shown below.
Under JSON key "tags" are two uniq keys which I'm interested, "as7c785324" and "h7g4Esbg_09002616803700".
Under these keys are multiple keys and their values for which I want to apply correct mapping.

So, if I want to apply epoch_second format to created key under tags.as7c785324, how define it when "as7c785324" can be anything in the next data insertion?

I've tested this simple example but it doesn't provide the result that I want to achieve here :slight_smile:

{
	"mapping": {
		"dynamic": "false",
		"properties": {
			"created": {
				"type": "date",
				"format": "epoch_second"
			}
		}
	}
}

Example data 1:

{
	"SSRC": {
		"1168346465": {},
		"1696704987": {}
	},
	"created": 1604395279,
	"created_us": 778888,
	"last signal": 1604395280,
	"result": "ok",
	"tags": {
		"as7c785324": {
			"created": 1604395279,
			"in dialogue with": "h7g4Esbg_09002616803700",
			"medias": [{
				"flags": [
					"initialized",
					"send",
					"recv"
				],
				"index": 1,
				"protocol": "RTP/AVP",
				"streams": [{
						"advertised endpoint": {
							"address": "10.20.160.241",
							"family": "IPv4",
							"port": 45492
						},
						"endpoint": {
							"address": "10.20.160.241",
							"family": "IPv4",
							"port": 45492
						},
						"flags": [
							"RTP",
							"filled"
						],
						"last packet": 1604395279,
						"local port": 42068,
						"stats": {
							"bytes": 0,
							"errors": 0,
							"packets": 0
						}
					},
					{
						"SSRC": 1168346465,
						"advertised endpoint": {
							"address": "10.20.160.241",
							"family": "IPv4",
							"port": 45493
						},
						"endpoint": {
							"address": "10.20.160.241",
							"family": "IPv4",
							"port": 45493
						},
						"flags": [
							"RTCP",
							"filled",
							"confirmed"
						],
						"last packet": 1604395300,
						"local port": 42069,
						"stats": {
							"bytes": 176,
							"errors": 0,
							"packets": 4
						}
					}
				],
				"type": "audio"
			}],
			"tag": "as7c785324"
		},
		"h7g4Esbg_09002616803700": {
			"created": 1604395279,
			"in dialogue with": "as7c785324",
			"medias": [{
				"flags": [
					"initialized",
					"send",
					"recv",
					"ICE controlling"
				],
				"index": 1,
				"protocol": "RTP/AVP",
				"streams": [{
						"SSRC": 1696704987,
						"advertised endpoint": {
							"address": "10.20.183.158",
							"family": "IPv4",
							"port": 6904
						},
						"endpoint": {
							"address": "10.20.183.158",
							"family": "IPv4",
							"port": 6904
						},
						"flags": [
							"RTP",
							"filled",
							"confirmed"
						],
						"last packet": 1604395303,
						"local port": 42056,
						"stats": {
							"bytes": 193500,
							"errors": 0,
							"packets": 1125
						}
					},
					{
						"advertised endpoint": {
							"address": "10.20.183.158",
							"family": "IPv4",
							"port": 6905
						},
						"endpoint": {
							"address": "10.20.183.158",
							"family": "IPv4",
							"port": 6905
						},
						"flags": [
							"RTCP",
							"filled"
						],
						"last packet": 1604395279,
						"local port": 42057,
						"stats": {
							"bytes": 0,
							"errors": 0,
							"packets": 0
						}
					}
				],
				"type": "audio"
			}],
			"tag": "h7g4Esbg_09002616803700"
		}
	},
	"totals": {
		"RTCP": {
			"bytes": 176,
			"errors": 0,
			"packets": 4
		},
		"RTP": {
			"bytes": 193500,
			"errors": 0,
			"packets": 1125
		}
	}
}

Example data 2:

{
	"SSRC": {
		"2358751276": {},
		"67241221": {}
	},
	"created": 1604396290,
	"created_us": 366908,
	"last signal": 1604396313,
	"result": "ok",
	"tags": {
		"gK08bdb4fb": {
			"created": 1604396290,
			"in dialogue with": "iuJWdXGleyA.",
			"medias": [{
				"flags": [
					"initialized",
					"send",
					"recv",
					"ICE controlling"
				],
				"index": 1,
				"protocol": "RTP/AVP",
				"streams": [{
						"SSRC": 2358751276,
						"advertised endpoint": {
							"address": "10.20.56.201",
							"family": "IPv4",
							"port": 51902
						},
						"endpoint": {
							"address": "10.20.56.201",
							"family": "IPv4",
							"port": 51902
						},
						"flags": [
							"RTP",
							"filled",
							"confirmed",
							"kernelized"
						],
						"last packet": 1604396345,
						"local port": 54552,
						"stats": {
							"bytes": 471968,
							"errors": 0,
							"packets": 2744
						}
					},
					{
						"advertised endpoint": {
							"address": "10.20.56.201",
							"family": "IPv4",
							"port": 51903
						},
						"endpoint": {
							"address": "10.20.56.201",
							"family": "IPv4",
							"port": 51903
						},
						"flags": [
							"RTCP",
							"filled"
						],
						"last packet": 1604396290,
						"local port": 54553,
						"stats": {
							"bytes": 0,
							"errors": 0,
							"packets": 0
						}
					}
				],
				"type": "audio"
			}],
			"tag": "gK08bdb4fb"
		},
		"iuJWdXGleyA.": {
			"created": 1604396290,
			"in dialogue with": "gK08bdb4fb",
			"medias": [{
				"flags": [
					"initialized",
					"send",
					"recv"
				],
				"index": 1,
				"protocol": "RTP/AVP",
				"streams": [{
						"SSRC": 67241221,
						"advertised endpoint": {
							"address": "10.20.127.194",
							"family": "IPv4",
							"port": 50064
						},
						"endpoint": {
							"address": "10.20.127.194",
							"family": "IPv4",
							"port": 50064
						},
						"flags": [
							"RTP",
							"filled",
							"confirmed",
							"kernelized"
						],
						"last packet": 1604396345,
						"local port": 54566,
						"stats": {
							"bytes": 271932,
							"errors": 0,
							"packets": 1581
						}
					},
					{
						"advertised endpoint": {
							"address": "10.20.127.194",
							"family": "IPv4",
							"port": 50065
						},
						"endpoint": {
							"address": "10.20.127.194",
							"family": "IPv4",
							"port": 50065
						},
						"flags": [
							"RTCP",
							"filled"
						],
						"last packet": 1604396290,
						"local port": 54567,
						"stats": {
							"bytes": 0,
							"errors": 0,
							"packets": 0
						}
					}
				],
				"type": "audio"
			}],
			"tag": "iuJWdXGleyA."
		}
	},
	"totals": {
		"RTCP": {
			"bytes": 0,
			"errors": 0,
			"packets": 0
		},
		"RTP": {
			"bytes": 743900,
			"errors": 0,
			"packets": 4325
		}
	}
}

Thank you in advance!

Br,
Aki

2

Having dynamic fields like that will lead to mapping explosion and a lot of problems. I would recommend restructuring your data and instead storing the tag name in the struct, e.g.:

"tags": [
  {
    "tag": "gK08bdb4fb",
    ...
  }
]
3

Hey Christian,

Thanks for your prompt answer!
Thats what I expected for answer. So just to be sure, there isn't a sneaky way to define wildcard for field name in the mapping?

Br,
Aki

4

No, there is not.

5

Thanks again for your prompt answer.
It's highly appreciated :slight_smile:

Br,
Aki

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.