Hi folks! We are on Elastic version 8.17 and have an existing ES index that we are now introducing a dynamic attribute using dynamic template to. The dynamic fields would be added within an existing nested field.
Our bridged schema definition -
...
"mappings": {
"dynamic": "false",
"dynamic_templates": [
{
"variant_vectors_by_model": {
"path_match": "variants.item_embeddings.*",
"mapping": {
"type": "dense_vector",
"index": true,
"similarity": "cosine"
}
}
}
],
"properties": {
"group_id": {
"type": "keyword"
},
"codes": {
"type": "keyword"
},
"variants": {
"dynamic": false,
"type": "nested",
"properties": {
"code": {
"type": "keyword",
"store": true,
"fields": {
"search_keyword": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
}
}
},
"item_embeddings": {
"type": "object",
"dynamic": true
}
}
}
}
}
...
A sample document we are trying to index -
{
"group_id": "random_group",
"codes": [
"1234",
"5678"
],
"variants": [
{
"item_embeddings": {
"model_v1": [
0.05667318031191826,
0.03676224127411842,
0.03171849250793457,
0.01680068112909794,
-0.05972050502896309,
-0.0732097253203392,
0.007977733388543129,
0.024116408079862595,
0.05233846604824066,
0.007091963663697243,
0.018920909613370895,
0.00994688831269741,
-0.03117530047893524,
-0.03920264169573784,
0.0024608306121081114,
0.08436288684606552,
0.09416638314723969,
-0.02230188064277172,
-0.04007192701101303,
-0.04768383875489235,
-0.05513140186667442,
0.04384570196270943,
-0.027235502377152443,
0.002757014473900199,
0.03788439929485321,
0.009888852946460247,
0.05454133078455925,
-0.013158638961613178,
0.01422518864274025,
-0.014908150769770145,
-0.012250110507011414,
-0.006658331025391817,
-0.009525774046778679,
0.00005040172618464567,
0.027454864233732224,
0.026344215497374535,
-0.014987037517130375,
-0.019977308809757233,
0.014925945550203323,
0.02228066883981228,
0.037019744515419006,
-0.1255623698234558,
-0.033644549548625946,
-0.028516236692667007,
0.044139184057712555,
-0.03505230322480202,
-0.03997375816106796,
0.011344539932906628,
0.014801067300140858,
0.015361261554062366,
0.03306423872709274,
-0.022899985313415527,
-0.024189596995711327,
-0.03816773369908333,
0.01008186861872673,
-0.00008663388143759221,
0.0514993779361248,
-0.002248553093522787,
-0.0419624000787735,
0.04228736460208893,
0.05774098262190819,
0.010159933939576149,
-0.007106570526957512,
0.02951483242213726,
0.05634437873959541,
-0.00033239895128645003,
-0.0293535478413105,
0.023926932364702225,
0.0348331555724144,
0.008085130713880062,
-0.00501627754420042,
-0.01947554387152195,
-0.07726071029901505,
-0.011533509008586407,
-0.026907769963145256,
-0.004003502428531647,
0.024635018780827522,
-0.003822702681645751,
-0.03705618157982826,
-0.0758415162563324,
-0.019427645951509476,
-0.06774822622537613,
0.006928539369255304,
-0.001465015928260982,
-0.03276766091585159,
-0.02698662504553795,
0.002840219996869564,
0.016294054687023163,
-0.004398120101541281,
-0.05081615969538689,
0.06606673449277878,
0.051079269498586655,
-0.00797408539801836,
-0.03056160919368267,
-0.07847192138433456,
0.04809975251555443,
0.02581951580941677,
-0.015610875561833382,
-0.0003428397758398205,
-0.01234754640609026,
0.0007668399484828115,
-0.003478564554825425,
0.05073230341076851,
0.026372309774160385,
-0.02552119456231594,
0.003984949551522732,
0.047887172549963,
0.052004192024469376,
-0.08150003105401993,
0.02950786054134369,
0.017233461141586304,
0.017632029950618744,
0.03181365877389908,
-0.0010408436646685004,
0.03340645134449005,
0.03351975604891777,
0.05431903153657913,
0.03496186062693596,
0.05294286459684372,
-0.03482921048998833
],
"model_v2": [
-0.011533509008586407,
-0.026907769963145256,
-0.004003502428531647,
0.024635018780827522,
-0.003822702681645751,
-0.03705618157982826,
-0.0758415162563324,
-0.019427645951509476,
-0.06774822622537613,
0.006928539369255304,
-0.001465015928260982,
-0.03276766091585159,
-0.02698662504553795,
0.002840219996869564,
0.016294054687023163,
-0.004398120101541281,
-0.05081615969538689,
0.06606673449277878,
0.051079269498586655,
-0.00797408539801836,
-0.03056160919368267,
-0.07847192138433456,
0.04809975251555443,
0.02581951580941677,
-0.015610875561833382,
-0.0003428397758398205,
-0.01234754640609026,
0.0007668399484828115,
-0.003478564554825425,
0.05073230341076851,
0.026372309774160385,
-0.02552119456231594,
0.003984949551522732,
0.047887172549963,
0.052004192024469376,
-0.08150003105401993,
0.02950786054134369,
0.017233461141586304,
0.017632029950618744,
0.03181365877389908,
-0.0010408436646685004,
0.03340645134449005,
0.03351975604891777,
0.05431903153657913,
0.03496186062693596,
0.05294286459684372,
-0.03482921048998833,
0.05645470693707466,
-0.017539020627737045,
0.05545613169670105,
0.026850489899516106,
-0.03160329908132553,
-0.014252671040594578,
-0.0055099185556173325,
-0.06456445902585983,
-0.0026039292570203543,
0.00016189702728297561,
-0.016720732674002647,
-0.03572947531938553,
-0.03525441512465477,
0.018726889044046402,
-0.020997267216444016,
-0.002859925152733922,
0.028501175343990326,
-0.08033537864685059,
-0.05114390328526497,
0.06307440251111984,
0.03973092883825302,
-0.021966882050037384,
0.012732028029859066,
0.005040609743446112,
-0.032527245581150055,
-0.05229898542165756,
0.018316911533474922,
-0.076856829226017,
-0.04725717753171921,
0.0217917338013649,
0.06270810216665268,
0.04003457725048065,
0.0505179762840271,
0.0009315540082752705,
0.017734821885824203,
0.03280230984091759,
-0.0018475797260180116,
0.026251157745718956,
0.08345657587051392,
0.07981608062982559,
-0.009289843030273914,
-0.0002385903790127486,
-0.0595359280705452,
-0.023921649903059006,
0.012957212515175343,
0.012339817360043526,
-0.025282321497797966,
-0.025736678391695023,
0.011146784760057926,
-0.012356351129710674,
-0.054246608167886734,
-0.07149291038513184,
0.008146672509610653,
-0.02918223664164543,
-0.006232865620404482,
-0.011239223182201385,
-0.11836868524551392,
0.032120078802108765,
-0.025980276986956596,
-0.015189899131655693,
0.03470538556575775,
0.08147629350423813,
0.024406176060438156,
-0.0012608955148607492,
0.010617654770612717,
0.0153469517827034,
-0.0022471777629107237,
0.012003668583929539,
-0.031690362840890884,
-0.03480655327439308,
-0.04548567160964012,
0.0572417713701725,
-0.038150712847709656,
-0.0019421266624704003,
0.012202783487737179,
0.007642634678632021,
0.027861179783940315,
0.024305002763867378,
0.0045427256263792515,
0.05376886576414108,
0.03932100906968117,
0.042523521929979324,
0.03893957659602165,
-0.04955670237541199,
-0.054524924606084824,
0.039282187819480896,
0.02744641713798046,
0.07789070904254913,
0.011605729348957539,
-0.07004345208406448,
-0.03383749723434448,
-0.03189760074019432,
0.034358154982328415
]
},
"code": "1234"
},
{
"item_embeddings": {
"model_v1": [
0.05667318031191826,
0.03676224127411842,
0.03171849250793457,
0.01680068112909794,
-0.05972050502896309,
-0.0732097253203392,
0.007977733388543129,
0.024116408079862595,
0.05233846604824066,
0.007091963663697243,
0.018920909613370895,
0.00994688831269741,
-0.03117530047893524,
-0.03920264169573784,
0.0024608306121081114,
0.08436288684606552,
0.09416638314723969,
-0.02230188064277172,
-0.04007192701101303,
-0.04768383875489235,
-0.05513140186667442,
0.04384570196270943,
-0.027235502377152443,
0.002757014473900199,
0.03788439929485321,
0.009888852946460247,
0.05454133078455925,
-0.013158638961613178,
0.01422518864274025,
-0.014908150769770145,
-0.012250110507011414,
-0.006658331025391817,
-0.009525774046778679,
0.00005040172618464567,
0.027454864233732224,
0.026344215497374535,
-0.014987037517130375,
-0.019977308809757233,
0.014925945550203323,
0.02228066883981228,
0.037019744515419006,
-0.1255623698234558,
-0.033644549548625946,
-0.028516236692667007,
0.044139184057712555,
-0.03505230322480202,
-0.03997375816106796,
0.011344539932906628,
0.014801067300140858,
0.015361261554062366,
0.03306423872709274,
-0.022899985313415527,
-0.024189596995711327,
-0.03816773369908333,
0.01008186861872673,
-0.00008663388143759221,
0.0514993779361248,
-0.002248553093522787,
-0.0419624000787735,
0.04228736460208893,
0.05774098262190819,
0.010159933939576149,
-0.007106570526957512,
0.02951483242213726,
0.05634437873959541,
-0.00033239895128645003,
-0.0293535478413105,
0.023926932364702225,
0.0348331555724144,
0.008085130713880062,
-0.00501627754420042,
-0.01947554387152195,
-0.07726071029901505,
-0.011533509008586407,
-0.026907769963145256,
-0.004003502428531647,
0.024635018780827522,
-0.003822702681645751,
-0.03705618157982826,
-0.0758415162563324,
-0.019427645951509476,
-0.06774822622537613,
0.006928539369255304,
-0.001465015928260982,
-0.03276766091585159,
-0.02698662504553795,
0.002840219996869564,
0.016294054687023163,
-0.004398120101541281,
-0.05081615969538689,
0.06606673449277878,
0.051079269498586655,
-0.00797408539801836,
-0.03056160919368267,
-0.07847192138433456,
0.04809975251555443,
0.02581951580941677,
-0.015610875561833382,
-0.0003428397758398205,
-0.01234754640609026,
0.0007668399484828115,
-0.003478564554825425,
0.05073230341076851,
0.026372309774160385,
-0.02552119456231594,
0.003984949551522732,
0.047887172549963,
0.052004192024469376,
-0.08150003105401993,
0.02950786054134369,
0.017233461141586304,
0.017632029950618744,
0.03181365877389908,
-0.0010408436646685004,
0.03340645134449005,
0.03351975604891777,
0.05431903153657913,
0.03496186062693596,
0.05294286459684372,
-0.03482921048998833
],
"model_v2": [
-0.011533509008586407,
-0.026907769963145256,
-0.004003502428531647,
0.024635018780827522,
-0.003822702681645751,
-0.03705618157982826,
-0.0758415162563324,
-0.019427645951509476,
-0.06774822622537613,
0.006928539369255304,
-0.001465015928260982,
-0.03276766091585159,
-0.02698662504553795,
0.002840219996869564,
0.016294054687023163,
-0.004398120101541281,
-0.05081615969538689,
0.06606673449277878,
0.051079269498586655,
-0.00797408539801836,
-0.03056160919368267,
-0.07847192138433456,
0.04809975251555443,
0.02581951580941677,
-0.015610875561833382,
-0.0003428397758398205,
-0.01234754640609026,
0.0007668399484828115,
-0.003478564554825425,
0.05073230341076851,
0.026372309774160385,
-0.02552119456231594,
0.003984949551522732,
0.047887172549963,
0.052004192024469376,
-0.08150003105401993,
0.02950786054134369,
0.017233461141586304,
0.017632029950618744,
0.03181365877389908,
-0.0010408436646685004,
0.03340645134449005,
0.03351975604891777,
0.05431903153657913,
0.03496186062693596,
0.05294286459684372,
-0.03482921048998833,
0.05645470693707466,
-0.017539020627737045,
0.05545613169670105,
0.026850489899516106,
-0.03160329908132553,
-0.014252671040594578,
-0.0055099185556173325,
-0.06456445902585983,
-0.0026039292570203543,
0.00016189702728297561,
-0.016720732674002647,
-0.03572947531938553,
-0.03525441512465477,
0.018726889044046402,
-0.020997267216444016,
-0.002859925152733922,
0.028501175343990326,
-0.08033537864685059,
-0.05114390328526497,
0.06307440251111984,
0.03973092883825302,
-0.021966882050037384,
0.012732028029859066,
0.005040609743446112,
-0.032527245581150055,
-0.05229898542165756,
0.018316911533474922,
-0.076856829226017,
-0.04725717753171921,
0.0217917338013649,
0.06270810216665268,
0.04003457725048065,
0.0505179762840271,
0.0009315540082752705,
0.017734821885824203,
0.03280230984091759,
-0.0018475797260180116,
0.026251157745718956,
0.08345657587051392,
0.07981608062982559,
-0.009289843030273914,
-0.0002385903790127486,
-0.0595359280705452,
-0.023921649903059006,
0.012957212515175343,
0.012339817360043526,
-0.025282321497797966,
-0.025736678391695023,
0.011146784760057926,
-0.012356351129710674,
-0.054246608167886734,
-0.07149291038513184,
0.008146672509610653,
-0.02918223664164543,
-0.006232865620404482,
-0.011239223182201385,
-0.11836868524551392,
0.032120078802108765,
-0.025980276986956596,
-0.015189899131655693,
0.03470538556575775,
0.08147629350423813,
0.024406176060438156,
-0.0012608955148607492,
0.010617654770612717,
0.0153469517827034,
-0.0022471777629107237,
0.012003668583929539,
-0.031690362840890884,
-0.03480655327439308,
-0.04548567160964012,
0.0572417713701725,
-0.038150712847709656,
-0.0019421266624704003,
0.012202783487737179,
0.007642634678632021,
0.027861179783940315,
0.024305002763867378,
0.0045427256263792515,
0.05376886576414108,
0.03932100906968117,
0.042523521929979324,
0.03893957659602165,
-0.04955670237541199,
-0.054524924606084824,
0.039282187819480896,
0.02744641713798046,
0.07789070904254913,
0.011605729348957539,
-0.07004345208406448,
-0.03383749723434448,
-0.03189760074019432,
0.034358154982328415
]
},
"code": "5678"
}
]
}
Elastic throws the following error -
{
"error": {
"root_cause": [
{
"type": "null_pointer_exception",
"reason": null
}
],
"type": "null_pointer_exception",
"reason": null
},
"status": 500
}
Indexing works fine if we modify the document and remove one nested variant like so -
{
"group_id": "random_group",
"codes": [
"1234",
"5678"
],
"variants": [
{
"item_embeddings": {
"model_v1": [
0.05667318031191826,
0.03676224127411842,
0.03171849250793457,
0.01680068112909794,
-0.05972050502896309,
-0.0732097253203392,
0.007977733388543129,
0.024116408079862595,
0.05233846604824066,
0.007091963663697243,
0.018920909613370895,
0.00994688831269741,
-0.03117530047893524,
-0.03920264169573784,
0.0024608306121081114,
0.08436288684606552,
0.09416638314723969,
-0.02230188064277172,
-0.04007192701101303,
-0.04768383875489235,
-0.05513140186667442,
0.04384570196270943,
-0.027235502377152443,
0.002757014473900199,
0.03788439929485321,
0.009888852946460247,
0.05454133078455925,
-0.013158638961613178,
0.01422518864274025,
-0.014908150769770145,
-0.012250110507011414,
-0.006658331025391817,
-0.009525774046778679,
0.00005040172618464567,
0.027454864233732224,
0.026344215497374535,
-0.014987037517130375,
-0.019977308809757233,
0.014925945550203323,
0.02228066883981228,
0.037019744515419006,
-0.1255623698234558,
-0.033644549548625946,
-0.028516236692667007,
0.044139184057712555,
-0.03505230322480202,
-0.03997375816106796,
0.011344539932906628,
0.014801067300140858,
0.015361261554062366,
0.03306423872709274,
-0.022899985313415527,
-0.024189596995711327,
-0.03816773369908333,
0.01008186861872673,
-0.00008663388143759221,
0.0514993779361248,
-0.002248553093522787,
-0.0419624000787735,
0.04228736460208893,
0.05774098262190819,
0.010159933939576149,
-0.007106570526957512,
0.02951483242213726,
0.05634437873959541,
-0.00033239895128645003,
-0.0293535478413105,
0.023926932364702225,
0.0348331555724144,
0.008085130713880062,
-0.00501627754420042,
-0.01947554387152195,
-0.07726071029901505,
-0.011533509008586407,
-0.026907769963145256,
-0.004003502428531647,
0.024635018780827522,
-0.003822702681645751,
-0.03705618157982826,
-0.0758415162563324,
-0.019427645951509476,
-0.06774822622537613,
0.006928539369255304,
-0.001465015928260982,
-0.03276766091585159,
-0.02698662504553795,
0.002840219996869564,
0.016294054687023163,
-0.004398120101541281,
-0.05081615969538689,
0.06606673449277878,
0.051079269498586655,
-0.00797408539801836,
-0.03056160919368267,
-0.07847192138433456,
0.04809975251555443,
0.02581951580941677,
-0.015610875561833382,
-0.0003428397758398205,
-0.01234754640609026,
0.0007668399484828115,
-0.003478564554825425,
0.05073230341076851,
0.026372309774160385,
-0.02552119456231594,
0.003984949551522732,
0.047887172549963,
0.052004192024469376,
-0.08150003105401993,
0.02950786054134369,
0.017233461141586304,
0.017632029950618744,
0.03181365877389908,
-0.0010408436646685004,
0.03340645134449005,
0.03351975604891777,
0.05431903153657913,
0.03496186062693596,
0.05294286459684372,
-0.03482921048998833
],
"model_v2": [
-0.011533509008586407,
-0.026907769963145256,
-0.004003502428531647,
0.024635018780827522,
-0.003822702681645751,
-0.03705618157982826,
-0.0758415162563324,
-0.019427645951509476,
-0.06774822622537613,
0.006928539369255304,
-0.001465015928260982,
-0.03276766091585159,
-0.02698662504553795,
0.002840219996869564,
0.016294054687023163,
-0.004398120101541281,
-0.05081615969538689,
0.06606673449277878,
0.051079269498586655,
-0.00797408539801836,
-0.03056160919368267,
-0.07847192138433456,
0.04809975251555443,
0.02581951580941677,
-0.015610875561833382,
-0.0003428397758398205,
-0.01234754640609026,
0.0007668399484828115,
-0.003478564554825425,
0.05073230341076851,
0.026372309774160385,
-0.02552119456231594,
0.003984949551522732,
0.047887172549963,
0.052004192024469376,
-0.08150003105401993,
0.02950786054134369,
0.017233461141586304,
0.017632029950618744,
0.03181365877389908,
-0.0010408436646685004,
0.03340645134449005,
0.03351975604891777,
0.05431903153657913,
0.03496186062693596,
0.05294286459684372,
-0.03482921048998833,
0.05645470693707466,
-0.017539020627737045,
0.05545613169670105,
0.026850489899516106,
-0.03160329908132553,
-0.014252671040594578,
-0.0055099185556173325,
-0.06456445902585983,
-0.0026039292570203543,
0.00016189702728297561,
-0.016720732674002647,
-0.03572947531938553,
-0.03525441512465477,
0.018726889044046402,
-0.020997267216444016,
-0.002859925152733922,
0.028501175343990326,
-0.08033537864685059,
-0.05114390328526497,
0.06307440251111984,
0.03973092883825302,
-0.021966882050037384,
0.012732028029859066,
0.005040609743446112,
-0.032527245581150055,
-0.05229898542165756,
0.018316911533474922,
-0.076856829226017,
-0.04725717753171921,
0.0217917338013649,
0.06270810216665268,
0.04003457725048065,
0.0505179762840271,
0.0009315540082752705,
0.017734821885824203,
0.03280230984091759,
-0.0018475797260180116,
0.026251157745718956,
0.08345657587051392,
0.07981608062982559,
-0.009289843030273914,
-0.0002385903790127486,
-0.0595359280705452,
-0.023921649903059006,
0.012957212515175343,
0.012339817360043526,
-0.025282321497797966,
-0.025736678391695023,
0.011146784760057926,
-0.012356351129710674,
-0.054246608167886734,
-0.07149291038513184,
0.008146672509610653,
-0.02918223664164543,
-0.006232865620404482,
-0.011239223182201385,
-0.11836868524551392,
0.032120078802108765,
-0.025980276986956596,
-0.015189899131655693,
0.03470538556575775,
0.08147629350423813,
0.024406176060438156,
-0.0012608955148607492,
0.010617654770612717,
0.0153469517827034,
-0.0022471777629107237,
0.012003668583929539,
-0.031690362840890884,
-0.03480655327439308,
-0.04548567160964012,
0.0572417713701725,
-0.038150712847709656,
-0.0019421266624704003,
0.012202783487737179,
0.007642634678632021,
0.027861179783940315,
0.024305002763867378,
0.0045427256263792515,
0.05376886576414108,
0.03932100906968117,
0.042523521929979324,
0.03893957659602165,
-0.04955670237541199,
-0.054524924606084824,
0.039282187819480896,
0.02744641713798046,
0.07789070904254913,
0.011605729348957539,
-0.07004345208406448,
-0.03383749723434448,
-0.03189760074019432,
0.034358154982328415
]
},
"code": "1234"
}
]
}
Subsequently, when we use the original document for indexing, it gets indexed correctly.
Can someone please help us understand if we are missing something in the index definition? The null pointer error with the original document doesn't have any details.
Thank you!