The es7.16.2 version inserts 3 million 768-dimensional sentence vectors, and the query results in 2-3 seconds. After the es8.1.1 version is used, the query time reaches 5 or 6 minutes. Is there any special configuration for the sentence vectors used in es8? Otherwise, what is the performance? will drop so much. I think es8 mainly optimizes this, why is the time-consuming far less than the previous version?
Here is my test request：
curl --location --request GET 'https://172.16.13.160:9200/faq_ch_dev/_search' \
--header 'Authorization: Basic ZWxhc3RpYzpvWTBPcGV5MGhzNUNIWWtjNlh2Rw==' \
--header 'Content-Type: application/json' \
--data-raw '{
"query": {
"script_score": {
"query": {
"bool": {
"must": {
"match": {
"question": {
"query": "明天上班有时间吗"
}
}
}
}
},
"script": {
"params": {
"query_vector": [
0.010329485,
-1.0314724,
0.31350482,
-0.71999794,
-0.7979984,
-0.24039888,
-0.22111988,
-0.24046914,
-0.025119293,
1.3060153,
-0.8122329,
1.0510491,
-0.86268514,
-0.075715765,
-0.18535855,
-0.08472172,
0.046924055,
-1.8892694,
-0.81752175,
-0.5597656,
0.004849773,
1.1892755,
-0.30277085,
0.7546594,
0.91824883,
0.38432047,
0.3484836,
-0.21091586,
-0.7094917,
-0.2757973,
0.9315029,
1.1413984,
0.46118253,
0.6521451,
1.0261195,
-0.69416606,
0.2507563,
1.402843,
0.29373214,
0.38195038,
0.9323526,
-0.24857846,
1.4017475,
0.18283814,
-0.5771189,
-0.16543747,
-0.376966,
-0.3270174,
-0.1621125,
2.4035234,
1.1000361,
-0.36589333,
-1.0298465,
-0.4254271,
1.1658672,
-0.2477906,
0.53854126,
-0.7975905,
0.7400645,
-1.4819657,
-0.3836806,
-0.597859,
-0.6167249,
-0.8867479,
0.10518749,
-0.85824394,
1.5182877,
-0.609238,
-1.730683,
0.23464464,
-0.9274167,
0.11148444,
0.94489914,
0.54216737,
-1.3429525,
0.10833995,
0.91599244,
0.72394645,
0.76251763,
0.28294218,
-0.40563786,
-0.29411787,
-1.3712219,
0.014758753,
0.37882993,
0.048286214,
0.17863101,
-0.5000006,
-0.020216681,
-0.3514441,
-0.9883788,
0.8736482,
0.9640754,
0.56725174,
-0.6080551,
-0.027788982,
0.5451085,
-0.6650821,
-0.032294318,
-1.152484,
0.6411357,
0.14839825,
-0.46550238,
0.81347215,
0.3444899,
0.22954789,
-0.1511423,
1.303952,
0.28728977,
-0.707758,
-0.64934206,
-0.9151875,
-0.4447411,
-0.7072869,
-0.25608173,
-0.5738975,
-0.45547575,
0.68242955,
-0.16973183,
1.2418166,
0.35363978,
0.33805412,
1.4853101,
0.84610665,
-0.3822353,
0.3241642,
-0.4176706,
0.9836666,
-0.19126874,
0.41624358,
-0.7381928,
-0.024816047,
-0.02959944,
0.7669304,
1.2546049,
-0.30695534,
0.7850483,
0.5262517,
-0.8369022,
0.87626225,
1.307143,
0.31688192,
-0.083201885,
1.2479573,
-0.83256656,
-1.031245,
0.9839985,
1.717865,
0.18363291,
-0.58953357,
0.30260047,
0.3958338,
-2.1218336,
-0.17503545,
-0.044611424,
0.2852384,
-0.04908388,
-2.0410593,
-1.0556458,
-0.3170435,
-1.0668851,
0.4123376,
-0.7242725,
0.8483758,
0.5035886,
-0.3985103,
1.5327017,
1.6686517,
0.27160448,
-0.025205702,
0.22292107,
-0.1731393,
-0.60968393,
-0.27733248,
-0.2653622,
-0.14685334,
-0.543602,
1.0432088,
-0.2790042,
0.12813734,
1.0996274,
0.2110098,
-1.1199118,
-0.6861326,
-1.5712746,
0.28520152,
-0.30266583,
-0.35432038,
-0.92575085,
0.582495,
0.10838801,
-0.17390299,
-1.1034501,
-1.2074463,
1.0877943,
1.3105795,
0.39330757,
-0.98344624,
0.052993223,
-0.6157517,
-0.5074349,
-1.5064768,
0.36039004,
0.4660583,
0.22219652,
0.23412953,
0.84571075,
-0.8031976,
1.3124471,
-0.45950583,
0.27604178,
-0.8065795,
-0.49213183,
0.16161779,
0.5550939,
-0.342804,
-0.7789892,
-0.9715067,
0.089212686,
0.1220874,
-1.0342392,
-0.4990942,
0.7125993,
-0.7577175,
-0.55543214,
-0.78037214,
0.57821697,
0.5951024,
0.21433035,
-0.26237428,
0.30689886,
-0.082823575,
0.5027089,
-1.1946809,
0.4142968,
0.7526753,
0.51895356,
-0.46459872,
0.22102097,
-0.28103834,
0.06643723,
-0.4565078,
-1.0303307,
-0.6899306,
0.092356876,
-0.24727662,
-0.7563828,
0.37626225,
-0.4472326,
0.44199967,
-0.49688596,
-1.0105863,
-0.34226167,
0.026881656,
0.6999551,
-0.11150711,
-2.1131206,
-0.8627628,
-0.35264802,
-0.07324137,
0.75301665,
-0.17812112,
0.1622595,
0.651941,
0.15390718,
-0.8250144,
-2.0174675,
0.2247178,
1.3202461,
0.21756569,
0.8750901,
0.9544593,
0.67208433,
0.4643662,
0.66332996,
-0.36294228,
-0.39646634,
-0.29128844,
-0.1570884,
-1.1479312,
0.4450886,
0.27323902,
-1.4466976,
-1.9144548,
-0.30192462,
1.6299244,
0.48825377,
-0.31238052,
0.61995673,
0.07692933,
-0.40910733,
0.2625625,
1.0126518,
-1.0620153,
0.29102132,
1.1273003,
0.6571528,
0.062974475,
0.91025144,
0.11154106,
0.9812876,
0.3215018,
1.1008499,
-1.4926254,
-1.15243,
1.4426559,
1.2686318,
0.999547,
1.0159988,
0.14626081,
0.60896987,
0.2805326,
2.1416953,
0.31353772,
0.000636667,
-0.77156055,
0.15893543,
-0.1320257,
-0.55203694,
0.55114794,
-0.6989647,
-0.47289237,
1.1702394,
-0.32676882,
-1.725369,
0.59965175,
0.72737247,
0.45177287,
-1.2118751,
1.4449062,
-0.5460473,
-0.022658817,
0.6686659,
0.08758924,
0.63785434,
-0.4374438,
0.2885223,
1.7778388,
-0.476678,
-0.95256525,
-0.8627278,
-0.5400367,
1.6201599,
1.653852,
-1.2912824,
-0.784659,
0.73575044,
0.18215726,
-0.08303544,
0.32194924,
0.80651736,
-0.3639654,
-0.39226082,
0.69298023,
1.0337205,
-0.3438353,
-1.2105362,
-0.43043652,
0.007367611,
0.5978887,
-0.7727904,
1.3620038,
0.76552236,
-0.7185271,
-1.927837,
-0.37863865,
0.08171803,
0.030110866,
-0.7466214,
-0.3496337,
-1.0597417,
-1.3736948,
0.012633711,
-0.9578191,
-0.64180344,
-0.5638537,
0.5489379,
0.8850684,
-0.89422065,
-0.031924788,
-0.2865706,
-0.1594104,
0.9390342,
1.1800237,
-0.35543123,
-0.82134664,
0.6411096,
-2.019386,
-0.2515307,
-0.29401672,
0.18430434,
-0.04934302,
0.050965473,
1.3223277,
0.01661924,
0.14249752,
-0.4292144,
-1.1903692,
1.5051063,
0.34804338,
-1.0402509,
0.71336627,
-0.82286197,
1.0731728,
1.0550898,
-0.11808388,
0.57263505,
0.26311472,
1.9997151,
-0.3519437,
-0.5267635,
-0.008324724,
0.42252693,
-0.8296619,
-0.24853021,
-0.3213937,
0.20485908,
0.5389338,
-0.22002059,
-1.6279026,
-0.118709445,
-1.5438352,
0.2478208,
-0.57374495,
0.63476866,
-0.5311868,
-1.0212721,
-0.44534114,
-1.7623811,
0.7054437,
-1.2091599,
0.5980951,
-1.0726758,
-0.11484855,
0.43085378,
-0.38339448,
-0.0209084,
-0.75595313,
-1.0984604,
-0.1224476,
-0.37561023,
1.530599,
1.0908906,
-0.49905246,
-0.8890929,
-0.06557762,
-0.28972223,
-0.4158352,
0.07390341,
1.7343174,
1.3893682,
0.5426272,
0.57399553,
0.38814044,
0.5800039,
0.22446024,
-0.15531316,
0.36831734,
0.22948289,
-1.8587363,
0.09448462,
1.5125706,
-0.5820074,
-0.59382606,
0.22109911,
0.35425502,
-1.0354018,
0.8727867,
-1.0695435,
0.97709846,
0.5308511,
0.8471424,
0.061469745,
-0.32849437,
-0.7009784,
0.48455542,
0.20521197,
-1.9330746,
0.14666834,
0.25043344,
-0.08396344,
-1.4202118,
-0.05160434,
-0.5426044,
-1.4614595,
0.83621305,
0.27521437,
-0.25394058,
-0.5883911,
-0.7245514,
-0.032757223,
-0.8640257,
0.14066927,
0.3959335,
0.5284248,
0.6695812,
-0.045278434,
-1.224313,
0.11283187,
-0.49749732,
0.9437068,
-1.6660881,
-0.582008,
0.881871,
0.6269966,
-0.79437435,
-1.0594375,
-0.07695232,
-0.5502082,
-0.50699145,
1.4708852,
1.6352711,
0.19442332,
1.6322093,
-1.747189,
0.29255706,
-0.8084675,
-0.34041893,
0.8864984,
0.50641024,
0.1492911,
1.1626067,
0.043899,
0.81105614,
0.8001353,
0.02907744,
-0.2733998,
-1.2666523,
-0.18303707,
-0.13028657,
-0.81676334,
-0.02309005,
0.7218813,
1.1409216,
0.9762101,
0.9515153,
0.0711052,
-0.13531964,
1.2930957,
0.7725485,
0.45737615,
0.7235968,
0.29437327,
0.62361574,
-1.1948808,
-0.2617061,
0.5395009,
-0.6104019,
-1.2574573,
-0.4661629,
-0.62591165,
-0.69023085,
0.03769734,
-0.11966968,
0.23391648,
1.0397823,
0.3080787,
0.17389917,
0.7423614,
-0.3265019,
-0.35315114,
0.00940273,
1.4403484,
-0.50201064,
-0.45365012,
0.29843518,
-0.072572775,
1.439951,
0.41480297,
0.8242403,
0.8235711,
0.31489378,
0.43930456,
0.20253256,
-0.24281654,
-0.68321145,
-0.8886796,
-0.3365065,
0.03624785,
0.2857151,
-0.35634422,
-0.6925299,
-0.27867,
-0.46527135,
0.8855008,
-0.018824615,
-0.32146904,
0.58566844,
-0.5657964,
0.70942885,
0.45365623,
-1.1542832,
-0.4152354,
-0.86195093,
0.20316243,
0.5341433,
-1.0733242,
1.4336886,
0.05841941,
0.96720177,
-0.8772945,
0.4883608,
0.70287174,
-1.4780267,
0.44414967,
0.008786201,
-1.1714286,
0.5347028,
0.5575583,
0.7068649,
0.8486873,
-0.36921537,
0.5202114,
0.6492906,
1.1640294,
-0.037097342,
-0.6336245,
-0.09900784,
-1.1895727,
0.43330002,
-1.7285976,
0.53675926,
1.2102298,
0.6622673,
-0.7664504,
-1.337621,
0.1592482,
-0.6904898,
0.22743128,
-0.4405301,
-0.56703585,
1.0286334,
0.2940408,
-1.5802904,
-0.7741747,
0.2961075,
1.447117,
-0.06948343,
2.1361446,
-0.17249364,
-0.46094155,
0.3633664,
0.59741133,
-0.44843313,
0.6986479,
-0.004787704,
-1.6842028,
1.0712221,
0.84774005,
0.7857606,
0.12381548,
-0.790225,
0.47535488,
0.4828004,
-1.4895383,
0.9247639,
0.36851728,
0.8688196,
-0.56295055,
0.009660453,
-0.0114329085,
-1.3264066,
0.5280587,
0.6698756,
-0.96509,
0.87214154,
-0.07235037,
0.25847083,
-2.466216,
-0.45204133,
-0.6391735,
-0.36796126,
-0.80855143,
0.34528363,
-0.9821234,
-0.71735114,
0.59643376,
0.09146099,
-1.2508098,
-0.63636357,
-0.29722223,
1.4821436,
-0.48694652,
-1.2795027,
-0.0015369663,
-0.0341951,
0.33503756,
0.15662207,
-0.20246565,
0.10003914,
1.5094347,
0.20367572,
-0.8740792,
0.91328675,
0.85201746,
-0.6895189,
-0.46802336,
-0.3328716,
0.3298049,
-0.39262238,
-0.8348563,
-0.38266587,
-1.0120201,
-0.21353945,
0.5047768,
-0.9873547,
-0.30378252,
-0.5180256,
0.2996903,
1.2072086,
0.4798299,
-0.37315854,
0.0060760155,
1.3278753,
-0.5014809,
-0.74786586,
1.530333,
0.48171622,
0.7588974,
0.6396381,
-0.47385064,
-0.14228714,
1.1538125,
0.059573412,
-0.8887416,
0.21537594,
-0.04542114,
1.3107146,
-0.42538345,
-0.023974448,
0.8807867,
-1.5797807,
0.34728146,
-1.1940956,
0.27013978,
1.7251315,
0.31565756,
0.59606075,
0.69962275,
0.30981082,
0.27450034,
0.6783993,
0.16227822,
0.5487558,
-0.70968133,
0.30777934,
0.37068415,
0.6988609,
-0.8681915,
-0.050575852,
0.23372984,
-0.27745628,
0.95542914,
0.22391495,
-2.0054002,
-1.9100155,
0.17662272,
-0.6328653,
-0.96898735,
-0.24372292,
-0.015774779,
0.7684885,
-0.9253229,
0.55917215,
0.9857328,
0.44582298,
0.43133885,
0.6236032
]
},
"source": "if ('\''question_vector'\'' != null) {cosineSimilarity(params.query_vector, '\''question_vector'\'')+1.0}"
}
}
}
}'
The rules defined are as follows：
"faq_ch_base": {
"order": 0,
"index_patterns": [
"faq_ch_*"
],
"settings": {
"index": {
"number_of_shards": "2",
"number_of_replicas": "2"
}
},
"mappings": {
"properties": {
"question_vector": {
"dims": 768,
"similarity": "cosine",
"index": true,
"type": "dense_vector"
},
"question": {
"type": "text"
},
"cat_id": {
"type": "long"
},
"qgroupId": {
"type": "long"
}
}
},
"aliases": {}
},
Time taken to test the query: 14m46s
