Can someone explain to me the mystery of the Query Norm?
I have two Date Histogram Queries built using a DisMaxQuery over a
Constant Score Filter that I think should produce IDENTICAL results.
In one case, the DisMaxQuery uses one Filter. In the second case, the
DisMaxQuery has a second Filter that produces NO results. So why do
the scores differ? Because Lucene/ES applies a "Query Norm" factor to
the scores.
Here are the queries and their results! Is there a way to set the
Query Norm to be 1.0? Thanks!
$ curl -XGET 'http://localhost:9200/minimal-stanford-2009-06/STANFORD/
_search?pretty=true' -d '
{
"query" : {
"dis_max" : {
"queries" : [ {
"constant_score" : {
"filter" : {
"term" : {
"text" : "calm"
}
},
"boost" : -0.5
}
} ]
}
},
"explain" : true,
"fields" : "text",
"facets" : {
"histo1" : {
"date_histogram" : {
"field" : "created_at",
"value_script" : "doc.score",
"interval" : "day"
}
}
}
}'
{
"took" : 5,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"failed" : 0
},
"hits" : {
"total" : 1605,
"max_score" : -1.0,
"hits" : [ {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "ac8c3b6be5930c7efe56853f7684d6e1",
"_score" : -1.0,
"_explanation" : {
"value" : -1.0,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.5, product
of:",
"details" : [ {
"value" : -0.5,
"description" : "boost"
}, {
"value" : 2.0,
"description" : "queryNorm"
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "7d789e55c170290c166818f2bf51bbaa",
"_score" : -1.0,
"_explanation" : {
"value" : -1.0,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.5, product
of:",
"details" : [ {
"value" : -0.5,
"description" : "boost"
}, {
"value" : 2.0,
"description" : "queryNorm"
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "87fb1713f50729f05bb15303135551cb",
"_score" : -1.0,
"_explanation" : {
"value" : -1.0,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.5, product
of:",
"details" : [ {
"value" : -0.5,
"description" : "boost"
}, {
"value" : 2.0,
"description" : "queryNorm"
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "bdc4e5446c4dfe2ceb90529161ae4614",
"_score" : -1.0,
"_explanation" : {
"value" : -1.0,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.5, product
of:",
"details" : [ {
"value" : -0.5,
"description" : "boost"
}, {
"value" : 2.0,
"description" : "queryNorm"
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "cb4028c29b04cec905d7ad4b8e55ea76",
"_score" : -1.0,
"_explanation" : {
"value" : -1.0,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.5, product
of:",
"details" : [ {
"value" : -0.5,
"description" : "boost"
}, {
"value" : 2.0,
"description" : "queryNorm"
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "fd0880d691741065a9b3a8d84561bc00",
"_score" : -1.0,
"_explanation" : {
"value" : -1.0,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.5, product
of:",
"details" : [ {
"value" : -0.5,
"description" : "boost"
}, {
"value" : 2.0,
"description" : "queryNorm"
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "fd11471567565d384d91ef4a97654741",
"_score" : -1.0,
"_explanation" : {
"value" : -1.0,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.5, product
of:",
"details" : [ {
"value" : -0.5,
"description" : "boost"
}, {
"value" : 2.0,
"description" : "queryNorm"
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "1907a018ebf3ea4a6db61658e3b2f1c0",
"_score" : -1.0,
"_explanation" : {
"value" : -1.0,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.5, product
of:",
"details" : [ {
"value" : -0.5,
"description" : "boost"
}, {
"value" : 2.0,
"description" : "queryNorm"
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "03993c95db70c222322b653bf541c354",
"_score" : -1.0,
"_explanation" : {
"value" : -1.0,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.5, product
of:",
"details" : [ {
"value" : -0.5,
"description" : "boost"
}, {
"value" : 2.0,
"description" : "queryNorm"
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "ed3046c47a248424c8adddb4ec97f6bb",
"_score" : -1.0,
"_explanation" : {
"value" : -1.0,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.5, product
of:",
"details" : [ {
"value" : -0.5,
"description" : "boost"
}, {
"value" : 2.0,
"description" : "queryNorm"
} ]
}
} ]
},
"facets" : {
"histo1" : {
"_type" : "date_histogram",
"entries" : [ {
"time" : 1244764800000,
"count" : 54,
"min" : -1.0,
"max" : -1.0,
"total" : -54.0,
"total_count" : 54,
"mean" : -1.0
}, {
"time" : 1244851200000,
"count" : 67,
"min" : -1.0,
"max" : -1.0,
"total" : -67.0,
"total_count" : 67,
"mean" : -1.0
}, {
"time" : 1244937600000,
"count" : 140,
"min" : -1.0,
"max" : -1.0,
"total" : -140.0,
"total_count" : 140,
"mean" : -1.0
}, {
"time" : 1245024000000,
"count" : 150,
"min" : -1.0,
"max" : -1.0,
"total" : -150.0,
"total_count" : 150,
"mean" : -1.0
}, {
"time" : 1245110400000,
"count" : 107,
"min" : -1.0,
"max" : -1.0,
"total" : -107.0,
"total_count" : 107,
"mean" : -1.0
}, {
"time" : 1245196800000,
"count" : 62,
"min" : -1.0,
"max" : -1.0,
"total" : -62.0,
"total_count" : 62,
"mean" : -1.0
}, {
"time" : 1245283200000,
"count" : 67,
"min" : -1.0,
"max" : -1.0,
"total" : -67.0,
"total_count" : 67,
"mean" : -1.0
}, {
"time" : 1245369600000,
"count" : 51,
"min" : -1.0,
"max" : -1.0,
"total" : -51.0,
"total_count" : 51,
"mean" : -1.0
}, {
"time" : 1245456000000,
"count" : 87,
"min" : -1.0,
"max" : -1.0,
"total" : -87.0,
"total_count" : 87,
"mean" : -1.0
}, {
"time" : 1245542400000,
"count" : 70,
"min" : -1.0,
"max" : -1.0,
"total" : -70.0,
"total_count" : 70,
"mean" : -1.0
}, {
"time" : 1245628800000,
"count" : 121,
"min" : -1.0,
"max" : -1.0,
"total" : -121.0,
"total_count" : 121,
"mean" : -1.0
}, {
"time" : 1245715200000,
"count" : 60,
"min" : -1.0,
"max" : -1.0,
"total" : -60.0,
"total_count" : 60,
"mean" : -1.0
}, {
"time" : 1245801600000,
"count" : 29,
"min" : -1.0,
"max" : -1.0,
"total" : -29.0,
"total_count" : 29,
"mean" : -1.0
}, {
"time" : 1245888000000,
"count" : 92,
"min" : -1.0,
"max" : -1.0,
"total" : -92.0,
"total_count" : 92,
"mean" : -1.0
}, {
"time" : 1245974400000,
"count" : 129,
"min" : -1.0,
"max" : -1.0,
"total" : -129.0,
"total_count" : 129,
"mean" : -1.0
}, {
"time" : 1246060800000,
"count" : 70,
"min" : -1.0,
"max" : -1.0,
"total" : -70.0,
"total_count" : 70,
"mean" : -1.0
}, {
"time" : 1246147200000,
"count" : 70,
"min" : -1.0,
"max" : -1.0,
"total" : -70.0,
"total_count" : 70,
"mean" : -1.0
}, {
"time" : 1246233600000,
"count" : 53,
"min" : -1.0,
"max" : -1.0,
"total" : -53.0,
"total_count" : 53,
"mean" : -1.0
}, {
"time" : 1246320000000,
"count" : 99,
"min" : -1.0,
"max" : -1.0,
"total" : -99.0,
"total_count" : 99,
"mean" : -1.0
}, {
"time" : 1246406400000,
"count" : 27,
"min" : -1.0,
"max" : -1.0,
"total" : -27.0,
"total_count" : 27,
"mean" : -1.0
} ]
}
}
}
$ curl -XGET 'http://localhost:9200/minimal-stanford-2009-06/STANFORD/
_search?pretty=true' -d '
{
"query" : {
"dis_max" : {
"queries" : [ {
"constant_score" : {
"filter" : {
"term" : {
"text" : "calm"
}
},
"boost" : -0.35
}
}, {
"constant_score" : {
"filter" : {
"term" : {
"text" : "iquweropiuq"
}
},
"boost" : 1.0
}
} ]
}
},
"explain" : true,
"fields" : "text",
"facets" : {
"histo1" : {
"date_histogram" : {
"field" : "created_at",
"value_script" : "doc.score",
"interval" : "day"
}
}
}
}'
{
"took" : 6,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"failed" : 0
},
"hits" : {
"total" : 1605,
"max_score" : -0.35,
"hits" : [ {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "ac8c3b6be5930c7efe56853f7684d6e1",
"_score" : -0.35,
"_explanation" : {
"value" : 0.0,
"description" : "max of:",
"details" : [ {
"value" : -0.35,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.35, product
of:",
"details" : [ {
"value" : -0.35,
"description" : "boost"
}, {
"value" : 1.0,
"description" : "queryNorm"
} ]
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "7d789e55c170290c166818f2bf51bbaa",
"_score" : -0.35,
"_explanation" : {
"value" : 0.0,
"description" : "max of:",
"details" : [ {
"value" : -0.35,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.35, product
of:",
"details" : [ {
"value" : -0.35,
"description" : "boost"
}, {
"value" : 1.0,
"description" : "queryNorm"
} ]
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "87fb1713f50729f05bb15303135551cb",
"_score" : -0.35,
"_explanation" : {
"value" : 0.0,
"description" : "max of:",
"details" : [ {
"value" : -0.35,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.35, product
of:",
"details" : [ {
"value" : -0.35,
"description" : "boost"
}, {
"value" : 1.0,
"description" : "queryNorm"
} ]
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "bdc4e5446c4dfe2ceb90529161ae4614",
"_score" : -0.35,
"_explanation" : {
"value" : 0.0,
"description" : "max of:",
"details" : [ {
"value" : -0.35,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.35, product
of:",
"details" : [ {
"value" : -0.35,
"description" : "boost"
}, {
"value" : 1.0,
"description" : "queryNorm"
} ]
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "cb4028c29b04cec905d7ad4b8e55ea76",
"_score" : -0.35,
"_explanation" : {
"value" : 0.0,
"description" : "max of:",
"details" : [ {
"value" : -0.35,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.35, product
of:",
"details" : [ {
"value" : -0.35,
"description" : "boost"
}, {
"value" : 1.0,
"description" : "queryNorm"
} ]
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "fd0880d691741065a9b3a8d84561bc00",
"_score" : -0.35,
"_explanation" : {
"value" : 0.0,
"description" : "max of:",
"details" : [ {
"value" : -0.35,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.35, product
of:",
"details" : [ {
"value" : -0.35,
"description" : "boost"
}, {
"value" : 1.0,
"description" : "queryNorm"
} ]
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "fd11471567565d384d91ef4a97654741",
"_score" : -0.35,
"_explanation" : {
"value" : 0.0,
"description" : "max of:",
"details" : [ {
"value" : -0.35,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.35, product
of:",
"details" : [ {
"value" : -0.35,
"description" : "boost"
}, {
"value" : 1.0,
"description" : "queryNorm"
} ]
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "1907a018ebf3ea4a6db61658e3b2f1c0",
"_score" : -0.35,
"_explanation" : {
"value" : 0.0,
"description" : "max of:",
"details" : [ {
"value" : -0.35,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.35, product
of:",
"details" : [ {
"value" : -0.35,
"description" : "boost"
}, {
"value" : 1.0,
"description" : "queryNorm"
} ]
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "03993c95db70c222322b653bf541c354",
"_score" : -0.35,
"_explanation" : {
"value" : 0.0,
"description" : "max of:",
"details" : [ {
"value" : -0.35,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.35, product
of:",
"details" : [ {
"value" : -0.35,
"description" : "boost"
}, {
"value" : 1.0,
"description" : "queryNorm"
} ]
} ]
}
}, {
"_shard" : 0,
"_node" : "OgmXoqzxQV2s1DXRwVg6Gg",
"_index" : "minimal-stanford-2009-06",
"_type" : "STANFORD",
"_id" : "ed3046c47a248424c8adddb4ec97f6bb",
"_score" : -0.35,
"_explanation" : {
"value" : 0.0,
"description" : "max of:",
"details" : [ {
"value" : -0.35,
"description" :
"ConstantScore(FilterCacheFilterWrapper(text:calm))^-0.35, product
of:",
"details" : [ {
"value" : -0.35,
"description" : "boost"
}, {
"value" : 1.0,
"description" : "queryNorm"
} ]
} ]
}
} ]
},
"facets" : {
"histo1" : {
"_type" : "date_histogram",
"entries" : [ {
"time" : 1244764800000,
"count" : 54,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -18.899999678134918,
"total_count" : 54,
"mean" : -0.3499999940395355
}, {
"time" : 1244851200000,
"count" : 67,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -23.44999960064888,
"total_count" : 67,
"mean" : -0.3499999940395355
}, {
"time" : 1244937600000,
"count" : 140,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -48.99999916553497,
"total_count" : 140,
"mean" : -0.3499999940395355
}, {
"time" : 1245024000000,
"count" : 150,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -52.49999910593033,
"total_count" : 150,
"mean" : -0.3499999940395355
}, {
"time" : 1245110400000,
"count" : 107,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -37.4499993622303,
"total_count" : 107,
"mean" : -0.3499999940395355
}, {
"time" : 1245196800000,
"count" : 62,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -21.699999630451202,
"total_count" : 62,
"mean" : -0.3499999940395355
}, {
"time" : 1245283200000,
"count" : 67,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -23.44999960064888,
"total_count" : 67,
"mean" : -0.3499999940395355
}, {
"time" : 1245369600000,
"count" : 51,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -17.84999969601631,
"total_count" : 51,
"mean" : -0.3499999940395355
}, {
"time" : 1245456000000,
"count" : 87,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -30.44999948143959,
"total_count" : 87,
"mean" : -0.3499999940395355
}, {
"time" : 1245542400000,
"count" : 70,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -24.499999582767487,
"total_count" : 70,
"mean" : -0.3499999940395355
}, {
"time" : 1245628800000,
"count" : 121,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -42.3499992787838,
"total_count" : 121,
"mean" : -0.3499999940395355
}, {
"time" : 1245715200000,
"count" : 60,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -20.99999964237213,
"total_count" : 60,
"mean" : -0.3499999940395355
}, {
"time" : 1245801600000,
"count" : 29,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -10.14999982714653,
"total_count" : 29,
"mean" : -0.3499999940395355
}, {
"time" : 1245888000000,
"count" : 92,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -32.19999945163727,
"total_count" : 92,
"mean" : -0.3499999940395355
}, {
"time" : 1245974400000,
"count" : 129,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -45.14999923110008,
"total_count" : 129,
"mean" : -0.3499999940395355
}, {
"time" : 1246060800000,
"count" : 70,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -24.499999582767487,
"total_count" : 70,
"mean" : -0.3499999940395355
}, {
"time" : 1246147200000,
"count" : 70,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -24.499999582767487,
"total_count" : 70,
"mean" : -0.3499999940395355
}, {
"time" : 1246233600000,
"count" : 53,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -18.549999684095383,
"total_count" : 53,
"mean" : -0.3499999940395355
}, {
"time" : 1246320000000,
"count" : 99,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -34.64999940991402,
"total_count" : 99,
"mean" : -0.3499999940395355
}, {
"time" : 1246406400000,
"count" : 27,
"min" : -0.3499999940395355,
"max" : -0.3499999940395355,
"total" : -9.449999839067459,
"total_count" : 27,
"mean" : -0.3499999940395355
} ]
}
}
}