Script_fields issue?

[Lukas_Vlcek1 (Lukáš Vlček)]

Hi,

I am experiencing script_fields issue. Below is a shell script with curl
examples. In short: I prepare an index, index three objects into it, verify
that I can query objects from index and that I can extract values of
particular "fields" (though script_fields does not require fields being
stored) and then use a script_fields query which fails. Not sure what is
wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XDELETE 'http://localhost:9200/foo/'
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards :
1, number_of_replicas : 0 }}'
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" }
}
}}'
curl -XGET 'http://localhost:9200/foo/bar/_mapping?pretty=1'
echo

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d
'{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d
'{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d
'{"date":"2010-01-03","string":"three"}'
curl -XPOST 'http://localhost:9200/foo/_refresh'
echo

echo Search
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
}
}'
echo

The following is output:

Index setup
{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
"foo" : {
"bar" : {
"_boost" : {
"name" : "_boost"
},
"dynamic" : true,
"enabled" : true,
"date_formats" : [ "dateOptionalTime", "yyyy/MM/dd
HH:mm:ss||yyyy/MM/dd" ],
"_source" : {
"enabled" : true,
"name" : "_source"
},
"_id" : {
"store" : "no"
},
"path" : "full",
"properties" : {
"string" : {
"omit_term_freq_and_positions" : false,
"index_name" : "string",
"index" : "not_analyzed",
"omit_norms" : false,
"store" : "yes",
"boost" : 1.0,
"term_vector" : "no",
"type" : "string"
},
"date" : {
"omit_term_freq_and_positions" : true,
"index_name" : "date",
"index" : "not_analyzed",
"omit_norms" : true,
"store" : "yes",
"boost" : 1.0,
"format" : "dateOptionalTime",
"precision_step" : 4,
"term_vector" : "no",
"type" : "date"
}
},
"_all" : {
"enabled" : true,
"store" : "no",
"term_vector" : "no"
},
"type" : "object"
}
}
}
Indexing
{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
"_shards" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [ {
"_index" : "foo",
"_type" : "bar",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"date" : "2010-01-01T00:00:00.000Z",
"string" : "one"
}
} ]
}
}
Search with script_fields

{
"_shards" : {
"total" : 1,
"successful" : 0,
"failed" : 1,
"failures" : [ {
"reason" : "CompileException[[Error: null]\n[Near : {... Unknown
....}]\n ^\n[Line: 1, Column: 0]]; nested: "
} ]
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [ ]
}
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata ] [Starshine] [foo] deleting
index
[11:45:12,818][INFO ][cluster.metadata ] [Starshine] [foo] creating
index, cause [api], shards [1]/[0], mappings []
[11:45:12,994][INFO ][cluster.metadata ] [Starshine] [foo]
create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type ] [Starshine] [13] Failed to
execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
^
[Line: 1, Column: 0]
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at
org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at
org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more

Should I open an issue?

Regards,
Lukas

[kimchy (Shay Banon)]

Hi,

I think the problem is with curl, and the fact that within the body of the
request the ' is used (within the script). It needs to be escaped somehow
(not sure how).

-shay.banon

On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček lukas.vlcek@gmail.com wrote:

Hi,

I am experiencing script_fields issue. Below is a shell script with curl
examples. In short: I prepare an index, index three objects into it, verify
that I can query objects from index and that I can extract values of
particular "fields" (though script_fields does not require fields being
stored) and then use a script_fields query which fails. Not sure what is
wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XDELETE 'http://localhost:9200/foo/'
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards :
1, number_of_replicas : 0 }}'
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" }
}
}}'
curl -XGET 'http://localhost:9200/foo/bar/_mapping?pretty=1'
echo

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d
'{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d
'{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d
'{"date":"2010-01-03","string":"three"}'
curl -XPOST 'http://localhost:9200/foo/_refresh'
echo

echo Search
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
}
}'
echo

The following is output:

Index setup

{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
"foo" : {
"bar" : {
"_boost" : {
"name" : "_boost"
},
"dynamic" : true,
"enabled" : true,
"date_formats" : [ "dateOptionalTime", "yyyy/MM/dd
HH:mm:ss||yyyy/MM/dd" ],
"_source" : {
"enabled" : true,
"name" : "_source"
},
"_id" : {
"store" : "no"
},
"path" : "full",
"properties" : {
"string" : {
"omit_term_freq_and_positions" : false,
"index_name" : "string",
"index" : "not_analyzed",
"omit_norms" : false,
"store" : "yes",
"boost" : 1.0,
"term_vector" : "no",
"type" : "string"
},
"date" : {
"omit_term_freq_and_positions" : true,
"index_name" : "date",
"index" : "not_analyzed",
"omit_norms" : true,
"store" : "yes",
"boost" : 1.0,
"format" : "dateOptionalTime",
"precision_step" : 4,
"term_vector" : "no",
"type" : "date"
}
},
"_all" : {
"enabled" : true,
"store" : "no",
"term_vector" : "no"
},
"type" : "object"
}
}
}
Indexing

{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
"_shards" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [ {
"_index" : "foo",
"_type" : "bar",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"date" : "2010-01-01T00:00:00.000Z",
"string" : "one"
}
} ]
}
}
Search with script_fields

{
"_shards" : {
"total" : 1,
"successful" : 0,
"failed" : 1,
"failures" : [ {
"reason" : "CompileException[[Error: null]\n[Near : {... Unknown
....}]\n ^\n[Line: 1, Column: 0]]; nested: "
} ]
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" :
}
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata ] [Starshine] [foo] deleting
index
[11:45:12,818][INFO ][cluster.metadata ] [Starshine] [foo] creating
index, cause [api], shards [1]/[0], mappings
[11:45:12,994][INFO ][cluster.metadata ] [Starshine] [foo]
create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type ] [Starshine] [13] Failed to
execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
^
[Line: 1, Column: 0]
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at
org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at
org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more

Should I open an issue?

Regards,
Lukas

[Lukas_Vlcek1 (Lukáš Vlček)]

Oh yeah, that's it.
BUT, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d
@script_fields.json

I put the following query into script_fields.json file:
{
"from" : 0, "size" : 3,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
},
"sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them (note
the "sort" part) it fails:
{
"error" : "SearchPhaseExecutionException[Failed to execute phase [query],
total failure; shardFailures
{[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure
[Failed to parse [{ "from" : 0, "size" : 3, "fields" :
["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" :
"doc['date'].date.millis" }, "test2" : { "script" :
"doc['string'].stringValue" } }, "sort" : ["test1"]}]]]; nested:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure [No
mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type ] [Starshine] [foo][0],
node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to
execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [Failed to parse [{ "from" : 0,
"size" : 3, "fields" : ["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" } }, "sort" :
["test1"]}]]
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at
org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at
org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at
org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted field?

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon shay.banon@elasticsearch.comwrote:

Hi,

I think the problem is with curl, and the fact that within the body of
the request the ' is used (within the script). It needs to be escaped
somehow (not sure how).

-shay.banon

On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Hi,

I am experiencing script_fields issue. Below is a shell script with curl
examples. In short: I prepare an index, index three objects into it, verify
that I can query objects from index and that I can extract values of
particular "fields" (though script_fields does not require fields being
stored) and then use a script_fields query which fails. Not sure what is
wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XDELETE 'http://localhost:9200/foo/'
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards
: 1, number_of_replicas : 0 }}'
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" }
}
}}'
curl -XGET 'http://localhost:9200/foo/bar/_mapping?pretty=1'
echo

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d
'{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d
'{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d
'{"date":"2010-01-03","string":"three"}'
curl -XPOST 'http://localhost:9200/foo/_refresh'
echo

echo Search
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
}
}'
echo

The following is output:

Index setup

{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
"foo" : {
"bar" : {
"_boost" : {
"name" : "_boost"
},
"dynamic" : true,
"enabled" : true,
"date_formats" : [ "dateOptionalTime", "yyyy/MM/dd
HH:mm:ss||yyyy/MM/dd" ],
"_source" : {
"enabled" : true,
"name" : "_source"
},
"_id" : {
"store" : "no"
},
"path" : "full",
"properties" : {
"string" : {
"omit_term_freq_and_positions" : false,
"index_name" : "string",
"index" : "not_analyzed",
"omit_norms" : false,
"store" : "yes",
"boost" : 1.0,
"term_vector" : "no",
"type" : "string"
},
"date" : {
"omit_term_freq_and_positions" : true,
"index_name" : "date",
"index" : "not_analyzed",
"omit_norms" : true,
"store" : "yes",
"boost" : 1.0,
"format" : "dateOptionalTime",
"precision_step" : 4,
"term_vector" : "no",
"type" : "date"
}
},
"_all" : {
"enabled" : true,
"store" : "no",
"term_vector" : "no"
},
"type" : "object"
}
}
}
Indexing

{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
"_shards" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [ {
"_index" : "foo",
"_type" : "bar",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"date" : "2010-01-01T00:00:00.000Z",
"string" : "one"
}
} ]
}
}
Search with script_fields

{
"_shards" : {
"total" : 1,
"successful" : 0,
"failed" : 1,
"failures" : [ {
"reason" : "CompileException[[Error: null]\n[Near : {... Unknown
....}]\n ^\n[Line: 1, Column: 0]]; nested: "
} ]
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" :
}
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata ] [Starshine] [foo]
deleting index
[11:45:12,818][INFO ][cluster.metadata ] [Starshine] [foo]
creating index, cause [api], shards [1]/[0], mappings
[11:45:12,994][INFO ][cluster.metadata ] [Starshine] [foo]
create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type ] [Starshine] [13] Failed
to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
^
[Line: 1, Column: 0]
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at
org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at
org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at
org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more

Should I open an issue?

Regards,
Lukas

[Lukas_Vlcek1 (Lukáš Vlček)]

So, I found that the
SortFieldhttp://lucene.apache.org/java/3_0_2/api/all/org/apache/lucene/search/SortField.htmlneeds
field type here:
http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118That
is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" },
"test1" : { "type" : "long" },
"test2" : { "type" : "string" }
}
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note: I
was not able to make it reverse:true or desc) but "sort" : ["test2"] still
fails with simple message:
{
"error" : "ReduceSearchPhaseException[Failed to execute phase [query],
[reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me.
Second issue is that I do not want to provide explicit mapping in advance
(fixing it to specific type) for script_fields. I think this is little
against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in order
to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of
script_fields definiton per query. Something like:
"script_fields" : {
"test1" : {
"script" : "some_fancy_math",
"params" : { ... },
"return_type" : "long"
}
}

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček lukas.vlcek@gmail.com wrote:

Oh yeah, that's it.
BUT, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d
@script_fields.json

I put the following query into script_fields.json file:
{
"from" : 0, "size" : 3,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
},
"sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them (note
the "sort" part) it fails:
{
"error" : "SearchPhaseExecutionException[Failed to execute phase [query],
total failure; shardFailures
{[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure
[Failed to parse [{ "from" : 0, "size" : 3, "fields" :
["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" :
"doc['date'].date.millis" }, "test2" : { "script" :
"doc['string'].stringValue" } }, "sort" : ["test1"]}]]]; nested:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure [No
mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type ] [Starshine] [foo][0],
node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to
execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [Failed to parse [{ "from" : 0,
"size" : 3, "fields" : ["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" } }, "sort" :
["test1"]}]]
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at
org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at
org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at
org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted field?

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon shay.banon@elasticsearch.comwrote:

Hi,

I think the problem is with curl, and the fact that within the body of
the request the ' is used (within the script). It needs to be escaped
somehow (not sure how).

-shay.banon

On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Hi,

I am experiencing script_fields issue. Below is a shell script with curl
examples. In short: I prepare an index, index three objects into it, verify
that I can query objects from index and that I can extract values of
particular "fields" (though script_fields does not require fields being
stored) and then use a script_fields query which fails. Not sure what is
wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XDELETE 'http://localhost:9200/foo/'
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : { number_of_shards
: 1, number_of_replicas : 0 }}'
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" }
}
}}'
curl -XGET 'http://localhost:9200/foo/bar/_mapping?pretty=1'
echo

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d
'{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d
'{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d
'{"date":"2010-01-03","string":"three"}'
curl -XPOST 'http://localhost:9200/foo/_refresh'
echo

echo Search
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
}
}'
echo

The following is output:

Index setup

{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
"foo" : {
"bar" : {
"_boost" : {
"name" : "_boost"
},
"dynamic" : true,
"enabled" : true,
"date_formats" : [ "dateOptionalTime", "yyyy/MM/dd
HH:mm:ss||yyyy/MM/dd" ],
"_source" : {
"enabled" : true,
"name" : "_source"
},
"_id" : {
"store" : "no"
},
"path" : "full",
"properties" : {
"string" : {
"omit_term_freq_and_positions" : false,
"index_name" : "string",
"index" : "not_analyzed",
"omit_norms" : false,
"store" : "yes",
"boost" : 1.0,
"term_vector" : "no",
"type" : "string"
},
"date" : {
"omit_term_freq_and_positions" : true,
"index_name" : "date",
"index" : "not_analyzed",
"omit_norms" : true,
"store" : "yes",
"boost" : 1.0,
"format" : "dateOptionalTime",
"precision_step" : 4,
"term_vector" : "no",
"type" : "date"
}
},
"_all" : {
"enabled" : true,
"store" : "no",
"term_vector" : "no"
},
"type" : "object"
}
}
}
Indexing

{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
"_shards" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [ {
"_index" : "foo",
"_type" : "bar",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"date" : "2010-01-01T00:00:00.000Z",
"string" : "one"
}
} ]
}
}
Search with script_fields

{
"_shards" : {
"total" : 1,
"successful" : 0,
"failed" : 1,
"failures" : [ {
"reason" : "CompileException[[Error: null]\n[Near : {... Unknown
....}]\n ^\n[Line: 1, Column: 0]]; nested: "
} ]
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" :
}
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata ] [Starshine] [foo]
deleting index
[11:45:12,818][INFO ][cluster.metadata ] [Starshine] [foo]
creating index, cause [api], shards [1]/[0], mappings
[11:45:12,994][INFO ][cluster.metadata ] [Starshine] [foo]
create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type ] [Starshine] [13] Failed
to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
^
[Line: 1, Column: 0]
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at
org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at
org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at
org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at
org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at
org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more

Should I open an issue?

Regards,
Lukas

[Lukas_Vlcek1 (Lukáš Vlček)]

Or may be the "return_type" should make more sense to be added in "sort"
portion of the query rather then in "script_fields" portion.

Lukas

On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček lukas.vlcek@gmail.com wrote:

So, I found that the SortFieldhttp://lucene.apache.org/java/3_0_2/api/all/org/apache/lucene/search/SortField.htmlneeds field type here:
http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118That
is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" },
"test1" : { "type" : "long" },
"test2" : { "type" : "string" }
}
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note: I
was not able to make it reverse:true or desc) but "sort" : ["test2"] still
fails with simple message:
{
"error" : "ReduceSearchPhaseException[Failed to execute phase [query],
[reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me.
Second issue is that I do not want to provide explicit mapping in advance
(fixing it to specific type) for script_fields. I think this is little
against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in order
to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of
script_fields definiton per query. Something like:
"script_fields" : {
"test1" : {
"script" : "some_fancy_math",
"params" : { ... },
"return_type" : "long"
}
}

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Oh yeah, that's it.
BUT, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d
@script_fields.json

I put the following query into script_fields.json file:
{
"from" : 0, "size" : 3,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
},
"sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them (note
the "sort" part) it fails:
{
"error" : "SearchPhaseExecutionException[Failed to execute phase
[query], total failure; shardFailures
{[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure
[Failed to parse [{ "from" : 0, "size" : 3, "fields" :
["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" :
"doc['date'].date.millis" }, "test2" : { "script" :
"doc['string'].stringValue" } }, "sort" : ["test1"]}]]]; nested:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure [No
mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type ] [Starshine] [foo][0],
node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to
execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [Failed to parse [{ "from" : 0,
"size" : 3, "fields" : ["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" } }, "sort" :
["test1"]}]]
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at
org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at
org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at
org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted
field?

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <shay.banon@elasticsearch.com

wrote:

Hi,

I think the problem is with curl, and the fact that within the body of
the request the ' is used (within the script). It needs to be escaped
somehow (not sure how).

-shay.banon

On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Hi,

I am experiencing script_fields issue. Below is a shell script with curl
examples. In short: I prepare an index, index three objects into it, verify
that I can query objects from index and that I can extract values of
particular "fields" (though script_fields does not require fields being
stored) and then use a script_fields query which fails. Not sure what is
wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XDELETE 'http://localhost:9200/foo/'
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : {
number_of_shards : 1, number_of_replicas : 0 }}'
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" }
}
}}'
curl -XGET 'http://localhost:9200/foo/bar/_mapping?pretty=1'
echo

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d
'{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d
'{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d
'{"date":"2010-01-03","string":"three"}'
curl -XPOST 'http://localhost:9200/foo/_refresh'
echo

echo Search
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
}
}'
echo

The following is output:

Index setup

{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
"foo" : {
"bar" : {
"_boost" : {
"name" : "_boost"
},
"dynamic" : true,
"enabled" : true,
"date_formats" : [ "dateOptionalTime", "yyyy/MM/dd
HH:mm:ss||yyyy/MM/dd" ],
"_source" : {
"enabled" : true,
"name" : "_source"
},
"_id" : {
"store" : "no"
},
"path" : "full",
"properties" : {
"string" : {
"omit_term_freq_and_positions" : false,
"index_name" : "string",
"index" : "not_analyzed",
"omit_norms" : false,
"store" : "yes",
"boost" : 1.0,
"term_vector" : "no",
"type" : "string"
},
"date" : {
"omit_term_freq_and_positions" : true,
"index_name" : "date",
"index" : "not_analyzed",
"omit_norms" : true,
"store" : "yes",
"boost" : 1.0,
"format" : "dateOptionalTime",
"precision_step" : 4,
"term_vector" : "no",
"type" : "date"
}
},
"_all" : {
"enabled" : true,
"store" : "no",
"term_vector" : "no"
},
"type" : "object"
}
}
}
Indexing

{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
"_shards" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [ {
"_index" : "foo",
"_type" : "bar",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"date" : "2010-01-01T00:00:00.000Z",
"string" : "one"
}
} ]
}
}
Search with script_fields

{
"_shards" : {
"total" : 1,
"successful" : 0,
"failed" : 1,
"failures" : [ {
"reason" : "CompileException[[Error: null]\n[Near : {... Unknown
....}]\n ^\n[Line: 1, Column: 0]]; nested: "
} ]
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" :
}
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata ] [Starshine] [foo]
deleting index
[11:45:12,818][INFO ][cluster.metadata ] [Starshine] [foo]
creating index, cause [api], shards [1]/[0], mappings
[11:45:12,994][INFO ][cluster.metadata ] [Starshine] [foo]
create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type ] [Starshine] [13] Failed
to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
^
[Line: 1, Column: 0]
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at
org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at
org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at
org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at
org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at
org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more

Should I open an issue?

Regards,
Lukas

[kimchy (Shay Banon)]

I am not sure that you understood what script fields do. They basically
provide a way to fetch custom script evaluation values for hits returned.
They are not related to sorting. If you want to control sorting based on
scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with sorting
fields (they don't have to be stored).

-shay.banon

On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček lukas.vlcek@gmail.com wrote:

Or may be the "return_type" should make more sense to be added in "sort"
portion of the query rather then in "script_fields" portion.

Lukas

On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

So, I found that the SortFieldhttp://lucene.apache.org/java/3_0_2/api/all/org/apache/lucene/search/SortField.htmlneeds field type here:
http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118That
is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" },
"test1" : { "type" : "long" },
"test2" : { "type" : "string" }
}
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note: I
was not able to make it reverse:true or desc) but "sort" : ["test2"] still
fails with simple message:
{
"error" : "ReduceSearchPhaseException[Failed to execute phase [query],
[reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me.
Second issue is that I do not want to provide explicit mapping in advance
(fixing it to specific type) for script_fields. I think this is little
against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in order
to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of
script_fields definiton per query. Something like:
"script_fields" : {
"test1" : {
"script" : "some_fancy_math",
"params" : { ... },
"return_type" : "long"
}
}

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Oh yeah, that's it.
BUT, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d
@script_fields.json

I put the following query into script_fields.json file:
{
"from" : 0, "size" : 3,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
},
"sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them
(note the "sort" part) it fails:
{
"error" : "SearchPhaseExecutionException[Failed to execute phase
[query], total failure; shardFailures
{[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure
[Failed to parse [{ "from" : 0, "size" : 3, "fields" :
["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" :
"doc['date'].date.millis" }, "test2" : { "script" :
"doc['string'].stringValue" } }, "sort" : ["test1"]}]]]; nested:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure [No
mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type ] [Starshine] [foo][0],
node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to
execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [Failed to parse [{ "from" : 0,
"size" : 3, "fields" : ["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" } }, "sort" :
["test1"]}]]
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at
org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at
org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at
org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted
field?

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <
shay.banon@elasticsearch.com> wrote:

Hi,

I think the problem is with curl, and the fact that within the body of
the request the ' is used (within the script). It needs to be escaped
somehow (not sure how).

-shay.banon

On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Hi,

I am experiencing script_fields issue. Below is a shell script with
curl examples. In short: I prepare an index, index three objects into it,
verify that I can query objects from index and that I can extract values of
particular "fields" (though script_fields does not require fields being
stored) and then use a script_fields query which fails. Not sure what is
wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XDELETE 'http://localhost:9200/foo/'
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : {
number_of_shards : 1, number_of_replicas : 0 }}'
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" }
}
}}'
curl -XGET 'http://localhost:9200/foo/bar/_mapping?pretty=1'
echo

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d
'{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d
'{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d
'{"date":"2010-01-03","string":"three"}'
curl -XPOST 'http://localhost:9200/foo/_refresh'
echo

echo Search
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
}
}'
echo

The following is output:

Index setup

{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
"foo" : {
"bar" : {
"_boost" : {
"name" : "_boost"
},
"dynamic" : true,
"enabled" : true,
"date_formats" : [ "dateOptionalTime", "yyyy/MM/dd
HH:mm:ss||yyyy/MM/dd" ],
"_source" : {
"enabled" : true,
"name" : "_source"
},
"_id" : {
"store" : "no"
},
"path" : "full",
"properties" : {
"string" : {
"omit_term_freq_and_positions" : false,
"index_name" : "string",
"index" : "not_analyzed",
"omit_norms" : false,
"store" : "yes",
"boost" : 1.0,
"term_vector" : "no",
"type" : "string"
},
"date" : {
"omit_term_freq_and_positions" : true,
"index_name" : "date",
"index" : "not_analyzed",
"omit_norms" : true,
"store" : "yes",
"boost" : 1.0,
"format" : "dateOptionalTime",
"precision_step" : 4,
"term_vector" : "no",
"type" : "date"
}
},
"_all" : {
"enabled" : true,
"store" : "no",
"term_vector" : "no"
},
"type" : "object"
}
}
}
Indexing

{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
"_shards" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [ {
"_index" : "foo",
"_type" : "bar",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"date" : "2010-01-01T00:00:00.000Z",
"string" : "one"
}
} ]
}
}
Search with script_fields

{
"_shards" : {
"total" : 1,
"successful" : 0,
"failed" : 1,
"failures" : [ {
"reason" : "CompileException[[Error: null]\n[Near : {... Unknown
....}]\n ^\n[Line: 1, Column: 0]]; nested: "
} ]
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" :
}
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata ] [Starshine] [foo]
deleting index
[11:45:12,818][INFO ][cluster.metadata ] [Starshine] [foo]
creating index, cause [api], shards [1]/[0], mappings
[11:45:12,994][INFO ][cluster.metadata ] [Starshine] [foo]
create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type ] [Starshine] [13]
Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
^
[Line: 1, Column: 0]
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at
org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at
org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at
org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at
org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at
org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more

Should I open an issue?

Regards,
Lukas

[Lukas_Vlcek1 (Lukáš Vlček)]

I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols. The
pattern after the pipe is a number. Now I want to get the following order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed (string)
and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string chain
first and then by the number value. But because simple string sort would not
work on the string chain (because of the variable length and different
number of items in it). Then I think it can be easily done using
scritpt_values (the number of items in string chain - in this case 3 - is
different for each query). In this case I would built four script fileds
f1-4 (three for string and one for number). The result would be (for
original order of entities, I am simply right-padding the chain of strings):

f1 f2 f3 f4

A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields
http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/
made
me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number value but
that is because I simplified it way too much)

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon shay.banon@elasticsearch.comwrote:

I am not sure that you understood what script fields do. They basically
provide a way to fetch custom script evaluation values for hits returned.
They are not related to sorting. If you want to control sorting based on
scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with sorting
fields (they don't have to be stored).

-shay.banon

On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Or may be the "return_type" should make more sense to be added in "sort"
portion of the query rather then in "script_fields" portion.

Lukas

On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

So, I found that the SortFieldhttp://lucene.apache.org/java/3_0_2/api/all/org/apache/lucene/search/SortField.htmlneeds field type here:
http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118That
is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" },
"test1" : { "type" : "long" },
"test2" : { "type" : "string" }
}
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note:
I was not able to make it reverse:true or desc) but "sort" : ["test2"] still
fails with simple message:
{
"error" : "ReduceSearchPhaseException[Failed to execute phase [query],
[reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me.
Second issue is that I do not want to provide explicit mapping in advance
(fixing it to specific type) for script_fields. I think this is little
against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in
order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of
script_fields definiton per query. Something like:
"script_fields" : {
"test1" : {
"script" : "some_fancy_math",
"params" : { ... },
"return_type" : "long"
}
}

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Oh yeah, that's it.
BUT, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d
@script_fields.json

I put the following query into script_fields.json file:
{
"from" : 0, "size" : 3,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
},
"sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them
(note the "sort" part) it fails:
{
"error" : "SearchPhaseExecutionException[Failed to execute phase
[query], total failure; shardFailures
{[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure
[Failed to parse [{ "from" : 0, "size" : 3, "fields" :
["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" :
"doc['date'].date.millis" }, "test2" : { "script" :
"doc['string'].stringValue" } }, "sort" : ["test1"]}]]]; nested:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure [No
mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type ] [Starshine] [foo][0],
node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to
execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [Failed to parse [{ "from" : 0,
"size" : 3, "fields" : ["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" } }, "sort" :
["test1"]}]]
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at
org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at
org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at
org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted
field?

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <
shay.banon@elasticsearch.com> wrote:

Hi,

I think the problem is with curl, and the fact that within the body
of the request the ' is used (within the script). It needs to be escaped
somehow (not sure how).

-shay.banon

On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Hi,

I am experiencing script_fields issue. Below is a shell script with
curl examples. In short: I prepare an index, index three objects into it,
verify that I can query objects from index and that I can extract values of
particular "fields" (though script_fields does not require fields being
stored) and then use a script_fields query which fails. Not sure what is
wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XDELETE 'http://localhost:9200/foo/'
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : {
number_of_shards : 1, number_of_replicas : 0 }}'
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" }
}
}}'
curl -XGET 'http://localhost:9200/foo/bar/_mapping?pretty=1'
echo

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d
'{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d
'{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d
'{"date":"2010-01-03","string":"three"}'
curl -XPOST 'http://localhost:9200/foo/_refresh'
echo

echo Search
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
}
}'
echo

The following is output:

Index setup

{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
"foo" : {
"bar" : {
"_boost" : {
"name" : "_boost"
},
"dynamic" : true,
"enabled" : true,
"date_formats" : [ "dateOptionalTime", "yyyy/MM/dd
HH:mm:ss||yyyy/MM/dd" ],
"_source" : {
"enabled" : true,
"name" : "_source"
},
"_id" : {
"store" : "no"
},
"path" : "full",
"properties" : {
"string" : {
"omit_term_freq_and_positions" : false,
"index_name" : "string",
"index" : "not_analyzed",
"omit_norms" : false,
"store" : "yes",
"boost" : 1.0,
"term_vector" : "no",
"type" : "string"
},
"date" : {
"omit_term_freq_and_positions" : true,
"index_name" : "date",
"index" : "not_analyzed",
"omit_norms" : true,
"store" : "yes",
"boost" : 1.0,
"format" : "dateOptionalTime",
"precision_step" : 4,
"term_vector" : "no",
"type" : "date"
}
},
"_all" : {
"enabled" : true,
"store" : "no",
"term_vector" : "no"
},
"type" : "object"
}
}
}
Indexing

{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
"_shards" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [ {
"_index" : "foo",
"_type" : "bar",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"date" : "2010-01-01T00:00:00.000Z",
"string" : "one"
}
} ]
}
}
Search with script_fields

{
"_shards" : {
"total" : 1,
"successful" : 0,
"failed" : 1,
"failures" : [ {
"reason" : "CompileException[[Error: null]\n[Near : {... Unknown
....}]\n ^\n[Line: 1, Column: 0]]; nested: "
} ]
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" :
}
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata ] [Starshine] [foo]
deleting index
[11:45:12,818][INFO ][cluster.metadata ] [Starshine] [foo]
creating index, cause [api], shards [1]/[0], mappings
[11:45:12,994][INFO ][cluster.metadata ] [Starshine] [foo]
create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type ] [Starshine] [13]
Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
^
[Line: 1, Column: 0]
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at
org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at
org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at
org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at
org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at
org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more

Should I open an issue?

Regards,
Lukas

[Lukas_Vlcek1 (Lukáš Vlček)]

On Thu, Jul 29, 2010 at 7:22 PM, Lukáš Vlček lukas.vlcek@gmail.com wrote:

I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols. The
pattern after the pipe is a number. Now I want to get the following order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed (string)
and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string chain
first and then by the number value. But because simple string sort would not
work on the string chain (because of the variable length and different
number of items in it). Then I think it can be easily done using
scritpt_values (the number of items in string chain - in this case 3 - is
different for each query). In this case I would built four script fileds
f1-4 (three for string and one for number). The result would be (for
original order of entities, I am simply right-padding the chain of strings):

f1 f2 f3 f4

A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields
http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/ made
me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number value
but that is because I simplified it way too much)

Just to fix my simplification: replace
A - E - . | 4
with
A - D - . | 4
and you get the something that needs to be sorted by both string chain and
number value.

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon shay.banon@elasticsearch.comwrote:

I am not sure that you understood what script fields do. They basically
provide a way to fetch custom script evaluation values for hits returned.
They are not related to sorting. If you want to control sorting based on
scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with sorting
fields (they don't have to be stored).

-shay.banon

On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Or may be the "return_type" should make more sense to be added in "sort"
portion of the query rather then in "script_fields" portion.

Lukas

On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

So, I found that the SortFieldhttp://lucene.apache.org/java/3_0_2/api/all/org/apache/lucene/search/SortField.htmlneeds field type here:
http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118That
is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" },
"test1" : { "type" : "long" },
"test2" : { "type" : "string" }
}
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note:
I was not able to make it reverse:true or desc) but "sort" : ["test2"] still
fails with simple message:
{
"error" : "ReduceSearchPhaseException[Failed to execute phase [query],
[reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me.
Second issue is that I do not want to provide explicit mapping in advance
(fixing it to specific type) for script_fields. I think this is little
against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in
order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of
script_fields definiton per query. Something like:
"script_fields" : {
"test1" : {
"script" : "some_fancy_math",
"params" : { ... },
"return_type" : "long"
}
}

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Oh yeah, that's it.
BUT, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d
@script_fields.json

I put the following query into script_fields.json file:
{
"from" : 0, "size" : 3,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
},
"sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them
(note the "sort" part) it fails:
{
"error" : "SearchPhaseExecutionException[Failed to execute phase
[query], total failure; shardFailures
{[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure
[Failed to parse [{ "from" : 0, "size" : 3, "fields" :
["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" :
"doc['date'].date.millis" }, "test2" : { "script" :
"doc['string'].stringValue" } }, "sort" : ["test1"]}]]]; nested:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure [No
mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type ] [Starshine] [foo][0],
node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to
execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [Failed to parse [{ "from" : 0,
"size" : 3, "fields" : ["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" } }, "sort" :
["test1"]}]]
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at
org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at
org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at
org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted
field?

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <
shay.banon@elasticsearch.com> wrote:

Hi,

I think the problem is with curl, and the fact that within the body
of the request the ' is used (within the script). It needs to be escaped
somehow (not sure how).

-shay.banon

On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Hi,

I am experiencing script_fields issue. Below is a shell script with
curl examples. In short: I prepare an index, index three objects into it,
verify that I can query objects from index and that I can extract values of
particular "fields" (though script_fields does not require fields being
stored) and then use a script_fields query which fails. Not sure what is
wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XDELETE 'http://localhost:9200/foo/'
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : {
number_of_shards : 1, number_of_replicas : 0 }}'
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" }
}
}}'
curl -XGET 'http://localhost:9200/foo/bar/_mapping?pretty=1'
echo

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d
'{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d
'{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d
'{"date":"2010-01-03","string":"three"}'
curl -XPOST 'http://localhost:9200/foo/_refresh'
echo

echo Search
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
}
}'
echo

The following is output:

Index setup

{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
"foo" : {
"bar" : {
"_boost" : {
"name" : "_boost"
},
"dynamic" : true,
"enabled" : true,
"date_formats" : [ "dateOptionalTime", "yyyy/MM/dd
HH:mm:ss||yyyy/MM/dd" ],
"_source" : {
"enabled" : true,
"name" : "_source"
},
"_id" : {
"store" : "no"
},
"path" : "full",
"properties" : {
"string" : {
"omit_term_freq_and_positions" : false,
"index_name" : "string",
"index" : "not_analyzed",
"omit_norms" : false,
"store" : "yes",
"boost" : 1.0,
"term_vector" : "no",
"type" : "string"
},
"date" : {
"omit_term_freq_and_positions" : true,
"index_name" : "date",
"index" : "not_analyzed",
"omit_norms" : true,
"store" : "yes",
"boost" : 1.0,
"format" : "dateOptionalTime",
"precision_step" : 4,
"term_vector" : "no",
"type" : "date"
}
},
"_all" : {
"enabled" : true,
"store" : "no",
"term_vector" : "no"
},
"type" : "object"
}
}
}
Indexing

{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
"_shards" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [ {
"_index" : "foo",
"_type" : "bar",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"date" : "2010-01-01T00:00:00.000Z",
"string" : "one"
}
} ]
}
}
Search with script_fields

{
"_shards" : {
"total" : 1,
"successful" : 0,
"failed" : 1,
"failures" : [ {
"reason" : "CompileException[[Error: null]\n[Near : {...
Unknown ....}]\n ^\n[Line: 1, Column: 0]]; nested: "
} ]
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" :
}
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata ] [Starshine] [foo]
deleting index
[11:45:12,818][INFO ][cluster.metadata ] [Starshine] [foo]
creating index, cause [api], shards [1]/[0], mappings
[11:45:12,994][INFO ][cluster.metadata ] [Starshine] [foo]
create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type ] [Starshine] [13]
Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
^
[Line: 1, Column: 0]
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at
org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at
org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at
org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at
org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at
org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more

Should I open an issue?

Regards,
Lukas

[kimchy (Shay Banon)]

I would, in this case, add another field to the document you index that does
the mentioned logic, index it, and then sort on it.

On Thu, Jul 29, 2010 at 8:22 PM, Lukáš Vlček lukas.vlcek@gmail.com wrote:

I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols. The
pattern after the pipe is a number. Now I want to get the following order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed (string)
and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string chain
first and then by the number value. But because simple string sort would not
work on the string chain (because of the variable length and different
number of items in it). Then I think it can be easily done using
scritpt_values (the number of items in string chain - in this case 3 - is
different for each query). In this case I would built four script fileds
f1-4 (three for string and one for number). The result would be (for
original order of entities, I am simply right-padding the chain of strings):

f1 f2 f3 f4

A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields
http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/ made
me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number value
but that is because I simplified it way too much)

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon shay.banon@elasticsearch.comwrote:

I am not sure that you understood what script fields do. They basically
provide a way to fetch custom script evaluation values for hits returned.
They are not related to sorting. If you want to control sorting based on
scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with sorting
fields (they don't have to be stored).

-shay.banon

On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Or may be the "return_type" should make more sense to be added in "sort"
portion of the query rather then in "script_fields" portion.

Lukas

On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

So, I found that the SortFieldhttp://lucene.apache.org/java/3_0_2/api/all/org/apache/lucene/search/SortField.htmlneeds field type here:
http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118That
is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" },
"test1" : { "type" : "long" },
"test2" : { "type" : "string" }
}
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works (note:
I was not able to make it reverse:true or desc) but "sort" : ["test2"] still
fails with simple message:
{
"error" : "ReduceSearchPhaseException[Failed to execute phase [query],
[reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to me.
Second issue is that I do not want to provide explicit mapping in advance
(fixing it to specific type) for script_fields. I think this is little
against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in
order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of
script_fields definiton per query. Something like:
"script_fields" : {
"test1" : {
"script" : "some_fancy_math",
"params" : { ... },
"return_type" : "long"
}
}

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Oh yeah, that's it.
BUT, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d
@script_fields.json

I put the following query into script_fields.json file:
{
"from" : 0, "size" : 3,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
},
"sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them
(note the "sort" part) it fails:
{
"error" : "SearchPhaseExecutionException[Failed to execute phase
[query], total failure; shardFailures
{[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure
[Failed to parse [{ "from" : 0, "size" : 3, "fields" :
["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" :
"doc['date'].date.millis" }, "test2" : { "script" :
"doc['string'].stringValue" } }, "sort" : ["test1"]}]]]; nested:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure [No
mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type ] [Starshine] [foo][0],
node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to
execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [Failed to parse [{ "from" : 0,
"size" : 3, "fields" : ["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" } }, "sort" :
["test1"]}]]
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at
org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at
org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at
org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted
field?

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <
shay.banon@elasticsearch.com> wrote:

Hi,

I think the problem is with curl, and the fact that within the body
of the request the ' is used (within the script). It needs to be escaped
somehow (not sure how).

-shay.banon

On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Hi,

I am experiencing script_fields issue. Below is a shell script with
curl examples. In short: I prepare an index, index three objects into it,
verify that I can query objects from index and that I can extract values of
particular "fields" (though script_fields does not require fields being
stored) and then use a script_fields query which fails. Not sure what is
wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XDELETE 'http://localhost:9200/foo/'
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : {
number_of_shards : 1, number_of_replicas : 0 }}'
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" }
}
}}'
curl -XGET 'http://localhost:9200/foo/bar/_mapping?pretty=1'
echo

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d
'{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d
'{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d
'{"date":"2010-01-03","string":"three"}'
curl -XPOST 'http://localhost:9200/foo/_refresh'
echo

echo Search
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
}
}'
echo

The following is output:

Index setup

{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
"foo" : {
"bar" : {
"_boost" : {
"name" : "_boost"
},
"dynamic" : true,
"enabled" : true,
"date_formats" : [ "dateOptionalTime", "yyyy/MM/dd
HH:mm:ss||yyyy/MM/dd" ],
"_source" : {
"enabled" : true,
"name" : "_source"
},
"_id" : {
"store" : "no"
},
"path" : "full",
"properties" : {
"string" : {
"omit_term_freq_and_positions" : false,
"index_name" : "string",
"index" : "not_analyzed",
"omit_norms" : false,
"store" : "yes",
"boost" : 1.0,
"term_vector" : "no",
"type" : "string"
},
"date" : {
"omit_term_freq_and_positions" : true,
"index_name" : "date",
"index" : "not_analyzed",
"omit_norms" : true,
"store" : "yes",
"boost" : 1.0,
"format" : "dateOptionalTime",
"precision_step" : 4,
"term_vector" : "no",
"type" : "date"
}
},
"_all" : {
"enabled" : true,
"store" : "no",
"term_vector" : "no"
},
"type" : "object"
}
}
}
Indexing

{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
"_shards" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [ {
"_index" : "foo",
"_type" : "bar",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"date" : "2010-01-01T00:00:00.000Z",
"string" : "one"
}
} ]
}
}
Search with script_fields

{
"_shards" : {
"total" : 1,
"successful" : 0,
"failed" : 1,
"failures" : [ {
"reason" : "CompileException[[Error: null]\n[Near : {...
Unknown ....}]\n ^\n[Line: 1, Column: 0]]; nested: "
} ]
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" :
}
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata ] [Starshine] [foo]
deleting index
[11:45:12,818][INFO ][cluster.metadata ] [Starshine] [foo]
creating index, cause [api], shards [1]/[0], mappings
[11:45:12,994][INFO ][cluster.metadata ] [Starshine] [foo]
create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type ] [Starshine] [13]
Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
^
[Line: 1, Column: 0]
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at
org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at
org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at
org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at
org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at
org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more

Should I open an issue?

Regards,
Lukas

[Lukas_Vlcek1 (Lukáš Vlček)]

I can not do this at index time because I do not know the length of the
string chain in advance.
Take for example document related to A | 0 entity. How can I know that I
want to right-padd it to A - . - . | 0 without knowing that there are other
related documents having A - B - C ?
As of know it seems that it will be easier to handle that logic on the
client side (I will sort document by client, in my case there won't be a lot
of related documents, I assume up to 30 max).

However, I am still curious if it is possible (or planned) to allow sort on
script_fields. As of now it seems to be possible only if explicit mapping is
provided for script_fields (which means it lacks a lot of flexibility
considering that script_fields are defined per query while mapping per
index).

Regards,
Lukas

2010/7/29 Shay Banon shay.banon@elasticsearch.com

I would, in this case, add another field to the document you index that
does the mentioned logic, index it, and then sort on it.

On Thu, Jul 29, 2010 at 8:22 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols. The
pattern after the pipe is a number. Now I want to get the following order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed (string)
and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string chain
first and then by the number value. But because simple string sort would not
work on the string chain (because of the variable length and different
number of items in it). Then I think it can be easily done using
scritpt_values (the number of items in string chain - in this case 3 - is
different for each query). In this case I would built four script fileds
f1-4 (three for string and one for number). The result would be (for
original order of entities, I am simply right-padding the chain of strings):

f1 f2 f3 f4

A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields
http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/ made
me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number value
but that is because I simplified it way too much)

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon <shay.banon@elasticsearch.com

wrote:

I am not sure that you understood what script fields do. They basically
provide a way to fetch custom script evaluation values for hits returned.
They are not related to sorting. If you want to control sorting based on
scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with sorting
fields (they don't have to be stored).

-shay.banon

On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Or may be the "return_type" should make more sense to be added in "sort"
portion of the query rather then in "script_fields" portion.

Lukas

On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

So, I found that the SortFieldhttp://lucene.apache.org/java/3_0_2/api/all/org/apache/lucene/search/SortField.htmlneeds field type here:
http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118That
is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" },
"test1" : { "type" : "long" },
"test2" : { "type" : "string" }
}
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works
(note: I was not able to make it reverse:true or desc) but "sort" :
["test2"] still fails with simple message:
{
"error" : "ReduceSearchPhaseException[Failed to execute phase
[query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to
me. Second issue is that I do not want to provide explicit mapping in
advance (fixing it to specific type) for script_fields. I think this is
little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in
order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of
script_fields definiton per query. Something like:
"script_fields" : {
"test1" : {
"script" : "some_fancy_math",
"params" : { ... },
"return_type" : "long"
}
}

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Oh yeah, that's it.
BUT, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d
@script_fields.json

I put the following query into script_fields.json file:
{
"from" : 0, "size" : 3,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
},
"sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them
(note the "sort" part) it fails:
{
"error" : "SearchPhaseExecutionException[Failed to execute phase
[query], total failure; shardFailures
{[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure
[Failed to parse [{ "from" : 0, "size" : 3, "fields" :
["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" :
"doc['date'].date.millis" }, "test2" : { "script" :
"doc['string'].stringValue" } }, "sort" : ["test1"]}]]]; nested:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure [No
mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type ] [Starshine] [foo][0],
node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]: Failed to
execute [org.elasticsearch.action.search.SearchRequest@406039a4]
org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [Failed to parse [{ "from" : 0,
"size" : 3, "fields" : ["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" } }, "sort" :
["test1"]}]]
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at
org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at
org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at
org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted
field?

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <
shay.banon@elasticsearch.com> wrote:

Hi,

I think the problem is with curl, and the fact that within the body
of the request the ' is used (within the script). It needs to be escaped
somehow (not sure how).

-shay.banon

On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <lukas.vlcek@gmail.com

wrote:

Hi,

I am experiencing script_fields issue. Below is a shell script with
curl examples. In short: I prepare an index, index three objects into it,
verify that I can query objects from index and that I can extract values of
particular "fields" (though script_fields does not require fields being
stored) and then use a script_fields query which fails. Not sure what is
wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XDELETE 'http://localhost:9200/foo/'
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : {
number_of_shards : 1, number_of_replicas : 0 }}'
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" }
}
}}'
curl -XGET 'http://localhost:9200/foo/bar/_mapping?pretty=1'
echo

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d
'{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d
'{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d
'{"date":"2010-01-03","string":"three"}'
curl -XPOST 'http://localhost:9200/foo/_refresh'
echo

echo Search
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
}
}'
echo

The following is output:

Index setup

{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
"foo" : {
"bar" : {
"_boost" : {
"name" : "_boost"
},
"dynamic" : true,
"enabled" : true,
"date_formats" : [ "dateOptionalTime", "yyyy/MM/dd
HH:mm:ss||yyyy/MM/dd" ],
"_source" : {
"enabled" : true,
"name" : "_source"
},
"_id" : {
"store" : "no"
},
"path" : "full",
"properties" : {
"string" : {
"omit_term_freq_and_positions" : false,
"index_name" : "string",
"index" : "not_analyzed",
"omit_norms" : false,
"store" : "yes",
"boost" : 1.0,
"term_vector" : "no",
"type" : "string"
},
"date" : {
"omit_term_freq_and_positions" : true,
"index_name" : "date",
"index" : "not_analyzed",
"omit_norms" : true,
"store" : "yes",
"boost" : 1.0,
"format" : "dateOptionalTime",
"precision_step" : 4,
"term_vector" : "no",
"type" : "date"
}
},
"_all" : {
"enabled" : true,
"store" : "no",
"term_vector" : "no"
},
"type" : "object"
}
}
}
Indexing

{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
"_shards" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [ {
"_index" : "foo",
"_type" : "bar",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"date" : "2010-01-01T00:00:00.000Z",
"string" : "one"
}
} ]
}
}
Search with script_fields

{
"_shards" : {
"total" : 1,
"successful" : 0,
"failed" : 1,
"failures" : [ {
"reason" : "CompileException[[Error: null]\n[Near : {...
Unknown ....}]\n ^\n[Line: 1, Column: 0]]; nested: "
} ]
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" :
}
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata ] [Starshine] [foo]
deleting index
[11:45:12,818][INFO ][cluster.metadata ] [Starshine] [foo]
creating index, cause [api], shards [1]/[0], mappings
[11:45:12,994][INFO ][cluster.metadata ] [Starshine] [foo]
create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type ] [Starshine] [13]
Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
^
[Line: 1, Column: 0]
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at
org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at
org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at
org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at
org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at
org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more

Should I open an issue?

Regards,
Lukas

[kimchy (Shay Banon)]

Script fields are not using any mappings. It pretty simple what they do, you
get 10 hits, for that 10 hits (docs), the script fields are evaluated. What
you want is to have the ability to sort based on several scripts, which is
possible.

-shay.banon

On Thu, Jul 29, 2010 at 10:09 PM, Lukáš Vlček lukas.vlcek@gmail.com wrote:

I can not do this at index time because I do not know the length of the
string chain in advance.
Take for example document related to A | 0 entity. How can I know that I
want to right-padd it to A - . - . | 0 without knowing that there are other
related documents having A - B - C ?
As of know it seems that it will be easier to handle that logic on the
client side (I will sort document by client, in my case there won't be a lot
of related documents, I assume up to 30 max).

However, I am still curious if it is possible (or planned) to allow sort on
script_fields. As of now it seems to be possible only if explicit mapping is
provided for script_fields (which means it lacks a lot of flexibility
considering that script_fields are defined per query while mapping per
index).

Regards,
Lukas

2010/7/29 Shay Banon shay.banon@elasticsearch.com

I would, in this case, add another field to the document you index that

does the mentioned logic, index it, and then sort on it.

On Thu, Jul 29, 2010 at 8:22 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols.
The pattern after the pipe is a number. Now I want to get the following
order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed (string)
and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string
chain first and then by the number value. But because simple string sort
would not work on the string chain (because of the variable length and
different number of items in it). Then I think it can be easily done using
scritpt_values (the number of items in string chain - in this case 3 - is
different for each query). In this case I would built four script fileds
f1-4 (three for string and one for number). The result would be (for
original order of entities, I am simply right-padding the chain of strings):

f1 f2 f3 f4

A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields
http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/ made
me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number value
but that is because I simplified it way too much)

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon <
shay.banon@elasticsearch.com> wrote:

I am not sure that you understood what script fields do. They basically
provide a way to fetch custom script evaluation values for hits returned.
They are not related to sorting. If you want to control sorting based on
scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with
sorting fields (they don't have to be stored).

-shay.banon

On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Or may be the "return_type" should make more sense to be added in
"sort" portion of the query rather then in "script_fields" portion.

Lukas

On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

So, I found that the SortFieldhttp://lucene.apache.org/java/3_0_2/api/all/org/apache/lucene/search/SortField.htmlneeds field type here:
http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118That
is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" },
"test1" : { "type" : "long" },
"test2" : { "type" : "string" }
}
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works
(note: I was not able to make it reverse:true or desc) but "sort" :
["test2"] still fails with simple message:
{
"error" : "ReduceSearchPhaseException[Failed to execute phase
[query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to
me. Second issue is that I do not want to provide explicit mapping in
advance (fixing it to specific type) for script_fields. I think this is
little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in
order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of
script_fields definiton per query. Something like:
"script_fields" : {
"test1" : {
"script" : "some_fancy_math",
"params" : { ... },
"return_type" : "long"
}
}

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Oh yeah, that's it.
BUT, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d
@script_fields.json

I put the following query into script_fields.json file:
{
"from" : 0, "size" : 3,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
},
"sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them
(note the "sort" part) it fails:
{
"error" : "SearchPhaseExecutionException[Failed to execute phase
[query], total failure; shardFailures
{[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure
[Failed to parse [{ "from" : 0, "size" : 3, "fields" :
["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" :
"doc['date'].date.millis" }, "test2" : { "script" :
"doc['string'].stringValue" } }, "sort" : ["test1"]}]]]; nested:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure [No
mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type ] [Starshine]
[foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]:
Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4
]
org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [Failed to parse [{ "from" : 0,
"size" : 3, "fields" : ["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" } }, "sort" :
["test1"]}]]
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at
org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at
org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at
org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted
field?

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <
shay.banon@elasticsearch.com> wrote:

Hi,

I think the problem is with curl, and the fact that within the
body of the request the ' is used (within the script). It needs to be
escaped somehow (not sure how).

-shay.banon

On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <
lukas.vlcek@gmail.com> wrote:

Hi,

I am experiencing script_fields issue. Below is a shell script with
curl examples. In short: I prepare an index, index three objects into it,
verify that I can query objects from index and that I can extract values of
particular "fields" (though script_fields does not require fields being
stored) and then use a script_fields query which fails. Not sure what is
wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XDELETE 'http://localhost:9200/foo/'
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : {
number_of_shards : 1, number_of_replicas : 0 }}'
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" }
}
}}'
curl -XGET 'http://localhost:9200/foo/bar/_mapping?pretty=1'
echo

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d
'{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d
'{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d
'{"date":"2010-01-03","string":"three"}'
curl -XPOST 'http://localhost:9200/foo/_refresh'
echo

echo Search
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
}
}'
echo

The following is output:

Index setup

{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
"foo" : {
"bar" : {
"_boost" : {
"name" : "_boost"
},
"dynamic" : true,
"enabled" : true,
"date_formats" : [ "dateOptionalTime", "yyyy/MM/dd
HH:mm:ss||yyyy/MM/dd" ],
"_source" : {
"enabled" : true,
"name" : "_source"
},
"_id" : {
"store" : "no"
},
"path" : "full",
"properties" : {
"string" : {
"omit_term_freq_and_positions" : false,
"index_name" : "string",
"index" : "not_analyzed",
"omit_norms" : false,
"store" : "yes",
"boost" : 1.0,
"term_vector" : "no",
"type" : "string"
},
"date" : {
"omit_term_freq_and_positions" : true,
"index_name" : "date",
"index" : "not_analyzed",
"omit_norms" : true,
"store" : "yes",
"boost" : 1.0,
"format" : "dateOptionalTime",
"precision_step" : 4,
"term_vector" : "no",
"type" : "date"
}
},
"_all" : {
"enabled" : true,
"store" : "no",
"term_vector" : "no"
},
"type" : "object"
}
}
}
Indexing

{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
"_shards" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [ {
"_index" : "foo",
"_type" : "bar",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"date" : "2010-01-01T00:00:00.000Z",
"string" : "one"
}
} ]
}
}
Search with script_fields

{
"_shards" : {
"total" : 1,
"successful" : 0,
"failed" : 1,
"failures" : [ {
"reason" : "CompileException[[Error: null]\n[Near : {...
Unknown ....}]\n ^\n[Line: 1, Column: 0]]; nested: "
} ]
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" :
}
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata ] [Starshine] [foo]
deleting index
[11:45:12,818][INFO ][cluster.metadata ] [Starshine] [foo]
creating index, cause [api], shards [1]/[0], mappings
[11:45:12,994][INFO ][cluster.metadata ] [Starshine] [foo]
create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type ] [Starshine] [13]
Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
^
[Line: 1, Column: 0]
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at
org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at
org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at
org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at
org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at
org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more

Should I open an issue?

Regards,
Lukas

[kimchy (Shay Banon)]

I meant possible to implement ;).

I had another look at your problem. Maybe one way to solve it, if you know
in advance the maximum number of levels you will have in the string field,
is to have empty markers for empty levels, that will still perserve sorting?
Adding multiple sort scripts feature can be done, but it will be slower.
First, because score based sorting is faster (which you get with custom
score query), and second because the mentioned scripts will need to be
evaluated for every doc the query hits in order to do the sorting correctly.
This can be mitigated by having more shards and more machines, basically
resulting in distributing the search load across machines (if the perf is
really a problem).

-shay.banon

On Fri, Jul 30, 2010 at 12:15 AM, Shay Banon
shay.banon@elasticsearch.comwrote:

Script fields are not using any mappings. It pretty simple what they do,
you get 10 hits, for that 10 hits (docs), the script fields are evaluated.
What you want is to have the ability to sort based on several scripts, which
is possible.

-shay.banon

On Thu, Jul 29, 2010 at 10:09 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

I can not do this at index time because I do not know the length of the
string chain in advance.
Take for example document related to A | 0 entity. How can I know that I
want to right-padd it to A - . - . | 0 without knowing that there are other
related documents having A - B - C ?
As of know it seems that it will be easier to handle that logic on the
client side (I will sort document by client, in my case there won't be a lot
of related documents, I assume up to 30 max).

However, I am still curious if it is possible (or planned) to allow sort
on script_fields. As of now it seems to be possible only if explicit mapping
is provided for script_fields (which means it lacks a lot of flexibility
considering that script_fields are defined per query while mapping per
index).

Regards,
Lukas

2010/7/29 Shay Banon shay.banon@elasticsearch.com

I would, in this case, add another field to the document you index that

does the mentioned logic, index it, and then sort on it.

On Thu, Jul 29, 2010 at 8:22 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols.
The pattern after the pipe is a number. Now I want to get the following
order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed
(string) and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string
chain first and then by the number value. But because simple string sort
would not work on the string chain (because of the variable length and
different number of items in it). Then I think it can be easily done using
scritpt_values (the number of items in string chain - in this case 3 - is
different for each query). In this case I would built four script fileds
f1-4 (three for string and one for number). The result would be (for
original order of entities, I am simply right-padding the chain of strings):

f1 f2 f3 f4

A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields
http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/ made
me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number value
but that is because I simplified it way too much)

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon <
shay.banon@elasticsearch.com> wrote:

I am not sure that you understood what script fields do. They basically
provide a way to fetch custom script evaluation values for hits returned.
They are not related to sorting. If you want to control sorting based on
scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with
sorting fields (they don't have to be stored).

-shay.banon

On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Or may be the "return_type" should make more sense to be added in
"sort" portion of the query rather then in "script_fields" portion.

Lukas

On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

So, I found that the SortFieldhttp://lucene.apache.org/java/3_0_2/api/all/org/apache/lucene/search/SortField.htmlneeds field type here:
http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118That
is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" },
"test1" : { "type" : "long" },
"test2" : { "type" : "string" }
}
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works
(note: I was not able to make it reverse:true or desc) but "sort" :
["test2"] still fails with simple message:
{
"error" : "ReduceSearchPhaseException[Failed to execute phase
[query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to
me. Second issue is that I do not want to provide explicit mapping in
advance (fixing it to specific type) for script_fields. I think this is
little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in
order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of
script_fields definiton per query. Something like:
"script_fields" : {
"test1" : {
"script" : "some_fancy_math",
"params" : { ... },
"return_type" : "long"
}
}

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Oh yeah, that's it.
BUT, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d
@script_fields.json

I put the following query into script_fields.json file:
{
"from" : 0, "size" : 3,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
},
"sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by them
(note the "sort" part) it fails:
{
"error" : "SearchPhaseExecutionException[Failed to execute phase
[query], total failure; shardFailures
{[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure
[Failed to parse [{ "from" : 0, "size" : 3, "fields" :
["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" :
"doc['date'].date.millis" }, "test2" : { "script" :
"doc['string'].stringValue" } }, "sort" : ["test1"]}]]]; nested:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure [No
mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type ] [Starshine]
[foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]:
Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4
]
org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [Failed to parse [{ "from" : 0,
"size" : 3, "fields" : ["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" } }, "sort" :
["test1"]}]]
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at
org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at
org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at
org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on scripted
field?

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <
shay.banon@elasticsearch.com> wrote:

Hi,

I think the problem is with curl, and the fact that within the
body of the request the ' is used (within the script). It needs to be
escaped somehow (not sure how).

-shay.banon

On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <
lukas.vlcek@gmail.com> wrote:

Hi,

I am experiencing script_fields issue. Below is a shell script
with curl examples. In short: I prepare an index, index three objects into
it, verify that I can query objects from index and that I can extract values
of particular "fields" (though script_fields does not require fields being
stored) and then use a script_fields query which fails. Not sure what is
wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XDELETE 'http://localhost:9200/foo/'
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : {
number_of_shards : 1, number_of_replicas : 0 }}'
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" }
}
}}'
curl -XGET 'http://localhost:9200/foo/bar/_mapping?pretty=1'
echo

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d
'{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d
'{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d
'{"date":"2010-01-03","string":"three"}'
curl -XPOST 'http://localhost:9200/foo/_refresh'
echo

echo Search
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
}
}'
echo

The following is output:

Index setup

{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
"foo" : {
"bar" : {
"_boost" : {
"name" : "_boost"
},
"dynamic" : true,
"enabled" : true,
"date_formats" : [ "dateOptionalTime", "yyyy/MM/dd
HH:mm:ss||yyyy/MM/dd" ],
"_source" : {
"enabled" : true,
"name" : "_source"
},
"_id" : {
"store" : "no"
},
"path" : "full",
"properties" : {
"string" : {
"omit_term_freq_and_positions" : false,
"index_name" : "string",
"index" : "not_analyzed",
"omit_norms" : false,
"store" : "yes",
"boost" : 1.0,
"term_vector" : "no",
"type" : "string"
},
"date" : {
"omit_term_freq_and_positions" : true,
"index_name" : "date",
"index" : "not_analyzed",
"omit_norms" : true,
"store" : "yes",
"boost" : 1.0,
"format" : "dateOptionalTime",
"precision_step" : 4,
"term_vector" : "no",
"type" : "date"
}
},
"_all" : {
"enabled" : true,
"store" : "no",
"term_vector" : "no"
},
"type" : "object"
}
}
}
Indexing

{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
"_shards" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [ {
"_index" : "foo",
"_type" : "bar",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"date" : "2010-01-01T00:00:00.000Z",
"string" : "one"
}
} ]
}
}
Search with script_fields

{
"_shards" : {
"total" : 1,
"successful" : 0,
"failed" : 1,
"failures" : [ {
"reason" : "CompileException[[Error: null]\n[Near : {...
Unknown ....}]\n ^\n[Line: 1, Column: 0]]; nested: "
} ]
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" :
}
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata ] [Starshine] [foo]
deleting index
[11:45:12,818][INFO ][cluster.metadata ] [Starshine] [foo]
creating index, cause [api], shards [1]/[0], mappings
[11:45:12,994][INFO ][cluster.metadata ] [Starshine] [foo]
create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type ] [Starshine] [13]
Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
^
[Line: 1, Column: 0]
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at
org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at
org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at
org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at
org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at
org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more

Should I open an issue?

Regards,
Lukas

[Lukas_Vlcek1 (Lukáš Vlček)]

I will do the sorting on the client side because the set of document I need
to sort is small (<30 documents in most cases). I just got impression that
ES could have done it for me in simple and elegant way (again, the pref
would not be a problem due to small number of documents in the set to sort).

However, it seems to me that there is a room for improvement when it comes
to script fields and sorting, and may be sorting in general because it could
be useful to allow users to provide custom sort function as well (
FieldComparatorSourcehttp://lucene.apache.org/java/3_0_2/api/all/org/apache/lucene/search/FieldComparatorSource.html),
a nice-to-have feature for some next release :-).

I red the script_fields docs again and I think I understand now what the
note regarding sorting and faceting means (it means that if the document
field have been already used for faceting or sorting then there is no
additional memory cost when used also in script_fields).

Regards,
Lukas

2010/7/30 Shay Banon shay.banon@elasticsearch.com

I meant possible to implement ;).

I had another look at your problem. Maybe one way to solve it, if you know
in advance the maximum number of levels you will have in the string field,
is to have empty markers for empty levels, that will still perserve sorting?
Adding multiple sort scripts feature can be done, but it will be slower.
First, because score based sorting is faster (which you get with custom
score query), and second because the mentioned scripts will need to be
evaluated for every doc the query hits in order to do the sorting correctly.
This can be mitigated by having more shards and more machines, basically
resulting in distributing the search load across machines (if the perf is
really a problem).

-shay.banon

On Fri, Jul 30, 2010 at 12:15 AM, Shay Banon <shay.banon@elasticsearch.com

wrote:

Script fields are not using any mappings. It pretty simple what they do,
you get 10 hits, for that 10 hits (docs), the script fields are evaluated.
What you want is to have the ability to sort based on several scripts, which
is possible.

-shay.banon

On Thu, Jul 29, 2010 at 10:09 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

I can not do this at index time because I do not know the length of the
string chain in advance.
Take for example document related to A | 0 entity. How can I know that I
want to right-padd it to A - . - . | 0 without knowing that there are other
related documents having A - B - C ?
As of know it seems that it will be easier to handle that logic on the
client side (I will sort document by client, in my case there won't be a lot
of related documents, I assume up to 30 max).

However, I am still curious if it is possible (or planned) to allow sort
on script_fields. As of now it seems to be possible only if explicit mapping
is provided for script_fields (which means it lacks a lot of flexibility
considering that script_fields are defined per query while mapping per
index).

Regards,
Lukas

2010/7/29 Shay Banon shay.banon@elasticsearch.com

I would, in this case, add another field to the document you index that

does the mentioned logic, index it, and then sort on it.

On Thu, Jul 29, 2010 at 8:22 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols.
The pattern after the pipe is a number. Now I want to get the following
order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed
(string) and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string
chain first and then by the number value. But because simple string sort
would not work on the string chain (because of the variable length and
different number of items in it). Then I think it can be easily done using
scritpt_values (the number of items in string chain - in this case 3 - is
different for each query). In this case I would built four script fileds
f1-4 (three for string and one for number). The result would be (for
original order of entities, I am simply right-padding the chain of strings):

f1 f2 f3 f4

A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields
http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/ made
me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number
value but that is because I simplified it way too much)

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon <
shay.banon@elasticsearch.com> wrote:

I am not sure that you understood what script fields do. They
basically provide a way to fetch custom script evaluation values for hits
returned. They are not related to sorting. If you want to control sorting
based on scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with
sorting fields (they don't have to be stored).

-shay.banon

On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Or may be the "return_type" should make more sense to be added in
"sort" portion of the query rather then in "script_fields" portion.

Lukas

On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

So, I found that the SortFieldhttp://lucene.apache.org/java/3_0_2/api/all/org/apache/lucene/search/SortField.htmlneeds field type here:
http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118That
is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" },
"test1" : { "type" : "long" },
"test2" : { "type" : "string" }
}
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works
(note: I was not able to make it reverse:true or desc) but "sort" :
["test2"] still fails with simple message:
{
"error" : "ReduceSearchPhaseException[Failed to execute phase
[query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues to
me. Second issue is that I do not want to provide explicit mapping in
advance (fixing it to specific type) for script_fields. I think this is
little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script in
order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of
script_fields definiton per query. Something like:
"script_fields" : {
"test1" : {
"script" : "some_fancy_math",
"params" : { ... },
"return_type" : "long"
}
}

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček <lukas.vlcek@gmail.com

wrote:

Oh yeah, that's it.
BUT, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d
@script_fields.json

I put the following query into script_fields.json file:
{
"from" : 0, "size" : 3,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
},
"sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by
them (note the "sort" part) it fails:
{
"error" : "SearchPhaseExecutionException[Failed to execute phase
[query], total failure; shardFailures
{[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure
[Failed to parse [{ "from" : 0, "size" : 3, "fields" :
["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" :
"doc['date'].date.millis" }, "test2" : { "script" :
"doc['string'].stringValue" } }, "sort" : ["test1"]}]]]; nested:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure [No
mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type ] [Starshine]
[foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]:
Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4
]
org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [Failed to parse [{ "from" : 0,
"size" : 3, "fields" : ["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" } }, "sort" :
["test1"]}]]
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at
org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [No mapping found for [test1]]
at
org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at
org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on
scripted field?

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <
shay.banon@elasticsearch.com> wrote:

Hi,

I think the problem is with curl, and the fact that within the
body of the request the ' is used (within the script). It needs to be
escaped somehow (not sure how).

-shay.banon

On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <
lukas.vlcek@gmail.com> wrote:

Hi,

I am experiencing script_fields issue. Below is a shell script
with curl examples. In short: I prepare an index, index three objects into
it, verify that I can query objects from index and that I can extract values
of particular "fields" (though script_fields does not require fields being
stored) and then use a script_fields query which fails. Not sure what is
wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XDELETE 'http://localhost:9200/foo/'
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : {
number_of_shards : 1, number_of_replicas : 0 }}'
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" }
}
}}'
curl -XGET 'http://localhost:9200/foo/bar/_mapping?pretty=1'
echo

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d
'{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d
'{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d
'{"date":"2010-01-03","string":"three"}'
curl -XPOST 'http://localhost:9200/foo/_refresh'
echo

echo Search
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d '
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
}
}'
echo

The following is output:

Index setup

{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
"foo" : {
"bar" : {
"_boost" : {
"name" : "_boost"
},
"dynamic" : true,
"enabled" : true,
"date_formats" : [ "dateOptionalTime", "yyyy/MM/dd
HH:mm:ss||yyyy/MM/dd" ],
"_source" : {
"enabled" : true,
"name" : "_source"
},
"_id" : {
"store" : "no"
},
"path" : "full",
"properties" : {
"string" : {
"omit_term_freq_and_positions" : false,
"index_name" : "string",
"index" : "not_analyzed",
"omit_norms" : false,
"store" : "yes",
"boost" : 1.0,
"term_vector" : "no",
"type" : "string"
},
"date" : {
"omit_term_freq_and_positions" : true,
"index_name" : "date",
"index" : "not_analyzed",
"omit_norms" : true,
"store" : "yes",
"boost" : 1.0,
"format" : "dateOptionalTime",
"precision_step" : 4,
"term_vector" : "no",
"type" : "date"
}
},
"_all" : {
"enabled" : true,
"store" : "no",
"term_vector" : "no"
},
"type" : "object"
}
}
}
Indexing

{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
"_shards" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [ {
"_index" : "foo",
"_type" : "bar",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"date" : "2010-01-01T00:00:00.000Z",
"string" : "one"
}
} ]
}
}
Search with script_fields

{
"_shards" : {
"total" : 1,
"successful" : 0,
"failed" : 1,
"failures" : [ {
"reason" : "CompileException[[Error: null]\n[Near : {...
Unknown ....}]\n ^\n[Line: 1, Column: 0]]; nested: "
} ]
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" :
}
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata ] [Starshine]
[foo] deleting index
[11:45:12,818][INFO ][cluster.metadata ] [Starshine]
[foo] creating index, cause [api], shards [1]/[0], mappings
[11:45:12,994][INFO ][cluster.metadata ] [Starshine]
[foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type ] [Starshine] [13]
Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
^
[Line: 1, Column: 0]
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at
org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at
org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at
org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at
org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at
org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more

Should I open an issue?

Regards,
Lukas

[kimchy (Shay Banon)]

Agreed. Open an issue, it should be simple to provide complete script based
sorting. Note that sorting get much more complicated in distributed search
since it need to happen twice, both on each shard, and then on the "reduce"
phase.

-shay.banon

On Fri, Jul 30, 2010 at 10:40 AM, Lukáš Vlček lukas.vlcek@gmail.com wrote:

I will do the sorting on the client side because the set of document I need
to sort is small (<30 documents in most cases). I just got impression that
ES could have done it for me in simple and elegant way (again, the pref
would not be a problem due to small number of documents in the set to sort).

However, it seems to me that there is a room for improvement when it comes
to script fields and sorting, and may be sorting in general because it could
be useful to allow users to provide custom sort function as well (
FieldComparatorSourcehttp://lucene.apache.org/java/3_0_2/api/all/org/apache/lucene/search/FieldComparatorSource.html),
a nice-to-have feature for some next release :-).

I red the script_fields docs again and I think I understand now what the
note regarding sorting and faceting means (it means that if the document
field have been already used for faceting or sorting then there is no
additional memory cost when used also in script_fields).

Regards,
Lukas

2010/7/30 Shay Banon shay.banon@elasticsearch.com

I meant possible to implement ;).

I had another look at your problem. Maybe one way to solve it, if you know
in advance the maximum number of levels you will have in the string field,
is to have empty markers for empty levels, that will still perserve sorting?
Adding multiple sort scripts feature can be done, but it will be slower.
First, because score based sorting is faster (which you get with custom
score query), and second because the mentioned scripts will need to be
evaluated for every doc the query hits in order to do the sorting correctly.
This can be mitigated by having more shards and more machines, basically
resulting in distributing the search load across machines (if the perf is
really a problem).

-shay.banon

On Fri, Jul 30, 2010 at 12:15 AM, Shay Banon <
shay.banon@elasticsearch.com> wrote:

Script fields are not using any mappings. It pretty simple what they do,
you get 10 hits, for that 10 hits (docs), the script fields are evaluated.
What you want is to have the ability to sort based on several scripts, which
is possible.

-shay.banon

On Thu, Jul 29, 2010 at 10:09 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

I can not do this at index time because I do not know the length of the
string chain in advance.
Take for example document related to A | 0 entity. How can I know that I
want to right-padd it to A - . - . | 0 without knowing that there are other
related documents having A - B - C ?
As of know it seems that it will be easier to handle that logic on the
client side (I will sort document by client, in my case there won't be a lot
of related documents, I assume up to 30 max).

However, I am still curious if it is possible (or planned) to allow sort
on script_fields. As of now it seems to be possible only if explicit mapping
is provided for script_fields (which means it lacks a lot of flexibility
considering that script_fields are defined per query while mapping per
index).

Regards,
Lukas

2010/7/29 Shay Banon shay.banon@elasticsearch.com

I would, in this case, add another field to the document you index that

does the mentioned logic, index it, and then sort on it.

On Thu, Jul 29, 2010 at 8:22 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

I have the following five entities:

A - D | 2
A - B - C | 3
A | 0
A - B | 1
A - E | 4

There the pattern before pipe is some ordered chain of string symbols.
The pattern after the pipe is a number. Now I want to get the following
order:

A | 0
A - B | 1
A - B - C | 3
A - D | 2
A - E | 4

In fact the part before the pipe is content of multivalued filed
(string) and the part after pipe is a numeric filed.

It would be easy to get desired order if we could sort by the string
chain first and then by the number value. But because simple string sort
would not work on the string chain (because of the variable length and
different number of items in it). Then I think it can be easily done using
scritpt_values (the number of items in string chain - in this case 3 - is
different for each query). In this case I would built four script fileds
f1-4 (three for string and one for number). The result would be (for
original order of entities, I am simply right-padding the chain of strings):

f1 f2 f3 f4

A - D - . | 2
A - B - C | 3
A - . - . | 0
A - B - . | 1
A - E - . | 4

Now we can simply "sort":[f1,f2,f3,f4] and get:

A - . - . | 0
A - B - . | 1
A - B - C | 3
A - D - . | 2
A - E - . | 4

I don't think it is that easy to get this by custom scoring. Is it?
Moreover, docs for script_fields
http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/script_fields/ made
me think that sorting can be used with them.

(Note: In my example it is not really needed to sort by the number
value but that is because I simplified it way too much)

Regards,
Lukas

On Thu, Jul 29, 2010 at 6:28 PM, Shay Banon <
shay.banon@elasticsearch.com> wrote:

I am not sure that you understood what script fields do. They
basically provide a way to fetch custom script evaluation values for hits
returned. They are not related to sorting. If you want to control sorting
based on scripts, the simplest way is to use the custom score query.

Other than that, I got a bit confused, did you have problems with
sorting fields (they don't have to be stored).

-shay.banon

On Thu, Jul 29, 2010 at 6:14 PM, Lukáš Vlček lukas.vlcek@gmail.comwrote:

Or may be the "return_type" should make more sense to be added in
"sort" portion of the query rather then in "script_fields" portion.

Lukas

On Thu, Jul 29, 2010 at 4:52 PM, Lukáš Vlček <lukas.vlcek@gmail.com

wrote:

So, I found that the SortFieldhttp://lucene.apache.org/java/3_0_2/api/all/org/apache/lucene/search/SortField.htmlneeds field type here:
http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118

http://github.com/elasticsearch/elasticsearch/blob/master/modules/elasticsearch/src/main/java/org/elasticsearch/search/query/SortParseElement.java#L118That
is why it is looking for mappings.

OK, I tried to provide mapping for script_fields:
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" },
"test1" : { "type" : "long" },
"test2" : { "type" : "string" }
}
}}'

Now, "sort" : ["test1"] (test1 is date converted to millis) works
(note: I was not able to make it reverse:true or desc) but "sort" :
["test2"] still fails with simple message:
{
"error" : "ReduceSearchPhaseException[Failed to execute phase
[query], [reduce] ]; nested: "
}
(no error in server log).

So, sorting on script_fields does not seem to work without issues
to me. Second issue is that I do not want to provide explicit mapping in
advance (fixing it to specific type) for script_fields. I think this is
little against ES approach because script_fields can be defined per query.

Would it be possible to ask mvel for return type for given script
in order to determine type for sort function on the fly?

Or how about if the client can provide the type directly as part of
script_fields definiton per query. Something like:
"script_fields" : {
"test1" : {
"script" : "some_fancy_math",
"params" : { ... },
"return_type" : "long"
}
}

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:33 PM, Lukáš Vlček <
lukas.vlcek@gmail.com> wrote:

Oh yeah, that's it.
BUT, here comes my original problem:

I changed my last query curl statement to:
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d
@script_fields.json

I put the following query into script_fields.json file:
{
"from" : 0, "size" : 3,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
},
"sort" : ["test1"]
}

Now, the script_fields work as expected but if I try to sort by
them (note the "sort" part) it fails:
{
"error" : "SearchPhaseExecutionException[Failed to execute phase
[query], total failure; shardFailures
{[0e8840a6-f98e-44a6-a66f-5ac08514ce1d][foo][0]:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure
[Failed to parse [{ "from" : 0, "size" : 3, "fields" :
["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" :
"doc['date'].date.millis" }, "test2" : { "script" :
"doc['string'].stringValue" } }, "sort" : ["test1"]}]]]; nested:
SearchParseException[[foo][0]: query[:],from[0],size[3]: Parse Failure [No
mapping found for [test1]]]; }]"
}

On server:
[15:30:14,689][DEBUG][action.search.type ] [Starshine]
[foo][0], node[0e8840a6-f98e-44a6-a66f-5ac08514ce1d], [P], s[STARTED]:
Failed to execute [org.elasticsearch.action.search.SearchRequest@406039a4
]
org.elasticsearch.search.SearchParseException: [foo][0]:
query[:],from[0],size[3]: Parse Failure [Failed to parse [{ "from" : 0,
"size" : 3, "fields" : ["date","string"], "query" : { "match_all" : { } },
"script_fields" : { "test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" } }, "sort" :
["test1"]}]]
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:365)
at
org.elasticsearch.search.SearchService.createContext(SearchService.java:298)
at
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:166)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteQuery(SearchServiceTransportAction.java:129)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.sendExecuteFirstPhase(TransportSearchQueryThenFetchAction.java:77)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:194)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.access$000(TransportSearchTypeAction.java:80)
at
org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction$1.run(TransportSearchTypeAction.java:153)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.elasticsearch.search.SearchParseException:
[foo][0]: query[:],from[0],size[3]: Parse Failure [No mapping found for
[test1]]
at
org.elasticsearch.search.query.SortParseElement.addSortField(SortParseElement.java:116)
at
org.elasticsearch.search.query.SortParseElement.parse(SortParseElement.java:58)
at
org.elasticsearch.search.SearchService.parseSource(SearchService.java:359)
... 10 more

So my original question I wanted to ask was: How to sort on
scripted field?

Regards,
Lukas

On Thu, Jul 29, 2010 at 3:08 PM, Shay Banon <
shay.banon@elasticsearch.com> wrote:

Hi,

I think the problem is with curl, and the fact that within the
body of the request the ' is used (within the script). It needs to be
escaped somehow (not sure how).

-shay.banon

On Thu, Jul 29, 2010 at 12:52 PM, Lukáš Vlček <
lukas.vlcek@gmail.com> wrote:

Hi,

I am experiencing script_fields issue. Below is a shell script
with curl examples. In short: I prepare an index, index three objects into
it, verify that I can query objects from index and that I can extract values
of particular "fields" (though script_fields does not require fields being
stored) and then use a script_fields query which fails. Not sure what is
wrong:

The following is my shell script:
#!/bin/sh

echo Index setup
curl -XDELETE 'http://localhost:9200/foo/'
curl -XPUT 'http://localhost:9200/foo/' -d '{ index : {
number_of_shards : 1, number_of_replicas : 0 }}'
curl -XPUT 'http://localhost:9200/foo/bar/_mapping' -d '
{ "bar" : {
"properties" : {
"date" : { "type" : "date", "store" : "yes" },
"string" : { "type" : "string", "store" : "yes", "index" :
"not_analyzed" }
}
}}'
curl -XGET 'http://localhost:9200/foo/bar/_mapping?pretty=1'
echo

echo Indexing
curl -XPUT 'http://localhost:9200/foo/bar/1' -d
'{"date":"2010-01-01","string":"one"}'
curl -XPUT 'http://localhost:9200/foo/bar/2' -d
'{"date":"2010-01-02","string":"two"}'
curl -XPUT 'http://localhost:9200/foo/bar/3' -d
'{"date":"2010-01-03","string":"three"}'
curl -XPOST 'http://localhost:9200/foo/_refresh'
echo

echo Search
curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d
'
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } }
}'
echo
echo Search with script_fields
echo

curl -XGET 'http://localhost:9200/foo/bar/_search?pretty=1' -d
'
{
"from" : 0, "size" : 1,
"fields" : ["date","string"],
"query" : { "match_all" : { } },
"script_fields" : {
"test1" : { "script" : "doc['date'].date.millis" },
"test2" : { "script" : "doc['string'].stringValue" }
}
}'
echo

The following is output:

Index setup

{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{"ok":true,"acknowledged":true}{
"foo" : {
"bar" : {
"_boost" : {
"name" : "_boost"
},
"dynamic" : true,
"enabled" : true,
"date_formats" : [ "dateOptionalTime", "yyyy/MM/dd
HH:mm:ss||yyyy/MM/dd" ],
"_source" : {
"enabled" : true,
"name" : "_source"
},
"_id" : {
"store" : "no"
},
"path" : "full",
"properties" : {
"string" : {
"omit_term_freq_and_positions" : false,
"index_name" : "string",
"index" : "not_analyzed",
"omit_norms" : false,
"store" : "yes",
"boost" : 1.0,
"term_vector" : "no",
"type" : "string"
},
"date" : {
"omit_term_freq_and_positions" : true,
"index_name" : "date",
"index" : "not_analyzed",
"omit_norms" : true,
"store" : "yes",
"boost" : 1.0,
"format" : "dateOptionalTime",
"precision_step" : 4,
"term_vector" : "no",
"type" : "date"
}
},
"_all" : {
"enabled" : true,
"store" : "no",
"term_vector" : "no"
},
"type" : "object"
}
}
}
Indexing

{"ok":true,"_index":"foo","_type":"bar","_id":"1"}{"ok":true,"_index":"foo","_type":"bar","_id":"2"}{"ok":true,"_index":"foo","_type":"bar","_id":"3"}{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
Search
{
"_shards" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [ {
"_index" : "foo",
"_type" : "bar",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"date" : "2010-01-01T00:00:00.000Z",
"string" : "one"
}
} ]
}
}
Search with script_fields

{
"_shards" : {
"total" : 1,
"successful" : 0,
"failed" : 1,
"failures" : [ {
"reason" : "CompileException[[Error: null]\n[Near : {...
Unknown ....}]\n ^\n[Line: 1, Column: 0]]; nested: "
} ]
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" :
}
}

And the following is what I see in server node log:

[11:45:12,773][INFO ][cluster.metadata ] [Starshine]
[foo] deleting index
[11:45:12,818][INFO ][cluster.metadata ] [Starshine]
[foo] creating index, cause [api], shards [1]/[0], mappings
[11:45:12,994][INFO ][cluster.metadata ] [Starshine]
[foo] create_mapping [bar]
[11:45:13,109][DEBUG][action.search.type ] [Starshine]
[13] Failed to execute fetch phase
[Error: null]
[Near : {... Unknown ....}]
^
[Line: 1, Column: 0]
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:402)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.compiler.ExecutableAccessor.getValue(ExecutableAccessor.java:37)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getCollectionProperty(ReflectiveAccessorOptimizer.java:655)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:332)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.optimizeAccessor(ReflectiveAccessorOptimizer.java:152)
at
org.elasticsearch.common.mvel2.optimizers.dynamic.DynamicOptimizer.optimizeAccessor(DynamicOptimizer.java:63)
at
org.elasticsearch.common.mvel2.ast.ASTNode.getReducedValueAccelerated(ASTNode.java:139)
at
org.elasticsearch.common.mvel2.MVELRuntime.execute(MVELRuntime.java:85)
at
org.elasticsearch.common.mvel2.compiler.CompiledExpression.getValue(CompiledExpression.java:104)
at
org.elasticsearch.common.mvel2.MVEL.executeExpression(MVEL.java:1053)
at
org.elasticsearch.script.ScriptService.execute(ScriptService.java:77)
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.execute(ScriptFieldsFunction.java:79)
at
org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:136)
at
org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:265)
at
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteFetch(SearchServiceTransportAction.java:309)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.executeFetch(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$200(TransportSearchQueryThenFetchAction.java:60)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.run(TransportSearchQueryThenFetchAction.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.lang.UnsupportedOperationException
at
org.elasticsearch.index.field.function.script.ScriptFieldsFunction.containsKey(ScriptFieldsFunction.java:112)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.getBeanProperty(ReflectiveAccessorOptimizer.java:545)
at
org.elasticsearch.common.mvel2.optimizers.impl.refl.ReflectiveAccessorOptimizer.compileGetChain(ReflectiveAccessorOptimizer.java:326)
... 23 more

Should I open an issue?

Regards,
Lukas