Scan and Scroll Problem

Hello,

here is my start scroll method.

public SearchResponse startScroll(String searchText, long journalId, String
indexName) {

    QueryBuilder queryBuilder = QueryBuilders.queryString(searchText)
            .field("email").field("firstName")
            .field("lastName").field("authorOfAbstracts")
            .defaultOperator(QueryStringQueryBuilder.Operator.AND);

    SearchRequestBuilder searchRequestBuilder = 

client.prepareSearch(indexName);
searchRequestBuilder.setSearchType(SearchType.SCAN);
searchRequestBuilder.setQuery(queryBuilder);
searchRequestBuilder.setSize(BATCH_SIZE);
searchRequestBuilder.setExplain(false);
searchRequestBuilder.setNoFields();
searchRequestBuilder.setVersion(true);

searchRequestBuilder.setScroll(TimeValue.timeValueMillis(SCROLL_TIME_IN_MILIS));

    return searchRequestBuilder.execute().actionGet();
}

and startIteration and returning the List of searchResponse

public List<SearchResponse> startIterationOnScroll(String scrollId){
    List<SearchResponse> responseList = new ArrayList<SearchResponse>();
    SearchResponse batchSearchResponse = null;
    do {
        batchSearchResponse = 

client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMillis(SCROLL_TIME_IN_MILIS)).execute().actionGet();
if(batchSearchResponse.getHits().getHits().length > 0)
responseList.add(batchSearchResponse);
//System.out.println("hit : " +
batchSearchResponse.getHits().getHits().length);
scrollId = batchSearchResponse.getScrollId();
} while(batchSearchResponse.getHits().getHits().length > 0);

    return responseList;
}

now once i have the list of responses i am converting all the results to
java object as below code using jackson as david's code.
but i am getting hit.getSourceAsString() null all the time in interation.
i do have data in SearchResponse as below.

Total Hits : 2
contact size : {
"_scroll_id" : "c2NhbjswOzE7dG90YWxfaGl0czoyOw==",
"took" : 21,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"failed" : 0
},
"hits" : {
"total" : 2,
"max_score" : 0.0,
"hits" : [ {
"_index" : "contactsearch",
"_type" : "2037",
"_id" : "9959080154290929",
"_version" : 1,
"_score" : 0.0
}, {
"_index" : "contactsearch",
"_type" : "9001",
"_id" : "9959080154290929",
"_version" : 1,
"_score" : 0.0
} ]
}
}

public List getContactList(List searchResponses)
throws IOException {
List contactList = new ArrayList();
String source ="";
for(SearchResponse searchResponse : searchResponses)
{
for(SearchHit hit : searchResponse.getHits()){
source = hit.getSourceAsString();
if(source != null)
{
Contact contact = fromJSON(source);
contactList.add(contact);
}
}
System.out.println("contact size : " + searchResponse);
}

    return contactList;
}

private Contact fromJSON(String json) throws IOException {
    ObjectMapper mapper = new ObjectMapper();
    return mapper.readValue(json.getBytes(), Contact.class);
}

please please suggest some help

Thanks

--

This line prevents elasticsearch from returning any fields in the
response except id and type: searchRequestBuilder.setNoFields();

On Thursday, November 8, 2012 6:16:42 AM UTC-5, mohsin husen wrote:

Hello,

here is my start scroll method.

public SearchResponse startScroll(String searchText, long journalId,
String indexName) {

    QueryBuilder queryBuilder = QueryBuilders.queryString(searchText)
            .field("email").field("firstName")
            .field("lastName").field("authorOfAbstracts")
            .defaultOperator(QueryStringQueryBuilder.Operator.AND);

    SearchRequestBuilder searchRequestBuilder = 

client.prepareSearch(indexName);
searchRequestBuilder.setSearchType(SearchType.SCAN);
searchRequestBuilder.setQuery(queryBuilder);
searchRequestBuilder.setSize(BATCH_SIZE);
searchRequestBuilder.setExplain(false);
searchRequestBuilder.setNoFields();
searchRequestBuilder.setVersion(true);

searchRequestBuilder.setScroll(TimeValue.timeValueMillis(SCROLL_TIME_IN_MILIS));

    return searchRequestBuilder.execute().actionGet();
}

and startIteration and returning the List of searchResponse

public List<SearchResponse> startIterationOnScroll(String scrollId){
    List<SearchResponse> responseList = new 

ArrayList();
SearchResponse batchSearchResponse = null;
do {
batchSearchResponse =
client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMillis(SCROLL_TIME_IN_MILIS)).execute().actionGet();
if(batchSearchResponse.getHits().getHits().length > 0)
responseList.add(batchSearchResponse);
//System.out.println("hit : " +
batchSearchResponse.getHits().getHits().length);
scrollId = batchSearchResponse.getScrollId();
} while(batchSearchResponse.getHits().getHits().length > 0);

    return responseList;
}

now once i have the list of responses i am converting all the results to
java object as below code using jackson as david's code.
but i am getting hit.getSourceAsString() null all the time in interation.
i do have data in SearchResponse as below.

Total Hits : 2
contact size : {
"_scroll_id" : "c2NhbjswOzE7dG90YWxfaGl0czoyOw==",
"took" : 21,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"failed" : 0
},
"hits" : {
"total" : 2,
"max_score" : 0.0,
"hits" : [ {
"_index" : "contactsearch",
"_type" : "2037",
"_id" : "9959080154290929",
"_version" : 1,
"_score" : 0.0
}, {
"_index" : "contactsearch",
"_type" : "9001",
"_id" : "9959080154290929",
"_version" : 1,
"_score" : 0.0
} ]
}
}

public List getContactList(List searchResponses)
throws IOException {
List contactList = new ArrayList();
String source ="";
for(SearchResponse searchResponse : searchResponses)
{
for(SearchHit hit : searchResponse.getHits()){
source = hit.getSourceAsString();
if(source != null)
{
Contact contact = fromJSON(source);
contactList.add(contact);
}
}
System.out.println("contact size : " + searchResponse);
}

    return contactList;
}

private Contact fromJSON(String json) throws IOException {
    ObjectMapper mapper = new ObjectMapper();
    return mapper.readValue(json.getBytes(), Contact.class);
}

please please suggest some help

Thanks

--

stupid meeeeeeee !
Thanks Igor :slight_smile:

On Friday, 9 November 2012 18:34:06 UTC, Igor Motov wrote:

This line prevents elasticsearch from returning any fields in the
response except id and type: searchRequestBuilder.setNoFields();

On Thursday, November 8, 2012 6:16:42 AM UTC-5, mohsin husen wrote:

Hello,

here is my start scroll method.

public SearchResponse startScroll(String searchText, long journalId,
String indexName) {

    QueryBuilder queryBuilder = QueryBuilders.queryString(searchText)
            .field("email").field("firstName")
            .field("lastName").field("authorOfAbstracts")
            .defaultOperator(QueryStringQueryBuilder.Operator.AND);

    SearchRequestBuilder searchRequestBuilder = 

client.prepareSearch(indexName);
searchRequestBuilder.setSearchType(SearchType.SCAN);
searchRequestBuilder.setQuery(queryBuilder);
searchRequestBuilder.setSize(BATCH_SIZE);
searchRequestBuilder.setExplain(false);
searchRequestBuilder.setNoFields();
searchRequestBuilder.setVersion(true);

searchRequestBuilder.setScroll(TimeValue.timeValueMillis(SCROLL_TIME_IN_MILIS));

    return searchRequestBuilder.execute().actionGet();
}

and startIteration and returning the List of searchResponse

public List<SearchResponse> startIterationOnScroll(String scrollId){
    List<SearchResponse> responseList = new 

ArrayList();
SearchResponse batchSearchResponse = null;
do {
batchSearchResponse =
client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMillis(SCROLL_TIME_IN_MILIS)).execute().actionGet();
if(batchSearchResponse.getHits().getHits().length > 0)
responseList.add(batchSearchResponse);
//System.out.println("hit : " +
batchSearchResponse.getHits().getHits().length);
scrollId = batchSearchResponse.getScrollId();
} while(batchSearchResponse.getHits().getHits().length > 0);

    return responseList;
}

now once i have the list of responses i am converting all the results to
java object as below code using jackson as david's code.
but i am getting hit.getSourceAsString() null all the time in interation.
i do have data in SearchResponse as below.

Total Hits : 2
contact size : {
"_scroll_id" : "c2NhbjswOzE7dG90YWxfaGl0czoyOw==",
"took" : 21,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"failed" : 0
},
"hits" : {
"total" : 2,
"max_score" : 0.0,
"hits" : [ {
"_index" : "contactsearch",
"_type" : "2037",
"_id" : "9959080154290929",
"_version" : 1,
"_score" : 0.0
}, {
"_index" : "contactsearch",
"_type" : "9001",
"_id" : "9959080154290929",
"_version" : 1,
"_score" : 0.0
} ]
}
}

public List getContactList(List searchResponses)
throws IOException {
List contactList = new ArrayList();
String source ="";
for(SearchResponse searchResponse : searchResponses)
{
for(SearchHit hit : searchResponse.getHits()){
source = hit.getSourceAsString();
if(source != null)
{
Contact contact = fromJSON(source);
contactList.add(contact);
}
}
System.out.println("contact size : " + searchResponse);
}

    return contactList;
}

private Contact fromJSON(String json) throws IOException {
    ObjectMapper mapper = new ObjectMapper();
    return mapper.readValue(json.getBytes(), Contact.class);
}

please please suggest some help

Thanks

--