Hi everyone,
Firstly i sorry for my bad english. I have one node in my server.
My index stats below this :
{
"_shards" : {
"total" : 4,
"successful" : 4,
"failed" : 0
},
"_all" : {
"primaries" : {
"docs" : {
"count" : 1951593,
"deleted" : 0
},
"store" : {
"size_in_bytes" : 594806126,
"reserved_in_bytes" : 0
},
"indexing" : {
"index_total" : 0,
"index_time_in_millis" : 0,
"index_current" : 0,
"index_failed" : 0,
"delete_total" : 0,
"delete_time_in_millis" : 0,
"delete_current" : 0,
"noop_update_total" : 0,
"is_throttled" : false,
"throttle_time_in_millis" : 0
},
"get" : {
"total" : 0,
"time_in_millis" : 0,
"exists_total" : 0,
"exists_time_in_millis" : 0,
"missing_total" : 0,
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"open_contexts" : 0,
"query_total" : 5900,
"query_time_in_millis" : 5539,
"query_current" : 0,
"fetch_total" : 1464,
"fetch_time_in_millis" : 488659,
"fetch_current" : 0,
"scroll_total" : 44,
"scroll_time_in_millis" : 6017059,
"scroll_current" : 0,
"suggest_total" : 0,
"suggest_time_in_millis" : 0,
"suggest_current" : 0
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size_in_bytes" : 0,
"total" : 0,
"total_time_in_millis" : 0,
"total_docs" : 0,
"total_size_in_bytes" : 0,
"total_stopped_time_in_millis" : 0,
"total_throttled_time_in_millis" : 0,
"total_auto_throttle_in_bytes" : 83886080
},
"refresh" : {
"total" : 8,
"total_time_in_millis" : 0,
"external_total" : 8,
"external_total_time_in_millis" : 0,
"listeners" : 0
},
"flush" : {
"total" : 4,
"periodic" : 0,
"total_time_in_millis" : 0
},
"warmer" : {
"current" : 0,
"total" : 4,
"total_time_in_millis" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 0,
"total_count" : 1668,
"hit_count" : 0,
"miss_count" : 1668,
"cache_size" : 0,
"cache_count" : 0,
"evictions" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 45,
"memory_in_bytes" : 317076,
"terms_memory_in_bytes" : 174240,
"stored_fields_memory_in_bytes" : 22984,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 25920,
"points_memory_in_bytes" : 0,
"doc_values_memory_in_bytes" : 93932,
"index_writer_memory_in_bytes" : 0,
"version_map_memory_in_bytes" : 0,
"fixed_bit_set_memory_in_bytes" : 0,
"max_unsafe_auto_id_timestamp" : -1,
"file_sizes" : { }
},
"translog" : {
"operations" : 0,
"size_in_bytes" : 220,
"uncommitted_operations" : 0,
"uncommitted_size_in_bytes" : 220,
"earliest_last_modified_age" : 5016454
},
"request_cache" : {
"memory_size_in_bytes" : 0,
"evictions" : 0,
"hit_count" : 0,
"miss_count" : 0
},
"recovery" : {
"current_as_source" : 0,
"current_as_target" : 0,
"throttle_time_in_millis" : 0
}
}
and my mappings below this :
{
"marka_listesi" : {
"mappings" : {
"properties" : {
"basvuruNo" : {
"type" : "keyword"
},
"basvuruTarihi" : {
"type" : "date"
},
"durum" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"evrakNo" : {
"type" : "keyword"
},
"ilanBultenNo" : {
"type" : "keyword"
},
"ilanBultenTarihi" : {
"type" : "date"
},
"korumaTarihi" : {
"type" : "date"
},
"markaAdi" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"markaID" : {
"type" : "integer"
},
"markaLogoUrl" : {
"type" : "text"
},
"sahip" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"siniflar" : {
"type" : "keyword"
},
"sonDegisiklik" : {
"type" : "date"
},
"sonIslem" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"sonIslemBaslik" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"sonIslemTarihi" : {
"type" : "date"
},
"tescilBultenNo" : {
"type" : "keyword"
},
"tescilBultenTarihi" : {
"type" : "date"
},
"tescilNo" : {
"type" : "keyword"
},
"tescilTarihi" : {
"type" : "date"
},
"tur" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"uTescilNo" : {
"type" : "keyword"
},
"vekil" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
},
"vekilFirma" : {
"type" : "text",
"analyzer" : "turkish_analyzer"
}
}
}
}
}
This index have 2 million record but size average 400-500 mb.
I need get all document in index with dynamic filter and compare in my class method. I'am using scroll.all api from nest but documents coming and adding all documents to my model list very very slow(1 minutes). I seen Maximum cpu percentage for query processing 20%.
I need several second for query complete. This is my nest query :
public static List<Models.Model.Sonuc.MarkaListesi>GetDocumentsOnSelectedClasses(string _indexName,ElasticClient _client,string[] _siniflar)
{
var _docs = new List<Models.Model.Sonuc.MarkaListesi>();
// number of slices in slice scroll
var scrollObserver = _client.ScrollAll<Models.Model.Sonuc.MarkaListesi>("1m", 4, s => s
.MaxDegreeOfParallelism(4)
.Search(se => se
.Index(_indexName)
.Query(q => q.Bool(b => b.Filter(fq=>fq.Terms(t=>t.Field("siniflar").Terms(_siniflar)))))
.Size(1000)
)
);
var waitHandle = new ManualResetEvent(false);
Exception exception = null;
var scrollAllObserver = new ScrollAllObserver<Models.Model.Sonuc.MarkaListesi>(
onNext: response =>
{
// do something with the documents
_docs.AddRange(response.SearchResponse.Documents);
},
onError: e =>
{
exception = e;
waitHandle.Set();
},
onCompleted: () => waitHandle.Set()
);
scrollObserver.Subscribe(scrollAllObserver);
waitHandle.WaitOne();
if (exception != null)
{
throw exception;
}
return _docs;
}
Finally my question is : how can i increase my search speed ?