Elastic Search Nest Slow Scroll Speed

Hi everyone,
Firstly i sorry for my bad english. I have one node in my server.
My index stats below this :

{
  "_shards" : {
    "total" : 4,
    "successful" : 4,
    "failed" : 0
  },
  "_all" : {
    "primaries" : {
      "docs" : {
        "count" : 1951593,
        "deleted" : 0
      },
      "store" : {
        "size_in_bytes" : 594806126,
        "reserved_in_bytes" : 0
      },
      "indexing" : {
        "index_total" : 0,
        "index_time_in_millis" : 0,
        "index_current" : 0,
        "index_failed" : 0,
        "delete_total" : 0,
        "delete_time_in_millis" : 0,
        "delete_current" : 0,
        "noop_update_total" : 0,
        "is_throttled" : false,
        "throttle_time_in_millis" : 0
      },
      "get" : {
        "total" : 0,
        "time_in_millis" : 0,
        "exists_total" : 0,
        "exists_time_in_millis" : 0,
        "missing_total" : 0,
        "missing_time_in_millis" : 0,
        "current" : 0
      },
      "search" : {
        "open_contexts" : 0,
        "query_total" : 5900,
        "query_time_in_millis" : 5539,
        "query_current" : 0,
        "fetch_total" : 1464,
        "fetch_time_in_millis" : 488659,
        "fetch_current" : 0,
        "scroll_total" : 44,
        "scroll_time_in_millis" : 6017059,
        "scroll_current" : 0,
        "suggest_total" : 0,
        "suggest_time_in_millis" : 0,
        "suggest_current" : 0
      },
      "merges" : {
        "current" : 0,
        "current_docs" : 0,
        "current_size_in_bytes" : 0,
        "total" : 0,
        "total_time_in_millis" : 0,
        "total_docs" : 0,
        "total_size_in_bytes" : 0,
        "total_stopped_time_in_millis" : 0,
        "total_throttled_time_in_millis" : 0,
        "total_auto_throttle_in_bytes" : 83886080
      },
      "refresh" : {
        "total" : 8,
        "total_time_in_millis" : 0,
        "external_total" : 8,
        "external_total_time_in_millis" : 0,
        "listeners" : 0
      },
      "flush" : {
        "total" : 4,
        "periodic" : 0,
        "total_time_in_millis" : 0
      },
      "warmer" : {
        "current" : 0,
        "total" : 4,
        "total_time_in_millis" : 0
      },
      "query_cache" : {
        "memory_size_in_bytes" : 0,
        "total_count" : 1668,
        "hit_count" : 0,
        "miss_count" : 1668,
        "cache_size" : 0,
        "cache_count" : 0,
        "evictions" : 0
      },
      "fielddata" : {
        "memory_size_in_bytes" : 0,
        "evictions" : 0
      },
      "completion" : {
        "size_in_bytes" : 0
      },
      "segments" : {
        "count" : 45,
        "memory_in_bytes" : 317076,
        "terms_memory_in_bytes" : 174240,
        "stored_fields_memory_in_bytes" : 22984,
        "term_vectors_memory_in_bytes" : 0,
        "norms_memory_in_bytes" : 25920,
        "points_memory_in_bytes" : 0,
        "doc_values_memory_in_bytes" : 93932,
        "index_writer_memory_in_bytes" : 0,
        "version_map_memory_in_bytes" : 0,
        "fixed_bit_set_memory_in_bytes" : 0,
        "max_unsafe_auto_id_timestamp" : -1,
        "file_sizes" : { }
      },
      "translog" : {
        "operations" : 0,
        "size_in_bytes" : 220,
        "uncommitted_operations" : 0,
        "uncommitted_size_in_bytes" : 220,
        "earliest_last_modified_age" : 5016454
      },
      "request_cache" : {
        "memory_size_in_bytes" : 0,
        "evictions" : 0,
        "hit_count" : 0,
        "miss_count" : 0
      },
      "recovery" : {
        "current_as_source" : 0,
        "current_as_target" : 0,
        "throttle_time_in_millis" : 0
      }
    }

and my mappings below this :

{
  "marka_listesi" : {
    "mappings" : {
      "properties" : {
        "basvuruNo" : {
          "type" : "keyword"
        },
        "basvuruTarihi" : {
          "type" : "date"
        },
        "durum" : {
          "type" : "text",
          "analyzer" : "turkish_analyzer"
        },
        "evrakNo" : {
          "type" : "keyword"
        },
        "ilanBultenNo" : {
          "type" : "keyword"
        },
        "ilanBultenTarihi" : {
          "type" : "date"
        },
        "korumaTarihi" : {
          "type" : "date"
        },
        "markaAdi" : {
          "type" : "text",
          "analyzer" : "turkish_analyzer"
        },
        "markaID" : {
          "type" : "integer"
        },
        "markaLogoUrl" : {
          "type" : "text"
        },
        "sahip" : {
          "type" : "text",
          "analyzer" : "turkish_analyzer"
        },
        "siniflar" : {
          "type" : "keyword"
        },
        "sonDegisiklik" : {
          "type" : "date"
        },
        "sonIslem" : {
          "type" : "text",
          "analyzer" : "turkish_analyzer"
        },
        "sonIslemBaslik" : {
          "type" : "text",
          "analyzer" : "turkish_analyzer"
        },
        "sonIslemTarihi" : {
          "type" : "date"
        },
        "tescilBultenNo" : {
          "type" : "keyword"
        },
        "tescilBultenTarihi" : {
          "type" : "date"
        },
        "tescilNo" : {
          "type" : "keyword"
        },
        "tescilTarihi" : {
          "type" : "date"
        },
        "tur" : {
          "type" : "text",
          "analyzer" : "turkish_analyzer"
        },
        "uTescilNo" : {
          "type" : "keyword"
        },
        "vekil" : {
          "type" : "text",
          "analyzer" : "turkish_analyzer"
        },
        "vekilFirma" : {
          "type" : "text",
          "analyzer" : "turkish_analyzer"
        }
      }
    }
  }
}

This index have 2 million record but size average 400-500 mb.
I need get all document in index with dynamic filter and compare in my class method. I'am using scroll.all api from nest but documents coming and adding all documents to my model list very very slow(1 minutes). I seen Maximum cpu percentage for query processing 20%.
I need several second for query complete. This is my nest query :

public static List<Models.Model.Sonuc.MarkaListesi>GetDocumentsOnSelectedClasses(string _indexName,ElasticClient _client,string[] _siniflar)
        {

            
            var _docs = new List<Models.Model.Sonuc.MarkaListesi>();

            // number of slices in slice scroll

            var scrollObserver = _client.ScrollAll<Models.Model.Sonuc.MarkaListesi>("1m", 4, s => s
                .MaxDegreeOfParallelism(4)
                .Search(se => se
                    .Index(_indexName)
                    .Query(q => q.Bool(b => b.Filter(fq=>fq.Terms(t=>t.Field("siniflar").Terms(_siniflar)))))
                    .Size(1000)
                )
            );
            var waitHandle = new ManualResetEvent(false);
            Exception exception = null;

            var scrollAllObserver = new ScrollAllObserver<Models.Model.Sonuc.MarkaListesi>(
                onNext: response =>
                {
        // do something with the documents
        _docs.AddRange(response.SearchResponse.Documents);
                },
                onError: e =>
                {
                    exception = e;
                    waitHandle.Set();
                },
                onCompleted: () => waitHandle.Set()
            );


            scrollObserver.Subscribe(scrollAllObserver);

            waitHandle.WaitOne();

            if (exception != null)
            {
                throw exception;
            }
            return _docs;
            

        }

Finally my question is : how can i increase my search speed ?

Query performance can also be limited by the heap size or the performance of the storage.

What is your heap size? Do you see anything around long or frequent garbage collection in the Elasticsearch logs?

What type of storage are you using? Can you check disk I/O and iowait while you are running a query?

My Heap Size : -Xms10g -Xmx10g
And Memory Lock : bootstrap.memory_lock: true

Server have: 64GB Ram and 1TB SSD
Disk usage for query process very low (Under : 100KB/SEC)

How many shards are you searching?

Index have 4 shard

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.