# An issue , in implmenting the scroll function

**URL:** <https://discuss.elastic.co/t/an-issue-in-implmenting-the-scroll-function/335813>\
**Category:** Elasticsearch\
**Tags:** docker\
**Created:** [June 12, 2023, 9:48pm UTC](https://discuss.elastic.co/t/an-issue-in-implmenting-the-scroll-function/335813 "2023-06-12T21:48:43Z")\
**Posts on this page:** 2\
**Page:** 1

<div class="post-metadata">

**Author:** ![Bettaieb\_Walid](https://sea2.discourse-cdn.com/elastic/user_avatar/discuss.elastic.co/bettaieb_walid/32/121149_2.png) [@Bettaieb\_Walid](https://discuss.elastic.co/u/Bettaieb_Walid)\
**Post date:** [June 12, 2023, 9:48pm UTC](https://discuss.elastic.co/t/an-issue-in-implmenting-the-scroll-function/335813/1 "2023-06-12T21:48:43Z")

</div>

hello ,  
i am developing a scroll api , in order to be able to fetch a large amount of data , i am able to console log the scroll id , but i am having an empty response , here is my code snippet ```const { Client } = require('@elastic/elasticsearch');

const client = new Client({  
node: 'http: // localhost: 9200', // Replace with your Elasticsearch endpoint  
// Add authentication options if necessary  
});

// POST /search  
exports.search = async (req, res) =\> {  
try {  
const { attribute, measuringPoint, beginDate, endDate } = req.body; // Get the selected attribute, measuring point, begin date, and end date from the request body

```
// Check if beginDate and endDate are valid date values
if (!isValidDate(beginDate) || !isValidDate(endDate)) {
  throw new Error('Invalid date format');
}

let query = {
  bool: {
    must: [
      {
        exists: {
          field: attribute
        }
      },
      {
        range: {
          createdAt: {
            gte: beginDate,
            lte: endDate
          }
        }
      }
    ]
  }
};

if (measuringPoint) {
  query.bool.must.push({
    match: {
      measuringPoint: measuringPoint
    }
  });
}

const initialSearchResponse = await client.search({
  index: 'index', // Replace with your index name
  body: {
    query: query,
    size: 1000, // Set the initial batch size
    sort: [{ createdAt: { order: 'asc' } }] // Sort by createdAt field in ascending order
  },
  scroll: '10s' // Enable scrolling and set the scroll time
});

console.log("Initial search response:", initialSearchResponse);

let formattedData = [];

let scrollId = initialSearchResponse._scroll_id;

while (true) {
  const hits = initialSearchResponse.hits.hits;

  console.log("Processing", hits.length, "hits...");

  formattedData.push(
    ...hits.map(hit => {
      const source = hit._source;
      const value = source[attribute];

      // Check if the value exists and is not null or empty
      if (value !== undefined && value !== null && value !== '') {
        return {
          phase: source.phase || '',
          name: attribute,
          measuringPoint: source.measuringPoint || '',
          value: parseFloat(value), // Convert value to a number
          createdAt: source.createdAt || '' // Include the createdAt attribute
        };
      }
      return null;
    }).filter(data => data !== null)
  );

  console.log("Formatted data size:", formattedData.length);

  if (hits.length === 0) {
    // If there are no more hits to retrieve, break the loop
    break;
  }

  const scrollResponse = await client.scroll({
    scroll: '10s',
    scroll_id: scrollId
  });

  console.log("Scroll response:", scrollResponse);

  scrollId = scrollResponse.body._scroll_id;
}

console.log("Final formatted data:", formattedData);
res.json(formattedData);

```

} catch (error) {  
console.error(error);  
res.status(500).json({ error: 'An error occurred while fetching data from Elasticsearch.' });  
}  
};

// Helper function to validate the date format  
function isValidDate(dateString) {  
const dateRegex = /^\d{4}-\d{2}-\d{2}$/;  
return dateRegex.test(dateString);  
}

```auto
and i would like , also, taking into consideration , that this code snippet , is working , implementing a simple serach with hardcoded 100000 values , ( 

```

const { Client } = require('@elastic/elasticsearch');

const client = new Client({  
node: 'http :// localhost: 9200', });

// POST /search  
exports.search = async (req, res) =\> {  
try {  
const { attribute, measuringPoint, beginDate, endDate } = req.body; // Get the selected attribute, measuring point, begin date, and end date from the request body

```
let query = {
  bool: {
    must: [
      {
        exists: {
          field: attribute
        }
      },
      {
        range: {
          createdAt: {
            gte: beginDate,
            lte: endDate
          }
        }
      }
    ]
  }
};

if (measuringPoint) {
  query.bool.must.push({
    match: {
      measuringPoint: measuringPoint
    }
  });
}

const response = await client.search({
  index: 'index',
  body: {
    query: query,
    size: 10000, // Retrieve 1000 files
    sort: [
      { createdAt: { order: 'asc' } } // Sort by createdAt field in ascending order
    ]
  }
});

const formattedData = response.hits.hits.map(hit => {
  const source = hit._source;
  return {
    phase: source.name,
    name: attribute,
    measuringPoint: source.measuringPoint,
    value: source[attribute],
    createdAt: source.createdAt // Include the createdAt attribute
  };
}).filter(data => data.value !== null);

console.log("This is the formatted data:", formattedData);
res.json(formattedData);

```

} catch (error) {  
console.error(error);  
res.status(500).json({ error: 'An error occurred while fetching data from Elasticsearch.' });  
}  
};

```auto
how to solve this issue here ? and thank you in advance !
```

---

<div class="post-metadata">

**Author:** ![system](https://us1.discourse-cdn.com/elastic/original/3X/1/a/1ac57faf039f6b580b3f104ef42a2a89e41014de.png) [@system](https://discuss.elastic.co/u/system)\
**Post date:** [July 10, 2023, 9:48pm UTC](https://discuss.elastic.co/t/an-issue-in-implmenting-the-scroll-function/335813/2 "2023-07-10T21:48:56Z")

</div>

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.