An issue , in implmenting the scroll function

[Bettaieb_Walid]

hello ,
i am developing a scroll api , in order to be able to fetch a large amount of data , i am able to console log the scroll id , but i am having an empty response , here is my code snippet ```const { Client } = require('@elastic/elasticsearch');

const client = new Client({
node: 'http: // localhost: 9200', // Replace with your Elasticsearch endpoint
// Add authentication options if necessary
});

// POST /search
exports.search = async (req, res) => {
try {
const { attribute, measuringPoint, beginDate, endDate } = req.body; // Get the selected attribute, measuring point, begin date, and end date from the request body

// Check if beginDate and endDate are valid date values
if (!isValidDate(beginDate) || !isValidDate(endDate)) {
  throw new Error('Invalid date format');
}

let query = {
  bool: {
    must: [
      {
        exists: {
          field: attribute
        }
      },
      {
        range: {
          createdAt: {
            gte: beginDate,
            lte: endDate
          }
        }
      }
    ]
  }
};

if (measuringPoint) {
  query.bool.must.push({
    match: {
      measuringPoint: measuringPoint
    }
  });
}

const initialSearchResponse = await client.search({
  index: 'index', // Replace with your index name
  body: {
    query: query,
    size: 1000, // Set the initial batch size
    sort: [{ createdAt: { order: 'asc' } }] // Sort by createdAt field in ascending order
  },
  scroll: '10s' // Enable scrolling and set the scroll time
});

console.log("Initial search response:", initialSearchResponse);

let formattedData = [];

let scrollId = initialSearchResponse._scroll_id;

while (true) {
  const hits = initialSearchResponse.hits.hits;

  console.log("Processing", hits.length, "hits...");

  formattedData.push(
    ...hits.map(hit => {
      const source = hit._source;
      const value = source[attribute];

      // Check if the value exists and is not null or empty
      if (value !== undefined && value !== null && value !== '') {
        return {
          phase: source.phase || '',
          name: attribute,
          measuringPoint: source.measuringPoint || '',
          value: parseFloat(value), // Convert value to a number
          createdAt: source.createdAt || '' // Include the createdAt attribute
        };
      }
      return null;
    }).filter(data => data !== null)
  );

  console.log("Formatted data size:", formattedData.length);

  if (hits.length === 0) {
    // If there are no more hits to retrieve, break the loop
    break;
  }

  const scrollResponse = await client.scroll({
    scroll: '10s',
    scroll_id: scrollId
  });

  console.log("Scroll response:", scrollResponse);

  scrollId = scrollResponse.body._scroll_id;
}

console.log("Final formatted data:", formattedData);
res.json(formattedData);

} catch (error) {
console.error(error);
res.status(500).json({ error: 'An error occurred while fetching data from Elasticsearch.' });
}
};

// Helper function to validate the date format
function isValidDate(dateString) {
const dateRegex = /^\d{4}-\d{2}-\d{2}$/;
return dateRegex.test(dateString);
}

and i would like , also, taking into consideration , that this code snippet , is working , implementing a simple serach with hardcoded 100000 values , ( 

const { Client } = require('@elastic/elasticsearch');

const client = new Client({
node: 'http :// localhost: 9200', });

// POST /search
exports.search = async (req, res) => {
try {
const { attribute, measuringPoint, beginDate, endDate } = req.body; // Get the selected attribute, measuring point, begin date, and end date from the request body

let query = {
  bool: {
    must: [
      {
        exists: {
          field: attribute
        }
      },
      {
        range: {
          createdAt: {
            gte: beginDate,
            lte: endDate
          }
        }
      }
    ]
  }
};

if (measuringPoint) {
  query.bool.must.push({
    match: {
      measuringPoint: measuringPoint
    }
  });
}

const response = await client.search({
  index: 'index',
  body: {
    query: query,
    size: 10000, // Retrieve 1000 files
    sort: [
      { createdAt: { order: 'asc' } } // Sort by createdAt field in ascending order
    ]
  }
});

const formattedData = response.hits.hits.map(hit => {
  const source = hit._source;
  return {
    phase: source.name,
    name: attribute,
    measuringPoint: source.measuringPoint,
    value: source[attribute],
    createdAt: source.createdAt // Include the createdAt attribute
  };
}).filter(data => data.value !== null);

console.log("This is the formatted data:", formattedData);
res.json(formattedData);

} catch (error) {
console.error(error);
res.status(500).json({ error: 'An error occurred while fetching data from Elasticsearch.' });
}
};

how to solve this issue here ? and thank you in advance !