Express.js not sending compressed files to front-end after compression enabled

[Akaash_Mukherjee]

I have an express.js server that's pulling data from Elasticsearch and serving it to the browser. I was under the impression that all I needed to do for the express app to send compressed responses was activating compression on the express app with the npm compression library. I did that, but when I check my browser, the response header from the server does not have Content-Encoding:gzip. Below I've shown my server set up and below that the request and response from chrome devtools on the client side. Any help would be greatly appreciated!

import {} from 'dotenv/config';
import logger from 'morgan';
import cookieParser from 'cookie-parser';
import bodyParser from 'body-parser';
import express from 'express';
import cors from 'cors';
import compression from 'compression';
import routes from './api';
import { hydrate } from './handlers/Hydrate';
import {
  onAuthorization,
  onNotFound,
  onError,
  onStart,
} from './utils/Server';


Context.enabled = true;


/**
 * HTTP ROUTER
 */

const app = express();

app.use(compression({ threshold: 0 }));

// add cors policy
app.use(cors({
  methods: ['POST'],
  origin: process.env.NODE_ENV === 'development' ? '*' : 'paragon-erp.com',
  maxAge: 600, // caches the results of the preflight -  helps avoid excessive preflight requests
}));
// 'cors' package only seem to set the necessary cors headers for preflight requests
// this makes sure it is added to the actual request
app.use((req, res, next) => {
  res.header('Access-Control-Allow-Origin', process.env.NODE_ENV === 'development' ? '*' : 'paragon-erp.com');
  res.header('Access-Control-Allow-Methods', 'POST');

  next();
});

app.use(logger('dev'));
app.use(cookieParser());

// parse application/x-www-form-urlencoded
app.use(bodyParser.urlencoded({ extended: true }));

// parse application/json
app.use(bodyParser.json());

// authentication handler
app.use(onAuthorization);

// the defined routes
app.use(routes);

// endpoint not found handler
app.use(onNotFound);

// error handler
app.use(onError);

// start server
app.listen(process.env.PORT, onStart);

The response from the express server as seen by the client:

HTTP/1.1 200 OK
X-Powered-By: Express
Access-Control-Allow-Origin: *
Access-Control-Allow-Methods: POST
Date: Mon, 03 Apr 2023 23:34:02 GMT
Connection: keep-alive
Transfer-Encoding: chunked

The Request by the client:

POST /data/transactions HTTP/1.1
Accept: */*
Accept-Encoding: gzip, deflate, br
Accept-Language: fr-CA,fr;q=0.9,fr-FR;q=0.8,gu;q=0.7,hi;q=0.6,es;q=0.5,pt;q=0.4,en-US;q=0.3,en;q=0.2
Connection: keep-alive
Content-Length: 108
Host: localhost:3000
Origin: http://localhost:3001
Referer: http://localhost:3001/
Sec-Fetch-Dest: empty
Sec-Fetch-Mode: cors
Sec-Fetch-Site: same-site
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36
authorization: TOKEN_PASSED_IN
content-type: application/json
sec-ch-ua: "Google Chrome";v="111", "Not(A:Brand";v="8", "Chromium";v="111"
sec-ch-ua-mobile: ?0
sec-ch-ua-platform: "Windows"

The code that's piping request from Elasticsearch to the front-end in the context of a recursive function as long as there is a new scroll_id:

  const options = {
    method: 'POST',
    headers: new fetch.Headers({
      Authorization: `Basic ${Buffer.from(`${process.env.ELASTIC_API_USERNAME}:${process.env.ELASTIC_API_PASSWORD}`).toString('base64')}`,
      'Content-Type': 'application/json',
    }),
    body: JSON.stringify(body),
  };

  // request data from ES
  const request = await fetch(uri, options);
  const esStream = request.body;

  // stream ES's response back to the client
  esStream.pipe(response, { end: false });

  // listen for scroll_id if we need we need to paginate
  esStream.once('data', async (chunk) => {
    const decodedChunk = bin2string(chunk);
    const isError = decodedChunk.includes('"error":');
    const isLastChunk = decodedChunk.indexOf('hits') === -1;

    if (isError) {
      let error;

      try {
        error = JSON.parse(decodedChunk)?.error;
      // eslint-disable-next-line no-empty
      } catch (e) {}

      console.error(error?.reason || error || decodedChunk);
    }

    // Retrieving scroll ID from chunk
    nextScrollId = decodedChunk.match('"_scroll_id":"([^"]*)"')?.[1];
    isLastCall = isLastChunk || isError;
  });

  esStream.on('end', async () => {
    // recursion base case: close stream
    if (isLastCall) {
      response.end();
    } else {
      response.write('\n');
      await fetchElasticSearchData(index, subdomain, modules, ids, response, filters, nextScrollId);
    }
  });