Modern Implementation of Sequential HTTP Requests in Node.js: From Callback Hell to Promises and Async/Await

Limitations of Traditional Callback Patterns

In early versions of Node.js, handling sequential HTTP requests typically involved nested callbacks, leading to what is known as "Callback Hell." The original code example illustrates the typical problems with this pattern:

http.get({ host: 'www.example.com', path: '/api_1.php' }, function(res) {
  res.on('data', function(d) {
    http.get({ host: 'www.example.com', path: '/api_2.php' }, function(res) {
      res.on('data', function(d) {
        http.get({ host: 'www.example.com', path: '/api_3.php' }, function(res) {
          res.on('data', function(d) {
            // Process third API response
          });
        });
      });
    });
  });
});

This code structure presents multiple issues: deep nesting makes code difficult to read and maintain, error handling becomes complex, and there is a lack of clear flow control. Each callback function creates a new scope, making variable management and data passing challenging.

Modern Solutions with Promises and Async/Await

With the evolution of JavaScript, Promises have become the standard way to handle asynchronous operations. Node.js has provided the util.promisify utility function since v8.0.0, which can convert callback-based APIs into Promise-returning functions:

const util = require('util');
const http = require('http');

// Convert http.get to a Promise-returning function
const getAsync = util.promisify(http.get);

async function fetchSequentialAPIs() {
  try {
    // Execute three API requests sequentially
    const response1 = await getAsync({ 
      host: 'www.example.com', 
      path: '/api_1.php' 
    });
    
    const data1 = await collectResponseData(response1);
    console.log('API 1 response:', data1);
    
    const response2 = await getAsync({ 
      host: 'www.example.com', 
      path: '/api_2.php' 
    });
    
    const data2 = await collectResponseData(response2);
    console.log('API 2 response:', data2);
    
    const response3 = await getAsync({ 
      host: 'www.example.com', 
      path: '/api_3.php' 
    });
    
    const data3 = await collectResponseData(response3);
    console.log('API 3 response:', data3);
    
    return { data1, data2, data3 };
  } catch (error) {
    console.error('Request failed:', error);
    throw error;
  }
}

// Helper function: collect response data
function collectResponseData(response) {
  return new Promise((resolve, reject) => {
    let data = '';
    response.on('data', (chunk) => {
      data += chunk;
    });
    response.on('end', () => {
      resolve(data);
    });
    response.on('error', reject);
  });
}

// Execute the function
fetchSequentialAPIs()
  .then(results => {
    console.log('All API requests completed:', results);
  })
  .catch(error => {
    console.error('Error during execution:', error);
  });

The advantages of this implementation approach include: flattened code structure, centralized error handling, and clear logical flow. The async/await syntax makes asynchronous code appear synchronous, significantly improving readability.

Alternative Solutions with Third-Party Libraries

In addition to native Promise support, the community provides various solutions. The async.js library offers methods like series and waterfall:

const async = require('async');
const http = require('http');

const endpoints = [
  { host: 'www.example.com', path: '/api_1.php' },
  { host: 'www.example.com', path: '/api_2.php' },
  { host: 'www.example.com', path: '/api_3.php' }
];

// Use async.series for sequential execution
async.series(
  endpoints.map(endpoint => {
    return function(callback) {
      http.get(endpoint, (res) => {
        let data = '';
        res.on('data', (chunk) => {
          data += chunk;
        });
        res.on('end', () => {
          callback(null, data);
        });
        res.on('error', (err) => {
          callback(err);
        });
      }).on('error', (err) => {
        callback(err);
      });
    };
  }),
  (err, results) => {
    if (err) {
      console.error('Request failed:', err);
    } else {
      console.log('All API responses:', results);
    }
  }
);

For scenarios requiring intermediate result passing, the waterfall method can be used. Recursive functions represent another classic solution, particularly suitable for handling dynamic API lists:

function callAPIsSequentially(apis, index = 0, results = []) {
  if (index >= apis.length) {
    return Promise.resolve(results);
  }
  
  return new Promise((resolve, reject) => {
    http.get(apis[index], (res) => {
      let data = '';
      res.on('data', (chunk) => {
        data += chunk;
      });
      res.on('end', () => {
        results.push(data);
        // Recursively call the next API
        callAPIsSequentially(apis, index + 1, results)
          .then(resolve)
          .catch(reject);
      });
      res.on('error', reject);
    }).on('error', reject);
  });
}

const apiList = [
  { host: 'www.example.com', path: '/api_1.php' },
  { host: 'www.example.com', path: '/api_2.php' },
  { host: 'www.example.com', path: '/api_3.php' }
];

callAPIsSequentially(apiList)
  .then(results => {
    console.log('Recursive approach completed:', results);
  })
  .catch(error => {
    console.error('Recursive request failed:', error);
  });

Concurrency Control and Error Handling

In practical applications, mixed sequential and concurrent requests may be necessary. Promises provide various concurrency control methods:

async function mixedRequests() {
  try {
    // Execute first two APIs sequentially
    const result1 = await fetchAPI('/api_1.php');
    const result2 = await fetchAPI('/api_2.php');
    
    // Execute next two APIs concurrently
    const [result3, result4] = await Promise.all([
      fetchAPI('/api_3.php'),
      fetchAPI('/api_4.php')
    ]);
    
    // Use Promise.race to get the fastest response
    const fastestResult = await Promise.race([
      fetchAPI('/api_5.php'),
      timeoutPromise(5000) // 5-second timeout
    ]);
    
    return { result1, result2, result3, result4, fastestResult };
  } catch (error) {
    // Unified error handling
    if (error.name === 'TimeoutError') {
      console.warn('Request timeout');
    } else {
      console.error('Other error:', error);
    }
    throw error;
  }
}

// Timeout Promise helper function
function timeoutPromise(ms) {
  return new Promise((_, reject) => {
    setTimeout(() => {
      reject(new Error('TimeoutError'));
    }, ms);
  });
}

For scenarios requiring concurrency limitation, libraries like p-limit can be used. For error handling, it is recommended to wrap await expressions in try-catch blocks to ensure all possible exceptions are caught.

Performance Considerations and Best Practices

When choosing an implementation approach for sequential requests, consider the following factors:

1. Code Readability: async/await typically provides the clearest code structure
2. Error Handling: Promise catch mechanisms and async/await try-catch provide unified error handling
3. Performance Impact: Sequential execution significantly increases total request time; consider if strict sequencing is truly necessary
4. Memory Usage: Recursive implementations need to consider call stack depth to avoid stack overflow
5. Compatibility: Ensure Node.js version supports the features being used

For most modern Node.js applications, the recommended approach is async/await combined with util.promisify. This combination provides good development experience, clear code structure, and reliable error handling. For legacy code or specific scenarios, libraries like async.js remain valid choices.