Building Resilient Systems:
What Happens When Things Fail?

// Without timeout - DANGEROUS
const response = await fetch('https://api.payment.com/charge');
// If API hangs, your request hangs forever
// Connection pool exhausted. Everything dies.

// With timeout - SAFE
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 5000);

try {
  const response = await fetch('https://api.payment.com/charge', {
    signal: controller.signal
  });
  // Process response
} catch (e) {
  if (e.name === 'AbortError') {
    // Handle timeout - fail gracefully
    console.log('Request timed out after 5 seconds');
  }
} finally {
  clearTimeout(timeout);
}

async function fetchWithRetry(url, maxRetries = 3) {
  for (let attempt = 0; attempt < maxRetries; attempt++) {
    try {
      return await fetch(url);
    } catch (error) {
      // Don't retry permanent failures
      if (!isTransient(error)) throw error;

      // Exponential backoff with jitter
      const delay = Math.min(1000 * Math.pow(2, attempt), 10000);
      const jitter = delay * 0.5 * Math.random();
      await sleep(delay + jitter);
    }
  }
  throw new Error(`Failed after ${maxRetries} attempts`);
}

function isTransient(error) {
  // Only retry transient errors
  return error.code === 'ECONNRESET' ||
         error.code === 'ETIMEDOUT' ||
         error.status === 503 ||
         error.status === 429;
}

class CircuitBreaker {
  constructor(options) {
    this.failureThreshold = options.failureThreshold || 5;
    this.resetTimeout = options.resetTimeout || 30000;
    this.state = 'CLOSED';
    this.failures = 0;
    this.lastFailure = null;
  }

  async call(fn) {
    // If OPEN, check if we should try half-open
    if (this.state === 'OPEN') {
      if (Date.now() - this.lastFailure > this.resetTimeout) {
        this.state = 'HALF_OPEN';
      } else {
        throw new Error('Circuit breaker is open');
      }
    }

    try {
      const result = await fn();
      this.onSuccess();
      return result;
    } catch (error) {
      this.onFailure();
      throw error;
    }
  }

  onSuccess() {
    this.failures = 0;
    this.state = 'CLOSED';
  }

  onFailure() {
    this.failures++;
    this.lastFailure = Date.now();
    if (this.failures >= this.failureThreshold) {
      this.state = 'OPEN';
    }
  }
}

// Usage
const paymentCircuit = new CircuitBreaker({
  failureThreshold: 5,
  resetTimeout: 30000
});

async function chargeUser(amount) {
  return paymentCircuit.call(() => paymentAPI.charge(amount));
}

app.get('/health', async (req, res) => {
  const health = {
    status: 'healthy',
    timestamp: new Date().toISOString(),
    checks: {}
  };

  // Check database
  try {
    await db.query('SELECT 1');
    health.checks.database = 'healthy';
  } catch (e) {
    health.checks.database = 'unhealthy';
    health.status = 'unhealthy';
  }

  // Check Redis
  try {
    await redis.ping();
    health.checks.cache = 'healthy';
  } catch (e) {
    health.checks.cache = 'unhealthy';
    health.status = 'unhealthy';
  }

  const statusCode = health.status === 'healthy' ? 200 : 503;
  res.status(statusCode).json(health);
});