MongoDB Replica Sets and Fault Tolerance: SQL-Style High Availability with Automatic Failover

-- Traditional SQL high availability setup challenges

-- Master-Slave replication requires manual failover
-- Primary database server
CREATE SERVER primary_db
  CONNECTION 'host=db-master.example.com port=5432 dbname=production';

-- Read replica servers  
CREATE SERVER replica_db_1
  CONNECTION 'host=db-replica-1.example.com port=5432 dbname=production';
CREATE SERVER replica_db_2  
  CONNECTION 'host=db-replica-2.example.com port=5432 dbname=production';

-- Application must handle connection routing
-- Read queries to replicas
SELECT customer_id, name, email 
FROM customers@replica_db_1
WHERE status = 'active';

-- Write queries to master
INSERT INTO orders (customer_id, product_id, quantity)
VALUES (123, 456, 2)
-- Must go to primary_db

-- Problems with traditional approaches:
-- - Manual failover when primary fails
-- - Complex connection string management
-- - Application-level routing logic required
-- - No automatic primary election
-- - Split-brain scenarios possible
-- - Expensive clustering solutions
-- - Recovery requires manual intervention
-- - No built-in consistency guarantees

// MongoDB replica set - automatic high availability
// Single connection string handles all routing
const mongoUrl = 'mongodb://db1.example.com,db2.example.com,db3.example.com/production?replicaSet=prodRS';

const client = new MongoClient(mongoUrl, {
  // Automatic failover and reconnection
  maxPoolSize: 10,
  serverSelectionTimeoutMS: 5000,
  heartbeatFrequencyMS: 10000,

  // Write concern for durability
  writeConcern: {
    w: 'majority',      // Write to majority of replica set members
    j: true,            // Ensure write to journal
    wtimeout: 5000      // Timeout for write acknowledgment
  },

  // Read preference for load distribution
  readPreference: 'secondaryPreferred', // Use secondaries when available
  readConcern: { level: 'majority' }     // Consistent reads
});

// Application code remains unchanged - replica set handles routing
const db = client.db('production');
const orders = db.collection('orders');

// Writes automatically go to primary
await orders.insertOne({
  customerId: ObjectId('64f1a2c4567890abcdef1234'),
  productId: ObjectId('64f1a2c4567890abcdef5678'),
  quantity: 2,
  orderDate: new Date(),
  status: 'pending'
});

// Reads can use secondaries based on read preference
const recentOrders = await orders.find({
  orderDate: { $gte: new Date(Date.now() - 24 * 60 * 60 * 1000) }
}).toArray();

// Benefits:
// - Automatic primary election on failure
// - Transparent failover (no connection string changes)
// - Built-in data consistency guarantees
// - Distributed consensus prevents split-brain
// - Hot standby replicas ready immediately
// - Rolling updates without downtime
// - Geographic distribution support
// - No additional clustering software needed

// Replica set configuration and management
class ReplicaSetManager {
  constructor() {
    this.replicaSetConfig = {
      _id: 'prodRS',
      version: 1,
      members: [
        {
          _id: 0,
          host: 'db-primary.example.com:27017',
          priority: 10,        // High priority for primary preference
          arbiterOnly: false,
          buildIndexes: true,
          hidden: false,
          slaveDelay: 0,
          votes: 1
        },
        {
          _id: 1, 
          host: 'db-secondary-1.example.com:27017',
          priority: 5,         // Medium priority for secondary
          arbiterOnly: false,
          buildIndexes: true,
          hidden: false,
          slaveDelay: 0,
          votes: 1
        },
        {
          _id: 2,
          host: 'db-secondary-2.example.com:27017', 
          priority: 5,         // Medium priority for secondary
          arbiterOnly: false,
          buildIndexes: true,
          hidden: false,
          slaveDelay: 0,
          votes: 1
        },
        {
          _id: 3,
          host: 'db-arbiter.example.com:27017',
          priority: 0,         // Arbiters cannot become primary
          arbiterOnly: true,   // Voting member but no data
          buildIndexes: false,
          hidden: false,
          slaveDelay: 0,
          votes: 1
        }
      ],
      settings: {
        chainingAllowed: true,              // Allow secondary-to-secondary sync
        heartbeatIntervalMillis: 2000,      // Heartbeat frequency
        heartbeatTimeoutSecs: 10,           // Heartbeat timeout
        electionTimeoutMillis: 10000,       // Election timeout
        catchUpTimeoutMillis: 60000,        // Catchup period for new primary
        getLastErrorModes: {
          'datacenter': {                   // Custom write concern mode
            'dc1': 1,
            'dc2': 1
          }
        },
        getLastErrorDefaults: {
          w: 'majority',
          wtimeout: 5000
        }
      }
    };
  }

  async initializeReplicaSet(primaryConnection) {
    try {
      // Initialize replica set on primary node
      const result = await primaryConnection.db('admin').runCommand({
        replSetInitiate: this.replicaSetConfig
      });

      console.log('Replica set initialization result:', result);

      // Wait for replica set to stabilize
      await this.waitForReplicaSetReady(primaryConnection);

      return { success: true, config: this.replicaSetConfig };

    } catch (error) {
      throw new Error(`Replica set initialization failed: ${error.message}`);
    }
  }

  async waitForReplicaSetReady(connection, maxWaitMs = 60000) {
    const startTime = Date.now();

    while (Date.now() - startTime < maxWaitMs) {
      try {
        const status = await connection.db('admin').runCommand({ replSetGetStatus: 1 });

        const primaryCount = status.members.filter(member => member.state === 1).length;
        const secondaryCount = status.members.filter(member => member.state === 2).length;

        if (primaryCount === 1 && secondaryCount >= 1) {
          console.log('Replica set is ready:', {
            primary: primaryCount,
            secondaries: secondaryCount,
            total: status.members.length
          });
          return true;
        }

        console.log('Waiting for replica set to stabilize...', {
          primary: primaryCount,
          secondaries: secondaryCount
        });

      } catch (error) {
        console.log('Replica set not ready yet:', error.message);
      }

      await new Promise(resolve => setTimeout(resolve, 2000));
    }

    throw new Error('Replica set failed to become ready within timeout');
  }

  async addReplicaSetMember(primaryConnection, newMemberConfig) {
    try {
      // Get current configuration
      const currentConfig = await primaryConnection.db('admin').runCommand({
        replSetGetConfig: 1
      });

      // Add new member to configuration
      const updatedConfig = currentConfig.config;
      updatedConfig.version += 1;
      updatedConfig.members.push(newMemberConfig);

      // Reconfigure replica set
      const result = await primaryConnection.db('admin').runCommand({
        replSetReconfig: updatedConfig
      });

      console.log('Member added successfully:', result);
      return result;

    } catch (error) {
      throw new Error(`Failed to add replica set member: ${error.message}`);
    }
  }

  async removeReplicaSetMember(primaryConnection, memberId) {
    try {
      const currentConfig = await primaryConnection.db('admin').runCommand({
        replSetGetConfig: 1
      });

      const updatedConfig = currentConfig.config;
      updatedConfig.version += 1;
      updatedConfig.members = updatedConfig.members.filter(member => member._id !== memberId);

      const result = await primaryConnection.db('admin').runCommand({
        replSetReconfig: updatedConfig
      });

      console.log('Member removed successfully:', result);
      return result;

    } catch (error) {
      throw new Error(`Failed to remove replica set member: ${error.message}`);
    }
  }

  async performStepDown(primaryConnection, stepDownSecs = 60) {
    try {
      // Force primary to step down (useful for maintenance)
      const result = await primaryConnection.db('admin').runCommand({
        replSetStepDown: stepDownSecs,
        secondaryCatchUpPeriodSecs: 15
      });

      console.log('Primary step down initiated:', result);
      return result;

    } catch (error) {
      // Step down command typically causes connection error as primary changes
      if (error.message.includes('connection') || error.message.includes('network')) {
        console.log('Step down successful - connection closed as expected');
        return { success: true, message: 'Primary stepped down successfully' };
      }
      throw error;
    }
  }
}

// Advanced write concern and read preference management
class ReplicaSetConsistencyManager {
  constructor(client) {
    this.client = client;
    this.db = client.db();

    // Define write concern levels for different operations
    this.writeConcerns = {
      critical: {
        w: 'majority',        // Wait for majority acknowledgment
        j: true,              // Wait for journal sync
        wtimeout: 10000       // 10 second timeout
      },
      standard: {
        w: 'majority',
        j: false,             // Don't wait for journal (faster)
        wtimeout: 5000
      },
      fast: {
        w: 1,                 // Only primary acknowledgment
        j: false,
        wtimeout: 1000
      },
      datacenter: {
        w: 'datacenter',      // Custom write concern mode
        j: true,
        wtimeout: 15000
      }
    };

    // Define read preference strategies
    this.readPreferences = {
      primaryOnly: { mode: 'primary' },
      secondaryPreferred: { 
        mode: 'secondaryPreferred',
        tagSets: [
          { region: 'us-west', datacenter: 'dc1' },  // Prefer specific tags
          { region: 'us-west' },                     // Fall back to region
          {}                                         // Fall back to any
        ],
        maxStalenessSeconds: 90  // Max replication lag
      },
      nearestRead: {
        mode: 'nearest',
        tagSets: [{ region: 'us-west' }],
        maxStalenessSeconds: 60
      },
      analyticsReads: {
        mode: 'secondary',
        tagSets: [{ usage: 'analytics' }],  // Dedicated analytics secondaries
        maxStalenessSeconds: 300
      }
    };
  }

  async performCriticalWrite(collection, operation, data, options = {}) {
    // High consistency write for critical data
    try {
      const session = this.client.startSession();

      const result = await session.withTransaction(async () => {
        const coll = this.db.collection(collection).withOptions({
          writeConcern: this.writeConcerns.critical,
          readPreference: this.readPreferences.primaryOnly
        });

        let operationResult;
        switch (operation) {
          case 'insert':
            operationResult = await coll.insertOne(data, { session });
            break;
          case 'update':
            operationResult = await coll.updateOne(data.filter, data.update, { 
              session, 
              ...options 
            });
            break;
          case 'replace':
            operationResult = await coll.replaceOne(data.filter, data.replacement, { 
              session, 
              ...options 
            });
            break;
          case 'delete':
            operationResult = await coll.deleteOne(data.filter, { session });
            break;
          default:
            throw new Error(`Unsupported operation: ${operation}`);
        }

        // Verify write was acknowledged by majority
        if (operationResult.acknowledged && 
            (operationResult.insertedId || operationResult.modifiedCount || operationResult.deletedCount)) {

          // Add audit log for critical operations
          await this.db.collection('audit_log').insertOne({
            operation: operation,
            collection: collection,
            timestamp: new Date(),
            writeConcern: 'critical',
            sessionId: session.id,
            result: {
              acknowledged: operationResult.acknowledged,
              insertedId: operationResult.insertedId,
              modifiedCount: operationResult.modifiedCount,
              deletedCount: operationResult.deletedCount
            }
          }, { session });
        }

        return operationResult;
      }, {
        readConcern: { level: 'majority' },
        writeConcern: this.writeConcerns.critical
      });

      await session.endSession();
      return result;

    } catch (error) {
      throw new Error(`Critical write failed: ${error.message}`);
    }
  }

  async performFastWrite(collection, operation, data, options = {}) {
    // Fast write for non-critical data
    const coll = this.db.collection(collection).withOptions({
      writeConcern: this.writeConcerns.fast
    });

    switch (operation) {
      case 'insert':
        return await coll.insertOne(data, options);
      case 'insertMany':
        return await coll.insertMany(data, options);
      case 'update':
        return await coll.updateOne(data.filter, data.update, options);
      case 'updateMany':
        return await coll.updateMany(data.filter, data.update, options);
      default:
        throw new Error(`Unsupported fast write operation: ${operation}`);
    }
  }

  async performConsistentRead(collection, query, options = {}) {
    // Read with strong consistency
    const coll = this.db.collection(collection).withOptions({
      readPreference: this.readPreferences.primaryOnly,
      readConcern: { level: 'majority' }
    });

    if (options.findOne) {
      return await coll.findOne(query, options);
    } else {
      return await coll.find(query, options).toArray();
    }
  }

  async performEventuallyConsistentRead(collection, query, options = {}) {
    // Read from secondaries for better performance
    const coll = this.db.collection(collection).withOptions({
      readPreference: this.readPreferences.secondaryPreferred,
      readConcern: { level: 'local' }
    });

    if (options.findOne) {
      return await coll.findOne(query, options);
    } else {
      return await coll.find(query, options).toArray();
    }
  }

  async performAnalyticsRead(collection, pipeline, options = {}) {
    // Long-running analytics queries on dedicated secondaries
    const coll = this.db.collection(collection).withOptions({
      readPreference: this.readPreferences.analyticsReads,
      readConcern: { level: 'available' }  // Fastest read concern
    });

    return await coll.aggregate(pipeline, {
      ...options,
      allowDiskUse: true,           // Allow large aggregations
      maxTimeMS: 300000,            // 5 minute timeout
      batchSize: 1000              // Optimize batch size
    }).toArray();
  }

  async checkReplicationLag() {
    // Monitor replication lag across replica set
    try {
      const status = await this.db.admin().command({ replSetGetStatus: 1 });
      const primary = status.members.find(member => member.state === 1);
      const secondaries = status.members.filter(member => member.state === 2);

      if (!primary) {
        return { error: 'No primary found in replica set' };
      }

      const lagInfo = secondaries.map(secondary => {
        const lagMs = primary.optimeDate.getTime() - secondary.optimeDate.getTime();
        return {
          member: secondary.name,
          lagSeconds: Math.round(lagMs / 1000),
          health: secondary.health,
          state: secondary.stateStr,
          lastHeartbeat: secondary.lastHeartbeat
        };
      });

      const maxLag = Math.max(...lagInfo.map(info => info.lagSeconds));

      return {
        primary: primary.name,
        secondaries: lagInfo,
        maxLagSeconds: maxLag,
        healthy: maxLag < 10, // Consider healthy if under 10 seconds lag
        timestamp: new Date()
      };

    } catch (error) {
      return { error: `Failed to check replication lag: ${error.message}` };
    }
  }

  async adaptWriteConcernBasedOnLag() {
    // Dynamically adjust write concern based on replication lag
    const lagInfo = await this.checkReplicationLag();

    if (lagInfo.error || !lagInfo.healthy) {
      console.warn('Replication issues detected, using primary-only writes');
      return this.writeConcerns.fast; // Fallback to primary-only
    }

    if (lagInfo.maxLagSeconds < 5) {
      return this.writeConcerns.critical; // Normal high consistency
    } else if (lagInfo.maxLagSeconds < 30) {
      return this.writeConcerns.standard; // Medium consistency
    } else {
      return this.writeConcerns.fast; // Primary-only for performance
    }
  }

  async performAdaptiveWrite(collection, operation, data, options = {}) {
    // Automatically choose write concern based on replica set health
    const adaptedWriteConcern = await this.adaptWriteConcernBasedOnLag();

    const coll = this.db.collection(collection).withOptions({
      writeConcern: adaptedWriteConcern
    });

    console.log(`Using adaptive write concern:`, adaptedWriteConcern);

    switch (operation) {
      case 'insert':
        return await coll.insertOne(data, options);
      case 'update':
        return await coll.updateOne(data.filter, data.update, options);
      case 'replace':
        return await coll.replaceOne(data.filter, data.replacement, options);
      case 'delete':
        return await coll.deleteOne(data.filter, options);
      default:
        throw new Error(`Unsupported adaptive write operation: ${operation}`);
    }
  }
}

// Failover testing and disaster recovery automation
class FailoverTestingManager {
  constructor(replicaSetUrl) {
    this.replicaSetUrl = replicaSetUrl;
    this.testResults = [];
  }

  async simulateNetworkPartition(duration = 30000) {
    // Simulate network partition by stepping down primary
    console.log('Starting network partition simulation...');

    const client = new MongoClient(this.replicaSetUrl);
    await client.connect();

    try {
      const startTime = Date.now();

      // Record initial replica set status
      const initialStatus = await this.getReplicaSetStatus(client);
      console.log('Initial status:', {
        primary: initialStatus.primary,
        secondaries: initialStatus.secondaries.length
      });

      // Force primary step down
      await client.db('admin').command({
        replSetStepDown: Math.ceil(duration / 1000),
        secondaryCatchUpPeriodSecs: 10
      });

      // Monitor failover process
      const failoverResult = await this.monitorFailover(client, duration);

      const testResult = {
        testType: 'network_partition',
        startTime: new Date(startTime),
        duration: duration,
        initialPrimary: initialStatus.primary,
        failoverTime: failoverResult.failoverTime,
        newPrimary: failoverResult.newPrimary,
        dataConsistency: await this.verifyDataConsistency(client),
        success: failoverResult.success
      };

      this.testResults.push(testResult);
      return testResult;

    } finally {
      await client.close();
    }
  }

  async simulateSecondaryFailure(secondaryHost) {
    // Simulate secondary node failure
    console.log(`Simulating failure of secondary: ${secondaryHost}`);

    const client = new MongoClient(this.replicaSetUrl);
    await client.connect();

    try {
      const startTime = Date.now();
      const initialStatus = await this.getReplicaSetStatus(client);

      // Simulate removing secondary from replica set
      const config = await client.db('admin').command({ replSetGetConfig: 1 });
      const targetMember = config.config.members.find(m => m.host.includes(secondaryHost));

      if (!targetMember) {
        throw new Error(`Secondary ${secondaryHost} not found in replica set`);
      }

      const updatedConfig = { ...config.config };
      updatedConfig.version += 1;
      updatedConfig.members = updatedConfig.members.filter(m => m._id !== targetMember._id);

      await client.db('admin').command({ replSetReconfig: updatedConfig });

      // Wait for configuration change to take effect
      await this.waitForConfigurationChange(client, 30000);

      // Test write operations during reduced redundancy
      const writeTestResult = await this.testWriteOperations(client);

      // Restore secondary after test period
      setTimeout(async () => {
        await this.restoreSecondary(client, targetMember);
      }, 60000);

      const testResult = {
        testType: 'secondary_failure',
        startTime: new Date(startTime),
        failedSecondary: secondaryHost,
        initialSecondaries: initialStatus.secondaries.length,
        writeTestResult: writeTestResult,
        success: writeTestResult.success
      };

      this.testResults.push(testResult);
      return testResult;

    } finally {
      await client.close();
    }
  }

  async testWriteOperations(client, testCount = 100) {
    // Test write operations during failure scenarios
    const testCollection = client.db('test').collection('failover_test');
    const results = {
      attempted: testCount,
      successful: 0,
      failed: 0,
      errors: [],
      averageLatency: 0,
      success: false
    };

    const latencies = [];

    for (let i = 0; i < testCount; i++) {
      const startTime = Date.now();

      try {
        await testCollection.insertOne({
          testId: i,
          timestamp: new Date(),
          data: `Test document ${i}`,
          failoverTest: true
        }, {
          writeConcern: { w: 'majority', wtimeout: 5000 }
        });

        const latency = Date.now() - startTime;
        latencies.push(latency);
        results.successful++;

      } catch (error) {
        results.failed++;
        results.errors.push({
          testId: i,
          error: error.message,
          timestamp: new Date()
        });
      }

      // Small delay between writes
      await new Promise(resolve => setTimeout(resolve, 100));
    }

    if (latencies.length > 0) {
      results.averageLatency = latencies.reduce((a, b) => a + b, 0) / latencies.length;
    }

    results.success = results.successful >= (testCount * 0.95); // 95% success rate

    // Clean up test data
    await testCollection.deleteMany({ failoverTest: true });

    return results;
  }

  async monitorFailover(client, maxWaitTime) {
    // Monitor replica set during failover
    const startTime = Date.now();
    let newPrimary = null;
    let failoverTime = null;

    while (Date.now() - startTime < maxWaitTime) {
      try {
        const status = await this.getReplicaSetStatus(client);

        if (status.primary && status.primary !== 'No primary') {
          newPrimary = status.primary;
          failoverTime = Date.now() - startTime;
          console.log(`New primary elected: ${newPrimary} (${failoverTime}ms)`);
          break;
        }

        console.log('Waiting for new primary election...');
        await new Promise(resolve => setTimeout(resolve, 1000));

      } catch (error) {
        console.log('Error during failover monitoring:', error.message);
        await new Promise(resolve => setTimeout(resolve, 2000));
      }
    }

    return {
      success: newPrimary !== null,
      newPrimary: newPrimary,
      failoverTime: failoverTime
    };
  }

  async getReplicaSetStatus(client) {
    // Get current replica set status
    try {
      const status = await client.db('admin').command({ replSetGetStatus: 1 });

      const primary = status.members.find(m => m.state === 1);
      const secondaries = status.members.filter(m => m.state === 2);
      const arbiters = status.members.filter(m => m.state === 7);

      return {
        primary: primary ? primary.name : 'No primary',
        secondaries: secondaries.map(s => ({ name: s.name, health: s.health })),
        arbiters: arbiters.map(a => ({ name: a.name, health: a.health })),
        ok: status.ok
      };

    } catch (error) {
      return {
        error: error.message,
        primary: 'Unknown',
        secondaries: [],
        arbiters: []
      };
    }
  }

  async verifyDataConsistency(client) {
    // Verify data consistency across replica set
    try {
      // Insert test document with strong consistency
      const testDoc = {
        _id: new ObjectId(),
        consistencyTest: true,
        timestamp: new Date(),
        randomValue: Math.random()
      };

      const testCollection = client.db('test').collection('consistency_test');

      await testCollection.insertOne(testDoc, {
        writeConcern: { w: 'majority', j: true }
      });

      // Wait for replication
      await new Promise(resolve => setTimeout(resolve, 2000));

      // Read from primary
      const primaryResult = await testCollection.findOne(
        { _id: testDoc._id },
        { readPreference: { mode: 'primary' } }
      );

      // Read from secondary
      const secondaryResult = await testCollection.findOne(
        { _id: testDoc._id },
        { 
          readPreference: { mode: 'secondaryPreferred' },
          maxStalenessSeconds: 10
        }
      );

      // Clean up
      await testCollection.deleteOne({ _id: testDoc._id });

      const consistent = primaryResult && secondaryResult && 
                        primaryResult.randomValue === secondaryResult.randomValue;

      return {
        consistent: consistent,
        primaryResult: primaryResult ? 'found' : 'not found',
        secondaryResult: secondaryResult ? 'found' : 'not found',
        timestamp: new Date()
      };

    } catch (error) {
      return {
        consistent: false,
        error: error.message,
        timestamp: new Date()
      };
    }
  }

  async generateFailoverReport() {
    // Generate comprehensive failover test report
    if (this.testResults.length === 0) {
      return { message: 'No failover tests have been run' };
    }

    const report = {
      totalTests: this.testResults.length,
      successfulTests: this.testResults.filter(t => t.success).length,
      failedTests: this.testResults.filter(t => !t.success).length,
      averageFailoverTime: 0,
      testTypes: {},
      consistency: {
        passed: 0,
        failed: 0
      },
      recommendations: []
    };

    // Calculate statistics
    const failoverTimes = this.testResults
      .filter(t => t.failoverTime)
      .map(t => t.failoverTime);

    if (failoverTimes.length > 0) {
      report.averageFailoverTime = failoverTimes.reduce((a, b) => a + b, 0) / failoverTimes.length;
    }

    // Group by test type
    this.testResults.forEach(result => {
      if (!report.testTypes[result.testType]) {
        report.testTypes[result.testType] = {
          count: 0,
          successful: 0,
          failed: 0
        };
      }

      report.testTypes[result.testType].count++;
      if (result.success) {
        report.testTypes[result.testType].successful++;
      } else {
        report.testTypes[result.testType].failed++;
      }
    });

    // Consistency check summary
    this.testResults.forEach(result => {
      if (result.dataConsistency) {
        if (result.dataConsistency.consistent) {
          report.consistency.passed++;
        } else {
          report.consistency.failed++;
        }
      }
    });

    // Generate recommendations
    if (report.averageFailoverTime > 30000) {
      report.recommendations.push('Consider tuning election timeout settings for faster failover');
    }

    if (report.consistency.failed > 0) {
      report.recommendations.push('Data consistency issues detected - review read/write concern settings');
    }

    if (report.failedTests > report.totalTests * 0.1) {
      report.recommendations.push('High failure rate detected - review replica set configuration');
    }

    report.generatedAt = new Date();
    return report;
  }

  // Utility methods
  async waitForConfigurationChange(client, maxWait) {
    const startTime = Date.now();
    while (Date.now() - startTime < maxWait) {
      try {
        await client.db('admin').command({ replSetGetStatus: 1 });
        return true;
      } catch (error) {
        await new Promise(resolve => setTimeout(resolve, 1000));
      }
    }
    return false;
  }

  async restoreSecondary(client, memberConfig) {
    try {
      const config = await client.db('admin').command({ replSetGetConfig: 1 });
      const updatedConfig = { ...config.config };
      updatedConfig.version += 1;
      updatedConfig.members.push(memberConfig);

      await client.db('admin').command({ replSetReconfig: updatedConfig });
      console.log('Secondary restored successfully');
    } catch (error) {
      console.error('Failed to restore secondary:', error.message);
    }
  }
}

// Multi-region replica set configuration
class GeographicReplicaSetManager {
  constructor() {
    this.multiRegionConfig = {
      _id: 'globalRS',
      version: 1,
      members: [
        // Primary region (US East)
        {
          _id: 0,
          host: 'db-primary-us-east.example.com:27017',
          priority: 10,
          tags: { 
            region: 'us-east',
            datacenter: 'dc1',
            usage: 'primary'
          }
        },
        {
          _id: 1,
          host: 'db-secondary-us-east.example.com:27017',
          priority: 8,
          tags: { 
            region: 'us-east',
            datacenter: 'dc1',
            usage: 'secondary'
          }
        },

        // Secondary region (US West)
        {
          _id: 2,
          host: 'db-secondary-us-west.example.com:27017',
          priority: 6,
          tags: { 
            region: 'us-west',
            datacenter: 'dc2',
            usage: 'secondary'
          }
        },
        {
          _id: 3,
          host: 'db-secondary-us-west-2.example.com:27017',
          priority: 5,
          tags: { 
            region: 'us-west',
            datacenter: 'dc2',
            usage: 'analytics'
          }
        },

        // Tertiary region (Europe)
        {
          _id: 4,
          host: 'db-secondary-eu-west.example.com:27017',
          priority: 4,
          tags: { 
            region: 'eu-west',
            datacenter: 'dc3',
            usage: 'secondary'
          }
        },

        // Arbiter for odd number voting
        {
          _id: 5,
          host: 'arbiter-us-central.example.com:27017',
          priority: 0,
          arbiterOnly: true,
          tags: { 
            region: 'us-central',
            usage: 'arbiter'
          }
        }
      ],
      settings: {
        getLastErrorModes: {
          'multiRegion': {
            'region': 2  // Require writes to reach 2 different regions
          },
          'crossDatacenter': {
            'datacenter': 2  // Require writes to reach 2 different datacenters
          }
        },
        getLastErrorDefaults: {
          w: 'multiRegion',
          wtimeout: 10000
        }
      }
    };
  }

  createRegionalReadPreferences() {
    return {
      // US East users - prefer local region
      usEastUsers: {
        mode: 'secondaryPreferred',
        tagSets: [
          { region: 'us-east' },
          { region: 'us-west' },
          {}
        ],
        maxStalenessSeconds: 90
      },

      // US West users - prefer local region
      usWestUsers: {
        mode: 'secondaryPreferred',
        tagSets: [
          { region: 'us-west' },
          { region: 'us-east' },
          {}
        ],
        maxStalenessSeconds: 90
      },

      // European users - prefer local region
      europeanUsers: {
        mode: 'secondaryPreferred',
        tagSets: [
          { region: 'eu-west' },
          { region: 'us-east' },
          {}
        ],
        maxStalenessSeconds: 120
      },

      // Analytics workloads - dedicated secondaries
      analytics: {
        mode: 'secondary',
        tagSets: [
          { usage: 'analytics' }
        ],
        maxStalenessSeconds: 300
      }
    };
  }

  async routeRequestByRegion(clientRegion, operation, collection, data) {
    const readPreferences = this.createRegionalReadPreferences();
    const regionPreference = readPreferences[`${clientRegion}Users`] || readPreferences.usEastUsers;

    // Create region-optimized connection
    const client = new MongoClient(this.globalReplicaSetUrl, {
      readPreference: regionPreference,
      writeConcern: { w: 'multiRegion', wtimeout: 10000 }
    });

    try {
      await client.connect();
      const db = client.db();
      const coll = db.collection(collection);

      switch (operation.type) {
        case 'read':
          return await coll.find(data.query).toArray();

        case 'write':
          // Ensure cross-region durability for writes
          return await coll.insertOne(data, {
            writeConcern: { w: 'multiRegion', j: true, wtimeout: 15000 }
          });

        case 'update':
          return await coll.updateOne(data.filter, data.update, {
            writeConcern: { w: 'crossDatacenter', j: true, wtimeout: 12000 }
          });

        default:
          throw new Error(`Unsupported operation: ${operation.type}`);
      }

    } finally {
      await client.close();
    }
  }

  async monitorCrossRegionLatency() {
    // Monitor latency between regions
    const regions = ['us-east', 'us-west', 'eu-west'];
    const latencyResults = {};

    for (const region of regions) {
      try {
        const startTime = Date.now();

        // Connect with region-specific preference
        const client = new MongoClient(this.globalReplicaSetUrl, {
          readPreference: {
            mode: 'secondary',
            tagSets: [{ region: region }]
          }
        });

        await client.connect();

        // Perform test read
        await client.db('test').collection('ping').findOne({});

        const latency = Date.now() - startTime;
        latencyResults[region] = {
          latency: latency,
          status: latency < 200 ? 'good' : latency < 500 ? 'acceptable' : 'poor'
        };

        await client.close();

      } catch (error) {
        latencyResults[region] = {
          latency: null,
          status: 'error',
          error: error.message
        };
      }
    }

    return {
      timestamp: new Date(),
      regions: latencyResults,
      averageLatency: Object.values(latencyResults)
        .filter(r => r.latency)
        .reduce((sum, r, _, arr) => sum + r.latency / arr.length, 0)
    };
  }
}