MongoDB Replica Sets and High Availability: Advanced Disaster Recovery and Fault Tolerance Strategies for Mission-Critical Applications

-- Traditional PostgreSQL streaming replication - manual failover and limited flexibility

-- Primary server configuration (postgresql.conf)
wal_level = replica
max_wal_senders = 3
wal_keep_segments = 64
archive_mode = on
archive_command = 'cp %p /var/lib/postgresql/wal_archive/%f'

-- Standby server configuration (recovery.conf)  
standby_mode = 'on'
primary_conninfo = 'host=primary-server port=5432 user=replicator'
restore_command = 'cp /var/lib/postgresql/wal_archive/%f %p'
trigger_file = '/tmp/postgresql.trigger.5432'

-- Manual failover process (complex and error-prone)
-- 1. Detect primary failure through monitoring
SELECT pg_is_in_recovery(); -- Check if server is in standby mode

-- 2. Promote standby to primary (manual intervention required)
-- Touch trigger file on standby server
-- $ touch /tmp/postgresql.trigger.5432

-- 3. Redirect application traffic (requires external load balancer configuration)
-- Update DNS/load balancer to point to new primary
-- Verify all applications can connect to new primary

-- 4. Reconfigure remaining servers (manual process)
-- Update primary_conninfo on other standby servers
-- Restart PostgreSQL services with new configuration

-- Complex query for checking replication lag
WITH replication_status AS (
  SELECT 
    client_addr,
    client_hostname,
    state,
    sent_lsn,
    write_lsn,
    flush_lsn,
    replay_lsn,
    write_lag,
    flush_lag,
    replay_lag,
    sync_priority,
    sync_state,

    -- Calculate replication delay in bytes
    pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn) as replay_delay_bytes,

    -- Check if standby is healthy
    CASE 
      WHEN state = 'streaming' AND pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn) < 16777216 THEN 'healthy'
      WHEN state = 'streaming' AND pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn) < 134217728 THEN 'lagging'
      WHEN state = 'streaming' THEN 'severely_lagging'
      ELSE 'disconnected'
    END as health_status,

    -- Estimate recovery time if primary fails
    CASE 
      WHEN replay_lag IS NOT NULL THEN 
        EXTRACT(EPOCH FROM replay_lag)::int
      ELSE 
        GREATEST(
          EXTRACT(EPOCH FROM flush_lag)::int,
          pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn) / 16777216 * 10
        )
    END as estimated_recovery_seconds

  FROM pg_stat_replication
  WHERE state IS NOT NULL
),

connection_health AS (
  SELECT 
    datname,
    usename,
    client_addr,
    state,
    query,
    state_change,

    -- Connection duration
    EXTRACT(EPOCH FROM (now() - backend_start))::int as connection_age_seconds,

    -- Query duration  
    CASE 
      WHEN state = 'active' THEN EXTRACT(EPOCH FROM (now() - query_start))::int
      ELSE 0
    END as active_query_duration_seconds,

    -- Identify potentially problematic connections
    CASE
      WHEN state = 'idle in transaction' AND (now() - state_change) > interval '5 minutes' THEN 'long_idle_transaction'
      WHEN state = 'active' AND (now() - query_start) > interval '10 minutes' THEN 'long_running_query'
      WHEN backend_type = 'walsender' THEN 'replication_connection'
      ELSE 'normal'
    END as connection_type

  FROM pg_stat_activity
  WHERE backend_type IN ('client backend', 'walsender')
    AND datname IS NOT NULL
)

-- Comprehensive replication monitoring query
SELECT 
  rs.client_addr as standby_server,
  rs.client_hostname as standby_hostname,
  rs.state as replication_state,
  rs.health_status,

  -- Lag information
  COALESCE(EXTRACT(EPOCH FROM rs.replay_lag)::int, 0) as replay_lag_seconds,
  ROUND(rs.replay_delay_bytes / 1048576.0, 2) as replay_delay_mb,
  rs.estimated_recovery_seconds,

  -- Sync configuration
  rs.sync_priority,
  rs.sync_state,

  -- Connection health
  ch.connection_age_seconds,
  ch.active_query_duration_seconds,

  -- Health assessment
  CASE 
    WHEN rs.health_status = 'healthy' AND rs.sync_state = 'sync' THEN 'excellent'
    WHEN rs.health_status = 'healthy' AND rs.sync_state = 'async' THEN 'good'
    WHEN rs.health_status = 'lagging' THEN 'warning'
    WHEN rs.health_status = 'severely_lagging' THEN 'critical'
    ELSE 'unknown'
  END as overall_health,

  -- Failover readiness
  CASE
    WHEN rs.health_status = 'healthy' AND rs.estimated_recovery_seconds < 30 THEN 'ready'
    WHEN rs.health_status IN ('healthy', 'lagging') AND rs.estimated_recovery_seconds < 120 THEN 'acceptable'
    ELSE 'not_ready'
  END as failover_readiness,

  -- Recommendations
  CASE
    WHEN rs.health_status = 'disconnected' THEN 'Check network connectivity and standby server status'
    WHEN rs.health_status = 'severely_lagging' THEN 'Investigate standby performance and network bandwidth'
    WHEN rs.replay_delay_bytes > 134217728 THEN 'Consider increasing wal_keep_segments or using replication slots'
    WHEN rs.sync_state != 'sync' AND rs.sync_priority > 0 THEN 'Review synchronous_standby_names configuration'
    ELSE 'Replication operating normally'
  END as recommendation

FROM replication_status rs
LEFT JOIN connection_health ch ON rs.client_addr = ch.client_addr 
                                AND ch.connection_type = 'replication_connection'
ORDER BY rs.sync_priority DESC, rs.replay_delay_bytes ASC;

-- Problems with traditional PostgreSQL replication:
-- 1. Manual failover process requiring human intervention and expertise
-- 2. Complex configuration management across multiple servers
-- 3. Limited built-in monitoring and health checking capabilities
-- 4. Potential for data loss during failover if not configured properly
-- 5. Application-level connection management complexity
-- 6. No automatic discovery of new primary after failover
-- 7. Split-brain scenarios possible without proper fencing mechanisms
-- 8. Limited geographic distribution capabilities for disaster recovery
-- 9. Difficulty in adding/removing replica servers without downtime
-- 10. Complex backup and point-in-time recovery coordination across replicas

-- Additional monitoring complexity
-- Check for replication slots to prevent WAL accumulation
SELECT 
  slot_name,
  plugin,
  slot_type,
  datoid,
  database,
  temporary,
  active,
  active_pid,
  xmin,
  catalog_xmin,
  restart_lsn,
  confirmed_flush_lsn,

  -- Calculate slot lag
  pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) as slot_lag_bytes,

  -- Check if slot is causing WAL retention
  CASE 
    WHEN active = false THEN 'inactive_slot'
    WHEN pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) > 1073741824 THEN 'excessive_lag'
    ELSE 'healthy'
  END as slot_status

FROM pg_replication_slots
ORDER BY slot_lag_bytes DESC;

-- MySQL replication (even more limited)
-- Master configuration
log-bin=mysql-bin
server-id=1
binlog-format=ROW
sync-binlog=1
innodb-flush-log-at-trx-commit=1

-- Slave configuration  
server-id=2
relay-log=mysql-relay
read-only=1

-- Basic replication status (limited information)
SHOW SLAVE STATUS\G

-- Manual failover process (basic and risky)
STOP SLAVE;
RESET SLAVE ALL;
-- Manually change master configuration

-- MySQL replication limitations:
-- - Even more manual failover process
-- - Limited monitoring and diagnostics
-- - Poor handling of network partitions
-- - Basic conflict resolution
-- - Limited geographic replication support
-- - Minimal built-in health checking
-- - Simple master-slave topology only

// MongoDB Replica Sets - automatic failover with advanced high availability features
const { MongoClient } = require('mongodb');

// Advanced MongoDB Replica Set Management and High Availability System
class MongoReplicaSetManager {
  constructor(connectionString) {
    this.connectionString = connectionString;
    this.client = null;
    this.db = null;

    // High availability configuration
    this.replicaSetConfig = {
      members: [],
      settings: {
        chainingAllowed: true,
        heartbeatIntervalMillis: 2000,
        heartbeatTimeoutSecs: 10,
        electionTimeoutMillis: 10000,
        catchUpTimeoutMillis: 60000,
        getLastErrorModes: {},
        getLastErrorDefaults: { w: 1, wtimeout: 0 }
      }
    };

    this.healthMetrics = new Map();
    this.failoverHistory = [];
    this.performanceTargets = {
      maxReplicationLagSeconds: 10,
      maxElectionTimeSeconds: 30,
      minHealthyMembers: 2
    };
  }

  async initializeReplicaSet(members, options = {}) {
    console.log('Initializing MongoDB replica set with advanced high availability...');

    const {
      replicaSetName = 'rs0',
      priority = { primary: 1, secondary: 0.5, arbiter: 0 },
      tags = {},
      writeConcern = { w: 'majority', j: true },
      readPreference = 'primaryPreferred'
    } = options;

    try {
      // Connect to the primary candidate
      this.client = new MongoClient(this.connectionString, {
        useNewUrlParser: true,
        useUnifiedTopology: true,
        replicaSet: replicaSetName,
        readPreference: readPreference,
        writeConcern: writeConcern,
        maxPoolSize: 10,
        serverSelectionTimeoutMS: 5000,
        socketTimeoutMS: 45000,
        heartbeatFrequencyMS: 10000,
        retryWrites: true,
        retryReads: true
      });

      await this.client.connect();
      this.db = this.client.db('admin');

      // Build replica set configuration
      const replicaSetConfig = {
        _id: replicaSetName,
        version: 1,
        members: members.map((member, index) => ({
          _id: index,
          host: member.host,
          priority: member.priority || priority[member.type] || 1,
          votes: member.type === 'arbiter' ? 1 : 1,
          arbiterOnly: member.type === 'arbiter',
          buildIndexes: member.type !== 'arbiter',
          hidden: member.hidden || false,
          slaveDelay: member.slaveDelay || 0,
          tags: { ...tags[member.type], region: member.region, datacenter: member.datacenter }
        })),
        settings: {
          chainingAllowed: true,
          heartbeatIntervalMillis: 2000,
          heartbeatTimeoutSecs: 10,
          electionTimeoutMillis: 10000,
          catchUpTimeoutMillis: 60000,

          // Advanced write concern configurations
          getLastErrorModes: {
            multiDataCenter: { datacenter: 2 },
            majority: { region: 2 }
          },
          getLastErrorDefaults: { 
            w: 'majority', 
            j: true,
            wtimeout: 10000 
          }
        }
      };

      // Initialize replica set
      const initResult = await this.db.runCommand({
        replSetInitiate: replicaSetConfig
      });

      if (initResult.ok === 1) {
        console.log('Replica set initialized successfully');

        // Wait for primary election
        await this.waitForPrimaryElection();

        // Perform initial health check
        const healthStatus = await this.performHealthCheck();

        // Setup monitoring
        await this.setupAdvancedMonitoring();

        console.log('Replica set ready for high availability operations');
        return {
          success: true,
          replicaSetName: replicaSetName,
          members: members,
          healthStatus: healthStatus
        };
      } else {
        throw new Error(`Replica set initialization failed: ${initResult.errmsg}`);
      }

    } catch (error) {
      console.error('Replica set initialization error:', error);
      return {
        success: false,
        error: error.message
      };
    }
  }

  async performComprehensiveHealthCheck() {
    console.log('Performing comprehensive replica set health assessment...');

    const healthReport = {
      timestamp: new Date(),
      replicaSetStatus: null,
      memberHealth: [],
      replicationLag: {},
      electionMetrics: {},
      networkConnectivity: {},
      performanceMetrics: {},
      alerts: [],
      recommendations: []
    };

    try {
      // Get replica set status
      const rsStatus = await this.db.runCommand({ replSetGetStatus: 1 });
      healthReport.replicaSetStatus = {
        name: rsStatus.set,
        primary: rsStatus.members.find(m => m.state === 1)?.name,
        memberCount: rsStatus.members.length,
        healthyMembers: rsStatus.members.filter(m => [1, 2, 7].includes(m.state)).length,
        state: rsStatus.myState
      };

      // Analyze each member
      for (const member of rsStatus.members) {
        const memberHealth = {
          name: member.name,
          state: member.state,
          stateStr: member.stateStr,
          health: member.health,
          uptime: member.uptime,
          lastHeartbeat: member.lastHeartbeat,
          lastHeartbeatRecv: member.lastHeartbeatRecv,
          pingMs: member.pingMs,
          syncSourceHost: member.syncingTo,

          // Calculate replication lag
          replicationLag: member.optimeDate && rsStatus.date ? 
            (rsStatus.date - member.optimeDate) / 1000 : null,

          // Member status assessment
          status: this.assessMemberStatus(member),

          // Performance metrics
          performanceMetrics: {
            heartbeatLatency: member.pingMs,
            connectionHealth: member.health === 1 ? 'healthy' : 'unhealthy',
            stateStability: this.assessStateStability(member)
          }
        };

        healthReport.memberHealth.push(memberHealth);

        // Track replication lag
        if (memberHealth.replicationLag !== null) {
          healthReport.replicationLag[member.name] = memberHealth.replicationLag;
        }
      }

      // Analyze election metrics
      healthReport.electionMetrics = await this.analyzeElectionMetrics(rsStatus);

      // Check network connectivity
      healthReport.networkConnectivity = await this.checkNetworkConnectivity(rsStatus.members);

      // Generate alerts based on thresholds
      healthReport.alerts = this.generateHealthAlerts(healthReport);

      // Generate recommendations
      healthReport.recommendations = this.generateHealthRecommendations(healthReport);

      console.log(`Health check completed: ${healthReport.memberHealth.length} members analyzed`);
      console.log(`Healthy members: ${healthReport.replicaSetStatus.healthyMembers}/${healthReport.replicaSetStatus.memberCount}`);
      console.log(`Alerts generated: ${healthReport.alerts.length}`);

      return healthReport;

    } catch (error) {
      console.error('Health check failed:', error);
      healthReport.error = error.message;
      return healthReport;
    }
  }

  assessMemberStatus(member) {
    const status = {
      overall: 'unknown',
      issues: [],
      strengths: []
    };

    // State-based assessment
    switch (member.state) {
      case 1: // PRIMARY
        status.overall = 'primary';
        status.strengths.push('Acting as primary, accepting writes');
        break;
      case 2: // SECONDARY
        status.overall = 'healthy';
        status.strengths.push('Healthy secondary, replicating data');
        if (member.optimeDate && Date.now() - member.optimeDate > 30000) {
          status.issues.push('Replication lag exceeds 30 seconds');
          status.overall = 'lagging';
        }
        break;
      case 3: // RECOVERING
        status.overall = 'recovering';
        status.issues.push('Member is in recovery state');
        break;
      case 5: // STARTUP2
        status.overall = 'starting';
        status.issues.push('Member is in startup phase');
        break;
      case 6: // UNKNOWN
        status.overall = 'unknown';
        status.issues.push('Member state is unknown');
        break;
      case 7: // ARBITER
        status.overall = 'arbiter';
        status.strengths.push('Functioning arbiter for elections');
        break;
      case 8: // DOWN
        status.overall = 'down';
        status.issues.push('Member is down or unreachable');
        break;
      case 9: // ROLLBACK
        status.overall = 'rollback';
        status.issues.push('Member is performing rollback');
        break;
      case 10: // REMOVED
        status.overall = 'removed';
        status.issues.push('Member has been removed from replica set');
        break;
      default:
        status.overall = 'unknown';
        status.issues.push(`Unexpected state: ${member.state}`);
    }

    // Health-based assessment
    if (member.health !== 1) {
      status.issues.push('Member health check failing');
      if (status.overall === 'healthy') {
        status.overall = 'unhealthy';
      }
    }

    // Network latency assessment
    if (member.pingMs && member.pingMs > 100) {
      status.issues.push(`High network latency: ${member.pingMs}ms`);
    } else if (member.pingMs && member.pingMs < 10) {
      status.strengths.push(`Low network latency: ${member.pingMs}ms`);
    }

    return status;
  }

  async implementAutomaticFailoverTesting() {
    console.log('Implementing automatic failover testing and validation...');

    const failoverTest = {
      testId: require('crypto').randomUUID(),
      timestamp: new Date(),
      phases: [],
      results: {
        success: false,
        totalTimeMs: 0,
        electionTimeMs: 0,
        dataConsistencyVerified: false,
        applicationConnectivityRestored: false
      }
    };

    try {
      // Phase 1: Pre-failover health check
      console.log('Phase 1: Pre-failover health assessment...');
      const preFailoverHealth = await this.performComprehensiveHealthCheck();
      failoverTest.phases.push({
        phase: 'pre_failover_health',
        timestamp: new Date(),
        status: 'completed',
        data: preFailoverHealth
      });

      if (preFailoverHealth.replicaSetStatus.healthyMembers < this.performanceTargets.minHealthyMembers + 1) {
        throw new Error('Insufficient healthy members for safe failover testing');
      }

      // Phase 2: Insert test data for consistency verification
      console.log('Phase 2: Inserting test data for consistency verification...');
      const testCollection = this.client.db('failover_test').collection('consistency_check');
      const testDocuments = Array.from({ length: 100 }, (_, i) => ({
        _id: `failover_test_${failoverTest.testId}_${i}`,
        timestamp: new Date(),
        sequenceNumber: i,
        testData: `Failover test data ${i}`,
        checksum: require('crypto').createHash('md5').update(`test_${i}`).digest('hex')
      }));

      await testCollection.insertMany(testDocuments, { writeConcern: { w: 'majority', j: true } });
      failoverTest.phases.push({
        phase: 'test_data_insertion',
        timestamp: new Date(),
        status: 'completed',
        data: { documentsInserted: testDocuments.length }
      });

      // Phase 3: Simulate primary failure (step down primary)
      console.log('Phase 3: Simulating primary failure...');
      const startTime = Date.now();

      await this.db.runCommand({ replSetStepDown: 60, force: true });

      failoverTest.phases.push({
        phase: 'primary_step_down',
        timestamp: new Date(),
        status: 'completed',
        data: { stepDownInitiated: true }
      });

      // Phase 4: Wait for new primary election
      console.log('Phase 4: Waiting for new primary election...');
      const electionStartTime = Date.now();

      const newPrimary = await this.waitForPrimaryElection(30000); // 30 second timeout
      const electionEndTime = Date.now();

      failoverTest.results.electionTimeMs = electionEndTime - electionStartTime;

      failoverTest.phases.push({
        phase: 'primary_election',
        timestamp: new Date(),
        status: 'completed',
        data: { 
          newPrimary: newPrimary,
          electionTimeMs: failoverTest.results.electionTimeMs
        }
      });

      // Phase 5: Verify data consistency
      console.log('Phase 5: Verifying data consistency...');

      // Reconnect to new primary
      await this.client.close();
      this.client = new MongoClient(this.connectionString, {
        useNewUrlParser: true,
        useUnifiedTopology: true,
        readPreference: 'primary'
      });
      await this.client.connect();

      const verificationCollection = this.client.db('failover_test').collection('consistency_check');
      const retrievedDocs = await verificationCollection.find({
        _id: { $regex: `^failover_test_${failoverTest.testId}_` }
      }).toArray();

      const consistencyCheck = {
        expectedCount: testDocuments.length,
        retrievedCount: retrievedDocs.length,
        dataIntegrityVerified: true,
        checksumMatches: 0
      };

      // Verify checksums
      for (const doc of retrievedDocs) {
        const expectedChecksum = require('crypto').createHash('md5')
          .update(`test_${doc.sequenceNumber}`).digest('hex');
        if (doc.checksum === expectedChecksum) {
          consistencyCheck.checksumMatches++;
        }
      }

      consistencyCheck.dataIntegrityVerified = 
        consistencyCheck.expectedCount === consistencyCheck.retrievedCount &&
        consistencyCheck.checksumMatches === consistencyCheck.expectedCount;

      failoverTest.results.dataConsistencyVerified = consistencyCheck.dataIntegrityVerified;

      failoverTest.phases.push({
        phase: 'data_consistency_verification',
        timestamp: new Date(),
        status: 'completed',
        data: consistencyCheck
      });

      // Phase 6: Test application connectivity
      console.log('Phase 6: Testing application connectivity...');

      try {
        // Simulate application operations
        await verificationCollection.insertOne({
          _id: `post_failover_${failoverTest.testId}`,
          timestamp: new Date(),
          message: 'Post-failover connectivity test'
        }, { writeConcern: { w: 'majority' } });

        const postFailoverDoc = await verificationCollection.findOne({
          _id: `post_failover_${failoverTest.testId}`
        });

        failoverTest.results.applicationConnectivityRestored = postFailoverDoc !== null;

      } catch (error) {
        console.error('Application connectivity test failed:', error);
        failoverTest.results.applicationConnectivityRestored = false;
      }

      failoverTest.phases.push({
        phase: 'application_connectivity_test',
        timestamp: new Date(),
        status: failoverTest.results.applicationConnectivityRestored ? 'completed' : 'failed',
        data: { connectivityRestored: failoverTest.results.applicationConnectivityRestored }
      });

      // Phase 7: Post-failover health check
      console.log('Phase 7: Post-failover health assessment...');
      const postFailoverHealth = await this.performComprehensiveHealthCheck();
      failoverTest.phases.push({
        phase: 'post_failover_health',
        timestamp: new Date(),
        status: 'completed',
        data: postFailoverHealth
      });

      // Calculate total test time
      failoverTest.results.totalTimeMs = Date.now() - startTime;

      // Determine overall success
      failoverTest.results.success = 
        failoverTest.results.electionTimeMs <= (this.performanceTargets.maxElectionTimeSeconds * 1000) &&
        failoverTest.results.dataConsistencyVerified &&
        failoverTest.results.applicationConnectivityRestored &&
        postFailoverHealth.replicaSetStatus.healthyMembers >= this.performanceTargets.minHealthyMembers;

      // Cleanup test data
      await verificationCollection.deleteMany({
        _id: { $regex: `^(failover_test_${failoverTest.testId}_|post_failover_${failoverTest.testId})` }
      });

      console.log(`Failover test completed: ${failoverTest.results.success ? 'SUCCESS' : 'PARTIAL_SUCCESS'}`);
      console.log(`Total failover time: ${failoverTest.results.totalTimeMs}ms`);
      console.log(`Election time: ${failoverTest.results.electionTimeMs}ms`);
      console.log(`Data consistency: ${failoverTest.results.dataConsistencyVerified ? 'VERIFIED' : 'FAILED'}`);
      console.log(`Application connectivity: ${failoverTest.results.applicationConnectivityRestored ? 'RESTORED' : 'FAILED'}`);

      // Record failover test in history
      this.failoverHistory.push(failoverTest);

      return failoverTest;

    } catch (error) {
      console.error('Failover test failed:', error);
      failoverTest.phases.push({
        phase: 'error',
        timestamp: new Date(),
        status: 'failed',
        error: error.message
      });
      failoverTest.results.success = false;
      return failoverTest;
    }
  }

  async setupAdvancedReadPreferences(applications) {
    console.log('Setting up advanced read preferences for optimal performance...');

    const readPreferenceConfigurations = {
      // Real-time dashboard - prefer primary for latest data
      realtime_dashboard: {
        readPreference: 'primary',
        maxStalenessSeconds: 0,
        tags: [],
        description: 'Real-time data requires primary reads',
        useCase: 'Live dashboards, real-time analytics'
      },

      // Reporting queries - can use secondaries with some lag tolerance
      reporting_analytics: {
        readPreference: 'secondaryPreferred',
        maxStalenessSeconds: 30,
        tags: [{ region: 'us-east', workload: 'analytics' }],
        description: 'Analytics workload can tolerate slight lag',
        useCase: 'Business intelligence, historical reports'
      },

      // Geographically distributed reads
      geographic_reads: {
        readPreference: 'nearest',
        maxStalenessSeconds: 60,
        tags: [],
        description: 'Prioritize network proximity for user-facing reads',
        useCase: 'User-facing applications, content delivery'
      },

      // Heavy analytical workloads
      heavy_analytics: {
        readPreference: 'secondary',
        maxStalenessSeconds: 120,
        tags: [{ workload: 'analytics', ssd: 'true' }],
        description: 'Dedicated secondary for heavy analytical queries',
        useCase: 'Data mining, complex aggregations, ML training'
      },

      // Backup and archival operations
      backup_operations: {
        readPreference: 'secondary',
        maxStalenessSeconds: 300,
        tags: [{ backup: 'true', priority: 'low' }],
        description: 'Use dedicated backup secondary',
        useCase: 'Backup operations, data archival, compliance exports'
      }
    };

    const clientConfigurations = {};

    for (const [appName, app] of Object.entries(applications)) {
      const config = readPreferenceConfigurations[app.readPattern] || readPreferenceConfigurations.geographic_reads;

      console.log(`Configuring read preferences for ${appName}:`);
      console.log(`  Pattern: ${app.readPattern}`);
      console.log(`  Read Preference: ${config.readPreference}`);
      console.log(`  Max Staleness: ${config.maxStalenessSeconds}s`);

      clientConfigurations[appName] = {
        connectionString: this.buildConnectionString(config),
        readPreference: config.readPreference,
        readPreferenceTags: config.tags,
        maxStalenessSeconds: config.maxStalenessSeconds,

        // Additional client options for optimization
        options: {
          maxPoolSize: app.connectionPoolSize || 10,
          minPoolSize: app.minConnectionPoolSize || 2,
          maxIdleTimeMS: 30000,
          serverSelectionTimeoutMS: 5000,
          socketTimeoutMS: 45000,
          connectTimeoutMS: 10000,

          // Retry configuration
          retryWrites: true,
          retryReads: true,

          // Write concern based on application requirements
          writeConcern: app.writeConcern || { w: 'majority', j: true },

          // Read concern for consistency requirements
          readConcern: { level: app.readConcern || 'majority' }
        },

        // Monitoring configuration
        monitoring: {
          commandMonitoring: true,
          serverMonitoring: true,
          topologyMonitoring: true
        },

        description: config.description,
        useCase: config.useCase,
        optimizationTips: this.generateReadOptimizationTips(config, app)
      };
    }

    // Setup monitoring for read preference effectiveness
    await this.setupReadPreferenceMonitoring(clientConfigurations);

    console.log(`Read preference configurations created for ${Object.keys(clientConfigurations).length} applications`);

    return clientConfigurations;
  }

  async implementDisasterRecoveryProcedures(options = {}) {
    console.log('Implementing comprehensive disaster recovery procedures...');

    const {
      backupSchedule = 'daily',
      retentionPolicy = { daily: 7, weekly: 4, monthly: 6 },
      geographicDistribution = true,
      automaticFailback = false,
      rtoTarget = 300, // Recovery Time Objective in seconds
      rpoTarget = 60   // Recovery Point Objective in seconds
    } = options;

    const disasterRecoveryPlan = {
      backupStrategy: await this.implementBackupStrategy(backupSchedule, retentionPolicy),
      failoverProcedures: await this.implementFailoverProcedures(rtoTarget),
      recoveryValidation: await this.implementRecoveryValidation(),
      monitoringAndAlerting: await this.setupDisasterRecoveryMonitoring(),
      documentationAndRunbooks: await this.generateDisasterRecoveryRunbooks(),
      testingSchedule: await this.createDisasterRecoveryTestSchedule()
    };

    // Geographic distribution setup
    if (geographicDistribution) {
      disasterRecoveryPlan.geographicDistribution = await this.setupGeographicDistribution();
    }

    // Automatic failback configuration
    if (automaticFailback) {
      disasterRecoveryPlan.automaticFailback = await this.configureAutomaticFailback();
    }

    console.log('Disaster recovery procedures implemented successfully');
    return disasterRecoveryPlan;
  }

  async implementBackupStrategy(schedule, retentionPolicy) {
    console.log('Implementing comprehensive backup strategy...');

    const backupStrategy = {
      hotBackups: {
        enabled: true,
        schedule: schedule,
        method: 'mongodump_with_oplog',
        compression: true,
        encryption: true,
        storageLocation: ['local', 's3', 'gcs'],
        retentionPolicy: retentionPolicy
      },

      continuousBackup: {
        enabled: true,
        oplogTailing: true,
        changeStreams: true,
        pointInTimeRecovery: true,
        maxRecoveryWindow: '7 days'
      },

      consistencyChecks: {
        enabled: true,
        frequency: 'daily',
        validationMethods: ['checksum', 'document_count', 'index_integrity']
      },

      crossRegionReplication: {
        enabled: true,
        regions: ['us-east-1', 'us-west-2', 'eu-west-1'],
        replicationLag: '< 60 seconds'
      }
    };

    // Implement backup automation
    const backupJobs = await this.createAutomatedBackupJobs(backupStrategy);

    return {
      ...backupStrategy,
      automationJobs: backupJobs,
      estimatedRPO: this.calculateEstimatedRPO(backupStrategy),
      storageRequirements: this.calculateStorageRequirements(backupStrategy)
    };
  }

  async waitForPrimaryElection(timeoutMs = 30000) {
    console.log('Waiting for primary election...');

    const startTime = Date.now();
    const pollInterval = 1000; // Check every second

    while (Date.now() - startTime < timeoutMs) {
      try {
        const status = await this.db.runCommand({ replSetGetStatus: 1 });
        const primary = status.members.find(member => member.state === 1);

        if (primary) {
          console.log(`Primary elected: ${primary.name}`);
          return primary.name;
        }

        await new Promise(resolve => setTimeout(resolve, pollInterval));
      } catch (error) {
        // Connection might be lost during election, continue polling
        await new Promise(resolve => setTimeout(resolve, pollInterval));
      }
    }

    throw new Error(`Primary election timeout after ${timeoutMs}ms`);
  }

  generateHealthAlerts(healthReport) {
    const alerts = [];

    // Check for unhealthy members
    const unhealthyMembers = healthReport.memberHealth.filter(m => 
      ['unhealthy', 'down', 'unknown'].includes(m.status.overall)
    );

    if (unhealthyMembers.length > 0) {
      alerts.push({
        severity: 'HIGH',
        type: 'UNHEALTHY_MEMBERS',
        message: `${unhealthyMembers.length} replica set members are unhealthy`,
        members: unhealthyMembers.map(m => m.name),
        impact: 'Reduced fault tolerance and potential for data inconsistency'
      });
    }

    // Check replication lag
    const laggedMembers = Object.entries(healthReport.replicationLag)
      .filter(([, lag]) => lag > this.performanceTargets.maxReplicationLagSeconds);

    if (laggedMembers.length > 0) {
      alerts.push({
        severity: 'MEDIUM',
        type: 'REPLICATION_LAG',
        message: `${laggedMembers.length} members have excessive replication lag`,
        details: Object.fromEntries(laggedMembers),
        impact: 'Potential data loss during failover'
      });
    }

    // Check minimum healthy members threshold
    if (healthReport.replicaSetStatus.healthyMembers < this.performanceTargets.minHealthyMembers) {
      alerts.push({
        severity: 'CRITICAL',
        type: 'INSUFFICIENT_HEALTHY_MEMBERS',
        message: `Only ${healthReport.replicaSetStatus.healthyMembers} healthy members (minimum: ${this.performanceTargets.minHealthyMembers})`,
        impact: 'Risk of complete service outage if another member fails'
      });
    }

    return alerts;
  }

  generateHealthRecommendations(healthReport) {
    const recommendations = [];

    // Analyze member distribution
    const membersByState = healthReport.memberHealth.reduce((acc, member) => {
      acc[member.stateStr] = (acc[member.stateStr] || 0) + 1;
      return acc;
    }, {});

    if (membersByState.SECONDARY < 2) {
      recommendations.push({
        priority: 'HIGH',
        category: 'REDUNDANCY',
        recommendation: 'Add additional secondary members for better fault tolerance',
        reasoning: 'Minimum of 2 secondary members recommended for high availability',
        implementation: 'Use rs.add() to add new replica set members'
      });
    }

    // Check for arbiter usage
    if (membersByState.ARBITER > 0) {
      recommendations.push({
        priority: 'MEDIUM',
        category: 'ARCHITECTURE',
        recommendation: 'Consider replacing arbiters with data-bearing members',
        reasoning: 'Data-bearing members provide better fault tolerance than arbiters',
        implementation: 'Add data-bearing member and remove arbiter when safe'
      });
    }

    // Check geographic distribution
    const regions = new Set(healthReport.memberHealth
      .map(m => m.tags?.region)
      .filter(r => r)
    );

    if (regions.size < 2) {
      recommendations.push({
        priority: 'MEDIUM',
        category: 'DISASTER_RECOVERY',
        recommendation: 'Implement geographic distribution of replica set members',
        reasoning: 'Multi-region deployment protects against datacenter-level failures',
        implementation: 'Deploy members across multiple availability zones or regions'
      });
    }

    return recommendations;
  }

  buildConnectionString(config) {
    // Build MongoDB connection string with read preference options
    const params = new URLSearchParams();

    params.append('readPreference', config.readPreference);

    if (config.maxStalenessSeconds > 0) {
      params.append('maxStalenessSeconds', config.maxStalenessSeconds.toString());
    }

    if (config.tags && config.tags.length > 0) {
      config.tags.forEach((tag, index) => {
        Object.entries(tag).forEach(([key, value]) => {
          params.append(`readPreferenceTags[${index}][${key}]`, value);
        });
      });
    }

    return `${this.connectionString}?${params.toString()}`;
  }

  generateReadOptimizationTips(config, app) {
    const tips = [];

    if (config.readPreference === 'secondary' || config.readPreference === 'secondaryPreferred') {
      tips.push('Consider using connection pooling to maintain connections to multiple secondaries');
      tips.push('Monitor secondary lag to ensure data freshness meets application requirements');
    }

    if (config.maxStalenessSeconds > 60) {
      tips.push('Verify that application logic can handle potentially stale data');
      tips.push('Implement application-level caching for frequently accessed but slow-changing data');
    }

    if (app.queryTypes && app.queryTypes.includes('aggregation')) {
      tips.push('Heavy aggregation workloads benefit from dedicated secondary members with optimized hardware');
      tips.push('Consider using $merge or $out stages to pre-compute results on secondaries');
    }

    return tips;
  }

  async createAutomatedBackupJobs(backupStrategy) {
    // Implementation would create actual backup automation
    // This is a simplified representation
    return {
      dailyHotBackup: {
        schedule: '0 2 * * *', // 2 AM daily
        retention: backupStrategy.hotBackups.retentionPolicy.daily,
        enabled: true
      },
      continuousOplogBackup: {
        enabled: backupStrategy.continuousBackup.enabled,
        method: 'changeStreams'
      },
      weeklyFullBackup: {
        schedule: '0 1 * * 0', // 1 AM Sunday
        retention: backupStrategy.hotBackups.retentionPolicy.weekly,
        enabled: true
      }
    };
  }

  calculateEstimatedRPO(backupStrategy) {
    if (backupStrategy.continuousBackup.enabled) {
      return '< 1 minute'; // With oplog tailing
    } else {
      return '24 hours'; // With daily backups only
    }
  }

  calculateStorageRequirements(backupStrategy) {
    // Simplified storage calculation
    return {
      daily: 'Database size × compression ratio × daily retention',
      weekly: 'Database size × compression ratio × weekly retention', 
      monthly: 'Database size × compression ratio × monthly retention',
      estimated: 'Contact administrator for detailed storage analysis'
    };
  }

  async close() {
    if (this.client) {
      await this.client.close();
    }
  }
}

// Benefits of MongoDB Replica Sets:
// - Automatic failover with intelligent primary election algorithms
// - Strong consistency with configurable write and read concerns
// - Geographic distribution support for disaster recovery
// - Built-in health monitoring and self-healing capabilities
// - Flexible read preference configuration for performance optimization
// - Comprehensive backup and point-in-time recovery options
// - Zero-downtime member addition and removal
// - Advanced replication monitoring and alerting
// - Split-brain prevention through majority-based decisions
// - SQL-compatible high availability management through QueryLeaf integration

module.exports = {
  MongoReplicaSetManager
};