MongoDB Time Series Data Storage and Optimization: Advanced Temporal Data Analytics and High-Performance Storage Strategies

-- Traditional PostgreSQL time series data handling - inefficient storage and limited optimization

-- Basic time series table with poor storage efficiency
CREATE TABLE sensor_readings (
    reading_id SERIAL PRIMARY KEY,
    device_id VARCHAR(50) NOT NULL,
    sensor_type VARCHAR(50) NOT NULL,
    location VARCHAR(100),
    timestamp TIMESTAMP NOT NULL,

    -- Measurements stored as separate columns (inflexible schema)
    temperature DECIMAL(5,2),
    humidity DECIMAL(5,2),
    pressure DECIMAL(7,2),
    battery_level INTEGER,
    signal_strength INTEGER,

    -- Limited metadata support
    device_metadata JSONB,

    -- Basic audit fields
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,

    -- Manual partitioning hint
    partition_key DATE GENERATED ALWAYS AS (DATE(timestamp)) STORED
);

-- Manual partitioning setup (complex maintenance overhead)
CREATE INDEX idx_sensor_readings_timestamp ON sensor_readings(timestamp DESC);
CREATE INDEX idx_sensor_readings_device_time ON sensor_readings(device_id, timestamp DESC);
CREATE INDEX idx_sensor_readings_type_time ON sensor_readings(sensor_type, timestamp DESC);

-- Attempt at time-based partitioning (limited automation)
DO $$
DECLARE
    start_date DATE;
    end_date DATE;
    partition_name TEXT;
BEGIN
    start_date := DATE_TRUNC('month', CURRENT_DATE - INTERVAL '6 months');

    WHILE start_date <= DATE_TRUNC('month', CURRENT_DATE + INTERVAL '3 months') LOOP
        end_date := start_date + INTERVAL '1 month';
        partition_name := 'sensor_readings_' || TO_CHAR(start_date, 'YYYY_MM');

        EXECUTE format('
            CREATE TABLE IF NOT EXISTS %I PARTITION OF sensor_readings
            FOR VALUES FROM (%L) TO (%L)',
            partition_name, start_date, end_date);

        start_date := end_date;
    END LOOP;
END;
$$;

-- Time series aggregation queries (inefficient for large datasets)
WITH hourly_averages AS (
    SELECT 
        device_id,
        sensor_type,
        DATE_TRUNC('hour', timestamp) as hour_bucket,

        -- Basic aggregations (limited analytical functions)
        COUNT(*) as reading_count,
        AVG(temperature) as avg_temperature,
        AVG(humidity) as avg_humidity,
        AVG(pressure) as avg_pressure,
        MIN(temperature) as min_temperature,
        MAX(temperature) as max_temperature,

        -- Standard deviation calculations (expensive)
        STDDEV(temperature) as temp_stddev,
        STDDEV(humidity) as humidity_stddev,

        -- Battery and connectivity metrics
        AVG(battery_level) as avg_battery,
        AVG(signal_strength) as avg_signal_strength,

        -- Data quality metrics
        COUNT(*) FILTER (WHERE temperature IS NOT NULL) as valid_temp_readings,
        COUNT(*) FILTER (WHERE humidity IS NOT NULL) as valid_humidity_readings

    FROM sensor_readings sr
    WHERE timestamp >= CURRENT_TIMESTAMP - INTERVAL '24 hours'
    AND timestamp < CURRENT_TIMESTAMP
    GROUP BY device_id, sensor_type, DATE_TRUNC('hour', timestamp)
),

daily_summaries AS (
    SELECT 
        device_id,
        sensor_type,
        DATE_TRUNC('day', hour_bucket) as day_bucket,

        -- Aggregation of aggregations (double computation overhead)
        SUM(reading_count) as total_readings_per_day,
        AVG(avg_temperature) as daily_avg_temperature,
        MIN(min_temperature) as daily_min_temperature,
        MAX(max_temperature) as daily_max_temperature,
        AVG(avg_humidity) as daily_avg_humidity,
        AVG(avg_pressure) as daily_avg_pressure,

        -- Battery consumption analysis
        MIN(avg_battery) as daily_min_battery,
        AVG(avg_battery) as daily_avg_battery,

        -- Connectivity quality
        AVG(avg_signal_strength) as daily_avg_signal,

        -- Data completeness metrics
        ROUND(
            (SUM(valid_temp_readings) * 100.0) / NULLIF(SUM(reading_count), 0), 2
        ) as temperature_data_completeness_percent,

        ROUND(
            (SUM(valid_humidity_readings) * 100.0) / NULLIF(SUM(reading_count), 0), 2
        ) as humidity_data_completeness_percent

    FROM hourly_averages
    GROUP BY device_id, sensor_type, DATE_TRUNC('day', hour_bucket)
),

device_health_analysis AS (
    -- Complex analysis requiring multiple scans
    SELECT 
        ds.device_id,
        ds.sensor_type,
        COUNT(*) as analysis_days,

        -- Temperature trend analysis (limited analytical capabilities)
        AVG(ds.daily_avg_temperature) as overall_avg_temperature,
        STDDEV(ds.daily_avg_temperature) as temperature_variability,

        -- Battery degradation analysis
        CASE 
            WHEN COUNT(*) > 1 THEN
                -- Simple linear trend approximation
                (MAX(ds.daily_avg_battery) - MIN(ds.daily_avg_battery)) / NULLIF(COUNT(*) - 1, 0)
            ELSE NULL
        END as daily_battery_degradation_rate,

        -- Connectivity stability
        AVG(ds.daily_avg_signal) as avg_connectivity,
        STDDEV(ds.daily_avg_signal) as connectivity_stability,

        -- Data quality assessment
        AVG(ds.temperature_data_completeness_percent) as avg_data_completeness,

        -- Device status classification
        CASE 
            WHEN AVG(ds.daily_avg_battery) < 20 THEN 'low_battery'
            WHEN AVG(ds.daily_avg_signal) < 30 THEN 'poor_connectivity'  
            WHEN AVG(ds.temperature_data_completeness_percent) < 80 THEN 'unreliable_data'
            ELSE 'healthy'
        END as device_status,

        -- Alert generation
        ARRAY[
            CASE WHEN AVG(ds.daily_avg_battery) < 15 THEN 'CRITICAL_BATTERY' END,
            CASE WHEN AVG(ds.daily_avg_signal) < 20 THEN 'CRITICAL_CONNECTIVITY' END,
            CASE WHEN AVG(ds.temperature_data_completeness_percent) < 50 THEN 'DATA_QUALITY_ISSUE' END,
            CASE WHEN STDDEV(ds.daily_avg_temperature) > 10 THEN 'TEMPERATURE_ANOMALY' END
        ]::TEXT[] as active_alerts

    FROM daily_summaries ds
    WHERE ds.day_bucket >= CURRENT_DATE - INTERVAL '7 days'
    GROUP BY ds.device_id, ds.sensor_type
)
SELECT 
    device_id,
    sensor_type,
    analysis_days,

    -- Performance metrics
    ROUND(overall_avg_temperature, 2) as avg_temp,
    ROUND(temperature_variability, 2) as temp_variability,
    ROUND(daily_battery_degradation_rate, 4) as battery_degradation_per_day,
    ROUND(avg_connectivity, 1) as avg_signal_strength,
    ROUND(avg_data_completeness, 1) as data_completeness_percent,

    -- Status and alerts
    device_status,
    ARRAY_REMOVE(active_alerts, NULL) as alerts,

    -- Recommendations
    CASE device_status
        WHEN 'low_battery' THEN 'Schedule battery replacement or reduce sampling frequency'
        WHEN 'poor_connectivity' THEN 'Check network coverage or relocate device'
        WHEN 'unreliable_data' THEN 'Inspect device sensors and calibration'
        ELSE 'Device operating normally'
    END as recommendation

FROM device_health_analysis
ORDER BY 
    CASE device_status 
        WHEN 'low_battery' THEN 1
        WHEN 'poor_connectivity' THEN 2  
        WHEN 'unreliable_data' THEN 3
        ELSE 4
    END,
    overall_avg_temperature DESC;

-- Traditional approach problems:
-- 1. Inefficient storage - no automatic compression for time series patterns
-- 2. Manual partitioning overhead with limited automation
-- 3. Poor query performance for time range analytics
-- 4. Complex aggregation logic requiring multiple query stages
-- 5. Limited schema flexibility for diverse sensor data
-- 6. No built-in time series analytical functions
-- 7. Expensive index maintenance for time-based queries
-- 8. Poor compression ratios leading to high storage costs
-- 9. Complex retention policy implementation
-- 10. Limited support for high-frequency data ingestion

-- Attempt at high-frequency data insertion (poor performance)
INSERT INTO sensor_readings (
    device_id, sensor_type, location, timestamp,
    temperature, humidity, pressure, battery_level, signal_strength
)
VALUES 
    ('device_001', 'environmental', 'warehouse_a', '2024-10-14 10:00:00', 23.5, 45.2, 1013.2, 85, 75),
    ('device_001', 'environmental', 'warehouse_a', '2024-10-14 10:00:10', 23.6, 45.1, 1013.3, 85, 76),
    ('device_001', 'environmental', 'warehouse_a', '2024-10-14 10:00:20', 23.4, 45.3, 1013.1, 85, 74),
    ('device_002', 'environmental', 'warehouse_b', '2024-10-14 10:00:00', 24.1, 42.8, 1012.8, 90, 82),
    ('device_002', 'environmental', 'warehouse_b', '2024-10-14 10:00:10', 24.2, 42.9, 1012.9, 90, 83);
-- Individual inserts are extremely inefficient for high-frequency data

-- Range queries with limited optimization
SELECT 
    device_id,
    AVG(temperature) as avg_temp,
    COUNT(*) as reading_count
FROM sensor_readings
WHERE timestamp BETWEEN '2024-10-14 09:00:00' AND '2024-10-14 11:00:00'
    AND sensor_type = 'environmental'
GROUP BY device_id
ORDER BY avg_temp DESC;

-- Problems:
-- 1. Full table scan for time range queries despite indexing
-- 2. No automatic data compression reducing storage efficiency
-- 3. Poor aggregation performance for time-based analytics
-- 4. Limited analytical functions for time series analysis
-- 5. Complex retention and archival policy implementation
-- 6. No built-in support for irregular time intervals
-- 7. Inefficient handling of sparse data and missing measurements
-- 8. Manual optimization required for high ingestion rates
-- 9. Limited support for multi-metric time series analysis
-- 10. Complex downsampling and data summarization requirements

// MongoDB Advanced Time Series Data Management
const { MongoClient, ObjectId } = require('mongodb');

const client = new MongoClient('mongodb://localhost:27017');
const db = client.db('advanced_time_series');

// Comprehensive MongoDB Time Series Manager
class AdvancedTimeSeriesManager {
  constructor(db, config = {}) {
    this.db = db;
    this.config = {
      // Time series collection configuration
      defaultGranularity: config.defaultGranularity || 'seconds',
      defaultExpiration: config.defaultExpiration || 86400 * 30, // 30 days
      enableCompression: config.enableCompression !== false,

      // Bucketing and storage optimization
      bucketMaxSpanSeconds: config.bucketMaxSpanSeconds || 3600, // 1 hour
      bucketRoundingSeconds: config.bucketRoundingSeconds || 60, // 1 minute

      // Performance optimization
      enablePreAggregation: config.enablePreAggregation || false,
      aggregationLevels: config.aggregationLevels || ['hourly', 'daily'],
      enableAutomaticIndexing: config.enableAutomaticIndexing !== false,

      // Data retention and lifecycle
      enableAutomaticExpiration: config.enableAutomaticExpiration !== false,
      retentionPolicies: config.retentionPolicies || {
        raw: 7 * 24 * 3600,      // 7 days
        hourly: 90 * 24 * 3600,  // 90 days  
        daily: 365 * 24 * 3600   // 1 year
      },

      // Quality and monitoring
      enableDataQualityTracking: config.enableDataQualityTracking || false,
      enableAnomalyDetection: config.enableAnomalyDetection || false,
      alertingThresholds: config.alertingThresholds || {}
    };

    this.collections = new Map();
    this.aggregationPipelines = new Map();

    this.initializeTimeSeriesSystem();
  }

  async initializeTimeSeriesSystem() {
    console.log('Initializing advanced time series system...');

    try {
      // Setup time series collections with optimization
      await this.setupTimeSeriesCollections();

      // Configure automatic aggregation pipelines  
      if (this.config.enablePreAggregation) {
        await this.setupPreAggregationPipelines();
      }

      // Setup data quality monitoring
      if (this.config.enableDataQualityTracking) {
        await this.setupDataQualityMonitoring();
      }

      // Initialize retention policies
      if (this.config.enableAutomaticExpiration) {
        await this.setupRetentionPolicies();
      }

      console.log('Time series system initialized successfully');

    } catch (error) {
      console.error('Error initializing time series system:', error);
      throw error;
    }
  }

  async createTimeSeriesCollection(collectionName, options = {}) {
    console.log(`Creating optimized time series collection: ${collectionName}`);

    try {
      const timeSeriesOptions = {
        timeseries: {
          timeField: options.timeField || 'timestamp',
          metaField: options.metaField || 'metadata',
          granularity: options.granularity || this.config.defaultGranularity,

          // Advanced bucketing configuration
          bucketMaxSpanSeconds: options.bucketMaxSpanSeconds || this.config.bucketMaxSpanSeconds,
          bucketRoundingSeconds: options.bucketRoundingSeconds || this.config.bucketRoundingSeconds
        },

        // Automatic expiration configuration
        expireAfterSeconds: options.expireAfterSeconds || this.config.defaultExpiration,

        // Storage optimization
        storageEngine: {
          wiredTiger: {
            configString: options.enableCompression ? 'block_compressor=zstd' : undefined
          }
        }
      };

      // Create the time series collection
      const collection = await this.db.createCollection(collectionName, timeSeriesOptions);

      // Store collection reference for management
      this.collections.set(collectionName, {
        collection: collection,
        config: timeSeriesOptions,
        createdAt: new Date()
      });

      // Create optimized indexes for time series queries
      await this.createTimeSeriesIndexes(collection, options);

      console.log(`Time series collection '${collectionName}' created successfully`);

      return {
        success: true,
        collectionName: collectionName,
        configuration: timeSeriesOptions,
        indexesCreated: true
      };

    } catch (error) {
      console.error(`Error creating time series collection '${collectionName}':`, error);
      return {
        success: false,
        error: error.message,
        collectionName: collectionName
      };
    }
  }

  async createTimeSeriesIndexes(collection, options = {}) {
    console.log('Creating optimized indexes for time series collection...');

    try {
      const indexes = [
        // Compound index for time range queries with metadata
        {
          key: { 
            [`${options.metaField || 'metadata'}.device_id`]: 1,
            [`${options.timeField || 'timestamp'}`]: -1 
          },
          name: 'device_time_idx',
          background: true
        },

        // Index for sensor type queries
        {
          key: { 
            [`${options.metaField || 'metadata'}.sensor_type`]: 1,
            [`${options.timeField || 'timestamp'}`]: -1 
          },
          name: 'sensor_time_idx',
          background: true
        },

        // Compound index for location-based queries
        {
          key: { 
            [`${options.metaField || 'metadata'}.location`]: 1,
            [`${options.metaField || 'metadata'}.device_id`]: 1,
            [`${options.timeField || 'timestamp'}`]: -1 
          },
          name: 'location_device_time_idx',
          background: true
        },

        // Index for data quality queries
        {
          key: { 
            [`${options.metaField || 'metadata'}.data_quality`]: 1,
            [`${options.timeField || 'timestamp'}`]: -1 
          },
          name: 'quality_time_idx',
          background: true,
          sparse: true
        }
      ];

      // Create all indexes
      await collection.createIndexes(indexes);

      console.log(`Created ${indexes.length} optimized indexes for time series collection`);

    } catch (error) {
      console.error('Error creating time series indexes:', error);
      throw error;
    }
  }

  async insertTimeSeriesData(collectionName, documents, options = {}) {
    console.log(`Inserting ${documents.length} time series documents into ${collectionName}...`);

    try {
      const collectionInfo = this.collections.get(collectionName);
      if (!collectionInfo) {
        throw new Error(`Time series collection '${collectionName}' not found`);
      }

      const collection = collectionInfo.collection;

      // Prepare documents for time series insertion
      const preparedDocuments = documents.map(doc => this.prepareTimeSeriesDocument(doc, options));

      // Execute optimized bulk insertion
      const insertOptions = {
        ordered: options.ordered !== undefined ? options.ordered : false,
        writeConcern: options.writeConcern || { w: 'majority', j: true },
        ...options.insertOptions
      };

      const insertResult = await collection.insertMany(preparedDocuments, insertOptions);

      // Update data quality metrics if enabled
      if (this.config.enableDataQualityTracking) {
        await this.updateDataQualityMetrics(collectionName, preparedDocuments);
      }

      // Trigger anomaly detection if enabled
      if (this.config.enableAnomalyDetection) {
        await this.checkForAnomalies(collectionName, preparedDocuments);
      }

      return {
        success: true,
        collectionName: collectionName,
        documentsInserted: insertResult.insertedCount,
        insertedIds: insertResult.insertedIds,

        // Performance metrics
        averageDocumentSize: this.calculateAverageDocumentSize(preparedDocuments),
        compressionEnabled: collectionInfo.config.timeseries.enableCompression,

        // Data quality summary
        dataQualityScore: options.trackQuality ? this.calculateDataQualityScore(preparedDocuments) : null
      };

    } catch (error) {
      console.error(`Error inserting time series data into '${collectionName}':`, error);
      return {
        success: false,
        error: error.message,
        collectionName: collectionName
      };
    }
  }

  prepareTimeSeriesDocument(document, options = {}) {
    // Ensure proper time series document structure
    const prepared = {
      timestamp: document.timestamp || new Date(),

      // Organize metadata for optimal bucketing
      metadata: {
        device_id: document.device_id || document.metadata?.device_id,
        sensor_type: document.sensor_type || document.metadata?.sensor_type,
        location: document.location || document.metadata?.location,

        // Device-specific metadata
        device_model: document.device_model || document.metadata?.device_model,
        firmware_version: document.firmware_version || document.metadata?.firmware_version,

        // Data quality indicators
        data_quality: options.calculateQuality ? this.assessDataQuality(document) : undefined,

        // Additional metadata preservation
        ...document.metadata
      },

      // Measurements with proper data types
      measurements: {
        // Environmental measurements
        temperature: this.validateMeasurement(document.temperature, 'temperature'),
        humidity: this.validateMeasurement(document.humidity, 'humidity'),
        pressure: this.validateMeasurement(document.pressure, 'pressure'),

        // Device status measurements
        battery_level: this.validateMeasurement(document.battery_level, 'battery'),
        signal_strength: this.validateMeasurement(document.signal_strength, 'signal'),

        // Custom measurements
        ...this.extractCustomMeasurements(document)
      }
    };

    // Remove undefined values to optimize storage
    this.removeUndefinedValues(prepared);

    return prepared;
  }

  validateMeasurement(value, measurementType) {
    if (value === null || value === undefined) return undefined;

    // Type-specific validation and normalization
    const validationRules = {
      temperature: { min: -50, max: 100, precision: 2 },
      humidity: { min: 0, max: 100, precision: 1 },
      pressure: { min: 900, max: 1100, precision: 1 },
      battery: { min: 0, max: 100, precision: 0 },
      signal: { min: 0, max: 100, precision: 0 }
    };

    const rule = validationRules[measurementType];
    if (!rule) return value; // No validation rule, return as-is

    const numericValue = Number(value);
    if (isNaN(numericValue)) return undefined;

    // Apply bounds checking
    const boundedValue = Math.max(rule.min, Math.min(rule.max, numericValue));

    // Apply precision rounding
    return Number(boundedValue.toFixed(rule.precision));
  }

  async performTimeSeriesAggregation(collectionName, aggregationRequest) {
    console.log(`Performing time series aggregation on ${collectionName}...`);

    try {
      const collectionInfo = this.collections.get(collectionName);
      if (!collectionInfo) {
        throw new Error(`Time series collection '${collectionName}' not found`);
      }

      const collection = collectionInfo.collection;

      // Build optimized aggregation pipeline
      const aggregationPipeline = this.buildTimeSeriesAggregationPipeline(aggregationRequest);

      // Execute aggregation with appropriate options
      const aggregationOptions = {
        allowDiskUse: true,
        maxTimeMS: aggregationRequest.maxTimeMS || 60000,
        hint: aggregationRequest.hint,
        comment: `time_series_aggregation_${Date.now()}`
      };

      const results = await collection.aggregate(aggregationPipeline, aggregationOptions).toArray();

      // Post-process results for enhanced analytics
      const processedResults = this.processAggregationResults(results, aggregationRequest);

      return {
        success: true,
        collectionName: collectionName,
        aggregationType: aggregationRequest.type,
        resultCount: results.length,

        // Aggregation results
        results: processedResults,

        // Execution metadata
        executionStats: {
          pipelineStages: aggregationPipeline.length,
          executionTime: Date.now(),
          dataPointsAnalyzed: this.estimateDataPointsAnalyzed(aggregationRequest)
        }
      };

    } catch (error) {
      console.error(`Error performing time series aggregation on '${collectionName}':`, error);
      return {
        success: false,
        error: error.message,
        collectionName: collectionName,
        aggregationType: aggregationRequest.type
      };
    }
  }

  buildTimeSeriesAggregationPipeline(request) {
    const pipeline = [];

    // Time range filtering (essential first stage for performance)
    if (request.timeRange) {
      pipeline.push({
        $match: {
          timestamp: {
            $gte: new Date(request.timeRange.start),
            $lte: new Date(request.timeRange.end)
          }
        }
      });
    }

    // Metadata filtering
    if (request.filters) {
      const matchConditions = {};

      if (request.filters.device_ids) {
        matchConditions['metadata.device_id'] = { $in: request.filters.device_ids };
      }

      if (request.filters.sensor_types) {
        matchConditions['metadata.sensor_type'] = { $in: request.filters.sensor_types };
      }

      if (request.filters.locations) {
        matchConditions['metadata.location'] = { $in: request.filters.locations };
      }

      if (Object.keys(matchConditions).length > 0) {
        pipeline.push({ $match: matchConditions });
      }
    }

    // Time-based grouping and aggregation
    switch (request.type) {
      case 'time_bucket_aggregation':
        pipeline.push(...this.buildTimeBucketAggregation(request));
        break;
      case 'device_summary':
        pipeline.push(...this.buildDeviceSummaryAggregation(request));
        break;
      case 'trend_analysis':
        pipeline.push(...this.buildTrendAnalysisAggregation(request));
        break;
      case 'anomaly_detection':
        pipeline.push(...this.buildAnomalyDetectionAggregation(request));
        break;
      default:
        pipeline.push(...this.buildDefaultAggregation(request));
    }

    // Result limiting and sorting
    if (request.sort) {
      pipeline.push({ $sort: request.sort });
    }

    if (request.limit) {
      pipeline.push({ $limit: request.limit });
    }

    return pipeline;
  }

  buildTimeBucketAggregation(request) {
    const bucketSize = request.bucketSize || 'hour';
    const bucketFormat = this.getBucketDateFormat(bucketSize);

    return [
      {
        $group: {
          _id: {
            time_bucket: {
              $dateFromString: {
                dateString: {
                  $dateToString: {
                    date: '$timestamp',
                    format: bucketFormat
                  }
                }
              }
            },
            device_id: '$metadata.device_id',
            sensor_type: '$metadata.sensor_type'
          },

          // Statistical aggregations
          measurement_count: { $sum: 1 },

          // Temperature statistics
          avg_temperature: { $avg: '$measurements.temperature' },
          min_temperature: { $min: '$measurements.temperature' },
          max_temperature: { $max: '$measurements.temperature' },
          temp_variance: { $stdDevPop: '$measurements.temperature' },

          // Humidity statistics
          avg_humidity: { $avg: '$measurements.humidity' },
          min_humidity: { $min: '$measurements.humidity' },
          max_humidity: { $max: '$measurements.humidity' },

          // Pressure statistics
          avg_pressure: { $avg: '$measurements.pressure' },
          pressure_range: {
            $subtract: [
              { $max: '$measurements.pressure' },
              { $min: '$measurements.pressure' }
            ]
          },

          // Device health metrics
          avg_battery_level: { $avg: '$measurements.battery_level' },
          min_battery_level: { $min: '$measurements.battery_level' },
          avg_signal_strength: { $avg: '$measurements.signal_strength' },

          // Data quality metrics
          data_completeness: {
            $avg: {
              $cond: {
                if: {
                  $and: [
                    { $ne: ['$measurements.temperature', null] },
                    { $ne: ['$measurements.humidity', null] },
                    { $ne: ['$measurements.pressure', null] }
                  ]
                },
                then: 1,
                else: 0
              }
            }
          },

          // Time range within bucket
          earliest_reading: { $min: '$timestamp' },
          latest_reading: { $max: '$timestamp' }
        }
      },

      // Post-processing and enrichment
      {
        $addFields: {
          time_bucket: '$_id.time_bucket',
          device_id: '$_id.device_id',
          sensor_type: '$_id.sensor_type',

          // Calculate additional metrics
          temperature_stability: {
            $cond: {
              if: { $gt: ['$temp_variance', 0] },
              then: { $divide: ['$temp_variance', '$avg_temperature'] },
              else: 0
            }
          },

          // Battery consumption rate (simplified)
          estimated_battery_consumption: {
            $subtract: [100, '$avg_battery_level']
          },

          // Data quality score
          data_quality_score: {
            $multiply: ['$data_completeness', 100]
          },

          // Bucket duration in minutes
          bucket_duration_minutes: {
            $divide: [
              { $subtract: ['$latest_reading', '$earliest_reading'] },
              60000
            ]
          }
        }
      },

      // Remove the grouped _id field
      {
        $project: { _id: 0 }
      }
    ];
  }

  buildDeviceSummaryAggregation(request) {
    return [
      {
        $group: {
          _id: '$metadata.device_id',

          // Basic metrics
          total_readings: { $sum: 1 },
          sensor_types: { $addToSet: '$metadata.sensor_type' },
          locations: { $addToSet: '$metadata.location' },

          // Time range
          first_reading: { $min: '$timestamp' },
          last_reading: { $max: '$timestamp' },

          // Environmental averages
          avg_temperature: { $avg: '$measurements.temperature' },
          avg_humidity: { $avg: '$measurements.humidity' },
          avg_pressure: { $avg: '$measurements.pressure' },

          // Environmental ranges
          temperature_range: {
            $subtract: [
              { $max: '$measurements.temperature' },
              { $min: '$measurements.temperature' }
            ]
          },
          humidity_range: {
            $subtract: [
              { $max: '$measurements.humidity' },
              { $min: '$measurements.humidity' }
            ]
          },

          // Device health metrics
          current_battery_level: { $last: '$measurements.battery_level' },
          min_battery_level: { $min: '$measurements.battery_level' },
          avg_signal_strength: { $avg: '$measurements.signal_strength' },
          min_signal_strength: { $min: '$measurements.signal_strength' },

          // Data quality assessment
          complete_readings: {
            $sum: {
              $cond: {
                if: {
                  $and: [
                    { $ne: ['$measurements.temperature', null] },
                    { $ne: ['$measurements.humidity', null] },
                    { $ne: ['$measurements.pressure', null] }
                  ]
                },
                then: 1,
                else: 0
              }
            }
          }
        }
      },

      {
        $addFields: {
          device_id: '$_id',

          // Operational duration
          operational_duration_hours: {
            $divide: [
              { $subtract: ['$last_reading', '$first_reading'] },
              3600000
            ]
          },

          // Reading frequency
          avg_reading_interval_minutes: {
            $cond: {
              if: { $gt: ['$total_readings', 1] },
              then: {
                $divide: [
                  { $subtract: ['$last_reading', '$first_reading'] },
                  { $multiply: [{ $subtract: ['$total_readings', 1] }, 60000] }
                ]
              },
              else: null
            }
          },

          // Data completeness percentage
          data_completeness_percent: {
            $multiply: [
              { $divide: ['$complete_readings', '$total_readings'] },
              100
            ]
          },

          // Device health status
          device_health_status: {
            $switch: {
              branches: [
                {
                  case: { $lt: ['$current_battery_level', 15] },
                  then: 'critical_battery'
                },
                {
                  case: { $lt: ['$avg_signal_strength', 30] },
                  then: 'poor_connectivity'
                },
                {
                  case: {
                    $lt: [
                      { $divide: ['$complete_readings', '$total_readings'] },
                      0.8
                    ]
                  },
                  then: 'data_quality_issues'
                }
              ],
              default: 'healthy'
            }
          }
        }
      },

      {
        $project: { _id: 0 }
      }
    ];
  }

  getBucketDateFormat(bucketSize) {
    const formats = {
      'minute': '%Y-%m-%d %H:%M:00',
      'hour': '%Y-%m-%d %H:00:00',
      'day': '%Y-%m-%d 00:00:00',
      'week': '%Y-%U 00:00:00', // Year-Week
      'month': '%Y-%m-01 00:00:00'
    };

    return formats[bucketSize] || formats['hour'];
  }

  async setupRetentionPolicies() {
    console.log('Setting up automatic data retention policies...');

    try {
      for (const [collectionName, collectionInfo] of this.collections.entries()) {
        // Configure TTL indexes for automatic expiration
        const collection = collectionInfo.collection;

        await collection.createIndex(
          { timestamp: 1 },
          {
            name: 'ttl_index',
            expireAfterSeconds: collectionInfo.config.expireAfterSeconds,
            background: true
          }
        );

        console.log(`Retention policy configured for ${collectionName}: ${collectionInfo.config.expireAfterSeconds} seconds`);
      }

    } catch (error) {
      console.error('Error setting up retention policies:', error);
      throw error;
    }
  }

  async setupPreAggregationPipelines() {
    console.log('Setting up pre-aggregation pipelines...');

    // This would typically involve setting up MongoDB change streams
    // or scheduled aggregation jobs for common query patterns

    for (const level of this.config.aggregationLevels) {
      const pipelineName = `pre_aggregation_${level}`;

      // Store pipeline configuration for later execution
      this.aggregationPipelines.set(pipelineName, {
        level: level,
        schedule: this.getAggregationSchedule(level),
        pipeline: this.buildPreAggregationPipeline(level)
      });

      console.log(`Pre-aggregation pipeline configured for ${level} level`);
    }
  }

  // Utility methods for time series management

  calculateAverageDocumentSize(documents) {
    if (!documents || documents.length === 0) return 0;

    const totalSize = documents.reduce((size, doc) => {
      return size + JSON.stringify(doc).length;
    }, 0);

    return Math.round(totalSize / documents.length);
  }

  assessDataQuality(document) {
    let qualityScore = 0;
    let totalChecks = 0;

    // Check for presence of key measurements
    const measurements = ['temperature', 'humidity', 'pressure'];
    for (const measurement of measurements) {
      totalChecks++;
      if (document[measurement] !== null && document[measurement] !== undefined) {
        qualityScore++;
      }
    }

    // Check for reasonable value ranges
    if (document.temperature !== null && document.temperature >= -50 && document.temperature <= 100) {
      qualityScore += 0.5;
    }
    totalChecks += 0.5;

    if (document.humidity !== null && document.humidity >= 0 && document.humidity <= 100) {
      qualityScore += 0.5;
    }
    totalChecks += 0.5;

    return totalChecks > 0 ? qualityScore / totalChecks : 0;
  }

  extractCustomMeasurements(document) {
    const customMeasurements = {};
    const standardFields = ['timestamp', 'device_id', 'sensor_type', 'location', 'metadata', 'temperature', 'humidity', 'pressure', 'battery_level', 'signal_strength'];

    for (const [key, value] of Object.entries(document)) {
      if (!standardFields.includes(key) && typeof value === 'number') {
        customMeasurements[key] = value;
      }
    }

    return customMeasurements;
  }

  removeUndefinedValues(obj) {
    Object.keys(obj).forEach(key => {
      if (obj[key] === undefined) {
        delete obj[key];
      } else if (typeof obj[key] === 'object' && obj[key] !== null) {
        this.removeUndefinedValues(obj[key]);

        // Remove empty objects
        if (Object.keys(obj[key]).length === 0) {
          delete obj[key];
        }
      }
    });
  }

  processAggregationResults(results, request) {
    // Add additional context and calculations to aggregation results
    return results.map(result => ({
      ...result,

      // Add computed fields based on aggregation type
      aggregation_metadata: {
        request_type: request.type,
        generated_at: new Date(),
        bucket_size: request.bucketSize,
        time_range: request.timeRange
      }
    }));
  }

  estimateDataPointsAnalyzed(request) {
    // Simplified estimation based on time range and expected frequency
    if (!request.timeRange) return 'unknown';

    const timeRangeMs = new Date(request.timeRange.end) - new Date(request.timeRange.start);
    const assumedFrequencyMs = 60000; // Assume 1 minute intervals

    return Math.round(timeRangeMs / assumedFrequencyMs);
  }

  getAggregationSchedule(level) {
    const schedules = {
      'hourly': '0 */1 * * * *',     // Every hour
      'daily': '0 0 */1 * * *',      // Every day at midnight
      'weekly': '0 0 0 */7 * *',     // Every week
      'monthly': '0 0 0 1 */1 *'     // Every month on 1st
    };

    return schedules[level] || schedules['daily'];
  }

  buildPreAggregationPipeline(level) {
    // Simplified pre-aggregation pipeline
    // In production, this would be much more sophisticated
    return [
      {
        $match: {
          timestamp: {
            $gte: new Date(Date.now() - this.getLevelTimeRange(level))
          }
        }
      },
      {
        $group: {
          _id: {
            device_id: '$metadata.device_id',
            time_bucket: this.getTimeBucketExpression(level)
          },
          avg_temperature: { $avg: '$measurements.temperature' },
          avg_humidity: { $avg: '$measurements.humidity' },
          count: { $sum: 1 }
        }
      }
    ];
  }

  getLevelTimeRange(level) {
    const ranges = {
      'hourly': 24 * 60 * 60 * 1000,      // 1 day
      'daily': 30 * 24 * 60 * 60 * 1000,  // 30 days
      'weekly': 12 * 7 * 24 * 60 * 60 * 1000, // 12 weeks
      'monthly': 12 * 30 * 24 * 60 * 60 * 1000 // 12 months
    };

    return ranges[level] || ranges['daily'];
  }

  getTimeBucketExpression(level) {
    const expressions = {
      'hourly': {
        $dateFromString: {
          dateString: {
            $dateToString: {
              date: '$timestamp',
              format: '%Y-%m-%d %H:00:00'
            }
          }
        }
      },
      'daily': {
        $dateFromString: {
          dateString: {
            $dateToString: {
              date: '$timestamp',
              format: '%Y-%m-%d 00:00:00'
            }
          }
        }
      }
    };

    return expressions[level] || expressions['hourly'];
  }
}

// Benefits of MongoDB Advanced Time Series Collections:
// - Purpose-built storage optimization with automatic compression
// - Intelligent bucketing for optimal query performance  
// - Built-in retention policies and automatic data expiration
// - Advanced indexing strategies optimized for temporal queries
// - Schema flexibility for diverse sensor and measurement data
// - Native aggregation capabilities for time series analytics
// - Automatic storage optimization and compression
// - High ingestion performance for IoT and monitoring workloads
// - Built-in support for metadata organization and filtering
// - SQL-compatible time series operations through QueryLeaf integration

module.exports = {
  AdvancedTimeSeriesManager
};