MongoDB GridFS and Binary Data Management: Advanced File Storage Patterns for Scalable Document-Based Applications

-- Traditional PostgreSQL file storage with external file system coordination challenges

-- File metadata table with limited integration capabilities  
CREATE TABLE file_metadata (
  file_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  filename VARCHAR(255) NOT NULL,
  original_filename VARCHAR(255) NOT NULL,
  file_path TEXT NOT NULL,
  file_size BIGINT NOT NULL,
  content_type VARCHAR(100),
  upload_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
  uploaded_by UUID REFERENCES users(user_id),
  file_hash VARCHAR(64),

  -- Basic metadata fields
  storage_location VARCHAR(50) DEFAULT 'local',
  is_public BOOLEAN DEFAULT FALSE,
  access_level VARCHAR(20) DEFAULT 'private',

  -- Versioning attempt (complex to manage)
  version_number INTEGER DEFAULT 1,
  parent_file_id UUID REFERENCES file_metadata(file_id),

  -- Status tracking
  processing_status VARCHAR(20) DEFAULT 'uploaded',
  last_accessed TIMESTAMP
);

-- Separate table for file associations (loose coupling problems)
CREATE TABLE document_files (
  document_id UUID NOT NULL,
  file_id UUID NOT NULL,
  relationship_type VARCHAR(50) NOT NULL, -- attachment, image, document, etc.
  display_order INTEGER,
  is_primary BOOLEAN DEFAULT FALSE,
  added_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
  PRIMARY KEY (document_id, file_id)
);

-- Complex query requiring multiple joins and external file system coordination
SELECT 
  d.document_id,
  d.title,
  d.content,
  d.created_at,

  -- File information requires complex aggregation
  json_agg(
    json_build_object(
      'file_id', fm.file_id,
      'filename', fm.filename,
      'original_filename', fm.original_filename,
      'file_size', fm.file_size,
      'content_type', fm.content_type,
      'file_path', fm.file_path, -- External file system path
      'relationship_type', df.relationship_type,
      'display_order', df.display_order,
      'is_primary', df.is_primary,
      'file_exists', (
        -- Expensive file system check required for each file
        CASE WHEN pg_stat_file(fm.file_path) IS NOT NULL 
             THEN TRUE ELSE FALSE END
      ),
      'accessible', (
        -- Additional access control complexity
        CASE WHEN fm.access_level = 'public' OR fm.uploaded_by = $2
             THEN TRUE ELSE FALSE END
      )
    ) ORDER BY df.display_order
  ) FILTER (WHERE fm.file_id IS NOT NULL) as files,

  -- File statistics aggregation
  COUNT(fm.file_id) as file_count,
  SUM(fm.file_size) as total_file_size,
  MAX(fm.upload_timestamp) as latest_file_upload

FROM documents d
LEFT JOIN document_files df ON d.document_id = df.document_id
LEFT JOIN file_metadata fm ON df.file_id = fm.file_id 
  AND fm.processing_status = 'completed'
WHERE d.user_id = $1 
  AND d.status = 'active'
  AND (fm.access_level = 'public' OR fm.uploaded_by = $1 OR $1 IN (
    SELECT user_id FROM document_permissions 
    WHERE document_id = d.document_id AND permission_level IN ('read', 'write', 'admin')
  ))
GROUP BY d.document_id, d.title, d.content, d.created_at
ORDER BY d.created_at DESC
LIMIT 50;

-- Problems with traditional file storage approaches:
-- 1. File system and database synchronization complexity
-- 2. Backup and replication coordination between file system and database
-- 3. Transactional integrity challenges across file system and database operations
-- 4. Complex access control implementation across multiple storage layers
-- 5. Difficulty implementing file versioning and history tracking
-- 6. Storage location management and migration complexity
-- 7. Limited file metadata search and indexing capabilities
-- 8. Performance bottlenecks with large numbers of files
-- 9. Scalability challenges with distributed file storage
-- 10. Complex error handling and recovery across multiple storage systems

-- File upload handling complexity (application layer)
/*
const multer = require('multer');
const path = require('path');
const fs = require('fs').promises;
const crypto = require('crypto');

class TraditionalFileStorage {
  constructor(storageConfig) {
    this.storagePath = storageConfig.storagePath;
    this.maxFileSize = storageConfig.maxFileSize || 10 * 1024 * 1024; // 10MB
    this.allowedTypes = storageConfig.allowedTypes || [];

    // Complex storage configuration
    this.storage = multer.diskStorage({
      destination: async (req, file, cb) => {
        const userDir = path.join(this.storagePath, req.user.id.toString());

        try {
          await fs.mkdir(userDir, { recursive: true });
          cb(null, userDir);
        } catch (error) {
          cb(error);
        }
      },

      filename: (req, file, cb) => {
        const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9);
        const fileExtension = path.extname(file.originalname);
        cb(null, file.fieldname + '-' + uniqueSuffix + fileExtension);
      }
    });
  }

  async handleFileUpload(req, res, next) {
    const upload = multer({
      storage: this.storage,
      limits: { fileSize: this.maxFileSize },
      fileFilter: (req, file, cb) => {
        if (this.allowedTypes.length > 0 && 
            !this.allowedTypes.includes(file.mimetype)) {
          cb(new Error('File type not allowed'));
          return;
        }
        cb(null, true);
      }
    }).array('files', 10);

    upload(req, res, async (err) => {
      if (err) {
        console.error('File upload error:', err);
        return res.status(400).json({ error: err.message });
      }

      try {
        // Complex file processing and database coordination
        const filePromises = req.files.map(async (file) => {
          // Calculate file hash
          const fileBuffer = await fs.readFile(file.path);
          const fileHash = crypto.createHash('sha256')
            .update(fileBuffer).digest('hex');

          // Database transaction complexity
          const client = await pool.connect();
          try {
            await client.query('BEGIN');

            // Insert file metadata
            const fileResult = await client.query(`
              INSERT INTO file_metadata (
                filename, original_filename, file_path, file_size, 
                content_type, uploaded_by, file_hash
              ) VALUES ($1, $2, $3, $4, $5, $6, $7)
              RETURNING file_id
            `, [
              file.filename,
              file.originalname,
              file.path,
              file.size,
              file.mimetype,
              req.user.id,
              fileHash
            ]);

            // Associate with document if specified
            if (req.body.document_id) {
              await client.query(`
                INSERT INTO document_files (
                  document_id, file_id, relationship_type, display_order
                ) VALUES ($1, $2, $3, $4)
              `, [
                req.body.document_id,
                fileResult.rows[0].file_id,
                req.body.relationship_type || 'attachment',
                parseInt(req.body.display_order) || 0
              ]);
            }

            await client.query('COMMIT');
            return {
              file_id: fileResult.rows[0].file_id,
              filename: file.filename,
              original_filename: file.originalname,
              file_size: file.size,
              content_type: file.mimetype
            };

          } catch (dbError) {
            await client.query('ROLLBACK');

            // Cleanup file on database error
            try {
              await fs.unlink(file.path);
            } catch (cleanupError) {
              console.error('File cleanup error:', cleanupError);
            }

            throw dbError;
          } finally {
            client.release();
          }
        });

        const uploadedFiles = await Promise.all(filePromises);
        res.json({ 
          success: true, 
          files: uploadedFiles,
          message: `${uploadedFiles.length} files uploaded successfully`
        });

      } catch (error) {
        console.error('File processing error:', error);
        res.status(500).json({ 
          error: 'File processing failed',
          details: error.message 
        });
      }
    });
  }

  async downloadFile(req, res) {
    try {
      const { file_id } = req.params;

      // Complex authorization and file access logic
      const fileQuery = `
        SELECT fm.*, dp.permission_level
        FROM file_metadata fm
        LEFT JOIN document_files df ON fm.file_id = df.file_id
        LEFT JOIN document_permissions dp ON df.document_id = dp.document_id 
          AND dp.user_id = $2
        WHERE fm.file_id = $1 
          AND (
            fm.is_public = true 
            OR fm.uploaded_by = $2 
            OR dp.permission_level IN ('read', 'write', 'admin')
          )
      `;

      const result = await pool.query(fileQuery, [file_id, req.user.id]);

      if (result.rows.length === 0) {
        return res.status(404).json({ error: 'File not found or access denied' });
      }

      const fileMetadata = result.rows[0];

      // Check if file exists on file system
      try {
        await fs.access(fileMetadata.file_path);
      } catch (error) {
        console.error('File missing from file system:', error);
        return res.status(404).json({ 
          error: 'File not found on storage system' 
        });
      }

      // Update access tracking
      await pool.query(
        'UPDATE file_metadata SET last_accessed = CURRENT_TIMESTAMP WHERE file_id = $1',
        [file_id]
      );

      // Set response headers
      res.setHeader('Content-Type', fileMetadata.content_type);
      res.setHeader('Content-Length', fileMetadata.file_size);
      res.setHeader('Content-Disposition', 
        `inline; filename="${fileMetadata.original_filename}"`);

      // Stream file
      const fileStream = require('fs').createReadStream(fileMetadata.file_path);
      fileStream.pipe(res);

      fileStream.on('error', (error) => {
        console.error('File streaming error:', error);
        if (!res.headersSent) {
          res.status(500).json({ error: 'File streaming failed' });
        }
      });

    } catch (error) {
      console.error('File download error:', error);
      res.status(500).json({ 
        error: 'File download failed',
        details: error.message 
      });
    }
  }
}

// Issues with traditional approach:
// 1. Complex file system and database coordination
// 2. Manual transaction management across storage layers
// 3. File cleanup complexity on errors
// 4. Limited streaming and chunking capabilities
// 5. Difficult backup and replication coordination
// 6. Complex access control across multiple systems
// 7. Manual file existence validation required
// 8. Limited metadata search capabilities
// 9. Scalability bottlenecks with large file counts
// 10. Error-prone manual file path management
*/

// MongoDB GridFS - comprehensive file storage with advanced patterns and seamless MongoDB integration
const { MongoClient, GridFSBucket } = require('mongodb');
const { Readable } = require('stream');
const crypto = require('crypto');
const mime = require('mime-types');

const client = new MongoClient('mongodb://localhost:27017');
const db = client.db('advanced_file_management_platform');

// Advanced MongoDB GridFS file storage and management system
class AdvancedGridFSManager {
  constructor(db, options = {}) {
    this.db = db;
    this.collections = {
      documents: db.collection('documents'),
      users: db.collection('users'),
      fileAccess: db.collection('file_access_logs'),
      fileVersions: db.collection('file_versions'),
      fileSharing: db.collection('file_sharing')
    };

    // GridFS configuration for different file types and use cases
    this.gridFSBuckets = {
      // General file storage
      files: new GridFSBucket(db, { 
        bucketName: 'files',
        chunkSizeBytes: 255 * 1024 // 255KB chunks for optimal performance
      }),

      // Image storage with different chunk size for better streaming
      images: new GridFSBucket(db, {
        bucketName: 'images', 
        chunkSizeBytes: 512 * 1024 // 512KB chunks for large images
      }),

      // Video storage optimized for streaming
      videos: new GridFSBucket(db, {
        bucketName: 'videos',
        chunkSizeBytes: 1024 * 1024 // 1MB chunks for video streaming
      }),

      // Document storage for PDFs, Office files, etc.
      documents: new GridFSBucket(db, {
        bucketName: 'documents',
        chunkSizeBytes: 128 * 1024 // 128KB chunks for document files
      }),

      // Archive storage for compressed files
      archives: new GridFSBucket(db, {
        bucketName: 'archives',
        chunkSizeBytes: 2048 * 1024 // 2MB chunks for large archives
      })
    };

    // Advanced file management configuration
    this.config = {
      maxFileSize: options.maxFileSize || 100 * 1024 * 1024, // 100MB default
      allowedMimeTypes: options.allowedMimeTypes || [],
      enableVersioning: options.enableVersioning !== false,
      enableThumbnails: options.enableThumbnails !== false,
      enableVirusScanning: options.enableVirusScanning || false,
      compressionEnabled: options.compressionEnabled || false,
      encryptionEnabled: options.encryptionEnabled || false,
      accessLogging: options.accessLogging !== false,
      automaticCleanup: options.automaticCleanup !== false
    };

    // File processing pipelines
    this.processors = new Map();
    this.thumbnailGenerators = new Map();
    this.metadataExtractors = new Map();

    this.setupFileProcessors();
    this.setupMetadataExtractors();
    this.setupThumbnailGenerators();
  }

  async uploadFile(fileStream, metadata, options = {}) {
    console.log(`Uploading file: ${metadata.filename}`);

    try {
      // Validate file and metadata
      const validationResult = await this.validateFileUpload(fileStream, metadata, options);
      if (!validationResult.valid) {
        throw new Error(`File validation failed: ${validationResult.errors.join(', ')}`);
      }

      // Determine appropriate GridFS bucket based on file type
      const bucketName = this.determineBucket(metadata.contentType);
      const gridFSBucket = this.gridFSBuckets[bucketName];

      // Enhanced metadata with comprehensive file information
      const enhancedMetadata = {
        // Original metadata
        filename: metadata.filename,
        contentType: metadata.contentType || mime.lookup(metadata.filename) || 'application/octet-stream',

        // File characteristics
        originalSize: metadata.size,
        uploadedAt: new Date(),
        uploadedBy: metadata.uploadedBy,

        // Advanced metadata
        fileHash: null, // Will be calculated during upload
        bucketName: bucketName,
        chunkSize: gridFSBucket.s.options.chunkSizeBytes,

        // Application context
        documentId: metadata.documentId,
        projectId: metadata.projectId,
        organizationId: metadata.organizationId,

        // Access control and permissions
        accessLevel: metadata.accessLevel || 'private',
        permissions: metadata.permissions || {},
        isPublic: metadata.isPublic || false,

        // File relationships and organization
        category: metadata.category || 'general',
        tags: metadata.tags || [],
        description: metadata.description,

        // Versioning information
        version: options.version || 1,
        parentFileId: metadata.parentFileId,
        versionHistory: [],

        // Processing status
        processingStatus: 'uploading',
        processingSteps: [],

        // Performance and optimization
        compressionApplied: false,
        encryptionApplied: false,
        thumbnailGenerated: false,
        metadataExtracted: false,

        // Usage tracking
        downloadCount: 0,
        lastAccessed: null,
        lastModified: new Date(),

        // Storage optimization
        storageOptimized: false,
        deduplicationChecked: false,
        duplicateOf: null,

        // Custom metadata fields
        customMetadata: metadata.customMetadata || {}
      };

      // Create upload stream with comprehensive error handling
      const uploadStream = gridFSBucket.openUploadStream(metadata.filename, {
        metadata: enhancedMetadata,

        // Advanced GridFS options
        chunkSizeBytes: gridFSBucket.s.options.chunkSizeBytes,
        disableMD5: false, // Enable MD5 for file integrity
      });

      // File processing pipeline setup
      let fileHash = crypto.createHash('sha256');
      let totalBytes = 0;
      let compressionStream = null;
      let encryptionStream = null;

      // Setup processing streams if enabled
      if (this.config.compressionEnabled && this.shouldCompress(metadata.contentType)) {
        compressionStream = this.createCompressionStream();
      }

      if (this.config.encryptionEnabled && metadata.encrypted) {
        encryptionStream = this.createEncryptionStream(metadata.encryptionKey);
      }

      // Promise-based upload handling with comprehensive progress tracking
      return new Promise((resolve, reject) => {
        let processingChain = fileStream;

        // Build processing chain
        if (compressionStream) {
          processingChain = processingChain.pipe(compressionStream);
        }

        if (encryptionStream) {
          processingChain = processingChain.pipe(encryptionStream);
        }

        // File data processing during upload
        processingChain.on('data', (chunk) => {
          fileHash.update(chunk);
          totalBytes += chunk.length;

          // Emit progress events
          this.emit('uploadProgress', {
            filename: metadata.filename,
            bytesUploaded: totalBytes,
            totalBytes: metadata.size,
            progress: metadata.size ? (totalBytes / metadata.size) * 100 : 0
          });
        });

        // Stream to GridFS
        processingChain.pipe(uploadStream);

        uploadStream.on('error', async (error) => {
          console.error(`GridFS upload error for ${metadata.filename}:`, error);

          // Cleanup partial upload
          try {
            await this.cleanupFailedUpload(uploadStream.id, bucketName);
          } catch (cleanupError) {
            console.error('Upload cleanup error:', cleanupError);
          }

          reject(error);
        });

        uploadStream.on('finish', async () => {
          try {
            console.log(`File upload completed: ${metadata.filename} (${uploadStream.id})`);

            // Update file metadata with calculated information
            const calculatedHash = fileHash.digest('hex');

            const updateResult = await this.collections.files.updateOne(
              { _id: uploadStream.id },
              {
                $set: {
                  'metadata.fileHash': calculatedHash,
                  'metadata.actualSize': totalBytes,
                  'metadata.compressionApplied': compressionStream !== null,
                  'metadata.encryptionApplied': encryptionStream !== null,
                  'metadata.processingStatus': 'uploaded',
                  'metadata.uploadCompletedAt': new Date()
                }
              }
            );

            // Check for duplicate files based on hash
            const duplicateCheck = await this.checkForDuplicates(calculatedHash, uploadStream.id);

            // Post-upload processing
            const postProcessingTasks = [
              this.extractFileMetadata(uploadStream.id, bucketName, metadata.contentType),
              this.generateThumbnails(uploadStream.id, bucketName, metadata.contentType),
              this.performVirusScanning(uploadStream.id, bucketName),
              this.logFileAccess(uploadStream.id, 'upload', metadata.uploadedBy),
              this.updateFileVersionHistory(uploadStream.id, metadata.parentFileId),
              this.triggerFileWebhooks(uploadStream.id, 'file.uploaded')
            ];

            // Execute post-processing tasks
            const processingResults = await Promise.allSettled(postProcessingTasks);

            // Track processing failures
            const failedProcessing = processingResults
              .filter(result => result.status === 'rejected')
              .map(result => result.reason);

            if (failedProcessing.length > 0) {
              console.warn(`Post-processing warnings for ${metadata.filename}:`, failedProcessing);
            }

            // Final file information
            const fileInfo = {
              fileId: uploadStream.id,
              filename: metadata.filename,
              contentType: metadata.contentType,
              size: totalBytes,
              uploadedAt: new Date(),
              fileHash: calculatedHash,
              bucketName: bucketName,
              gridFSId: uploadStream.id,

              // Processing results
              processingStatus: failedProcessing.length === 0 ? 'completed' : 'completed_with_warnings',
              processingWarnings: failedProcessing,

              // Duplication information
              duplicateInfo: duplicateCheck,

              // URLs for file access
              downloadUrl: this.generateDownloadUrl(uploadStream.id, bucketName),
              streamUrl: this.generateStreamUrl(uploadStream.id, bucketName),
              thumbnailUrl: metadata.contentType.startsWith('image/') ? 
                this.generateThumbnailUrl(uploadStream.id) : null,

              // Metadata
              metadata: enhancedMetadata
            };

            // Update processing status
            await this.collections.files.updateOne(
              { _id: uploadStream.id },
              {
                $set: {
                  'metadata.processingStatus': fileInfo.processingStatus,
                  'metadata.processingCompletedAt': new Date(),
                  'metadata.processingWarnings': failedProcessing
                }
              }
            );

            resolve(fileInfo);

          } catch (error) {
            console.error(`Post-upload processing error for ${metadata.filename}:`, error);
            reject(error);
          }
        });
      });

    } catch (error) {
      console.error(`File upload error for ${metadata.filename}:`, error);
      throw error;
    }
  }

  async downloadFile(fileId, options = {}) {
    console.log(`Downloading file: ${fileId}`);

    try {
      // Get file information
      const fileInfo = await this.getFileInfo(fileId);
      if (!fileInfo) {
        throw new Error(`File not found: ${fileId}`);
      }

      // Authorization check
      if (options.userId) {
        const authorized = await this.checkFileAccess(fileId, options.userId, 'read');
        if (!authorized) {
          throw new Error('Access denied');
        }
      }

      // Determine appropriate bucket
      const bucketName = fileInfo.metadata?.bucketName || 'files';
      const gridFSBucket = this.gridFSBuckets[bucketName];

      // Create download stream
      const downloadStream = gridFSBucket.openDownloadStream(fileId);

      // Setup stream processing if needed
      let processingChain = downloadStream;

      if (fileInfo.metadata?.encryptionApplied && options.decryptionKey) {
        const decryptionStream = this.createDecryptionStream(options.decryptionKey);
        processingChain = processingChain.pipe(decryptionStream);
      }

      if (fileInfo.metadata?.compressionApplied) {
        const decompressionStream = this.createDecompressionStream();
        processingChain = processingChain.pipe(decompressionStream);
      }

      // Log file access
      if (options.userId) {
        await this.logFileAccess(fileId, 'download', options.userId);

        // Update access statistics
        await this.collections.files.updateOne(
          { _id: fileId },
          {
            $inc: { 'metadata.downloadCount': 1 },
            $set: { 'metadata.lastAccessed': new Date() }
          }
        );
      }

      // Return stream with file information
      return {
        stream: processingChain,
        fileInfo: fileInfo,
        contentType: fileInfo.contentType,
        contentLength: fileInfo.length,
        filename: fileInfo.filename,

        // Additional headers for HTTP response
        headers: {
          'Content-Type': fileInfo.contentType,
          'Content-Length': fileInfo.length,
          'Content-Disposition': `${options.disposition || 'inline'}; filename="${fileInfo.filename}"`,
          'Cache-Control': options.cacheControl || 'private, max-age=3600',
          'ETag': fileInfo.metadata?.fileHash,
          'Last-Modified': fileInfo.uploadDate.toUTCString()
        }
      };

    } catch (error) {
      console.error(`File download error for ${fileId}:`, error);
      throw error;
    }
  }

  async searchFiles(query, options = {}) {
    console.log(`Searching files with query:`, query);

    try {
      // Build comprehensive search pipeline
      const searchPipeline = [
        // Stage 1: Initial filtering based on search criteria
        {
          $match: {
            ...this.buildFileSearchFilter(query, options),
            // Ensure we're searching in the files collection
            filename: { $exists: true }
          }
        },

        // Stage 2: Add computed fields for search relevance
        {
          $addFields: {
            // Text search relevance scoring
            textScore: {
              $cond: {
                if: { $ne: [query.text, null] },
                then: {
                  $add: [
                    // Filename match weight
                    { $cond: { 
                      if: { $regexMatch: { input: '$filename', regex: query.text, options: 'i' } },
                      then: 10, else: 0 
                    }},
                    // Description match weight
                    { $cond: { 
                      if: { $regexMatch: { input: '$metadata.description', regex: query.text, options: 'i' } },
                      then: 5, else: 0 
                    }},
                    // Tags match weight
                    { $cond: { 
                      if: { $in: [query.text, '$metadata.tags'] },
                      then: 8, else: 0 
                    }},
                    // Category match weight
                    { $cond: { 
                      if: { $regexMatch: { input: '$metadata.category', regex: query.text, options: 'i' } },
                      then: 3, else: 0 
                    }}
                  ]
                },
                else: 0
              }
            },

            // Recency scoring (newer files get higher scores)
            recencyScore: {
              $divide: [
                { $subtract: [new Date(), '$uploadDate'] },
                86400000 // Convert to days
              ]
            },

            // Popularity scoring based on download count
            popularityScore: {
              $multiply: [
                { $log10: { $add: ['$metadata.downloadCount', 1] } },
                2
              ]
            },

            // Size category for filtering
            sizeCategory: {
              $switch: {
                branches: [
                  { case: { $lt: ['$length', 1024 * 1024] }, then: 'small' }, // < 1MB
                  { case: { $lt: ['$length', 10 * 1024 * 1024] }, then: 'medium' }, // < 10MB
                  { case: { $lt: ['$length', 100 * 1024 * 1024] }, then: 'large' }, // < 100MB
                ],
                default: 'very_large'
              }
            }
          }
        },

        // Stage 3: Apply advanced filtering
        {
          $match: {
            ...(query.sizeCategory && { sizeCategory: query.sizeCategory }),
            ...(query.minScore && { textScore: { $gte: query.minScore } })
          }
        },

        // Stage 4: Lookup related document information if file is associated
        {
          $lookup: {
            from: 'documents',
            localField: 'metadata.documentId',
            foreignField: '_id',
            as: 'documentInfo',
            pipeline: [
              {
                $project: {
                  title: 1,
                  status: 1,
                  createdBy: 1,
                  projectId: 1
                }
              }
            ]
          }
        },

        // Stage 5: Lookup user information for uploaded_by
        {
          $lookup: {
            from: 'users',
            localField: 'metadata.uploadedBy',
            foreignField: '_id',
            as: 'uploaderInfo',
            pipeline: [
              {
                $project: {
                  name: 1,
                  email: 1,
                  avatar: 1
                }
              }
            ]
          }
        },

        // Stage 6: Calculate final relevance score
        {
          $addFields: {
            relevanceScore: {
              $add: [
                '$textScore',
                { $divide: ['$popularityScore', 4] },
                { $cond: { if: { $lt: ['$recencyScore', 30] }, then: 5, else: 0 } }, // Bonus for files < 30 days old
                { $cond: { if: { $gt: [{ $size: '$documentInfo' }, 0] }, then: 2, else: 0 } } // Bonus for associated files
              ]
            }
          }
        },

        // Stage 7: Project final result structure
        {
          $project: {
            fileId: '$_id',
            filename: 1,
            contentType: 1,
            length: 1,
            uploadDate: 1,

            // Metadata information
            category: '$metadata.category',
            tags: '$metadata.tags',
            description: '$metadata.description',
            accessLevel: '$metadata.accessLevel',
            isPublic: '$metadata.isPublic',

            // File characteristics
            fileHash: '$metadata.fileHash',
            bucketName: '$metadata.bucketName',
            downloadCount: '$metadata.downloadCount',
            lastAccessed: '$metadata.lastAccessed',

            // Processing status
            processingStatus: '$metadata.processingStatus',
            thumbnailGenerated: '$metadata.thumbnailGenerated',

            // Computed scores
            textScore: 1,
            popularityScore: 1,
            relevanceScore: 1,
            sizeCategory: 1,

            // Related information
            documentInfo: { $arrayElemAt: ['$documentInfo', 0] },
            uploaderInfo: { $arrayElemAt: ['$uploaderInfo', 0] },

            // Access URLs
            downloadUrl: {
              $concat: [
                '/api/files/',
                { $toString: '$_id' },
                '/download'
              ]
            },

            thumbnailUrl: {
              $cond: {
                if: { $eq: ['$metadata.thumbnailGenerated', true] },
                then: {
                  $concat: [
                    '/api/files/',
                    { $toString: '$_id' },
                    '/thumbnail'
                  ]
                },
                else: null
              }
            },

            // Formatted file information
            formattedSize: {
              $switch: {
                branches: [
                  { 
                    case: { $lt: ['$length', 1024] },
                    then: { $concat: [{ $toString: '$length' }, ' bytes'] }
                  },
                  { 
                    case: { $lt: ['$length', 1024 * 1024] },
                    then: { 
                      $concat: [
                        { $toString: { $round: [{ $divide: ['$length', 1024] }, 1] } },
                        ' KB'
                      ]
                    }
                  },
                  { 
                    case: { $lt: ['$length', 1024 * 1024 * 1024] },
                    then: { 
                      $concat: [
                        { $toString: { $round: [{ $divide: ['$length', 1024 * 1024] }, 1] } },
                        ' MB'
                      ]
                    }
                  }
                ],
                default: { 
                  $concat: [
                    { $toString: { $round: [{ $divide: ['$length', 1024 * 1024 * 1024] }, 1] } },
                    ' GB'
                  ]
                }
              }
            }
          }
        },

        // Stage 8: Sort by relevance and apply pagination
        { $sort: this.buildSearchSort(options.sortBy, options.sortOrder) },
        { $skip: options.skip || 0 },
        { $limit: options.limit || 20 }
      ];

      // Execute search pipeline
      const searchResults = await this.db.collection('fs.files').aggregate(searchPipeline).toArray();

      // Get total count for pagination
      const totalCountPipeline = [
        { $match: this.buildFileSearchFilter(query, options) },
        { $count: 'total' }
      ];

      const countResult = await this.db.collection('fs.files').aggregate(totalCountPipeline).toArray();
      const totalCount = countResult.length > 0 ? countResult[0].total : 0;

      return {
        files: searchResults,
        pagination: {
          total: totalCount,
          page: Math.floor((options.skip || 0) / (options.limit || 20)) + 1,
          limit: options.limit || 20,
          pages: Math.ceil(totalCount / (options.limit || 20))
        },
        query: query,
        searchTime: Date.now() - (options.startTime || Date.now()),

        // Search analytics
        analytics: {
          averageRelevanceScore: searchResults.length > 0 ? 
            searchResults.reduce((sum, file) => sum + file.relevanceScore, 0) / searchResults.length : 0,
          categoryDistribution: this.analyzeCategoryDistribution(searchResults),
          sizeDistribution: this.analyzeSizeDistribution(searchResults),
          contentTypeDistribution: this.analyzeContentTypeDistribution(searchResults)
        }
      };

    } catch (error) {
      console.error('File search error:', error);
      throw error;
    }
  }

  async manageFileVersions(fileId, operation, options = {}) {
    console.log(`Managing file versions for ${fileId}, operation: ${operation}`);

    try {
      switch (operation) {
        case 'create_version':
          return await this.createFileVersion(fileId, options);

        case 'list_versions':
          return await this.listFileVersions(fileId, options);

        case 'restore_version':
          return await this.restoreFileVersion(fileId, options.versionId);

        case 'delete_version':
          return await this.deleteFileVersion(fileId, options.versionId);

        case 'compare_versions':
          return await this.compareFileVersions(fileId, options.versionId1, options.versionId2);

        default:
          throw new Error(`Unknown version operation: ${operation}`);
      }
    } catch (error) {
      console.error(`File version management error for ${fileId}:`, error);
      throw error;
    }
  }

  async createFileVersion(originalFileId, options) {
    console.log(`Creating new version for file: ${originalFileId}`);

    const session = this.db.client.startSession();

    try {
      await session.withTransaction(async () => {
        // Get original file information
        const originalFile = await this.getFileInfo(originalFileId);
        if (!originalFile) {
          throw new Error(`Original file not found: ${originalFileId}`);
        }

        // Create new version metadata
        const versionMetadata = {
          ...originalFile.metadata,
          version: (originalFile.metadata?.version || 1) + 1,
          parentFileId: originalFileId,
          versionCreatedAt: new Date(),
          versionCreatedBy: options.userId,
          versionNotes: options.versionNotes,
          isCurrentVersion: true
        };

        // Upload new version
        const newVersionInfo = await this.uploadFile(options.fileStream, versionMetadata, {
          version: versionMetadata.version
        });

        // Update original file to mark as not current
        await this.collections.files.updateOne(
          { _id: originalFileId },
          {
            $set: { 'metadata.isCurrentVersion': false },
            $push: {
              'metadata.versionHistory': {
                versionId: newVersionInfo.fileId,
                version: versionMetadata.version,
                createdAt: versionMetadata.versionCreatedAt,
                createdBy: versionMetadata.versionCreatedBy,
                notes: versionMetadata.versionNotes
              }
            }
          }
        );

        // Update version references in related documents
        if (originalFile.metadata?.documentId) {
          await this.collections.documents.updateMany(
            { 'files.fileId': originalFileId },
            {
              $set: { 'files.$.fileId': newVersionInfo.fileId },
              $push: {
                'files.$.versionHistory': {
                  previousFileId: originalFileId,
                  newFileId: newVersionInfo.fileId,
                  versionedAt: new Date(),
                  versionedBy: options.userId
                }
              }
            }
          );
        }

        return newVersionInfo;
      });

    } catch (error) {
      console.error(`File version creation error for ${originalFileId}:`, error);
      throw error;
    } finally {
      await session.endSession();
    }
  }

  // Helper methods for advanced file processing

  determineBucket(contentType) {
    if (contentType.startsWith('image/')) return 'images';
    if (contentType.startsWith('video/')) return 'videos';
    if (contentType.includes('pdf') || 
        contentType.includes('document') || 
        contentType.includes('word') || 
        contentType.includes('excel') || 
        contentType.includes('powerpoint')) return 'documents';
    if (contentType.includes('zip') || 
        contentType.includes('tar') || 
        contentType.includes('compress')) return 'archives';
    return 'files';
  }

  async validateFileUpload(fileStream, metadata, options) {
    const errors = [];

    // Size validation
    if (metadata.size > this.config.maxFileSize) {
      errors.push(`File size ${metadata.size} exceeds maximum ${this.config.maxFileSize}`);
    }

    // MIME type validation
    if (this.config.allowedMimeTypes.length > 0 && 
        !this.config.allowedMimeTypes.includes(metadata.contentType)) {
      errors.push(`Content type ${metadata.contentType} is not allowed`);
    }

    // Filename validation
    if (!metadata.filename || metadata.filename.trim().length === 0) {
      errors.push('Filename is required');
    }

    return { valid: errors.length === 0, errors };
  }

  buildFileSearchFilter(query, options) {
    const filter = {};

    // Text search across multiple fields
    if (query.text) {
      filter.$or = [
        { filename: { $regex: query.text, $options: 'i' } },
        { 'metadata.description': { $regex: query.text, $options: 'i' } },
        { 'metadata.tags': { $in: [new RegExp(query.text, 'i')] } },
        { 'metadata.category': { $regex: query.text, $options: 'i' } }
      ];
    }

    // Content type filtering
    if (query.contentType) {
      if (Array.isArray(query.contentType)) {
        filter.contentType = { $in: query.contentType };
      } else {
        filter.contentType = { $regex: query.contentType, $options: 'i' };
      }
    }

    // Date range filtering
    if (query.uploadedAfter || query.uploadedBefore) {
      filter.uploadDate = {};
      if (query.uploadedAfter) filter.uploadDate.$gte = new Date(query.uploadedAfter);
      if (query.uploadedBefore) filter.uploadDate.$lte = new Date(query.uploadedBefore);
    }

    // Size range filtering
    if (query.minSize || query.maxSize) {
      filter.length = {};
      if (query.minSize) filter.length.$gte = query.minSize;
      if (query.maxSize) filter.length.$lte = query.maxSize;
    }

    // Category filtering
    if (query.category) {
      filter['metadata.category'] = query.category;
    }

    // Tags filtering
    if (query.tags) {
      const tags = Array.isArray(query.tags) ? query.tags : [query.tags];
      filter['metadata.tags'] = { $in: tags };
    }

    // Uploader filtering
    if (query.uploadedBy) {
      filter['metadata.uploadedBy'] = query.uploadedBy;
    }

    // Access level filtering
    if (query.accessLevel) {
      filter['metadata.accessLevel'] = query.accessLevel;
    }

    // Document association filtering
    if (query.documentId) {
      filter['metadata.documentId'] = query.documentId;
    }

    // Processing status filtering
    if (query.processingStatus) {
      filter['metadata.processingStatus'] = query.processingStatus;
    }

    return filter;
  }

  buildSearchSort(sortBy = 'relevance', sortOrder = 'desc') {
    const sortDirection = sortOrder === 'desc' ? -1 : 1;

    switch (sortBy) {
      case 'relevance':
        return { relevanceScore: -1, uploadDate: -1 };
      case 'name':
        return { filename: sortDirection };
      case 'size':
        return { length: sortDirection };
      case 'date':
        return { uploadDate: sortDirection };
      case 'popularity':
        return { popularityScore: -1, uploadDate: -1 };
      case 'type':
        return { contentType: sortDirection, filename: 1 };
      default:
        return { relevanceScore: -1, uploadDate: -1 };
    }
  }

  setupFileProcessors() {
    // Image processing
    this.processors.set('image', async (fileId, contentType) => {
      // Implement image processing (resize, optimize, etc.)
      console.log(`Processing image: ${fileId}`);
    });

    // Document processing
    this.processors.set('document', async (fileId, contentType) => {
      // Implement document processing (text extraction, etc.)
      console.log(`Processing document: ${fileId}`);
    });

    // Video processing
    this.processors.set('video', async (fileId, contentType) => {
      // Implement video processing (thumbnail, compression, etc.)
      console.log(`Processing video: ${fileId}`);
    });
  }

  setupThumbnailGenerators() {
    // Image thumbnail generation
    this.thumbnailGenerators.set('image', async (fileId) => {
      console.log(`Generating image thumbnail for: ${fileId}`);
      // Implement image thumbnail generation
    });

    // PDF thumbnail generation
    this.thumbnailGenerators.set('pdf', async (fileId) => {
      console.log(`Generating PDF thumbnail for: ${fileId}`);
      // Implement PDF thumbnail generation
    });
  }

  setupMetadataExtractors() {
    // Image metadata extraction (EXIF, etc.)
    this.metadataExtractors.set('image', async (fileId) => {
      console.log(`Extracting image metadata for: ${fileId}`);
      // Implement EXIF and other metadata extraction
    });

    // Document metadata extraction
    this.metadataExtractors.set('document', async (fileId) => {
      console.log(`Extracting document metadata for: ${fileId}`);
      // Implement document properties extraction
    });
  }
}

// Benefits of MongoDB GridFS Advanced File Management:
// - Seamless integration with MongoDB data model and queries
// - Automatic file chunking and streaming for large files
// - Built-in file versioning and history tracking
// - Comprehensive metadata management and search capabilities
// - Advanced file processing pipelines and thumbnail generation
// - Integrated access control and permission management
// - Automatic backup and replication with MongoDB cluster
// - Sophisticated file search with relevance scoring
// - Real-time file access logging and analytics
// - SQL-compatible file operations through QueryLeaf integration

module.exports = {
  AdvancedGridFSManager
};