MongoDB Search Indexing and Full-Text Search Optimization: Advanced Text Search Strategies for Modern Applications

-- Traditional PostgreSQL text search with performance and functionality limitations

-- Simple LIKE pattern matching (extremely slow on large datasets)
SELECT 
  product_id,
  product_name,
  description,
  category,
  price
FROM products 
WHERE product_name LIKE '%laptop%' 
   OR description LIKE '%laptop%'
   OR category LIKE '%laptop%';

-- Problems with LIKE pattern matching:
-- 1. No relevance scoring or ranking capabilities
-- 2. Extremely poor performance on large datasets (full table scans)
-- 3. Case sensitivity issues requiring additional LOWER() calls
-- 4. No stemming support (won't match "laptops", "laptop's")
-- 5. No stop word handling or language-specific features
-- 6. Limited wildcard and partial matching capabilities
-- 7. No phrase matching or proximity search
-- 8. Sequential scanning makes it unusable for real-time search

-- Improved PostgreSQL approach with full-text search (better but limited)
-- Create full-text search indexes
CREATE INDEX products_fts_idx ON products 
USING gin(to_tsvector('english', product_name || ' ' || description));

-- Full-text search query with ranking
WITH search_results AS (
  SELECT 
    product_id,
    product_name,
    description,
    category,
    price,

    -- Text search vector matching
    to_tsvector('english', product_name || ' ' || description) as document,
    plainto_tsquery('english', 'gaming laptop high performance') as query,

    -- Basic relevance ranking
    ts_rank(
      to_tsvector('english', product_name || ' ' || description),
      plainto_tsquery('english', 'gaming laptop high performance')
    ) as relevance_score,

    -- Headline generation for snippets
    ts_headline(
      'english',
      description,
      plainto_tsquery('english', 'gaming laptop high performance'),
      'StartSel=<b>, StopSel=</b>, MaxWords=35, MinWords=15, ShortWord=3'
    ) as search_snippet

  FROM products
  WHERE to_tsvector('english', product_name || ' ' || description) 
        @@ plainto_tsquery('english', 'gaming laptop high performance')
),

ranked_results AS (
  SELECT 
    sr.*,

    -- Additional ranking factors
    CASE 
      WHEN LOWER(product_name) LIKE '%gaming%' AND LOWER(product_name) LIKE '%laptop%' THEN 2.0
      WHEN LOWER(product_name) LIKE '%gaming%' OR LOWER(product_name) LIKE '%laptop%' THEN 1.5  
      ELSE 1.0
    END as title_boost,

    -- Category relevance boost
    CASE 
      WHEN LOWER(category) IN ('computers', 'laptops', 'gaming') THEN 1.3
      WHEN LOWER(category) IN ('electronics', 'hardware') THEN 1.1
      ELSE 1.0  
    END as category_boost,

    -- Price range consideration for commercial applications
    CASE 
      WHEN price BETWEEN 800 AND 2000 THEN 1.2  -- Sweet spot for gaming laptops
      WHEN price > 2000 THEN 1.1
      ELSE 1.0
    END as price_boost

  FROM search_results sr
)

SELECT 
  product_id,
  product_name,
  description,
  category,
  price,
  search_snippet,

  -- Combined relevance score
  ROUND(
    (relevance_score * title_boost * category_boost * price_boost), 
    4
  ) as final_relevance_score,

  -- Search result quality indicators
  CASE 
    WHEN relevance_score > 0.3 THEN 'high'
    WHEN relevance_score > 0.1 THEN 'medium'  
    ELSE 'low'
  END as result_quality,

  -- Match type classification
  CASE 
    WHEN LOWER(product_name) LIKE '%gaming laptop%' THEN 'exact_phrase_title'
    WHEN LOWER(product_name) LIKE '%gaming%' AND LOWER(product_name) LIKE '%laptop%' THEN 'all_terms_title'
    WHEN LOWER(description) LIKE '%gaming laptop%' THEN 'exact_phrase_description'
    ELSE 'partial_match'
  END as match_type

FROM ranked_results
WHERE relevance_score > 0.05  -- Filter out very low relevance results
ORDER BY final_relevance_score DESC, price ASC
LIMIT 50;

-- Advanced PostgreSQL search with multiple criteria and faceting
WITH search_base AS (
  SELECT 
    product_id,
    product_name,
    description,
    category,
    price,
    brand,
    specifications,  -- JSONB field
    created_at,

    -- Multiple search vector approaches for different field weights
    setweight(to_tsvector('english', product_name), 'A') ||
    setweight(to_tsvector('english', COALESCE(description, '')), 'B') ||
    setweight(to_tsvector('english', category), 'C') ||
    setweight(to_tsvector('english', brand), 'D') as search_vector,

    plainto_tsquery('english', 'gaming laptop RTX nvidia') as search_query

  FROM products
  WHERE 
    -- Pre-filter for performance
    category IN ('Computers', 'Laptops', 'Gaming', 'Electronics') AND
    price BETWEEN 300 AND 5000 AND
    created_at >= CURRENT_DATE - INTERVAL '2 years'
),

search_matches AS (
  SELECT 
    *,

    -- Text search ranking with field weighting
    ts_rank_cd(search_vector, search_query, 32) as base_relevance,

    -- Individual field matches for boost calculation
    to_tsvector('english', product_name) @@ search_query as title_match,
    to_tsvector('english', description) @@ search_query as description_match,
    to_tsvector('english', category) @@ search_query as category_match,
    to_tsvector('english', brand) @@ search_query as brand_match,

    -- JSON field search for specifications
    specifications::text ILIKE '%RTX%' OR specifications::text ILIKE '%nvidia%' as spec_match,

    -- Generate search snippets
    ts_headline(
      'english',
      description,
      search_query,
      'StartSel=<mark>, StopSel=</mark>, MaxWords=50, MinWords=20'
    ) as snippet

  FROM search_base
  WHERE search_vector @@ search_query
),

enhanced_results AS (
  SELECT 
    *,

    -- Calculate comprehensive relevance score
    base_relevance * 
    (CASE WHEN title_match THEN 3.0 ELSE 1.0 END) *
    (CASE WHEN brand_match THEN 1.5 ELSE 1.0 END) *
    (CASE WHEN spec_match THEN 1.8 ELSE 1.0 END) *
    (CASE WHEN category_match THEN 1.3 ELSE 1.0 END) as enhanced_relevance,

    -- Quality scoring
    CASE 
      WHEN title_match AND spec_match THEN 'excellent'
      WHEN title_match OR (description_match AND spec_match) THEN 'good'
      WHEN description_match OR category_match THEN 'fair'
      ELSE 'poor'
    END as result_quality_tier,

    -- Commercial relevance for e-commerce
    CASE 
      WHEN price BETWEEN 1000 AND 2500 AND spec_match THEN 'high_commercial_value'
      WHEN price BETWEEN 500 AND 1000 THEN 'moderate_commercial_value'
      ELSE 'standard_commercial_value'
    END as commercial_tier

  FROM search_matches
)

-- Final results with faceting information
SELECT 
  product_id,
  product_name,
  description,
  category,
  brand,
  price,
  snippet,
  ROUND(enhanced_relevance, 4) as relevance_score,
  result_quality_tier,
  commercial_tier,

  -- Match indicators for UI highlighting
  title_match,
  description_match,
  brand_match,
  spec_match,

  created_at

FROM enhanced_results
WHERE enhanced_relevance > 0.1
ORDER BY enhanced_relevance DESC, price ASC
LIMIT 100;

-- Search faceting query for filter options
WITH facet_data AS (
  SELECT 
    category,
    brand,

    -- Price range buckets
    CASE 
      WHEN price < 500 THEN 'Under $500'
      WHEN price BETWEEN 500 AND 999 THEN '$500-$999'
      WHEN price BETWEEN 1000 AND 1499 THEN '$1000-$1499'  
      WHEN price BETWEEN 1500 AND 2499 THEN '$1500-$2499'
      ELSE '$2500+'
    END as price_range,

    -- Extract key specs from JSON
    CASE 
      WHEN specifications::text ILIKE '%RTX 40%' THEN 'RTX 40 Series'
      WHEN specifications::text ILIKE '%RTX 30%' THEN 'RTX 30 Series'  
      WHEN specifications::text ILIKE '%RTX%' THEN 'RTX Graphics'
      WHEN specifications::text ILIKE '%GTX%' THEN 'GTX Graphics'
      ELSE 'Other Graphics'
    END as graphics_type

  FROM products
  WHERE to_tsvector('english', product_name || ' ' || description) 
        @@ plainto_tsquery('english', 'gaming laptop RTX nvidia')
)

SELECT 
  'category' as facet_type,
  category as facet_value,
  COUNT(*) as result_count
FROM facet_data
WHERE category IS NOT NULL
GROUP BY category

UNION ALL

SELECT 
  'brand' as facet_type,
  brand as facet_value,
  COUNT(*) as result_count  
FROM facet_data
WHERE brand IS NOT NULL
GROUP BY brand

UNION ALL

SELECT 
  'price_range' as facet_type,
  price_range as facet_value,
  COUNT(*) as result_count
FROM facet_data  
GROUP BY price_range

UNION ALL

SELECT 
  'graphics' as facet_type,
  graphics_type as facet_value,
  COUNT(*) as result_count
FROM facet_data
GROUP BY graphics_type

ORDER BY facet_type, result_count DESC;

-- PostgreSQL limitations for text search:
-- 1. Complex configuration and maintenance of text search indexes
-- 2. Limited language support compared to specialized search engines
-- 3. Basic relevance scoring with limited customization options
-- 4. No built-in support for auto-complete or suggestion features
-- 5. Difficulty handling real-time index updates with high write volumes
-- 6. Limited support for complex query syntax (phrase matching, proximity)
-- 7. Basic stemming and stop word handling
-- 8. No native support for search analytics or performance monitoring
-- 9. Difficult to implement features like "did you mean" or fuzzy matching
-- 10. Scalability limitations for very large text corpora

-- MySQL approach (even more limited)
SELECT 
  product_id,
  product_name,
  description,
  MATCH(product_name, description) AGAINST('gaming laptop' IN NATURAL LANGUAGE MODE) as relevance
FROM products 
WHERE MATCH(product_name, description) AGAINST('gaming laptop' IN NATURAL LANGUAGE MODE)
ORDER BY relevance DESC
LIMIT 20;

-- MySQL full-text search limitations:
-- - Very basic relevance scoring
-- - Limited query syntax support  
-- - Poor performance with large datasets
-- - Minimal customization options
-- - No advanced features like phrase matching or proximity search
-- - Basic language support
-- - Difficult to integrate with application logic

// MongoDB Advanced Text Search - comprehensive full-text search with optimization
const { MongoClient } = require('mongodb');

// Advanced MongoDB Text Search Manager
class MongoDBTextSearchManager {
  constructor(db) {
    this.db = db;
    this.searchMetrics = {
      queries: 0,
      averageResponseTime: 0,
      totalResponseTime: 0,
      cacheHits: 0,
      cacheMisses: 0
    };
    this.searchCache = new Map();
    this.indexOptimization = {
      lastOptimization: null,
      optimizationThreshold: 10000, // Optimize after 10k queries
      performanceTargetMs: 100
    };
  }

  async initializeTextSearchIndexes() {
    console.log('Initializing comprehensive text search indexes...');

    const products = this.db.collection('products');
    const articles = this.db.collection('articles');
    const users = this.db.collection('users');

    try {
      // Create compound text index for products with weighted fields
      await products.createIndex(
        {
          name: 'text',
          description: 'text',
          category: 'text',
          brand: 'text',
          'specifications.processor': 'text',
          'specifications.graphics': 'text',
          'specifications.memory': 'text',
          tags: 'text'
        },
        {
          weights: {
            name: 10,           // Highest weight for product names
            brand: 8,           // High weight for brand names
            category: 5,        // Medium-high weight for categories
            description: 3,     // Medium weight for descriptions
            'specifications.processor': 4,
            'specifications.graphics': 4,
            'specifications.memory': 2,
            tags: 6
          },
          name: 'products_comprehensive_text_search',
          default_language: 'english',
          language_override: 'searchLanguage',
          textIndexVersion: 3,
          background: true
        }
      );

      // Create text index for articles/blog content
      await articles.createIndex(
        {
          title: 'text',
          content: 'text',
          excerpt: 'text',
          tags: 'text',
          'author.name': 'text'
        },
        {
          weights: {
            title: 10,
            excerpt: 6,
            tags: 8,
            content: 2,
            'author.name': 4
          },
          name: 'articles_text_search',
          default_language: 'english',
          background: true
        }
      );

      // Create user search index for user discovery
      await users.createIndex(
        {
          'profile.first_name': 'text',
          'profile.last_name': 'text',
          'profile.bio': 'text',
          'profile.skills': 'text',
          'profile.company': 'text',
          'profile.location': 'text'
        },
        {
          weights: {
            'profile.first_name': 10,
            'profile.last_name': 10,
            'profile.company': 6,
            'profile.skills': 8,
            'profile.bio': 3,
            'profile.location': 4
          },
          name: 'users_profile_text_search',
          default_language: 'english',
          background: true
        }
      );

      // Create additional supporting indexes for search optimization
      await products.createIndex({ category: 1, price: 1 });
      await products.createIndex({ brand: 1, 'specifications.type': 1 });
      await products.createIndex({ price: 1 });
      await products.createIndex({ createdAt: -1 });
      await products.createIndex({ 'ratings.average': -1, 'ratings.count': -1 });

      // Support indexes for articles
      await articles.createIndex({ publishedAt: -1 });
      await articles.createIndex({ 'author.id': 1, publishedAt: -1 });
      await articles.createIndex({ status: 1, publishedAt: -1 });
      await articles.createIndex({ category: 1, publishedAt: -1 });

      console.log('Text search indexes created successfully');
      return {
        success: true,
        indexes: [
          'products_comprehensive_text_search',
          'articles_text_search', 
          'users_profile_text_search'
        ]
      };

    } catch (error) {
      console.error('Error creating text search indexes:', error);
      throw error;
    }
  }

  async performAdvancedTextSearch(collection, searchTerms, options = {}) {
    const startTime = Date.now();
    console.log(`Performing advanced text search on ${collection} for: "${searchTerms}"`);

    const {
      limit = 20,
      skip = 0,
      sortBy = 'relevance',
      includeScore = true,
      language = 'english',
      caseSensitive = false,
      diacriticSensitive = false,
      filters = {},
      facets = [],
      highlighting = true,
      minScore = 0.5
    } = options;

    try {
      const coll = this.db.collection(collection);

      // Build text search query
      const textQuery = this.buildTextSearchQuery(searchTerms, {
        language,
        caseSensitive, 
        diacriticSensitive
      });

      // Combine text search with additional filters
      const searchQuery = {
        $text: textQuery,
        ...filters
      };

      console.log('Search query:', JSON.stringify(searchQuery, null, 2));

      // Build aggregation pipeline for advanced search features
      const pipeline = [
        // Match stage with text search and filters
        { $match: searchQuery },

        // Add text score for relevance ranking
        { $addFields: { 
          searchScore: { $meta: 'textScore' },
          searchTimestamp: new Date()
        }},

        // Filter by minimum score threshold
        { $match: { searchScore: { $gte: minScore } } },

        // Add computed relevance factors
        { $addFields: this.buildRelevanceFactors(collection) },

        // Calculate final relevance score
        { $addFields: {
          finalRelevance: {
            $multiply: [
              '$searchScore',
              { $ifNull: ['$categoryBoost', 1] },
              { $ifNull: ['$priceBoost', 1] },
              { $ifNull: ['$brandBoost', 1] },
              { $ifNull: ['$qualityBoost', 1] }
            ]
          }
        }},

        // Sort by relevance or other criteria
        { $sort: this.buildSortCriteria(sortBy) },

        // Pagination
        { $skip: skip },
        { $limit: limit },

        // Project final results
        { $project: this.buildProjection(collection, includeScore, highlighting) }
      ];

      // Add faceting stages if requested
      if (facets.length > 0) {
        pipeline.push({
          $facet: {
            results: [{ $skip: 0 }], // Continue main pipeline
            facets: this.buildFacetPipeline(facets)
          }
        });
      }

      // Execute search query
      const results = await coll.aggregate(pipeline).toArray();

      const endTime = Date.now();
      const responseTime = endTime - startTime;

      // Update search metrics
      this.updateSearchMetrics(responseTime);

      // Process and format results
      const formattedResults = this.formatSearchResults(results, {
        collection,
        searchTerms,
        responseTime,
        facets: facets.length > 0
      });

      console.log(`Search completed in ${responseTime}ms, found ${formattedResults.results.length} results`);

      return formattedResults;

    } catch (error) {
      console.error('Text search failed:', error);
      throw error;
    }
  }

  buildTextSearchQuery(searchTerms, options) {
    const { language, caseSensitive, diacriticSensitive } = options;

    // Handle different search term formats
    let searchString;
    if (typeof searchTerms === 'string') {
      // Parse search string for advanced syntax
      searchString = this.parseSearchSyntax(searchTerms);
    } else if (Array.isArray(searchTerms)) {
      // Join array terms with AND logic
      searchString = searchTerms.map(term => `"${term}"`).join(' ');
    } else {
      searchString = searchTerms.toString();
    }

    const textQuery = {
      $search: searchString
    };

    if (language !== 'english') {
      textQuery.$language = language;
    }

    if (caseSensitive) {
      textQuery.$caseSensitive = true;
    }

    if (diacriticSensitive) {
      textQuery.$diacriticSensitive = true;
    }

    return textQuery;
  }

  parseSearchSyntax(searchString) {
    // Handle advanced search syntax
    let parsed = searchString.trim();

    // Convert common search patterns
    parsed = parsed
      .replace(/\bAND\b/gi, ' ')
      .replace(/\bOR\b/gi, ' | ')
      .replace(/\bNOT\b/gi, ' -')
      .replace(/\+([^\s]+)/g, '"$1"'); // Quoted terms for exact match

    return parsed;
  }

  buildRelevanceFactors(collection) {
    // Collection-specific relevance boosting
    const baseFactors = {
      categoryBoost: 1,
      priceBoost: 1,
      brandBoost: 1,
      qualityBoost: 1
    };

    if (collection === 'products') {
      return {
        ...baseFactors,
        categoryBoost: {
          $switch: {
            branches: [
              { case: { $in: ['$category', ['Gaming', 'Computers', 'Laptops']] }, then: 1.3 },
              { case: { $in: ['$category', ['Electronics', 'Hardware']] }, then: 1.1 }
            ],
            default: 1.0
          }
        },
        priceBoost: {
          $switch: {
            branches: [
              { case: { $and: [{ $gte: ['$price', 500] }, { $lte: ['$price', 2000] }] }, then: 1.2 },
              { case: { $gt: ['$price', 2000] }, then: 1.1 }
            ],
            default: 1.0
          }
        },
        qualityBoost: {
          $cond: {
            if: { $and: [
              { $gte: [{ $ifNull: ['$ratings.average', 0] }, 4.0] },
              { $gte: [{ $ifNull: ['$ratings.count', 0] }, 10] }
            ]},
            then: 1.4,
            else: 1.0
          }
        }
      };
    } else if (collection === 'articles') {
      return {
        ...baseFactors,
        qualityBoost: {
          $cond: {
            if: { $gte: [{ $ifNull: ['$views', 0] }, 1000] },
            then: 1.3,
            else: 1.0
          }
        },
        categoryBoost: {
          $switch: {
            branches: [
              { case: { $eq: ['$status', 'featured'] }, then: 1.5 },
              { case: { $eq: ['$status', 'published'] }, then: 1.0 }
            ],
            default: 0.8
          }
        }
      };
    }

    return baseFactors;
  }

  buildSortCriteria(sortBy) {
    switch (sortBy) {
      case 'relevance':
        return { finalRelevance: -1, searchScore: -1 };
      case 'price_low':
        return { price: 1, finalRelevance: -1 };
      case 'price_high':
        return { price: -1, finalRelevance: -1 };
      case 'newest':
        return { createdAt: -1, finalRelevance: -1 };
      case 'rating':
        return { 'ratings.average': -1, 'ratings.count': -1, finalRelevance: -1 };
      case 'popularity':
        return { views: -1, finalRelevance: -1 };
      default:
        return { finalRelevance: -1, searchScore: -1 };
    }
  }

  buildProjection(collection, includeScore, highlighting) {
    const baseProjection = {
      searchScore: includeScore ? 1 : 0,
      finalRelevance: includeScore ? 1 : 0,
      searchTimestamp: 0 // Remove from output
    };

    if (collection === 'products') {
      return {
        ...baseProjection,
        _id: 1,
        name: 1,
        description: 1,
        category: 1,
        brand: 1,
        price: 1,
        images: 1,
        specifications: 1,
        ratings: 1,
        availability: 1,
        createdAt: 1,
        // Add search highlighting if requested
        searchHighlight: highlighting ? {
          $literal: 'Highlighting would be implemented here'
        } : 0
      };
    } else if (collection === 'articles') {
      return {
        ...baseProjection,
        _id: 1,
        title: 1,
        excerpt: 1,
        content: highlighting ? 1 : 0,
        author: 1,
        publishedAt: 1,
        category: 1,
        tags: 1,
        views: 1,
        status: 1
      };
    }

    return baseProjection;
  }

  buildFacetPipeline(facets) {
    const facetStages = [];

    for (const facet of facets) {
      switch (facet.type) {
        case 'category':
          facetStages.push({
            $group: {
              _id: '$category',
              count: { $sum: 1 },
              avgPrice: { $avg: '$price' },
              avgRating: { $avg: '$ratings.average' }
            }
          });
          break;

        case 'price_range':
          facetStages.push({
            $bucket: {
              groupBy: '$price',
              boundaries: [0, 500, 1000, 1500, 2000, 5000],
              default: 'Other',
              output: {
                count: { $sum: 1 },
                avgRating: { $avg: '$ratings.average' },
                products: { $push: '$name' }
              }
            }
          });
          break;

        case 'brand':
          facetStages.push({
            $group: {
              _id: '$brand',
              count: { $sum: 1 },
              avgPrice: { $avg: '$price' },
              avgRating: { $avg: '$ratings.average' }
            }
          });
          break;
      }
    }

    return facetStages;
  }

  formatSearchResults(results, metadata) {
    const { collection, searchTerms, responseTime, facets } = metadata;

    if (facets && results[0] && results[0].facets) {
      // Handle faceted results
      return {
        results: results[0].results || [],
        facets: results[0].facets || [],
        metadata: {
          collection,
          searchTerms,
          responseTime,
          totalResults: results[0].results?.length || 0,
          searchType: 'faceted'
        }
      };
    } else {
      // Handle regular results
      return {
        results: results || [],
        metadata: {
          collection,
          searchTerms,
          responseTime,
          totalResults: results.length,
          searchType: 'standard'
        }
      };
    }
  }

  async performAutoComplete(collection, partialTerm, options = {}) {
    console.log(`Performing autocomplete for "${partialTerm}" on ${collection}`);

    const {
      limit = 10,
      fields = ['name'],
      minLength = 2
    } = options;

    if (partialTerm.length < minLength) {
      return { suggestions: [], metadata: { reason: 'Term too short' } };
    }

    const coll = this.db.collection(collection);

    try {
      // Use regex for prefix matching combined with text search
      const regexPattern = new RegExp(`^${partialTerm}`, 'i');

      const suggestions = await coll.aggregate([
        {
          $match: {
            $or: fields.map(field => ({ [field]: regexPattern }))
          }
        },
        {
          $project: {
            suggestion: { $arrayElemAt: [fields.map(field => `$${field}`), 0] },
            score: { $literal: 1 }
          }
        },
        { $group: { _id: '$suggestion', score: { $first: '$score' } } },
        { $sort: { _id: 1 } },
        { $limit: limit },
        { $project: { _id: 0, suggestion: '$_id', score: 1 } }
      ]).toArray();

      return {
        suggestions: suggestions.map(s => s.suggestion),
        metadata: {
          term: partialTerm,
          collection,
          count: suggestions.length
        }
      };

    } catch (error) {
      console.error('Autocomplete failed:', error);
      return { suggestions: [], error: error.message };
    }
  }

  async performFuzzySearch(collection, searchTerm, options = {}) {
    console.log(`Performing fuzzy search for "${searchTerm}" on ${collection}`);

    const {
      maxDistance = 2,
      limit = 20,
      fields = ['name', 'description']
    } = options;

    const coll = this.db.collection(collection);

    try {
      // Use MongoDB's fuzzy text search capabilities
      const pipeline = [
        {
          $match: {
            $text: {
              $search: searchTerm,
              $caseSensitive: false,
              $diacriticSensitive: false
            }
          }
        },
        {
          $addFields: {
            score: { $meta: 'textScore' },
            fuzzyMatch: true
          }
        },
        { $sort: { score: -1 } },
        { $limit: limit }
      ];

      const results = await coll.aggregate(pipeline).toArray();

      return {
        results,
        metadata: {
          searchTerm,
          collection,
          fuzzy: true,
          maxDistance,
          count: results.length
        }
      };

    } catch (error) {
      console.error('Fuzzy search failed:', error);
      throw error;
    }
  }

  updateSearchMetrics(responseTime) {
    this.searchMetrics.queries++;
    this.searchMetrics.totalResponseTime += responseTime;
    this.searchMetrics.averageResponseTime = 
      this.searchMetrics.totalResponseTime / this.searchMetrics.queries;
  }

  async getSearchAnalytics(timeRange = '24h') {
    console.log(`Generating search analytics for ${timeRange}`);

    // In a real implementation, this would query a search analytics collection
    return {
      period: timeRange,
      totalQueries: this.searchMetrics.queries,
      averageResponseTime: Math.round(this.searchMetrics.averageResponseTime),
      cacheHitRate: this.searchMetrics.queries > 0 ? 
        (this.searchMetrics.cacheHits / this.searchMetrics.queries * 100) : 0,
      topSearchTerms: [
        'gaming laptop',
        'mongodb tutorial', 
        'javascript framework',
        'web development',
        'data visualization'
      ],
      performanceMetrics: {
        fastQueries: 85, // < 100ms
        mediumQueries: 12, // 100-500ms  
        slowQueries: 3 // > 500ms
      },
      recommendations: [
        'Consider adding more text indexes for frequently searched fields',
        'Monitor query patterns for optimization opportunities',
        'Implement search result caching for common queries'
      ]
    };
  }
}

// Example usage: Comprehensive text search implementation
async function demonstrateAdvancedTextSearch() {
  const client = new MongoClient('mongodb://localhost:27017');
  await client.connect();
  const db = client.db('ecommerce_platform');

  const searchManager = new MongoDBTextSearchManager(db);

  // Initialize text search indexes
  await searchManager.initializeTextSearchIndexes();

  // Advanced product search
  const productResults = await searchManager.performAdvancedTextSearch('products', 
    'gaming laptop RTX nvidia', {
      limit: 20,
      sortBy: 'relevance',
      filters: {
        price: { $gte: 800, $lte: 3000 },
        availability: { $ne: 'out_of_stock' }
      },
      facets: [
        { type: 'category' },
        { type: 'brand' }, 
        { type: 'price_range' }
      ],
      minScore: 0.3
    }
  );

  console.log('Product Search Results:', JSON.stringify(productResults, null, 2));

  // Autocomplete demonstration
  const autoCompleteResults = await searchManager.performAutoComplete('products', 'gam', {
    limit: 10,
    fields: ['name', 'category']
  });

  console.log('Autocomplete Results:', autoCompleteResults);

  // Fuzzy search for typos
  const fuzzyResults = await searchManager.performFuzzySearch('articles', 'javasript tutorial', {
    maxDistance: 2,
    limit: 15
  });

  console.log('Fuzzy Search Results:', fuzzyResults);

  // Search analytics
  const analytics = await searchManager.getSearchAnalytics('24h');
  console.log('Search Analytics:', analytics);

  await client.close();
}

// Production-ready text search with advanced optimization and monitoring
class EnterpriseTextSearchManager extends MongoDBTextSearchManager {
  constructor(db, enterpriseConfig = {}) {
    super(db);

    this.enterpriseConfig = {
      enableSearchCaching: enterpriseConfig.enableSearchCaching || true,
      enableSearchAnalytics: enterpriseConfig.enableSearchAnalytics || true,
      enableAutoOptimization: enterpriseConfig.enableAutoOptimization || true,
      cacheSize: enterpriseConfig.cacheSize || 10000,
      cacheTTL: enterpriseConfig.cacheTTL || 300000, // 5 minutes
      optimizationInterval: enterpriseConfig.optimizationInterval || 3600000 // 1 hour
    };

    this.setupEnterpriseFeatures();
  }

  async performSemanticSearch(collection, searchQuery, options = {}) {
    console.log(`Performing semantic search on ${collection}`);

    const {
      useEmbeddings = true,
      semanticWeight = 0.3,
      textWeight = 0.7,
      limit = 20
    } = options;

    // Combine traditional text search with semantic similarity
    const pipeline = [
      // Traditional text search
      {
        $match: {
          $text: {
            $search: searchQuery,
            $caseSensitive: false
          }
        }
      },

      // Add text search score
      {
        $addFields: {
          textScore: { $meta: 'textScore' }
        }
      }
    ];

    if (useEmbeddings) {
      // Add semantic similarity scoring (would integrate with vector search)
      pipeline.push({
        $addFields: {
          semanticScore: {
            // Placeholder for vector similarity calculation
            $literal: 0.8
          }
        }
      });

      // Combine scores
      pipeline.push({
        $addFields: {
          hybridScore: {
            $add: [
              { $multiply: ['$textScore', textWeight] },
              { $multiply: ['$semanticScore', semanticWeight] }
            ]
          }
        }
      });
    }

    // Sort and limit results
    pipeline.push(
      { $sort: { hybridScore: -1 } },
      { $limit: limit }
    );

    const coll = this.db.collection(collection);
    const results = await coll.aggregate(pipeline).toArray();

    return {
      results,
      searchType: 'semantic',
      metadata: {
        query: searchQuery,
        useEmbeddings,
        textWeight,
        semanticWeight
      }
    };
  }

  async performMultiCollectionSearch(searchQuery, collections, options = {}) {
    console.log(`Performing multi-collection search across: ${collections.join(', ')}`);

    const {
      limit = 50,
      rankAcrossCollections = true
    } = options;

    const searchPromises = collections.map(async (collectionName) => {
      const results = await this.performAdvancedTextSearch(collectionName, searchQuery, {
        limit: Math.ceil(limit / collections.length),
        includeScore: true
      });

      return {
        collection: collectionName,
        results: results.results.map(result => ({
          ...result,
          sourceCollection: collectionName,
          collectionType: this.getCollectionType(collectionName)
        }))
      };
    });

    const collectionResults = await Promise.all(searchPromises);

    if (rankAcrossCollections) {
      // Merge and rank results across all collections
      const allResults = collectionResults.flatMap(cr => cr.results);
      allResults.sort((a, b) => (b.finalRelevance || b.searchScore) - (a.finalRelevance || a.searchScore));

      return {
        results: allResults.slice(0, limit),
        metadata: {
          searchQuery,
          collections,
          totalCollections: collections.length,
          ranked: true
        }
      };
    } else {
      // Return results grouped by collection
      return {
        resultsByCollection: collectionResults,
        metadata: {
          searchQuery,
          collections,
          totalCollections: collections.length,
          ranked: false
        }
      };
    }
  }

  async implementSearchSuggestions(searchHistory, userBehavior, options = {}) {
    console.log('Generating intelligent search suggestions');

    const {
      maxSuggestions = 10,
      includePopular = true,
      includePersonalized = true
    } = options;

    const suggestions = [];

    // Popular search suggestions
    if (includePopular) {
      const popularQueries = await this.getPopularSearchQueries(5);
      suggestions.push(...popularQueries.map(q => ({
        query: q.query,
        type: 'popular',
        count: q.count,
        score: q.popularity_score
      })));
    }

    // Personalized suggestions based on user behavior
    if (includePersonalized && userBehavior) {
      const personalizedQueries = await this.getPersonalizedSuggestions(userBehavior, 5);
      suggestions.push(...personalizedQueries.map(q => ({
        query: q.query,
        type: 'personalized', 
        reason: q.reason,
        score: q.relevance_score
      })));
    }

    // Sort by score and return top suggestions
    suggestions.sort((a, b) => b.score - a.score);

    return {
      suggestions: suggestions.slice(0, maxSuggestions),
      metadata: {
        includePopular,
        includePersonalized,
        totalGenerated: suggestions.length
      }
    };
  }

  async performSearchWithCorrection(searchQuery, collection, options = {}) {
    console.log(`Search with spell correction for: "${searchQuery}"`);

    // First, try the original query
    let results = await this.performAdvancedTextSearch(collection, searchQuery, {
      ...options,
      minScore: 0.3
    });

    if (results.results.length < 3) {
      // Try to find spelling corrections
      const corrections = await this.generateSpellCorrections(searchQuery);

      if (corrections.length > 0) {
        // Try search with corrected query
        const correctedResults = await this.performAdvancedTextSearch(
          collection, 
          corrections[0].suggestion, 
          options
        );

        if (correctedResults.results.length > results.results.length) {
          return {
            ...correctedResults,
            searchCorrection: {
              originalQuery: searchQuery,
              correctedQuery: corrections[0].suggestion,
              confidence: corrections[0].confidence
            }
          };
        }
      }
    }

    return results;
  }

  async generateSpellCorrections(query) {
    // Simplified spell correction logic
    // In production, would use sophisticated algorithms like Levenshtein distance
    const commonCorrections = new Map([
      ['javasript', 'javascript'],
      ['phython', 'python'],
      ['reactjs', 'react'],
      ['mongdb', 'mongodb'],
      ['databse', 'database']
    ]);

    const words = query.toLowerCase().split(' ');
    const corrections = [];

    for (const word of words) {
      if (commonCorrections.has(word)) {
        corrections.push({
          original: word,
          suggestion: commonCorrections.get(word),
          confidence: 0.9
        });
      }
    }

    if (corrections.length > 0) {
      let correctedQuery = query;
      corrections.forEach(correction => {
        correctedQuery = correctedQuery.replace(
          new RegExp(correction.original, 'gi'), 
          correction.suggestion
        );
      });

      return [{
        suggestion: correctedQuery,
        confidence: corrections.reduce((sum, c) => sum + c.confidence, 0) / corrections.length,
        corrections: corrections
      }];
    }

    return [];
  }

  getCollectionType(collectionName) {
    const typeMap = {
      'products': 'Product',
      'articles': 'Article',
      'users': 'User Profile',
      'categories': 'Category',
      'brands': 'Brand'
    };

    return typeMap[collectionName] || 'Document';
  }

  async getPopularSearchQueries(limit) {
    // In production, would query search analytics collection
    return [
      { query: 'gaming laptop', count: 1250, popularity_score: 0.95 },
      { query: 'wireless headphones', count: 890, popularity_score: 0.87 },
      { query: 'smartphone deals', count: 756, popularity_score: 0.82 },
      { query: 'home office setup', count: 623, popularity_score: 0.78 },
      { query: 'fitness tracker', count: 567, popularity_score: 0.75 }
    ].slice(0, limit);
  }

  async getPersonalizedSuggestions(userBehavior, limit) {
    // Analyze user behavior for personalized suggestions
    const { searchHistory, purchases, browsingCategories } = userBehavior;

    const suggestions = [];

    // Based on search history
    if (searchHistory?.length > 0) {
      suggestions.push({
        query: `${searchHistory[0]} accessories`,
        reason: 'Based on recent searches',
        relevance_score: 0.8
      });
    }

    // Based on purchase history
    if (browsingCategories?.length > 0) {
      suggestions.push({
        query: `new ${browsingCategories[0]} products`,
        reason: 'Based on browsing history',
        relevance_score: 0.75
      });
    }

    return suggestions.slice(0, limit);
  }
}