MongoDB Vector Search for Semantic Applications: Building AI-Powered Search with SQL-Style Vector Operations

-- Traditional relational search - limited semantic understanding

-- PostgreSQL full-text search with performance and relevance challenges
CREATE TABLE documents (
  document_id SERIAL PRIMARY KEY,
  title VARCHAR(500) NOT NULL,
  content TEXT NOT NULL,
  category VARCHAR(100),
  tags TEXT[],
  author VARCHAR(200),
  created_at TIMESTAMP DEFAULT NOW(),

  -- Full-text search vector (keyword-based only)
  search_vector tsvector GENERATED ALWAYS AS (
    setweight(to_tsvector('english', title), 'A') ||
    setweight(to_tsvector('english', content), 'B') ||
    setweight(to_tsvector('english', array_to_string(tags, ' ')), 'C')
  ) STORED
);

-- Create full-text search index
CREATE INDEX idx_documents_fts ON documents USING GIN(search_vector);

-- Additional indexes for filtering
CREATE INDEX idx_documents_category ON documents(category);
CREATE INDEX idx_documents_created_at ON documents(created_at DESC);
CREATE INDEX idx_documents_author ON documents(author);

-- Traditional keyword-based search with limited semantic understanding
WITH search_query AS (
  SELECT 
    document_id,
    title,
    content,
    category,
    author,
    created_at,

    -- Basic relevance scoring (keyword-based only)
    ts_rank_cd(search_vector, plainto_tsquery('english', 'machine learning algorithms')) as relevance_score,

    -- Highlight matching text
    ts_headline('english', content, plainto_tsquery('english', 'machine learning algorithms'), 
                'MaxWords=50, MinWords=20, ShortWord=3, HighlightAll=false') as highlighted_content,

    -- Basic similarity using trigram matching (very limited)
    similarity(title, 'machine learning algorithms') as title_similarity,

    -- Category boosting (manual relevance adjustment)
    CASE category 
      WHEN 'AI' THEN 1.5 
      WHEN 'Technology' THEN 1.2 
      ELSE 1.0 
    END as category_boost

  FROM documents
  WHERE search_vector @@ plainto_tsquery('english', 'machine learning algorithms')
     OR similarity(title, 'machine learning algorithms') > 0.1
),

ranked_results AS (
  SELECT 
    *,
    -- Combined relevance scoring (still keyword-dependent)
    (relevance_score * category_boost * 
     CASE WHEN title_similarity > 0.3 THEN 2.0 ELSE 1.0 END) as final_score,

    -- Manual semantic grouping (limited effectiveness)
    CASE 
      WHEN content ILIKE '%neural network%' OR content ILIKE '%deep learning%' THEN 'Deep Learning'
      WHEN content ILIKE '%statistics%' OR content ILIKE '%data science%' THEN 'Data Science' 
      WHEN content ILIKE '%algorithm%' OR content ILIKE '%optimization%' THEN 'Algorithms'
      ELSE 'General'
    END as semantic_category,

    -- Time decay factor
    CASE 
      WHEN created_at >= NOW() - INTERVAL '30 days' THEN 1.2
      WHEN created_at >= NOW() - INTERVAL '90 days' THEN 1.0
      WHEN created_at >= NOW() - INTERVAL '1 year' THEN 0.8
      ELSE 0.6
    END as recency_boost

  FROM search_query
  WHERE relevance_score > 0.01
),

related_documents AS (
  -- Attempt to find related documents (very basic approach)
  SELECT DISTINCT
    r1.document_id,
    r2.document_id as related_id,
    r2.title as related_title,

    -- Basic relatedness calculation
    (array_length(array(SELECT UNNEST(r1.tags) INTERSECT SELECT UNNEST(r2.tags)), 1) / 
     GREATEST(array_length(r1.tags, 1), array_length(r2.tags, 1))::numeric) as tag_similarity,

    CASE WHEN r1.category = r2.category THEN 0.3 ELSE 0 END as category_match,
    CASE WHEN r1.author = r2.author THEN 0.2 ELSE 0 END as author_match

  FROM ranked_results r1
  JOIN documents r2 ON r1.document_id != r2.document_id
  WHERE r1.final_score > 0.5
),

final_results AS (
  SELECT 
    r.document_id,
    r.title,
    LEFT(r.content, 200) || '...' as content_preview,
    r.highlighted_content,
    r.category,
    r.semantic_category,
    r.author,
    r.created_at,

    -- Final ranking with all factors
    ROUND((r.final_score * r.recency_boost)::numeric, 4) as final_relevance_score,

    -- Related documents (limited by keyword overlap)
    COALESCE(
      (SELECT json_agg(json_build_object(
        'id', related_id,
        'title', related_title,
        'similarity', ROUND((tag_similarity + category_match + author_match)::numeric, 3)
      )) FROM related_documents rd 
       WHERE rd.document_id = r.document_id 
         AND (tag_similarity + category_match + author_match) > 0.1
       LIMIT 5),
      '[]'::json
    ) as related_documents

  FROM ranked_results r
)

SELECT 
  document_id,
  title,
  content_preview,
  highlighted_content,
  category,
  semantic_category,
  author,
  final_relevance_score,
  related_documents,

  -- Search result metadata
  COUNT(*) OVER () as total_results,
  ROW_NUMBER() OVER (ORDER BY final_relevance_score DESC) as result_rank

FROM final_results
ORDER BY final_relevance_score DESC, created_at DESC
LIMIT 20;

-- Problems with traditional keyword-based search:
-- 1. No understanding of semantic meaning or context
-- 2. Cannot handle synonyms, related concepts, or conceptual queries
-- 3. Limited relevance scoring based only on keyword frequency and position  
-- 4. Poor handling of multilingual content and cross-language search
-- 5. No support for similarity search across different content types
-- 6. Manual and error-prone relevance tuning with limited effectiveness
-- 7. Cannot understand user intent beyond explicit keyword matches
-- 8. Poor recommendation capabilities based only on metadata overlap
-- 9. Limited support for complex search patterns and AI-powered features
-- 10. No integration with modern machine learning and embedding models

-- MySQL approach (even more limited)
SELECT 
  document_id,
  title,
  content,
  category,

  -- Basic full-text search (MySQL limitations)
  MATCH(title, content) AGAINST('machine learning' IN NATURAL LANGUAGE MODE) as relevance,

  -- Simple keyword highlighting
  REPLACE(
    REPLACE(title, 'machine', '<mark>machine</mark>'), 
    'learning', '<mark>learning</mark>'
  ) as highlighted_title

FROM mysql_documents
WHERE MATCH(title, content) AGAINST('machine learning' IN NATURAL LANGUAGE MODE)
ORDER BY relevance DESC
LIMIT 10;

-- MySQL limitations:
-- - Very basic full-text search with limited relevance algorithms
-- - No semantic understanding or contextual matching
-- - Limited text processing and language support
-- - Basic relevance scoring without advanced ranking factors
-- - No support for vector embeddings or similarity search
-- - Limited customization of search behavior and ranking
-- - Poor performance with large text corpuses
-- - No integration with modern AI/ML search techniques

// MongoDB Atlas Vector Search - AI-powered semantic search and similarity matching
const { MongoClient } = require('mongodb');

const client = new MongoClient('mongodb+srv://your-cluster.mongodb.net/');
const db = client.db('intelligent_search_platform');

// Advanced vector search and semantic similarity platform
class VectorSearchManager {
  constructor(db) {
    this.db = db;
    this.collections = {
      documents: db.collection('documents'),
      vectorIndex: db.collection('vector_index_metadata'),
      searchAnalytics: db.collection('search_analytics'),
      userProfiles: db.collection('user_profiles'),
      recommendations: db.collection('recommendations')
    };

    // Vector search configuration
    this.vectorConfig = {
      dimensions: 1536, // OpenAI text-embedding-ada-002
      similarity: 'cosine',
      indexType: 'knnVector'
    };

    this.embeddingModel = 'text-embedding-ada-002'; // Can be configured for different models
  }

  async initializeVectorSearchIndexes() {
    console.log('Initializing Atlas Vector Search indexes...');

    // Create vector search index for document content
    const contentVectorIndex = {
      name: 'content_vector_index',
      definition: {
        fields: [
          {
            type: 'vector',
            path: 'contentVector',
            numDimensions: this.vectorConfig.dimensions,
            similarity: this.vectorConfig.similarity
          },
          {
            type: 'filter',
            path: 'category'
          },
          {
            type: 'filter', 
            path: 'tags'
          },
          {
            type: 'filter',
            path: 'publishedDate'
          },
          {
            type: 'filter',
            path: 'author'
          },
          {
            type: 'filter',
            path: 'contentType'
          }
        ]
      }
    };

    // Create vector search index for title embeddings
    const titleVectorIndex = {
      name: 'title_vector_index', 
      definition: {
        fields: [
          {
            type: 'vector',
            path: 'titleVector',
            numDimensions: this.vectorConfig.dimensions,
            similarity: this.vectorConfig.similarity
          }
        ]
      }
    };

    // Create hybrid search index combining vector and text search
    const hybridSearchIndex = {
      name: 'hybrid_search_index',
      definition: {
        fields: [
          {
            type: 'vector',
            path: 'contentVector',
            numDimensions: this.vectorConfig.dimensions,
            similarity: this.vectorConfig.similarity
          },
          {
            type: 'autocomplete',
            path: 'title',
            tokenization: 'edgeGram',
            minGrams: 2,
            maxGrams: 15
          },
          {
            type: 'text',
            path: 'content',
            analyzer: 'lucene.standard'
          },
          {
            type: 'text',
            path: 'tags',
            analyzer: 'lucene.keyword'
          }
        ]
      }
    };

    try {
      // Note: In practice, vector search indexes are created through MongoDB Atlas UI
      // or MongoDB CLI. This code shows the structure for reference.
      console.log('Vector search indexes configured:');
      console.log('- Content Vector Index:', contentVectorIndex.name);
      console.log('- Title Vector Index:', titleVectorIndex.name); 
      console.log('- Hybrid Search Index:', hybridSearchIndex.name);

      // Store index metadata for application reference
      await this.collections.vectorIndex.insertMany([
        { ...contentVectorIndex, createdAt: new Date(), status: 'active' },
        { ...titleVectorIndex, createdAt: new Date(), status: 'active' },
        { ...hybridSearchIndex, createdAt: new Date(), status: 'active' }
      ]);

      return {
        contentVectorIndex: contentVectorIndex.name,
        titleVectorIndex: titleVectorIndex.name,
        hybridSearchIndex: hybridSearchIndex.name
      };

    } catch (error) {
      console.error('Vector index initialization failed:', error);
      throw error;
    }
  }

  async ingestDocumentsWithVectorization(documents) {
    console.log(`Processing ${documents.length} documents for vector search ingestion...`);

    const processedDocuments = [];
    const batchSize = 10;

    // Process documents in batches to manage API rate limits
    for (let i = 0; i < documents.length; i += batchSize) {
      const batch = documents.slice(i, i + batchSize);

      console.log(`Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(documents.length / batchSize)}`);

      const batchPromises = batch.map(async (doc) => {
        try {
          // Generate embeddings for title and content
          const [titleEmbedding, contentEmbedding] = await Promise.all([
            this.generateEmbedding(doc.title),
            this.generateEmbedding(doc.content)
          ]);

          // Extract key phrases and entities for enhanced searchability
          const extractedEntities = await this.extractEntities(doc.content);
          const keyPhrases = await this.extractKeyPhrases(doc.content);

          // Calculate content characteristics for better matching
          const contentCharacteristics = this.analyzeContentCharacteristics(doc.content);

          return {
            _id: doc._id || new ObjectId(),

            // Original document content
            title: doc.title,
            content: doc.content,
            summary: doc.summary || this.generateSummary(doc.content),

            // Document metadata
            category: doc.category,
            tags: doc.tags || [],
            author: doc.author,
            publishedDate: doc.publishedDate || new Date(),
            contentType: doc.contentType || 'article',
            language: doc.language || 'en',

            // Vector embeddings for semantic search
            titleVector: titleEmbedding,
            contentVector: contentEmbedding,

            // Enhanced searchability features
            entities: extractedEntities,
            keyPhrases: keyPhrases,
            contentCharacteristics: contentCharacteristics,

            // Search optimization metadata
            searchMetadata: {
              wordCount: doc.content.split(/\s+/).length,
              readingTime: Math.ceil(doc.content.split(/\s+/).length / 200), // minutes
              complexity: contentCharacteristics.complexity,
              topicDistribution: contentCharacteristics.topics,
              sentimentScore: contentCharacteristics.sentiment
            },

            // Document quality and authority signals
            qualitySignals: {
              authorityScore: doc.authorityScore || 0.5,
              freshnessScore: this.calculateFreshnessScore(doc.publishedDate || new Date()),
              engagementScore: doc.engagementScore || 0.5,
              accuracyScore: doc.accuracyScore || 0.8
            },

            // Indexing and processing metadata
            indexed: true,
            indexedAt: new Date(),
            vectorModelVersion: this.embeddingModel,
            processingVersion: '1.0'
          };

        } catch (error) {
          console.error(`Failed to process document ${doc._id}:`, error);
          return null;
        }
      });

      const batchResults = await Promise.all(batchPromises);
      const validResults = batchResults.filter(result => result !== null);
      processedDocuments.push(...validResults);

      // Rate limiting pause between batches
      if (i + batchSize < documents.length) {
        await new Promise(resolve => setTimeout(resolve, 1000));
      }
    }

    // Bulk insert processed documents
    if (processedDocuments.length > 0) {
      const insertResult = await this.collections.documents.insertMany(processedDocuments, {
        ordered: false
      });

      console.log(`Successfully indexed ${insertResult.insertedCount} documents with vector embeddings`);

      return {
        totalProcessed: documents.length,
        successfullyIndexed: insertResult.insertedCount,
        failed: documents.length - processedDocuments.length,
        indexedDocuments: processedDocuments
      };
    }

    return {
      totalProcessed: documents.length,
      successfullyIndexed: 0,
      failed: documents.length,
      indexedDocuments: []
    };
  }

  async performSemanticSearch(query, options = {}) {
    console.log(`Performing semantic search for: "${query}"`);

    const {
      limit = 20,
      filters = {},
      includeScore = true,
      similarityThreshold = 0.7,
      searchType = 'semantic', // 'semantic', 'hybrid', 'keyword'
      userContext = null
    } = options;

    try {
      // Generate query embedding for semantic search
      const queryEmbedding = await this.generateEmbedding(query);

      let pipeline = [];

      if (searchType === 'semantic' || searchType === 'hybrid') {
        // Vector similarity search stage
        pipeline.push({
          $vectorSearch: {
            index: 'content_vector_index',
            path: 'contentVector',
            queryVector: queryEmbedding,
            numCandidates: limit * 10, // Search more candidates for better results
            limit: limit * 2, // Get more results for reranking
            filter: this.buildFilterExpression(filters)
          }
        });

        // Add vector search score
        pipeline.push({
          $addFields: {
            vectorScore: { $meta: 'vectorSearchScore' },
            searchMethod: 'vector'
          }
        });
      }

      if (searchType === 'hybrid') {
        // Combine with text search for hybrid approach
        pipeline.push({
          $unionWith: {
            coll: 'documents',
            pipeline: [
              {
                $search: {
                  index: 'hybrid_search_index',
                  compound: {
                    should: [
                      {
                        text: {
                          query: query,
                          path: ['title', 'content'],
                          score: { boost: { value: 2.0 } }
                        }
                      },
                      {
                        autocomplete: {
                          query: query,
                          path: 'title',
                          score: { boost: { value: 1.5 } }
                        }
                      }
                    ],
                    filter: this.buildSearchFilterClauses(filters)
                  }
                }
              },
              {
                $addFields: {
                  textScore: { $meta: 'searchScore' },
                  searchMethod: 'text'
                }
              },
              { $limit: limit }
            ]
          }
        });
      }

      // Enhanced result processing and ranking
      pipeline.push({
        $addFields: {
          // Calculate comprehensive relevance score
          relevanceScore: {
            $switch: {
              branches: [
                {
                  case: { $eq: ['$searchMethod', 'vector'] },
                  then: {
                    $multiply: [
                      { $ifNull: ['$vectorScore', 0] },
                      { $add: [
                        { $multiply: [{ $ifNull: ['$qualitySignals.authorityScore', 0.5] }, 0.2] },
                        { $multiply: [{ $ifNull: ['$qualitySignals.freshnessScore', 0.5] }, 0.1] },
                        { $multiply: [{ $ifNull: ['$qualitySignals.engagementScore', 0.5] }, 0.15] },
                        0.55 // Base score weight
                      ]}
                    ]
                  }
                },
                {
                  case: { $eq: ['$searchMethod', 'text'] },
                  then: {
                    $multiply: [
                      { $ifNull: ['$textScore', 0] },
                      0.8 // Weight text search lower than semantic
                    ]
                  }
                }
              ],
              default: 0
            }
          },

          // Extract relevant snippets
          contentSnippet: {
            $substrCP: [
              '$content', 
              0, 
              300
            ]
          },

          // Calculate query-document semantic similarity
          semanticRelevance: {
            $cond: {
              if: { $gt: [{ $ifNull: ['$vectorScore', 0] }, similarityThreshold] },
              then: 'high',
              else: {
                $cond: {
                  if: { $gt: [{ $ifNull: ['$vectorScore', 0] }, similarityThreshold * 0.8] },
                  then: 'medium',
                  else: 'low'
                }
              }
            }
          }
        }
      });

      // User personalization if context provided
      if (userContext) {
        pipeline.push({
          $addFields: {
            personalizedScore: {
              $multiply: [
                '$relevanceScore',
                {
                  $add: [
                    // Category preference boost
                    {
                      $cond: {
                        if: { $in: ['$category', userContext.preferredCategories || []] },
                        then: 0.2,
                        else: 0
                      }
                    },
                    // Author preference boost  
                    {
                      $cond: {
                        if: { $in: ['$author', userContext.followedAuthors || []] },
                        then: 0.15,
                        else: 0
                      }
                    },
                    // Language preference
                    {
                      $cond: {
                        if: { $eq: ['$language', userContext.preferredLanguage || 'en'] },
                        then: 0.1,
                        else: -0.05
                      }
                    },
                    1.0 // Base multiplier
                  ]
                }
              ]
            }
          }
        });
      }

      // Filter by similarity threshold and finalize results
      pipeline.push(
        {
          $match: {
            relevanceScore: { $gte: similarityThreshold * 0.5 }
          }
        },
        {
          $sort: {
            [userContext ? 'personalizedScore' : 'relevanceScore']: -1,
            publishedDate: -1
          }
        },
        {
          $limit: limit
        },
        {
          $project: {
            _id: 1,
            title: 1,
            contentSnippet: 1,
            category: 1,
            tags: 1,
            author: 1,
            publishedDate: 1,
            contentType: 1,
            language: 1,
            entities: 1,
            keyPhrases: 1,
            searchMetadata: 1,
            relevanceScore: includeScore ? 1 : 0,
            personalizedScore: (includeScore && userContext) ? 1 : 0,
            vectorScore: includeScore ? 1 : 0,
            textScore: includeScore ? 1 : 0,
            semanticRelevance: 1,
            searchMethod: 1
          }
        }
      );

      const searchStart = Date.now();
      const results = await this.collections.documents.aggregate(pipeline).toArray();
      const searchTime = Date.now() - searchStart;

      // Log search analytics
      await this.logSearchAnalytics({
        query: query,
        searchType: searchType,
        filters: filters,
        resultCount: results.length,
        searchTime: searchTime,
        userContext: userContext,
        timestamp: new Date()
      });

      console.log(`Semantic search completed in ${searchTime}ms, found ${results.length} results`);

      return {
        query: query,
        searchType: searchType,
        results: results,
        metadata: {
          totalResults: results.length,
          searchTime: searchTime,
          similarityThreshold: similarityThreshold,
          filtersApplied: Object.keys(filters).length > 0
        }
      };

    } catch (error) {
      console.error('Semantic search failed:', error);
      throw error;
    }
  }

  async findSimilarDocuments(documentId, options = {}) {
    console.log(`Finding documents similar to: ${documentId}`);

    const {
      limit = 10,
      similarityThreshold = 0.75,
      excludeCategories = [],
      includeScore = true
    } = options;

    // Get the source document and its vector
    const sourceDocument = await this.collections.documents.findOne(
      { _id: documentId },
      { projection: { contentVector: 1, title: 1, category: 1, tags: 1 } }
    );

    if (!sourceDocument || !sourceDocument.contentVector) {
      throw new Error('Source document not found or not vectorized');
    }

    // Find similar documents using vector search
    const pipeline = [
      {
        $vectorSearch: {
          index: 'content_vector_index',
          path: 'contentVector',
          queryVector: sourceDocument.contentVector,
          numCandidates: limit * 20,
          limit: limit * 2,
          filter: {
            $and: [
              { _id: { $ne: documentId } }, // Exclude source document
              excludeCategories.length > 0 ? 
                { category: { $not: { $in: excludeCategories } } } : 
                {}
            ]
          }
        }
      },
      {
        $addFields: {
          similarityScore: { $meta: 'vectorSearchScore' },

          // Calculate additional similarity factors
          tagSimilarity: {
            $let: {
              vars: {
                commonTags: {
                  $size: {
                    $setIntersection: ['$tags', sourceDocument.tags || []]
                  }
                },
                totalTags: {
                  $add: [
                    { $size: { $ifNull: ['$tags', []] } },
                    { $size: { $ifNull: [sourceDocument.tags, []] } }
                  ]
                }
              },
              in: {
                $cond: {
                  if: { $gt: ['$$totalTags', 0] },
                  then: { $divide: ['$$commonTags', '$$totalTags'] },
                  else: 0
                }
              }
            }
          },

          categorySimilarity: {
            $cond: {
              if: { $eq: ['$category', sourceDocument.category] },
              then: 0.2,
              else: 0
            }
          }
        }
      },
      {
        $addFields: {
          combinedSimilarity: {
            $add: [
              { $multiply: ['$similarityScore', 0.7] },
              { $multiply: ['$tagSimilarity', 0.2] },
              '$categorySimilarity'
            ]
          }
        }
      },
      {
        $match: {
          combinedSimilarity: { $gte: similarityThreshold }
        }
      },
      {
        $sort: { combinedSimilarity: -1 }
      },
      {
        $limit: limit
      },
      {
        $project: {
          _id: 1,
          title: 1,
          contentSnippet: { $substrCP: ['$content', 0, 200] },
          category: 1,
          tags: 1,
          author: 1,
          publishedDate: 1,
          similarityScore: includeScore ? 1 : 0,
          combinedSimilarity: includeScore ? 1 : 0,
          searchMetadata: 1
        }
      }
    ];

    const similarDocuments = await this.collections.documents.aggregate(pipeline).toArray();

    return {
      sourceDocumentId: documentId,
      sourceTitle: sourceDocument.title,
      similarDocuments: similarDocuments,
      metadata: {
        totalSimilar: similarDocuments.length,
        similarityThreshold: similarityThreshold,
        searchMethod: 'vector_similarity'
      }
    };
  }

  async generateRecommendations(userId, options = {}) {
    console.log(`Generating personalized recommendations for user: ${userId}`);

    const {
      limit = 15,
      diversityFactor = 0.3,
      includeExplanations = true
    } = options;

    // Get user profile and interaction history
    const userProfile = await this.collections.userProfiles.findOne({ userId: userId });

    if (!userProfile) {
      console.log('User profile not found, using general recommendations');
      return this.generateGeneralRecommendations(limit);
    }

    // Build user preference vector from interaction history
    const userVector = await this.buildUserPreferenceVector(userProfile);

    if (!userVector) {
      return this.generateGeneralRecommendations(limit);
    }

    // Find documents matching user preferences
    const pipeline = [
      {
        $vectorSearch: {
          index: 'content_vector_index',
          path: 'contentVector',
          queryVector: userVector,
          numCandidates: limit * 10,
          limit: limit * 3,
          filter: {
            $and: [
              // Exclude already read documents
              { _id: { $not: { $in: userProfile.readDocuments || [] } } },

              // Include preferred categories
              userProfile.preferredCategories && userProfile.preferredCategories.length > 0 ?
                { category: { $in: userProfile.preferredCategories } } :
                {},

              // Fresh content preference
              {
                publishedDate: {
                  $gte: new Date(Date.now() - 90 * 24 * 60 * 60 * 1000) // Last 90 days
                }
              }
            ]
          }
        }
      },
      {
        $addFields: {
          preferenceScore: { $meta: 'vectorSearchScore' },

          // Category affinity scoring
          categoryScore: {
            $switch: {
              branches: (userProfile.categoryAffinities || []).map(affinity => ({
                case: { $eq: ['$category', affinity.category] },
                then: affinity.score
              })),
              default: 0.5
            }
          },

          // Author following boost
          authorScore: {
            $cond: {
              if: { $in: ['$author', userProfile.followedAuthors || []] },
              then: 0.8,
              else: 0.4
            }
          },

          // Freshness scoring
          freshnessScore: {
            $divide: [
              { $subtract: [Date.now(), '$publishedDate'] },
              (30 * 24 * 60 * 60 * 1000) // 30 days in milliseconds
            ]
          }
        }
      },
      {
        $addFields: {
          recommendationScore: {
            $add: [
              { $multiply: ['$preferenceScore', 0.4] },
              { $multiply: ['$categoryScore', 0.25] },
              { $multiply: ['$authorScore', 0.2] },
              { $multiply: [{ $max: [0, { $subtract: [1, '$freshnessScore'] }] }, 0.15] }
            ]
          }
        }
      }
    ];

    // Apply diversity to avoid filter bubble
    if (diversityFactor > 0) {
      pipeline.push({
        $group: {
          _id: '$category',
          documents: {
            $push: {
              _id: '$_id',
              title: '$title',
              recommendationScore: '$recommendationScore',
              category: '$category',
              author: '$author',
              publishedDate: '$publishedDate',
              tags: '$tags'
            }
          },
          maxScore: { $max: '$recommendationScore' }
        }
      });

      pipeline.push({
        $sort: { maxScore: -1 }
      });

      // Select diverse recommendations
      pipeline.push({
        $project: {
          documents: {
            $slice: [
              { $sortArray: { input: '$documents', sortBy: { recommendationScore: -1 } } },
              Math.ceil(limit * diversityFactor)
            ]
          }
        }
      });

      pipeline.push({
        $unwind: '$documents'
      });

      pipeline.push({
        $replaceRoot: { newRoot: '$documents' }
      });
    }

    pipeline.push(
      {
        $sort: { recommendationScore: -1 }
      },
      {
        $limit: limit
      }
    );

    const recommendations = await this.collections.documents.aggregate(pipeline).toArray();

    // Generate explanations if requested
    if (includeExplanations) {
      for (const rec of recommendations) {
        rec.explanation = this.generateRecommendationExplanation(rec, userProfile);
      }
    }

    // Store recommendations for future analysis
    await this.collections.recommendations.insertOne({
      userId: userId,
      recommendations: recommendations.map(r => ({
        documentId: r._id,
        score: r.recommendationScore,
        explanation: r.explanation
      })),
      generatedAt: new Date(),
      algorithm: 'vector_preference_matching',
      diversityFactor: diversityFactor
    });

    return {
      userId: userId,
      recommendations: recommendations,
      metadata: {
        totalRecommendations: recommendations.length,
        algorithm: 'vector_preference_matching',
        diversityApplied: diversityFactor > 0,
        generatedAt: new Date()
      }
    };
  }

  // Helper methods for vector search operations

  async generateEmbedding(text) {
    // In production, this would call OpenAI API or other embedding service
    // For this example, we'll simulate embeddings

    // Simulate API call delay
    await new Promise(resolve => setTimeout(resolve, 100));

    // Generate mock embedding vector (in production, use actual embedding API)
    const mockEmbedding = Array.from({ length: this.vectorConfig.dimensions }, () => 
      Math.random() * 2 - 1 // Values between -1 and 1
    );

    return mockEmbedding;
  }

  async extractEntities(text) {
    // Simulate entity extraction (in production, use NLP service)
    const entities = [];

    // Basic keyword extraction simulation
    const words = text.toLowerCase().split(/\W+/);
    const entityKeywords = ['mongodb', 'database', 'javascript', 'python', 'ai', 'machine learning'];

    entityKeywords.forEach(keyword => {
      if (words.includes(keyword) || words.includes(keyword.replace(' ', ''))) {
        entities.push({
          text: keyword,
          type: 'technology',
          confidence: 0.8
        });
      }
    });

    return entities;
  }

  async extractKeyPhrases(text) {
    // Simulate key phrase extraction
    const sentences = text.split(/[.!?]+/);
    const keyPhrases = [];

    sentences.forEach(sentence => {
      const words = sentence.trim().split(/\s+/);
      if (words.length >= 3 && words.length <= 8) {
        keyPhrases.push({
          phrase: sentence.trim(),
          relevance: Math.random()
        });
      }
    });

    return keyPhrases.sort((a, b) => b.relevance - a.relevance).slice(0, 10);
  }

  analyzeContentCharacteristics(content) {
    const wordCount = content.split(/\s+/).length;
    const sentenceCount = content.split(/[.!?]+/).length;
    const avgWordsPerSentence = wordCount / sentenceCount;

    return {
      complexity: avgWordsPerSentence > 20 ? 'high' : avgWordsPerSentence > 15 ? 'medium' : 'low',
      topics: ['general'], // Would use topic modeling in production
      sentiment: Math.random() * 2 - 1, // -1 to 1 scale
      readabilityScore: Math.max(0, Math.min(100, 100 - (avgWordsPerSentence * 2)))
    };
  }

  calculateFreshnessScore(publishedDate) {
    const ageInDays = (Date.now() - publishedDate.getTime()) / (24 * 60 * 60 * 1000);
    return Math.max(0, Math.min(1, 1 - (ageInDays / 365))); // Decay over 1 year
  }

  generateSummary(content) {
    // Simple summary generation (first 200 characters)
    return content.length > 200 ? content.substring(0, 197) + '...' : content;
  }

  buildFilterExpression(filters) {
    const filterExpression = { $and: [] };

    if (filters.category) {
      filterExpression.$and.push({ category: { $eq: filters.category } });
    }

    if (filters.author) {
      filterExpression.$and.push({ author: { $eq: filters.author } });
    }

    if (filters.tags && filters.tags.length > 0) {
      filterExpression.$and.push({ tags: { $in: filters.tags } });
    }

    if (filters.dateRange) {
      filterExpression.$and.push({ 
        publishedDate: {
          $gte: new Date(filters.dateRange.start),
          $lte: new Date(filters.dateRange.end)
        }
      });
    }

    return filterExpression.$and.length > 0 ? filterExpression : {};
  }

  buildSearchFilterClauses(filters) {
    const clauses = [];

    if (filters.category) {
      clauses.push({ equals: { path: 'category', value: filters.category } });
    }

    if (filters.tags && filters.tags.length > 0) {
      clauses.push({ in: { path: 'tags', value: filters.tags } });
    }

    return clauses;
  }

  async logSearchAnalytics(analyticsData) {
    try {
      await this.collections.searchAnalytics.insertOne({
        ...analyticsData,
        sessionId: analyticsData.userContext?.sessionId,
        userId: analyticsData.userContext?.userId
      });
    } catch (error) {
      console.warn('Failed to log search analytics:', error.message);
    }
  }

  async buildUserPreferenceVector(userProfile) {
    if (!userProfile.interactionHistory || userProfile.interactionHistory.length === 0) {
      return null;
    }

    // Get vectors for user's previously interacted documents
    const interactedDocuments = await this.collections.documents.find(
      { 
        _id: { $in: userProfile.interactionHistory.slice(-20).map(h => h.documentId) } 
      },
      { projection: { contentVector: 1 } }
    ).toArray();

    if (interactedDocuments.length === 0) {
      return null;
    }

    // Calculate weighted average vector based on interaction types
    const weightedVectors = interactedDocuments.map((doc, index) => {
      const interaction = userProfile.interactionHistory.find(h => 
        h.documentId.toString() === doc._id.toString()
      );

      const weight = this.getInteractionWeight(interaction.type);
      return doc.contentVector.map(val => val * weight);
    });

    // Average the vectors
    const dimensions = weightedVectors[0].length;
    const avgVector = Array(dimensions).fill(0);

    weightedVectors.forEach(vector => {
      vector.forEach((val, i) => {
        avgVector[i] += val;
      });
    });

    return avgVector.map(val => val / weightedVectors.length);
  }

  getInteractionWeight(interactionType) {
    const weights = {
      'view': 0.1,
      'like': 0.3,
      'share': 0.5,
      'bookmark': 0.7,
      'comment': 0.8
    };
    return weights[interactionType] || 0.1;
  }

  generateRecommendationExplanation(recommendation, userProfile) {
    const explanations = [];

    if (userProfile.preferredCategories && userProfile.preferredCategories.includes(recommendation.category)) {
      explanations.push(`Matches your interest in ${recommendation.category}`);
    }

    if (userProfile.followedAuthors && userProfile.followedAuthors.includes(recommendation.author)) {
      explanations.push(`By ${recommendation.author}, an author you follow`);
    }

    if (recommendation.tags) {
      const matchingTags = recommendation.tags.filter(tag => 
        userProfile.interests && userProfile.interests.includes(tag)
      );
      if (matchingTags.length > 0) {
        explanations.push(`Related to ${matchingTags.slice(0, 2).join(' and ')}`);
      }
    }

    if (explanations.length === 0) {
      explanations.push('Similar to content you\'ve previously engaged with');
    }

    return explanations.join('; ');
  }

  async generateGeneralRecommendations(limit) {
    // Fallback recommendations based on popularity and quality
    const pipeline = [
      {
        $addFields: {
          popularityScore: {
            $add: [
              { $multiply: [{ $ifNull: ['$qualitySignals.engagementScore', 0.5] }, 0.4] },
              { $multiply: [{ $ifNull: ['$qualitySignals.authorityScore', 0.5] }, 0.3] },
              { $multiply: [{ $ifNull: ['$qualitySignals.freshnessScore', 0.5] }, 0.3] }
            ]
          }
        }
      },
      {
        $sort: { popularityScore: -1 }
      },
      {
        $limit: limit
      },
      {
        $project: {
          _id: 1,
          title: 1,
          contentSnippet: { $substrCP: ['$content', 0, 200] },
          category: 1,
          author: 1,
          publishedDate: 1,
          popularityScore: 1
        }
      }
    ];

    const recommendations = await this.collections.documents.aggregate(pipeline).toArray();

    return {
      recommendations: recommendations,
      metadata: {
        algorithm: 'popularity_based',
        totalRecommendations: recommendations.length
      }
    };
  }
}

// Benefits of MongoDB Atlas Vector Search:
// - Native vector database capabilities within MongoDB Atlas infrastructure
// - Seamless integration with existing MongoDB documents and operations  
// - Support for multiple vector similarity algorithms (cosine, euclidean, dot product)
// - Hybrid search combining vector similarity with traditional text search
// - Scalable vector indexing with automatic optimization and maintenance
// - Built-in filtering capabilities for combining semantic search with metadata filters
// - Real-time vector search with sub-second response times at scale
// - Integration with popular embedding models (OpenAI, Cohere, Hugging Face)
// - Support for multiple vector dimensions and embedding types
// - Advanced ranking and personalization capabilities for AI-powered applications

module.exports = {
  VectorSearchManager
};