MongoDB Text Search and Full-Text Indexing: SQL-Style Content Search with Advanced Language Processing

-- SQL basic text search - limited and inefficient
-- Simple pattern matching - no relevance scoring
SELECT 
  article_id,
  title,
  content,
  author,
  published_date
FROM articles
WHERE title LIKE '%mongodb%'
   OR content LIKE '%database%'
   OR content LIKE '%nosql%'
ORDER BY published_date DESC
LIMIT 20;

-- Problems with LIKE-based search:
-- - No relevance ranking or scoring
-- - Case-sensitive matching issues
-- - No stemming (search for "running" won't find "run")
-- - No language-specific processing
-- - Poor performance on large text fields
-- - No phrase matching or proximity search
-- - Cannot handle synonyms or related terms

-- More advanced SQL text search with full-text indexes
CREATE FULLTEXT INDEX article_search_idx ON articles(title, content);

SELECT 
  article_id,
  title,
  MATCH(title, content) AGAINST('mongodb database nosql' IN NATURAL LANGUAGE MODE) as relevance_score
FROM articles
WHERE MATCH(title, content) AGAINST('mongodb database nosql' IN NATURAL LANGUAGE MODE)
ORDER BY relevance_score DESC
LIMIT 20;

-- Limitations even with full-text indexes:
-- - Limited language support and customization
-- - Basic relevance scoring algorithms
-- - Difficulty combining with other query conditions
-- - Limited control over text analysis pipeline
-- - No support for complex document structures
-- - Separate indexing and maintenance overhead

// MongoDB native text search - comprehensive and efficient
// Create sophisticated text index with language-specific processing
db.articles.createIndex({
  title: "text",
  content: "text", 
  tags: "text",
  author: "text"
}, {
  weights: {
    title: 10,    // Title matches weighted higher
    content: 5,   // Content matches medium weight
    tags: 8,      // Tag matches high weight
    author: 3     // Author matches lower weight
  },
  default_language: "english",
  language_override: "language", // Per-document language specification
  textIndexVersion: 3,
  name: "comprehensive_text_search"
});

// Powerful text search with relevance scoring and ranking
const searchResults = await db.articles.find({
  $text: {
    $search: "mongodb database nosql performance",
    $language: "english",
    $caseSensitive: false,
    $diacriticSensitive: false
  }
}, {
  score: { $meta: "textScore" },
  title: 1,
  content: 1,
  author: 1,
  published_date: 1,
  tags: 1
}).sort({ 
  score: { $meta: "textScore" } 
}).limit(20);

// Benefits of MongoDB text search:
// - Built-in relevance scoring with customizable weights
// - Language-aware stemming and stop word processing
// - Phrase matching and proximity scoring
// - Case and diacritic insensitive search
// - Integration with other query conditions
// - Support for 15+ languages out of the box
// - Efficient indexing with document structure awareness
// - Real-time search without external dependencies

// Advanced text indexing for content-rich applications
class TextSearchManager {
  constructor(db) {
    this.db = db;
    this.searchConfig = {
      supportedLanguages: [
        'english', 'spanish', 'french', 'german', 'portuguese',
        'russian', 'arabic', 'chinese', 'japanese', 'korean'
      ],
      defaultWeights: {
        title: 10,
        summary: 8,
        content: 5,
        tags: 7,
        category: 6,
        author: 3
      }
    };
  }

  async setupComprehensiveTextIndex(collectionName, indexConfig) {
    const collection = this.db.collection(collectionName);

    // Build text index specification
    const indexSpec = {};
    const options = {
      weights: {},
      default_language: indexConfig.defaultLanguage || 'english',
      language_override: indexConfig.languageField || 'language',
      textIndexVersion: 3,
      name: `${collectionName}_comprehensive_text_search`
    };

    // Configure searchable fields with weights
    for (const [field, weight] of Object.entries(indexConfig.fields)) {
      indexSpec[field] = 'text';
      options.weights[field] = weight;
    }

    // Add partial filter for performance
    if (indexConfig.partialFilter) {
      options.partialFilterExpression = indexConfig.partialFilter;
    }

    // Create the text index
    await collection.createIndex(indexSpec, options);

    console.log(`Text index created for ${collectionName}:`, {
      fields: Object.keys(indexSpec),
      weights: options.weights,
      language: options.default_language
    });

    return {
      collection: collectionName,
      indexName: options.name,
      configuration: options
    };
  }

  async setupArticleTextSearch() {
    // Specialized text search for article/blog content
    const articleConfig = {
      fields: {
        title: 10,           // Highest priority for title matches
        summary: 8,          // High priority for summary/excerpt
        content: 5,          // Medium priority for body content
        tags: 7,             // High priority for tag matches
        category: 6,         // Medium-high for category matches
        author: 3            // Lower priority for author matches
      },
      defaultLanguage: 'english',
      languageField: 'language',
      partialFilter: { 
        status: 'published',
        deleted: { $ne: true }
      }
    };

    return await this.setupComprehensiveTextIndex('articles', articleConfig);
  }

  async setupProductTextSearch() {
    // E-commerce product search configuration
    const productConfig = {
      fields: {
        name: 10,            // Product name highest priority
        description: 6,      // Product description medium-high
        brand: 8,            // Brand name high priority
        category: 7,         // Category high priority
        tags: 8,             // Product tags high priority
        specifications: 4,   // Technical specs lower priority
        reviews: 3           // Customer review content lowest
      },
      defaultLanguage: 'english',
      languageField: 'language',
      partialFilter: { 
        active: true,
        in_stock: true
      }
    };

    return await this.setupComprehensiveTextIndex('products', productConfig);
  }

  async performTextSearch(collectionName, searchQuery, options = {}) {
    const collection = this.db.collection(collectionName);

    // Build text search query
    const textSearchFilter = {
      $text: {
        $search: searchQuery,
        $language: options.language || 'english',
        $caseSensitive: options.caseSensitive || false,
        $diacriticSensitive: options.diacriticSensitive || false
      }
    };

    // Combine with additional filters
    const combinedFilter = { ...textSearchFilter };
    if (options.additionalFilters) {
      Object.assign(combinedFilter, options.additionalFilters);
    }

    // Build projection with text score
    const projection = {
      score: { $meta: "textScore" }
    };

    if (options.fields) {
      options.fields.forEach(field => {
        projection[field] = 1;
      });
    }

    // Execute search with scoring and sorting
    const cursor = collection.find(combinedFilter, projection)
      .sort({ score: { $meta: "textScore" } });

    // Apply pagination
    if (options.skip) cursor.skip(options.skip);
    if (options.limit) cursor.limit(options.limit);

    const results = await cursor.toArray();

    // Enhance results with search metadata
    return {
      results: results.map(doc => ({
        ...doc,
        relevanceScore: doc.score,
        searchQuery: searchQuery,
        matchedTerms: this.extractMatchedTerms(doc, searchQuery)
      })),
      searchMetadata: {
        query: searchQuery,
        totalResults: results.length,
        language: options.language || 'english',
        searchTime: new Date(),
        facets: await this.generateSearchFacets(collectionName, combinedFilter, options)
      }
    };
  }

  async performAdvancedTextSearch(collectionName, searchConfig) {
    // Advanced search with multiple query types and aggregation
    const collection = this.db.collection(collectionName);

    const pipeline = [];

    // Stage 1: Text search matching
    if (searchConfig.textQuery) {
      pipeline.push({
        $match: {
          $text: {
            $search: searchConfig.textQuery,
            $language: searchConfig.language || 'english'
          }
        }
      });

      // Add text score to documents
      pipeline.push({
        $addFields: {
          textScore: { $meta: "textScore" }
        }
      });
    }

    // Stage 2: Additional filtering
    if (searchConfig.filters) {
      pipeline.push({
        $match: searchConfig.filters
      });
    }

    // Stage 3: Enhanced scoring with business logic
    if (searchConfig.customScoring) {
      pipeline.push({
        $addFields: {
          combinedScore: {
            $add: [
              { $multiply: ["$textScore", searchConfig.textWeight || 1] },
              { $multiply: [
                { $cond: [{ $gte: ["$popularity_score", 80] }, 2, 1] },
                searchConfig.popularityWeight || 0.5
              ]},
              { $multiply: [
                { $cond: [{ $gte: ["$recency_days", 0] }, 
                  { $subtract: [30, "$recency_days"] }, 0] },
                searchConfig.recencyWeight || 0.3
              ]}
            ]
          }
        }
      });
    }

    // Stage 4: Faceted search aggregation
    if (searchConfig.generateFacets) {
      pipeline.push({
        $facet: {
          results: [
            { $sort: { 
              [searchConfig.customScoring ? 'combinedScore' : 'textScore']: -1 
            }},
            { $skip: searchConfig.skip || 0 },
            { $limit: searchConfig.limit || 20 },
            {
              $project: {
                _id: 1,
                title: 1,
                content: { $substr: ["$content", 0, 200] }, // Excerpt
                author: 1,
                category: 1,
                tags: 1,
                published_date: 1,
                textScore: 1,
                combinedScore: searchConfig.customScoring ? 1 : 0,
                highlightedContent: {
                  $function: {
                    body: `function(content, query) {
                      const terms = query.split(' ');
                      let highlighted = content;
                      terms.forEach(term => {
                        const regex = new RegExp(term, 'gi');
                        highlighted = highlighted.replace(regex, '<mark>$&</mark>');
                      });
                      return highlighted.substring(0, 300);
                    }`,
                    args: ["$content", searchConfig.textQuery],
                    lang: "js"
                  }
                }
              }
            }
          ],
          categoryFacets: [
            { $group: { 
              _id: "$category", 
              count: { $sum: 1 },
              avgScore: { $avg: "$textScore" }
            }},
            { $sort: { count: -1 } },
            { $limit: 10 }
          ],
          authorFacets: [
            { $group: { 
              _id: "$author", 
              count: { $sum: 1 },
              avgScore: { $avg: "$textScore" }
            }},
            { $sort: { count: -1 } },
            { $limit: 10 }
          ],
          tagFacets: [
            { $unwind: "$tags" },
            { $group: { 
              _id: "$tags", 
              count: { $sum: 1 },
              avgScore: { $avg: "$textScore" }
            }},
            { $sort: { count: -1 } },
            { $limit: 15 }
          ],
          dateRangeFacets: [
            {
              $group: {
                _id: {
                  $dateToString: {
                    format: "%Y-%m",
                    date: "$published_date"
                  }
                },
                count: { $sum: 1 },
                avgScore: { $avg: "$textScore" }
              }
            },
            { $sort: { "_id": -1 } },
            { $limit: 12 }
          ]
        }
      });
    } else {
      // Simple results without faceting
      pipeline.push(
        { $sort: { 
          [searchConfig.customScoring ? 'combinedScore' : 'textScore']: -1 
        }},
        { $skip: searchConfig.skip || 0 },
        { $limit: searchConfig.limit || 20 }
      );
    }

    const searchResults = await collection.aggregate(pipeline).toArray();

    return {
      searchResults: searchConfig.generateFacets ? searchResults[0] : { results: searchResults },
      searchMetadata: {
        query: searchConfig.textQuery,
        filters: searchConfig.filters,
        language: searchConfig.language,
        searchTime: new Date(),
        configuration: searchConfig
      }
    };
  }

  async performPhraseSearch(collectionName, phrase, options = {}) {
    // Exact phrase and proximity matching
    const collection = this.db.collection(collectionName);

    const searchQueries = [
      // Exact phrase search (quoted)
      {
        query: `"${phrase}"`,
        weight: 3,
        type: 'exact_phrase'
      },
      // Proximity search (words near each other)
      {
        query: phrase.split(' ').join(' '),
        weight: 2,
        type: 'proximity'  
      },
      // Individual terms (fallback)
      {
        query: phrase,
        weight: 1,
        type: 'terms'
      }
    ];

    const results = [];

    for (const searchQuery of searchQueries) {
      const queryResults = await collection.find({
        $text: { $search: searchQuery.query }
      }, {
        score: { $meta: "textScore" },
        title: 1,
        content: 1,
        author: 1,
        published_date: 1
      }).sort({ 
        score: { $meta: "textScore" } 
      }).limit(options.limit || 10).toArray();

      // Weight and tag results by search type
      const weightedResults = queryResults.map(doc => ({
        ...doc,
        searchType: searchQuery.type,
        adjustedScore: doc.score * searchQuery.weight,
        originalScore: doc.score
      }));

      results.push(...weightedResults);
    }

    // Deduplicate and merge results
    const deduplicatedResults = this.deduplicateSearchResults(results);

    // Sort by adjusted score
    deduplicatedResults.sort((a, b) => b.adjustedScore - a.adjustedScore);

    return {
      results: deduplicatedResults.slice(0, options.limit || 20),
      phraseQuery: phrase,
      searchStrategies: searchQueries.map(q => q.type)
    };
  }

  async performMultiLanguageSearch(collectionName, searchQuery, languages = []) {
    // Search across multiple languages with language detection
    const collection = this.db.collection(collectionName);

    const searchPromises = languages.map(async language => {
      const results = await collection.find({
        $text: {
          $search: searchQuery,
          $language: language
        }
      }, {
        score: { $meta: "textScore" },
        title: 1,
        content: 1,
        language: 1,
        author: 1,
        published_date: 1
      }).sort({ 
        score: { $meta: "textScore" } 
      }).limit(10).toArray();

      return results.map(doc => ({
        ...doc,
        searchLanguage: language,
        languageScore: doc.score
      }));
    });

    const languageResults = await Promise.all(searchPromises);
    const allResults = languageResults.flat();

    // Group by document and select best language match
    const documentMap = new Map();

    allResults.forEach(result => {
      const docId = result._id.toString();
      if (!documentMap.has(docId) || 
          documentMap.get(docId).languageScore < result.languageScore) {
        documentMap.set(docId, result);
      }
    });

    const finalResults = Array.from(documentMap.values())
      .sort((a, b) => b.languageScore - a.languageScore);

    return {
      results: finalResults,
      searchQuery: searchQuery,
      languagesSearched: languages,
      languageDistribution: this.calculateLanguageDistribution(finalResults)
    };
  }

  extractMatchedTerms(document, searchQuery) {
    // Extract which search terms matched in the document
    const searchTerms = searchQuery.toLowerCase().split(/\s+/);
    const documentText = [
      document.title || '',
      document.content || '',
      (document.tags || []).join(' '),
      document.author || ''
    ].join(' ').toLowerCase();

    return searchTerms.filter(term => 
      documentText.includes(term)
    );
  }

  async generateSearchFacets(collectionName, baseFilter, options) {
    // Generate search facets for filtering
    const collection = this.db.collection(collectionName);

    const facetPipeline = [
      { $match: baseFilter },
      {
        $facet: {
          categories: [
            { $group: { _id: "$category", count: { $sum: 1 } } },
            { $sort: { count: -1 } }
          ],
          authors: [
            { $group: { _id: "$author", count: { $sum: 1 } } },
            { $sort: { count: -1 } }
          ],
          dateRanges: [
            {
              $group: {
                _id: {
                  $dateToString: { format: "%Y", date: "$published_date" }
                },
                count: { $sum: 1 }
              }
            },
            { $sort: { "_id": -1 } }
          ]
        }
      }
    ];

    const facetResults = await collection.aggregate(facetPipeline).toArray();
    return facetResults[0] || {};
  }

  deduplicateSearchResults(results) {
    // Remove duplicate documents, keeping highest scored version
    const seen = new Map();

    results.forEach(result => {
      const docId = result._id.toString();
      if (!seen.has(docId) || seen.get(docId).adjustedScore < result.adjustedScore) {
        seen.set(docId, result);
      }
    });

    return Array.from(seen.values());
  }

  calculateLanguageDistribution(results) {
    // Calculate distribution of results by language
    const distribution = {};

    results.forEach(result => {
      const lang = result.searchLanguage || result.language || 'unknown';
      distribution[lang] = (distribution[lang] || 0) + 1;
    });

    return distribution;
  }

  async createSearchSuggestions(collectionName, partialQuery, options = {}) {
    // Generate search suggestions based on partial input
    const collection = this.db.collection(collectionName);

    // Use aggregation to find common terms
    const suggestionPipeline = [
      {
        $match: {
          $or: [
            { title: { $regex: partialQuery, $options: 'i' } },
            { tags: { $regex: partialQuery, $options: 'i' } },
            { category: { $regex: partialQuery, $options: 'i' } }
          ]
        }
      },
      {
        $project: {
          words: {
            $concatArrays: [
              { $split: [{ $toLower: "$title" }, " "] },
              { $ifNull: ["$tags", []] },
              [{ $toLower: "$category" }]
            ]
          }
        }
      },
      { $unwind: "$words" },
      {
        $match: {
          words: { $regex: `^${partialQuery.toLowerCase()}`, $options: 'i' }
        }
      },
      {
        $group: {
          _id: "$words",
          frequency: { $sum: 1 }
        }
      },
      {
        $match: {
          frequency: { $gte: 2 } // Only suggest terms that appear multiple times
        }
      },
      { $sort: { frequency: -1 } },
      { $limit: options.maxSuggestions || 10 }
    ];

    const suggestions = await collection.aggregate(suggestionPipeline).toArray();

    return suggestions.map(s => ({
      term: s._id,
      frequency: s.frequency,
      suggestion: s._id
    }));
  }
}

// Advanced text analysis and custom processing
class TextAnalysisEngine {
  constructor(db) {
    this.db = db;
    this.stopWords = {
      english: ['the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by'],
      spanish: ['el', 'la', 'de', 'que', 'y', 'a', 'en', 'un', 'es', 'se', 'no', 'te'],
      french: ['le', 'de', 'et', 'à', 'un', 'il', 'être', 'et', 'en', 'avoir', 'que', 'pour']
    };
  }

  async analyzeDocumentContent(document, language = 'english') {
    // Comprehensive document analysis for search optimization
    const content = this.extractTextContent(document);

    const analysis = {
      documentId: document._id,
      language: language,

      // Basic statistics
      wordCount: this.countWords(content),
      characterCount: content.length,
      sentenceCount: this.countSentences(content),
      paragraphCount: this.countParagraphs(content),

      // Term analysis
      termFrequency: await this.calculateTermFrequency(content, language),
      keyPhrases: await this.extractKeyPhrases(content, language),
      namedEntities: await this.extractNamedEntities(content),

      // Content quality metrics
      readabilityScore: this.calculateReadabilityScore(content),
      contentDensity: this.calculateContentDensity(content),
      uniqueTermsRatio: await this.calculateUniqueTermsRatio(content, language),

      // Search optimization data
      suggestedTags: await this.generateSuggestedTags(content, language),
      searchKeywords: await this.extractSearchKeywords(content, language),
      contentSummary: await this.generateContentSummary(content),

      analyzedAt: new Date()
    };

    return analysis;
  }

  extractTextContent(document) {
    // Extract searchable text from document structure
    const textFields = [];

    if (document.title) textFields.push(document.title);
    if (document.content) textFields.push(document.content);
    if (document.summary) textFields.push(document.summary);
    if (document.description) textFields.push(document.description);
    if (document.tags) textFields.push(document.tags.join(' '));

    return textFields.join(' ');
  }

  async calculateTermFrequency(content, language) {
    // Calculate term frequency for search relevance
    const words = content.toLowerCase()
      .replace(/[^\w\s]/g, ' ')
      .split(/\s+/)
      .filter(word => word.length > 2);

    // Remove stop words
    const stopWords = this.stopWords[language] || this.stopWords.english;
    const filteredWords = words.filter(word => !stopWords.includes(word));

    // Calculate frequency
    const frequency = {};
    filteredWords.forEach(word => {
      frequency[word] = (frequency[word] || 0) + 1;
    });

    // Return sorted by frequency
    return Object.entries(frequency)
      .sort((a, b) => b[1] - a[1])
      .slice(0, 20)
      .map(([term, count]) => ({
        term,
        frequency: count,
        percentage: (count / filteredWords.length) * 100
      }));
  }

  async extractKeyPhrases(content, language, maxPhrases = 10) {
    // Extract key phrases (2-3 word combinations)
    const sentences = content.split(/[.!?]+/);
    const phrases = [];

    sentences.forEach(sentence => {
      const words = sentence.toLowerCase()
        .replace(/[^\w\s]/g, ' ')
        .split(/\s+/)
        .filter(word => word.length > 2);

      // Generate 2-word phrases
      for (let i = 0; i < words.length - 1; i++) {
        const phrase = `${words[i]} ${words[i + 1]}`;
        if (!this.isStopWordPhrase(phrase, language)) {
          phrases.push(phrase);
        }
      }

      // Generate 3-word phrases
      for (let i = 0; i < words.length - 2; i++) {
        const phrase = `${words[i]} ${words[i + 1]} ${words[i + 2]}`;
        if (!this.isStopWordPhrase(phrase, language)) {
          phrases.push(phrase);
        }
      }
    });

    // Count phrase frequency
    const phraseFreq = {};
    phrases.forEach(phrase => {
      phraseFreq[phrase] = (phraseFreq[phrase] || 0) + 1;
    });

    return Object.entries(phraseFreq)
      .filter(([phrase, count]) => count >= 2) // Only phrases that appear multiple times
      .sort((a, b) => b[1] - a[1])
      .slice(0, maxPhrases)
      .map(([phrase, frequency]) => ({ phrase, frequency }));
  }

  async extractNamedEntities(content) {
    // Simple named entity recognition
    const entities = {
      persons: [],
      organizations: [],
      locations: [],
      technologies: []
    };

    // Technology terms (common in technical content)
    const techTerms = [
      'mongodb', 'javascript', 'python', 'java', 'react', 'node.js', 'express',
      'angular', 'vue', 'postgresql', 'mysql', 'redis', 'docker', 'kubernetes',
      'aws', 'azure', 'google cloud', 'github', 'gitlab', 'jenkins'
    ];

    const lowerContent = content.toLowerCase();

    techTerms.forEach(term => {
      if (lowerContent.includes(term.toLowerCase())) {
        entities.technologies.push({
          entity: term,
          occurrences: (lowerContent.match(new RegExp(term.toLowerCase(), 'g')) || []).length
        });
      }
    });

    // Simple patterns for other entities (can be enhanced with NLP libraries)
    const capitalizedWords = content.match(/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b/g) || [];

    // Classify capitalized words (simplified heuristic)
    capitalizedWords.forEach(word => {
      if (word.length > 2 && !this.isCommonWord(word)) {
        // Simple classification based on context patterns
        if (content.includes(`${word} Inc`) || content.includes(`${word} Corp`) ||
            content.includes(`${word} Company`) || content.includes(`${word} Ltd`)) {
          entities.organizations.push({ entity: word, confidence: 0.7 });
        } else if (content.includes(`in ${word}`) || content.includes(`at ${word}`) ||
                   content.includes(`${word} city`) || content.includes(`${word} state`)) {
          entities.locations.push({ entity: word, confidence: 0.6 });
        } else {
          entities.persons.push({ entity: word, confidence: 0.5 });
        }
      }
    });

    return entities;
  }

  async generateSuggestedTags(content, language) {
    // Generate suggested tags based on content analysis
    const termFreq = await this.calculateTermFrequency(content, language);
    const keyPhrases = await this.extractKeyPhrases(content, language, 5);
    const entities = await this.extractNamedEntities(content);

    const suggestedTags = [];

    // Add high-frequency terms
    termFreq.slice(0, 5).forEach(term => {
      suggestedTags.push({
        tag: term.term,
        source: 'term_frequency',
        confidence: Math.min(term.percentage / 10, 1.0)
      });
    });

    // Add key phrases
    keyPhrases.forEach(phrase => {
      suggestedTags.push({
        tag: phrase.phrase.replace(/\s+/g, '-'),
        source: 'key_phrase',
        confidence: Math.min(phrase.frequency / 5, 1.0)
      });
    });

    // Add technology entities
    entities.technologies.forEach(tech => {
      suggestedTags.push({
        tag: tech.entity.toLowerCase().replace(/\s+/g, '-'),
        source: 'technology',
        confidence: Math.min(tech.occurrences / 3, 1.0)
      });
    });

    // Sort by confidence and remove duplicates
    return suggestedTags
      .filter(tag => tag.confidence > 0.3)
      .sort((a, b) => b.confidence - a.confidence)
      .slice(0, 10);
  }

  async generateContentSummary(content, maxLength = 200) {
    // Generate content summary for search previews
    const sentences = content.split(/[.!?]+/)
      .map(s => s.trim())
      .filter(s => s.length > 20);

    if (sentences.length === 0) {
      return content.substring(0, maxLength);
    }

    // Score sentences based on term frequency and position
    const termFreq = await this.calculateTermFrequency(content, 'english');
    const importantTerms = termFreq.slice(0, 10).map(t => t.term);

    const sentenceScores = sentences.map((sentence, index) => {
      let score = 0;

      // Position score (earlier sentences weighted higher)
      score += Math.max(0, 5 - index) * 0.2;

      // Important terms score
      const lowerSentence = sentence.toLowerCase();
      importantTerms.forEach(term => {
        if (lowerSentence.includes(term)) {
          score += 1;
        }
      });

      return { sentence, score, index };
    });

    // Select top sentences while maintaining order
    const selectedSentences = sentenceScores
      .sort((a, b) => b.score - a.score)
      .slice(0, 3)
      .sort((a, b) => a.index - b.index);

    const summary = selectedSentences
      .map(s => s.sentence)
      .join('. ');

    return summary.length > maxLength ? 
      summary.substring(0, maxLength - 3) + '...' : 
      summary;
  }

  calculateReadabilityScore(content) {
    // Simple readability score calculation
    const words = content.split(/\s+/).length;
    const sentences = content.split(/[.!?]+/).length;
    const characters = content.replace(/\s/g, '').length;

    if (sentences === 0 || words === 0) return 0;

    // Simplified Flesch Reading Ease formula
    const avgWordsPerSentence = words / sentences;
    const avgCharsPerWord = characters / words;

    const readabilityScore = 206.835 - 1.015 * avgWordsPerSentence - 84.6 * (avgCharsPerWord / 5);
    return Math.max(0, Math.min(100, readabilityScore));
  }

  countWords(content) {
    return content.split(/\s+/).filter(word => word.length > 0).length;
  }

  countSentences(content) {
    return content.split(/[.!?]+/).filter(s => s.trim().length > 0).length;
  }

  countParagraphs(content) {
    return content.split(/\n\s*\n/).filter(p => p.trim().length > 0).length;
  }

  isStopWordPhrase(phrase, language) {
    const stopWords = this.stopWords[language] || this.stopWords.english;
    const words = phrase.split(' ');
    return words.every(word => stopWords.includes(word));
  }

  isCommonWord(word) {
    const commonWords = ['The', 'This', 'That', 'With', 'From', 'They', 'Have', 'More'];
    return commonWords.includes(word);
  }
}

// Search performance optimization and caching
class SearchOptimizationService {
  constructor(db, cacheService) {
    this.db = db;
    this.cache = cacheService;
    this.searchStats = db.collection('search_statistics');
    this.popularQueries = new Map();
  }

  async setupSearchPerformanceMonitoring() {
    // Monitor and optimize search performance
    const searchCollections = ['articles', 'products', 'documents'];

    for (const collection of searchCollections) {
      await this.analyzeTextIndexPerformance(collection);
      await this.setupSearchQueryCaching(collection);
      await this.createSearchAnalytics(collection);
    }
  }

  async analyzeTextIndexPerformance(collectionName) {
    // Analyze text index performance and suggest optimizations
    const collection = this.db.collection(collectionName);

    // Get index statistics
    const indexStats = await collection.aggregate([
      { $indexStats: {} },
      { $match: { "key.title": "text" } } // Find text indexes
    ]).toArray();

    // Sample query performance
    const sampleQueries = [
      'database performance optimization',
      'mongodb indexing strategies', 
      'full text search implementation',
      'content management system',
      'real time analytics'
    ];

    const performanceResults = [];

    for (const query of sampleQueries) {
      const startTime = process.hrtime.bigint();

      const results = await collection.find({
        $text: { $search: query }
      }, {
        score: { $meta: "textScore" }
      }).sort({ 
        score: { $meta: "textScore" } 
      }).limit(20).toArray();

      const endTime = process.hrtime.bigint();
      const executionTime = Number(endTime - startTime) / 1000000; // Convert to milliseconds

      performanceResults.push({
        query,
        resultCount: results.length,
        executionTimeMs: executionTime,
        avgScore: results.length > 0 ? 
          results.reduce((sum, r) => sum + r.score, 0) / results.length : 0
      });
    }

    // Store performance analysis
    await this.db.collection('search_performance').insertOne({
      collection: collectionName,
      indexStats: indexStats,
      queryPerformance: performanceResults,
      averageExecutionTime: performanceResults.reduce((sum, r) => sum + r.executionTimeMs, 0) / performanceResults.length,
      analyzedAt: new Date(),
      recommendations: this.generatePerformanceRecommendations(performanceResults, indexStats)
    });

    return performanceResults;
  }

  async setupSearchQueryCaching(collectionName) {
    // Implement intelligent search result caching
    const originalFind = this.db.collection(collectionName).find;
    const collection = this.db.collection(collectionName);

    // Override find method to add caching for text searches
    collection.findWithCache = async function(query, projection, options = {}) {
      // Only cache text search queries
      if (query.$text) {
        const cacheKey = this.generateCacheKey(query, projection, options);
        const cachedResult = await this.cache.get(cacheKey);

        if (cachedResult) {
          console.log(`Cache hit for search query: ${query.$text.$search}`);
          return cachedResult;
        }

        // Execute query and cache results
        const results = await originalFind.call(collection, query, projection)
          .sort(options.sort || { score: { $meta: "textScore" } })
          .limit(options.limit || 20)
          .toArray();

        // Cache for 5 minutes
        await this.cache.set(cacheKey, results, 300);
        console.log(`Cached search results for: ${query.$text.$search}`);

        return results;
      }

      // Non-text searches use original method
      return originalFind.call(collection, query, projection);
    }.bind(this);
  }

  generateCacheKey(query, projection, options) {
    // Generate consistent cache key for search queries
    const keyData = {
      search: query.$text?.$search,
      language: query.$text?.$language,
      filters: Object.keys(query).filter(k => k !== '$text'),
      projection: projection,
      sort: options.sort,
      limit: options.limit
    };

    return `search_${Buffer.from(JSON.stringify(keyData)).toString('base64')}`;
  }

  async createSearchAnalytics(collectionName) {
    // Create comprehensive search analytics
    const analyticsData = {
      collection: collectionName,
      date: new Date(),

      // Query pattern analysis
      popularSearchTerms: await this.getPopularSearchTerms(collectionName),
      searchFrequency: await this.getSearchFrequencyStats(collectionName),
      noResultsQueries: await this.getNoResultsQueries(collectionName),

      // Performance metrics
      avgResponseTime: await this.getAverageResponseTime(collectionName),
      queryComplexityDistribution: await this.getQueryComplexityStats(collectionName),

      // Result quality metrics
      clickThroughRates: await this.getClickThroughRates(collectionName),
      searchAbandonmentRate: await this.getSearchAbandonmentRate(collectionName)
    };

    await this.db.collection('search_analytics').insertOne(analyticsData);
    return analyticsData;
  }

  async logSearchQuery(collectionName, query, results, executionTime, userId = null) {
    // Log search queries for analytics
    const searchLog = {
      collection: collectionName,
      query: query,
      resultCount: results.length,
      executionTimeMs: executionTime,
      userId: userId,
      timestamp: new Date(),

      // Extract search characteristics
      searchLength: query.$text?.$search?.length || 0,
      searchTerms: query.$text?.$search?.split(' ').length || 0,
      hasFilters: Object.keys(query).length > 1,
      language: query.$text?.$language || 'english'
    };

    await this.searchStats.insertOne(searchLog);

    // Update popular queries tracking
    const queryText = query.$text?.$search;
    if (queryText) {
      this.updatePopularQueries(queryText);
    }
  }

  updatePopularQueries(queryText) {
    // Track popular queries in memory for quick access
    const count = this.popularQueries.get(queryText) || 0;
    this.popularQueries.set(queryText, count + 1);

    // Keep only top 100 queries to manage memory
    if (this.popularQueries.size > 100) {
      const sorted = Array.from(this.popularQueries.entries())
        .sort((a, b) => b[1] - a[1]);

      this.popularQueries.clear();
      sorted.slice(0, 100).forEach(([query, count]) => {
        this.popularQueries.set(query, count);
      });
    }
  }

  async getPopularSearchTerms(collectionName, limit = 20) {
    // Get most popular search terms from analytics
    const pipeline = [
      { 
        $match: { 
          collection: collectionName,
          timestamp: { $gte: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) } // Last 7 days
        }
      },
      {
        $group: {
          _id: "$query.$text.$search",
          searchCount: { $sum: 1 },
          avgResults: { $avg: "$resultCount" },
          avgExecutionTime: { $avg: "$executionTimeMs" }
        }
      },
      { $sort: { searchCount: -1 } },
      { $limit: limit }
    ];

    return await this.searchStats.aggregate(pipeline).toArray();
  }

  async optimizeSearchIndexes(collectionName) {
    // Optimize search indexes based on query patterns
    const collection = this.db.collection(collectionName);
    const analytics = await this.getSearchFrequencyStats(collectionName);

    // Analyze frequently searched fields
    const fieldUsage = await this.searchStats.aggregate([
      { $match: { collection: collectionName } },
      {
        $group: {
          _id: null,
          totalSearches: { $sum: 1 },
          avgResultCount: { $avg: "$resultCount" },
          slowQueries: { 
            $sum: { $cond: [{ $gt: ["$executionTimeMs", 100] }, 1, 0] }
          }
        }
      }
    ]).toArray();

    const optimizationRecommendations = [];

    // Check if index optimization is needed
    if (fieldUsage[0]?.slowQueries > fieldUsage[0]?.totalSearches * 0.1) {
      optimizationRecommendations.push({
        type: 'performance',
        recommendation: 'Consider compound indexes for frequently combined query filters',
        priority: 'high'
      });
    }

    if (fieldUsage[0]?.avgResultCount < 5) {
      optimizationRecommendations.push({
        type: 'relevance', 
        recommendation: 'Review text index weights to improve result relevance',
        priority: 'medium'
      });
    }

    return {
      currentPerformance: fieldUsage[0],
      recommendations: optimizationRecommendations,
      suggestedActions: await this.generateOptimizationActions(collectionName)
    };
  }

  generatePerformanceRecommendations(performanceResults, indexStats) {
    const recommendations = [];
    const avgExecutionTime = performanceResults.reduce((sum, r) => sum + r.executionTimeMs, 0) / performanceResults.length;

    if (avgExecutionTime > 50) {
      recommendations.push({
        type: 'performance',
        message: 'Average query execution time is high. Consider index optimization.',
        priority: 'high'
      });
    }

    const lowScoreQueries = performanceResults.filter(r => r.avgScore < 1.0);
    if (lowScoreQueries.length > performanceResults.length * 0.3) {
      recommendations.push({
        type: 'relevance',
        message: 'Many queries return low relevance scores. Review index weights.',
        priority: 'medium'
      });
    }

    return recommendations;
  }

  // Placeholder methods for analytics
  async getSearchFrequencyStats(collection) { /* Implementation */ }
  async getNoResultsQueries(collection) { /* Implementation */ }
  async getAverageResponseTime(collection) { /* Implementation */ }
  async getQueryComplexityStats(collection) { /* Implementation */ }
  async getClickThroughRates(collection) { /* Implementation */ }
  async getSearchAbandonmentRate(collection) { /* Implementation */ }
  async generateOptimizationActions(collection) { /* Implementation */ }
}