server/storage.ts

import { 
  documents, 
  searchQueries, 
  searchResults, 
  citations,
  type Document, 
  type InsertDocument,
  type SearchQuery,
  type InsertSearchQuery,
  type SearchResult,
  type InsertSearchResult,
  type Citation,
  type InsertCitation,
  type SearchRequest,
  type SearchResponse,
  type DocumentWithContext
} from "@shared/schema";

export interface IStorage {
  // Document management
  getDocument(id: number): Promise<Document | undefined>;
  getDocuments(limit?: number, offset?: number): Promise<Document[]>;
  createDocument(document: InsertDocument): Promise<Document>;
  updateDocument(id: number, document: Partial<InsertDocument>): Promise<Document | undefined>;
  deleteDocument(id: number): Promise<boolean>;

  // Search functionality
  searchDocuments(request: SearchRequest): Promise<SearchResponse>;
  getDocumentsBySourceType(sourceType: string): Promise<Document[]>;
  
  // Search query history
  createSearchQuery(query: InsertSearchQuery): Promise<SearchQuery>;
  getSearchQueries(limit?: number): Promise<SearchQuery[]>;
  
  // Search results
  createSearchResult(result: InsertSearchResult): Promise<SearchResult>;
  getSearchResults(queryId: number): Promise<SearchResult[]>;
  
  // Citations
  createCitation(citation: InsertCitation): Promise<Citation>;
  getCitationsByDocument(documentId: number): Promise<Citation[]>;
  deleteCitation(id: number): Promise<boolean>;
}

export class MemStorage implements IStorage {
  private documents: Map<number, Document>;
  private searchQueries: Map<number, SearchQuery>;
  private searchResults: Map<number, SearchResult>;
  private citations: Map<number, Citation>;
  private currentDocumentId: number;
  private currentQueryId: number;
  private currentResultId: number;
  private currentCitationId: number;

  constructor() {
    this.documents = new Map();
    this.searchQueries = new Map();
    this.searchResults = new Map();
    this.citations = new Map();
    this.currentDocumentId = 1;
    this.currentQueryId = 1;
    this.currentResultId = 1;
    this.currentCitationId = 1;

    // Initialize with some sample documents for demonstration
    this.initializeSampleData();
  }

  private initializeSampleData() {
    const sampleDocs: InsertDocument[] = [
      {
        title: "GPT-5 Architecture and Capabilities",
        content: "GPT-5 represents a significant leap in language model capabilities with improved reasoning, multimodal understanding, and reduced hallucinations. The model features enhanced chain-of-thought reasoning, better factual accuracy, and native support for images, audio, and video processing. Key improvements include: 50% reduction in hallucination rates, 3x better mathematical reasoning, native multimodal processing without separate encoders, improved instruction following, and enhanced safety guardrails. The architecture incorporates mixture-of-experts scaling, improved attention mechanisms, and novel alignment techniques.",
        source: "OpenAI Research, 2025 • Technical Report • openai.com/research/gpt-5",
        sourceType: "research",
        url: "https://openai.com/research/gpt-5",
        metadata: {
          authors: ["OpenAI Research Team"],
          year: 2025,
          venue: "OpenAI",
          model_params: "1.8T",
          context_length: "1M tokens"
        },
        embedding: JSON.stringify([0.1, 0.2, 0.3])
      },
      {
        title: "Claude 4 Constitutional AI and Safety",
        content: "Claude 4 introduces advanced constitutional AI training with improved helpfulness, harmlessness, and honesty. The model demonstrates superior reasoning capabilities while maintaining strong safety guardrails. Key features include: constitutional training from human feedback, improved factual accuracy through retrieval augmentation, advanced reasoning chains for complex problems, enhanced code generation and debugging, multilingual capabilities across 95 languages, and robust safety measures against misuse. The training process incorporates novel techniques for alignment and reduces harmful outputs by 90% compared to previous versions.",
        source: "Anthropic Research, 2025 • AI Safety Paper • anthropic.com/claude-4",
        sourceType: "research",
        url: "https://anthropic.com/claude-4",
        metadata: {
          authors: ["Anthropic Safety Team"],
          year: 2025,
          venue: "Anthropic",
          safety_rating: "AAA",
          context_length: "2M tokens"
        },
        embedding: JSON.stringify([0.2, 0.1, 0.4])
      },
      {
        title: "Gemini Ultra 2.0: Multimodal AI Breakthrough",
        content: "Gemini Ultra 2.0 achieves state-of-the-art performance across text, image, audio, and video understanding tasks. The model demonstrates human-level performance on complex reasoning benchmarks and shows emergent capabilities in scientific discovery. Major breakthroughs include: unified multimodal architecture processing all input types simultaneously, breakthrough performance on MMLU (95.2%), advanced video understanding and generation, real-time multimodal conversation capabilities, integration with robotics applications, and novel attention mechanisms for cross-modal reasoning.",
        source: "Google DeepMind, 2025 • Nature AI • deepmind.google/gemini-ultra-2",
        sourceType: "research",
        url: "https://deepmind.google/gemini-ultra-2",
        metadata: {
          authors: ["Google DeepMind Team"],
          year: 2025,
          venue: "Nature AI",
          benchmark_scores: {"MMLU": 95.2, "GSM8K": 97.8},
          modalities: ["text", "image", "audio", "video"]
        },
        embedding: JSON.stringify([0.3, 0.2, 0.1])
      },
      {
        title: "LangChain 2.0: Advanced Agent Orchestration",
        content: "LangChain 2.0 introduces revolutionary agent orchestration capabilities with improved reasoning, tool use, and multi-agent collaboration. The framework enables complex AI workflows with autonomous planning and execution. New features include: advanced agent planning with hierarchical task decomposition, multi-agent collaboration frameworks, improved tool integration and API calling, enhanced memory systems for long-term context, robust error handling and recovery mechanisms, and native support for voice and multimodal interactions.",
        source: "LangChain Documentation, 2025 • python.langchain.com/v2.0",
        sourceType: "documentation",
        url: "https://python.langchain.com/v2.0/",
        metadata: {
          version: "2.0.0",
          year: 2025,
          type: "framework",
          language: "Python",
          github_stars: 85000
        },
        embedding: JSON.stringify([0.4, 0.3, 0.2])
      },
      {
        title: "Retrieval-Augmented Generation Advances in 2024",
        content: "Recent advances in RAG systems focus on improving retrieval accuracy, reducing hallucinations, and enhancing context integration. Key developments include: hybrid dense-sparse retrieval combining semantic and lexical matching, multi-hop reasoning for complex queries, improved chunking strategies with semantic boundaries, real-time knowledge updating and fact verification, advanced reranking using cross-encoders, and integration with knowledge graphs for structured reasoning. Performance improvements show 40% better factual accuracy and 60% reduction in hallucinations.",
        source: "AI Research Collective, 2024 • ICML Workshop • arxiv.org/abs/2024.rag.advances",
        sourceType: "research",
        url: "https://arxiv.org/abs/2301.00234",
        metadata: {
          authors: ["Various Research Teams"],
          year: 2024,
          venue: "ICML Workshop",
          improvements: {"accuracy": "40%", "hallucination_reduction": "60%"}
        },
        embedding: JSON.stringify([0.1, 0.4, 0.2])
      },
      {
        title: "Vector Database Optimization for Large-Scale AI",
        content: "Modern vector databases have evolved to handle billion-scale embeddings with sub-millisecond retrieval times. Innovations include: approximate nearest neighbor algorithms with 99.9% recall, distributed indexing across multiple nodes, dynamic embedding updates without full reindexing, GPU-accelerated similarity search, compression techniques reducing storage by 80%, and integration with real-time streaming data. Popular solutions include Pinecone, Weaviate, Qdrant, and Chroma, each optimized for different use cases and scale requirements.",
        source: "Vector Database Survey, 2024 • VLDB Conference • vldb.org/vector-db-2024",
        sourceType: "research",
        url: "https://vldb.org/pvldb/",
        metadata: {
          year: 2024,
          venue: "VLDB",
          performance: {"recall": "99.9%", "storage_reduction": "80%"},
          databases: ["Pinecone", "Weaviate", "Qdrant", "Chroma"]
        },
        embedding: JSON.stringify([0.2, 0.3, 0.4])
      },
      {
        title: "Fine-tuning Large Language Models: 2024 Best Practices",
        content: "Latest techniques for fine-tuning LLMs focus on parameter efficiency and task specialization. Key methods include: LoRA (Low-Rank Adaptation) for efficient parameter updates, QLoRA for quantized fine-tuning reducing memory by 75%, instruction tuning for better task following, reinforcement learning from human feedback (RLHF), constitutional AI training for safety, and multi-task learning for generalization. New approaches like AdaLoRA and DoRA show improved performance with even fewer parameters.",
        source: "NeurIPS 2024 • Fine-tuning Workshop • neurips.cc/finetuning-2024",
        sourceType: "research",
        url: "https://neurips.cc/",
        metadata: {
          year: 2024,
          venue: "NeurIPS",
          techniques: ["LoRA", "QLoRA", "RLHF", "Constitutional AI"],
          memory_reduction: "75%"
        },
        embedding: JSON.stringify([0.3, 0.1, 0.3])
      },
      {
        title: "Transformer Architecture Evolution: Beyond Attention",
        content: "New transformer variants address computational efficiency and long-context understanding. Innovations include: Mamba state-space models for linear scaling, mixture-of-experts for sparse computation, ring attention for distributed processing, retrieval-augmented architectures, improved positional encodings for long sequences, and hybrid CNN-transformer models. These advances enable processing of million-token contexts while maintaining efficiency and accuracy.",
        source: "Transformer Evolution Survey, 2024 • ICLR • iclr.cc/transformer-evolution",
        sourceType: "research",
        url: "https://iclr.cc/",
        metadata: {
          year: 2024,
          venue: "ICLR",
          architectures: ["Mamba", "MoE", "Ring Attention"],
          context_length: "1M tokens"
        },
        embedding: JSON.stringify([0.4, 0.2, 0.3])
      },
      {
        title: "AI Safety and Alignment Research 2024",
        content: "Significant progress in AI safety focuses on interpretability, robustness, and alignment. Key developments include: mechanistic interpretability revealing model internals, adversarial training for robustness, constitutional AI for value alignment, red teaming methodologies for vulnerability discovery, safety evaluation frameworks, and governance approaches for responsible deployment. Research shows improved safety metrics across multiple dimensions while maintaining model capabilities.",
        source: "AI Safety Research Consortium, 2024 • AI Safety Journal • aisafety.org/2024-report",
        sourceType: "research",
        url: "https://www.aisafety.org/",
        metadata: {
          year: 2024,
          focus_areas: ["interpretability", "robustness", "alignment"],
          safety_improvements: "significant"
        },
        embedding: JSON.stringify([0.1, 0.3, 0.4])
      },
      {
        title: "Multimodal AI: Vision-Language Integration",
        content: "Advances in multimodal AI enable seamless integration of text, images, audio, and video. Key breakthroughs include: unified embedding spaces for cross-modal retrieval, vision-language models with improved spatial reasoning, audio-visual understanding for comprehensive scene analysis, real-time multimodal interaction capabilities, and applications in robotics and autonomous systems. Models like GPT-4V, Gemini Vision, and Claude 3 demonstrate human-level performance on complex multimodal tasks.",
        source: "Multimodal AI Survey, 2024 • Computer Vision Conference • cvpr.org/multimodal-2024",
        sourceType: "research",
        url: "https://cvpr.thecvf.com/",
        metadata: {
          year: 2024,
          venue: "CVPR",
          modalities: ["text", "image", "audio", "video"],
          models: ["GPT-4V", "Gemini Vision", "Claude 3"]
        },
        embedding: JSON.stringify([0.2, 0.4, 0.1])
      },
      {
        title: "Edge AI and Model Compression Techniques",
        content: "Deployment of AI models on edge devices requires sophisticated compression and optimization. Latest techniques include: quantization to INT8 and INT4 precision, pruning redundant parameters, knowledge distillation from large to small models, neural architecture search for efficient designs, and specialized hardware optimization. These methods achieve 10x model size reduction while maintaining 95% of original performance.",
        source: "Edge AI Research, 2024 • Mobile Computing Conference • mobicom.org/edge-ai-2024",
        sourceType: "research",
        url: "https://www.sigmobile.org/mobicom/",
        metadata: {
          year: 2024,
          techniques: ["quantization", "pruning", "distillation"],
          size_reduction: "10x",
          performance_retention: "95%"
        },
        embedding: JSON.stringify([0.3, 0.4, 0.1])
      },
      {
        title: "AI Code Generation and Programming Assistants",
        content: "Code generation AI has evolved to support full software development workflows. Advanced capabilities include: multi-language code generation with context awareness, automated testing and debugging assistance, code review and optimization suggestions, natural language to code translation, integration with development environments, and support for complex software architectures. Tools like GitHub Copilot X, CodeT5+, and StarCoder show significant productivity improvements.",
        source: "Software Engineering AI, 2024 • ICSE Conference • icse.org/code-ai-2024",
        sourceType: "research",
        url: "https://conf.researchr.org/home/icse-2024",
        metadata: {
          year: 2024,
          venue: "ICSE",
          tools: ["GitHub Copilot X", "CodeT5+", "StarCoder"],
          productivity_gain: "significant"
        },
        embedding: JSON.stringify([0.4, 0.1, 0.3])
      }
    ];

    sampleDocs.forEach(doc => {
      this.createDocument(doc);
    });
  }

  async getDocument(id: number): Promise<Document | undefined> {
    return this.documents.get(id);
  }

  async getDocuments(limit = 50, offset = 0): Promise<Document[]> {
    const allDocs = Array.from(this.documents.values());
    return allDocs.slice(offset, offset + limit);
  }

  async createDocument(insertDocument: InsertDocument): Promise<Document> {
    const id = this.currentDocumentId++;
    const document: Document = {
      ...insertDocument,
      id,
      metadata: insertDocument.metadata || null,
      url: insertDocument.url || null,
      embedding: insertDocument.embedding || null,
      createdAt: new Date()
    };
    this.documents.set(id, document);
    return document;
  }

  async updateDocument(id: number, updateData: Partial<InsertDocument>): Promise<Document | undefined> {
    const existing = this.documents.get(id);
    if (!existing) return undefined;

    const updated: Document = { ...existing, ...updateData };
    this.documents.set(id, updated);
    return updated;
  }

  async deleteDocument(id: number): Promise<boolean> {
    return this.documents.delete(id);
  }

  async searchDocuments(request: SearchRequest): Promise<SearchResponse> {
    const startTime = Date.now();
    const allDocs = Array.from(this.documents.values());
    
    // Simple text-based search simulation
    // In a real implementation, this would use vector similarity
    const query = request.query.toLowerCase();
    const matchedDocs = allDocs
      .map(doc => {
        const content = doc.content.toLowerCase();
        const title = doc.title.toLowerCase();
        
        // Simple relevance scoring based on keyword matching
        let score = 0;
        const queryWords = query.split(' ');
        
        queryWords.forEach(word => {
          if (title.includes(word)) score += 0.3;
          if (content.includes(word)) score += 0.1;
        });

        // Add bonus for exact phrase matches
        if (content.includes(query)) score += 0.5;
        if (title.includes(query)) score += 0.8;

        // Filter by source type if specified
        if (request.filters?.sourceTypes?.length) {
          if (!request.filters.sourceTypes.includes(doc.sourceType)) {
            score = 0;
          }
        }

        return {
          ...doc,
          relevanceScore: Math.min(score, 1.0),
          snippet: this.extractSnippet(doc.content, query),
          rank: 0
        };
      })
      .filter(doc => doc.relevanceScore > 0)
      .sort((a, b) => b.relevanceScore - a.relevanceScore)
      .map((doc, index) => ({ ...doc, rank: index + 1 }))
      .slice(request.offset, request.offset + request.limit);

    const searchTime = (Date.now() - startTime) / 1000;

    // Save search query
    const searchQuery = await this.createSearchQuery({
      query: request.query,
      searchType: request.searchType,
      filters: request.filters,
      resultsCount: matchedDocs.length,
      searchTime
    });

    // Save search results
    for (const doc of matchedDocs) {
      await this.createSearchResult({
        queryId: searchQuery.id,
        documentId: doc.id,
        relevanceScore: doc.relevanceScore,
        snippet: doc.snippet,
        rank: doc.rank
      });
    }

    return {
      results: matchedDocs,
      totalCount: matchedDocs.length,
      searchTime,
      query: request.query,
      queryId: searchQuery.id
    };
  }

  private extractSnippet(content: string, query: string, maxLength = 200): string {
    const queryLower = query.toLowerCase();
    const contentLower = content.toLowerCase();
    
    const index = contentLower.indexOf(queryLower);
    if (index === -1) {
      return content.substring(0, maxLength) + (content.length > maxLength ? '...' : '');
    }

    const start = Math.max(0, index - 50);
    const end = Math.min(content.length, index + queryLower.length + 150);
    
    let snippet = content.substring(start, end);
    if (start > 0) snippet = '...' + snippet;
    if (end < content.length) snippet = snippet + '...';
    
    return snippet;
  }

  async getDocumentsBySourceType(sourceType: string): Promise<Document[]> {
    return Array.from(this.documents.values()).filter(doc => doc.sourceType === sourceType);
  }

  async createSearchQuery(insertQuery: InsertSearchQuery): Promise<SearchQuery> {
    const id = this.currentQueryId++;
    const query: SearchQuery = {
      ...insertQuery,
      id,
      searchType: insertQuery.searchType || 'semantic',
      filters: insertQuery.filters || null,
      resultsCount: insertQuery.resultsCount || null,
      searchTime: insertQuery.searchTime || null,
      createdAt: new Date()
    };
    this.searchQueries.set(id, query);
    return query;
  }

  async getSearchQueries(limit = 50): Promise<SearchQuery[]> {
    const allQueries = Array.from(this.searchQueries.values());
    return allQueries.slice(0, limit).sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
  }

  async createSearchResult(insertResult: InsertSearchResult): Promise<SearchResult> {
    const id = this.currentResultId++;
    const result: SearchResult = { ...insertResult, id };
    this.searchResults.set(id, result);
    return result;
  }

  async getSearchResults(queryId: number): Promise<SearchResult[]> {
    return Array.from(this.searchResults.values()).filter(result => result.queryId === queryId);
  }

  async createCitation(insertCitation: InsertCitation): Promise<Citation> {
    const id = this.currentCitationId++;
    const citation: Citation = {
      ...insertCitation,
      id,
      section: insertCitation.section || null,
      pageNumber: insertCitation.pageNumber || null,
      createdAt: new Date()
    };
    this.citations.set(id, citation);
    return citation;
  }

  async getCitationsByDocument(documentId: number): Promise<Citation[]> {
    return Array.from(this.citations.values()).filter(citation => citation.documentId === documentId);
  }

  async deleteCitation(id: number): Promise<boolean> {
    return this.citations.delete(id);
  }
}

import { SQLiteStorage } from './sqlite-storage';

// Use SQLite storage in production, keep MemStorage for testing
export const storage = process.env.NODE_ENV === 'test' ? new MemStorage() : new SQLiteStorage();