Tweak vector search

generate_all_embeddings.js

@@ -0,0 +1,68 @@
+// Using built-in fetch (Node.js 18+)
+
+async function generateAllEmbeddings() {
+  console.log('🚀 Starting to generate embeddings for all documents...');
+  
+  try {
+    // Get all documents
+    const response = await fetch('http://localhost:5000/api/documents');
+    const documents = await response.json();
+    
+    console.log(`📚 Found ${documents.length} documents`);
+    
+    // Generate embeddings for each document
+    for (const doc of documents) {
+      console.log(`\n📄 Processing: ${doc.title} (ID: ${doc.id})`);
+      
+      // Generate embedding using the content
+      const embeddingResponse = await fetch('http://localhost:5000/api/embeddings', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ input: doc.content.substring(0, 8000) }) // Limit content length
+      });
+      
+      if (embeddingResponse.ok) {
+        const embeddingResult = await embeddingResponse.json();
+        const embedding = embeddingResult.data[0].embedding;
+        
+        console.log(`✅ Generated embedding with ${embedding.length} dimensions`);
+        
+        // Update document with embedding
+        const updateResponse = await fetch(`http://localhost:5000/api/documents/process/${doc.id}`, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ 
+            operations: ['generate_embedding'],
+            embedding: embedding 
+          })
+        });
+        
+        if (updateResponse.ok) {
+          console.log(`✅ Updated document ${doc.id} with embedding`);
+        } else {
+          console.log(`❌ Failed to update document ${doc.id}`);
+        }
+        
+      } else {
+        console.log(`❌ Failed to generate embedding for ${doc.title}`);
+      }
+      
+      // Small delay to avoid overwhelming the API
+      await new Promise(resolve => setTimeout(resolve, 1000));
+    }
+    
+    console.log('\n🎉 Embedding generation completed!');
+    console.log('\n🔍 Now you can test vector search with these queries:');
+    console.log('- "attention mechanism transformer architecture"');
+    console.log('- "multimodal language model GPT"');
+    console.log('- "constitutional AI safety alignment"');
+    console.log('- "mixtral mixture of experts"');
+    console.log('- "retrieval augmented generation knowledge"');
+    
+  } catch (error) {
+    console.error('❌ Error:', error.message);
+  }
+}
+
+// Run the function
+generateAllEmbeddings().catch(console.error);

generate_embeddings_simple.js

@@ -0,0 +1,55 @@
+// Simple script to generate embeddings for key documents
+async function generateEmbeddings() {
+  const keyDocuments = [
+    {
+      id: 1,
+      title: "Attention Is All You Need",
+      content: "We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being more parallelizable and requiring significantly less time to train."
+    },
+    {
+      id: 11,
+      title: "Mixtral of Experts",
+      content: "We introduce Mixtral 8x7B, a Sparse Mixture of Experts (SMoE) language model. Mixtral has the same architecture as Mistral 7B, but each layer is composed of 8 feedforward blocks (i.e. experts). For every token, at each layer, a router network selects two experts to process the current state and combine their outputs."
+    },
+    {
+      id: 2,
+      title: "GPT-4 Technical Report",
+      content: "We report the development of GPT-4, a large-scale, multimodal model which can accept image and text inputs and produce text outputs. While less capable than humans in many real-world scenarios, GPT-4 exhibits human-level performance on various professional and academic benchmarks."
+    }
+  ];
+
+  console.log('🚀 Generating embeddings for key documents...');
+  
+  for (const doc of keyDocuments) {
+    try {
+      console.log(`\n📄 Processing: ${doc.title}`);
+      
+      // Generate embedding
+      const response = await fetch('http://localhost:5000/api/embeddings', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ input: doc.content })
+      });
+      
+      if (response.ok) {
+        const result = await response.json();
+        console.log(`✅ Generated embedding for ${doc.title} (${result.data[0].embedding.length} dimensions)`);
+      } else {
+        console.log(`❌ Failed to generate embedding for ${doc.title}`);
+      }
+      
+      // Small delay
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      
+    } catch (error) {
+      console.log(`❌ Error processing ${doc.title}: ${error.message}`);
+    }
+  }
+  
+  console.log('\n✅ Embedding generation test completed!');
+  console.log('\n📝 Note: These embeddings were generated but not stored in the database.');
+  console.log('To actually store embeddings, we need to use the proper document update endpoint.');
+}
+
+// Run the function
+generateEmbeddings().catch(console.error);