generate_all_embeddings.js

// Using built-in fetch (Node.js 18+)

async function generateAllEmbeddings() {
  console.log('🚀 Starting to generate embeddings for all documents...');
  
  try {
    // Get all documents
    const response = await fetch('http://localhost:5000/api/documents');
    const documents = await response.json();
    
    console.log(`📚 Found ${documents.length} documents`);
    
    // Generate embeddings for each document
    for (const doc of documents) {
      console.log(`\n📄 Processing: ${doc.title} (ID: ${doc.id})`);
      
      // Generate embedding using the content
      const embeddingResponse = await fetch('http://localhost:5000/api/embeddings', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ input: doc.content.substring(0, 8000) }) // Limit content length
      });
      
      if (embeddingResponse.ok) {
        const embeddingResult = await embeddingResponse.json();
        const embedding = embeddingResult.data[0].embedding;
        
        console.log(`✅ Generated embedding with ${embedding.length} dimensions`);
        
        // Update document with embedding
        const updateResponse = await fetch(`http://localhost:5000/api/documents/process/${doc.id}`, {
          method: 'POST',
          headers: { 'Content-Type': 'application/json' },
          body: JSON.stringify({ 
            operations: ['generate_embedding'],
            embedding: embedding 
          })
        });
        
        if (updateResponse.ok) {
          console.log(`✅ Updated document ${doc.id} with embedding`);
        } else {
          console.log(`❌ Failed to update document ${doc.id}`);
        }
        
      } else {
        console.log(`❌ Failed to generate embedding for ${doc.title}`);
      }
      
      // Small delay to avoid overwhelming the API
      await new Promise(resolve => setTimeout(resolve, 1000));
    }
    
    console.log('\n🎉 Embedding generation completed!');
    console.log('\n🔍 Now you can test vector search with these queries:');
    console.log('- "attention mechanism transformer architecture"');
    console.log('- "multimodal language model GPT"');
    console.log('- "constitutional AI safety alignment"');
    console.log('- "mixtral mixture of experts"');
    console.log('- "retrieval augmented generation knowledge"');
    
  } catch (error) {
    console.error('❌ Error:', error.message);
  }
}

// Run the function
generateAllEmbeddings().catch(console.error);