generate_embeddings_simple.js

// Simple script to generate embeddings for key documents
async function generateEmbeddings() {
  const keyDocuments = [
    {
      id: 1,
      title: "Attention Is All You Need",
      content: "We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being more parallelizable and requiring significantly less time to train."
    },
    {
      id: 11,
      title: "Mixtral of Experts",
      content: "We introduce Mixtral 8x7B, a Sparse Mixture of Experts (SMoE) language model. Mixtral has the same architecture as Mistral 7B, but each layer is composed of 8 feedforward blocks (i.e. experts). For every token, at each layer, a router network selects two experts to process the current state and combine their outputs."
    },
    {
      id: 2,
      title: "GPT-4 Technical Report",
      content: "We report the development of GPT-4, a large-scale, multimodal model which can accept image and text inputs and produce text outputs. While less capable than humans in many real-world scenarios, GPT-4 exhibits human-level performance on various professional and academic benchmarks."
    }
  ];

  console.log('🚀 Generating embeddings for key documents...');
  
  for (const doc of keyDocuments) {
    try {
      console.log(`\n📄 Processing: ${doc.title}`);
      
      // Generate embedding
      const response = await fetch('http://localhost:5000/api/embeddings', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ input: doc.content })
      });
      
      if (response.ok) {
        const result = await response.json();
        console.log(`✅ Generated embedding for ${doc.title} (${result.data[0].embedding.length} dimensions)`);
      } else {
        console.log(`❌ Failed to generate embedding for ${doc.title}`);
      }
      
      // Small delay
      await new Promise(resolve => setTimeout(resolve, 1000));
      
    } catch (error) {
      console.log(`❌ Error processing ${doc.title}: ${error.message}`);
    }
  }
  
  console.log('\n✅ Embedding generation test completed!');
  console.log('\n📝 Note: These embeddings were generated but not stored in the database.');
  console.log('To actually store embeddings, we need to use the proper document update endpoint.');
}

// Run the function
generateEmbeddings().catch(console.error);