process_data/run_pipeline.sh

#!/bin/bash

# Database to HuggingFace Pipeline
# Similar to FutureBench's to_csv.sh and to_benchmark.sh but combined

echo "🚀 Starting Database to HuggingFace Pipeline..."

# Check if HF_TOKEN is set
if [ -z "$HF_TOKEN" ]; then
    echo "⚠️  HF_TOKEN not set. Will save files locally instead of uploading."
    echo "   To upload to HuggingFace, set: export HF_TOKEN='your_token_here'"
    echo ""
fi

# Change to project root (same as to_csv.sh)
cd ../..

# Run the pipeline
python3 leaderboard/process_data/db_to_hf.py

# Check if it was successful
if [ $? -eq 0 ]; then
    echo ""
    echo "✅ Pipeline completed successfully!"
    echo ""
    echo "Next steps:"
    echo "1. Check your HuggingFace repositories for updated data"
    echo "2. Your leaderboard will automatically use the new data"
    echo "3. Consider setting up a cron job to run this regularly"
else
    echo ""
    echo "❌ Pipeline failed. Check the error messages above."
    exit 1
fi