File size: 2,024 Bytes
cdf8921 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 |
import os
import pandas as pd
from rest_clients.hs_evaluator_client import HsEvaluatorClient
from src.agent.base_agent import BasicAgent
class Evaluator:
def __init__(self, profile):
self.profile = profile
self.username = profile.username if profile else None
self.space_id = os.getenv("SPACE_ID")
self.agent = BasicAgent()
self.hs_evaluator_client: HsEvaluatorClient | None = None
def run_and_submit(self):
if not self.username:
return "Please Login to Hugging Face with the button.", None
questions = self.get_hs_evaluator_client().fetch_questions()
if not questions:
return "Fetched questions list is empty or invalid format.", None
results_log, answers_payload = self._run_agent(questions)
if not answers_payload:
return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
return self.get_hs_evaluator_client().submit_answers(answers_payload, results_log)
def _run_agent(self, questions):
results_log = []
answers_payload = []
for item in questions:
task_id = item.get("task_id")
question_text = item.get("question")
if not task_id or question_text is None:
continue
try:
submitted_answer = self.agent(question_text)
answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
except Exception as e:
results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
return results_log, answers_payload
def get_hs_evaluator_client(self):
if not self.hs_evaluator_client:
self.hs_evaluator_client = HsEvaluatorClient(self.username, self.space_id)
return self.hs_evaluator_client |