app.py

import os
import gradio as gr
import inspect
import pandas as pd
import requests
import logging 
import datetime
import json # Added for saving submission data


log_timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
log_file_name = f"evaluation_run_{log_timestamp}.log"


logger = logging.getLogger("eval_logger")
logger.setLevel(logging.INFO)
file_handler = logging.FileHandler(log_file_name)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(name)s - %(module)s - %(funcName)s - %(lineno)d - %(message)s')
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)


logger.info("Logging setup complete. Log file: %s", log_file_name)


from dataset_helper import fetch_all_questions, download_file # fetch_random_question is also available if needed


from agent import LangGraphAgent

# --- Constants ---
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"

# --- Basic Agent Definition ---
# ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
# class BasicAgent: # Moved to agent.py
#     def __init__(self, api_url: str):
#         print("BasicAgent initialized.")
#         self.api_url = api_url # Store api_url for potential use in downloading files
# 
#     def __call__(self, task_id: str, question: str, file_name: str | None) -> str:
#         print(f"Agent received task_id: {task_id}, question (first 50 chars): {question[:50]}...")
#         if file_name:
#             print(f"Question has an associated file: {file_name}")
#             # Example: Download the file if needed by the agent's logic
#             # local_file_path = download_file(self.api_url, task_id, file_name)
#             # if local_file_path:
#             #     print(f"File {file_name} downloaded to {local_file_path}")
#             #     # Agent would then process this file
#             # else:
#             #     print(f"Failed to download {file_name} for task {task_id}")
#             #     return "Error: Could not download associated file."
#         
#         # Current placeholder answer
#         fixed_answer = "This is a default answer from BasicAgent."
#         print(f"Agent returning fixed answer: {fixed_answer}")
#         return fixed_answer

def _submit_answers_to_api(submit_url: str, submission_data: dict, results_log: list, logger_instance: logging.Logger) -> tuple[str, pd.DataFrame]:
    """Handles the submission of answers to the API and processes the response."""
    try:
        submission_log_timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
        submission_file_name = f"submission_payload_{submission_log_timestamp}.json"
        
        # Create an 'submissions' directory if it doesn't exist
        submissions_dir = "submissions"
        if not os.path.exists(submissions_dir):
            os.makedirs(submissions_dir)
            logger_instance.info(f"Created directory: {submissions_dir}")

        submission_file_path = os.path.join(submissions_dir, submission_file_name)

        with open(submission_file_path, 'w') as f:
            json.dump(submission_data, f, indent=4)
        logger_instance.info(f"Submission payload saved to: {submission_file_path}")
    except Exception as e:
        logger_instance.error(f"Failed to save submission payload: {e}", exc_info=True)

    logger_instance.info(f"Submitting {len(submission_data.get('answers', []))} answers to: {submit_url}")
    try:
        response = requests.post(submit_url, json=submission_data, timeout=60)
        response.raise_for_status()
        result_data = response.json()
        final_status = (
            f"Submission Successful!\n"
            f"User: {result_data.get('username')}\n"
            f"Overall Score: {result_data.get('score', 'N/A')}% "
            f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
            f"Message: {result_data.get('message', 'No message received.')}"
        )
        logger_instance.info(f"Submission successful: {final_status}")
        results_df = pd.DataFrame(results_log)
        return final_status, results_df
    except requests.exceptions.HTTPError as e:
        error_detail = f"Server responded with status {e.response.status_code}."
        try:
            error_json = e.response.json()
            error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
        except requests.exceptions.JSONDecodeError:
            error_detail += f" Response: {e.response.text[:500]}"
        status_message = f"Submission Failed: {error_detail}"
        logger_instance.error(status_message, exc_info=True)
        results_df = pd.DataFrame(results_log)
        return status_message, results_df
    except requests.exceptions.Timeout:
        status_message = "Submission Failed: The request timed out."
        logger_instance.error(status_message, exc_info=True)
        results_df = pd.DataFrame(results_log)
        return status_message, results_df
    except requests.exceptions.RequestException as e:
        status_message = f"Submission Failed: Network error - {e}"
        logger_instance.error(status_message, exc_info=True)
        results_df = pd.DataFrame(results_log)
        return status_message, results_df
    except Exception as e:
        status_message = f"An unexpected error occurred during submission: {e}"
        logger_instance.error(status_message, exc_info=True)
        results_df = pd.DataFrame(results_log)
        return status_message, results_df

def run_and_submit_all(profile: gr.OAuthProfile | None):
    """
    Fetches all questions, runs the BasicAgent on them, submits all answers,
    and displays the results.
    """
    logger.info("run_and_submit_all started.")
    space_id = os.getenv("SPACE_ID")

    if profile:
        username = f"{profile.username}"
        logger.info(f"User logged in: {username}")
    else:
        logger.warning("User not logged in.")
        return "Please Login to Hugging Face with the button.", None

    api_url = DEFAULT_API_URL
    submit_url = f"{api_url}/submit"

    try:
        logger.info("Initializing agent...")
        global agent
        agent = LangGraphAgent(api_url=DEFAULT_API_URL, answers_dir="answers")
        logger.info("Agent initialized.")
    except Exception as e:
        logger.error(f"Error instantiating agent: {e}", exc_info=True)
        return f"Error initializing agent: {e}", None

    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
    logger.info(f"Agent code URL: {agent_code}")

    logger.info(f"Fetching questions using dataset_helper from: {api_url}")
    questions_data = fetch_all_questions(api_url)

    if questions_data is None:
        logger.error("Failed to fetch questions via dataset_helper. questions_data is None.")
        return "Error fetching questions. Please check the logs.", None
    
    total_questions_fetched = len(questions_data)
    logger.info(f"Fetched {total_questions_fetched} questions via dataset_helper.")
    if not questions_data:
        logger.warning("Fetched questions list is empty (0 questions).")
        return "Fetched questions list is empty. Nothing to process.", pd.DataFrame(results_log if 'results_log' in locals() else [])

    results_log = []
    answers_payload = []
    successful_answers_count = 0
    answers_from_cache_count = 0
    logger.info(f"Running agent on {total_questions_fetched} questions...")
    for item_index, item in enumerate(questions_data):
        task_id = item.get("task_id")
        question_text = item.get("question")
        file_name = item.get("file_name")
        logger.info(f"Processing question {item_index + 1}/{total_questions_fetched}, task_id: {task_id}")

        if not task_id or question_text is None:
            logger.warning(f"Skipping item {item_index + 1} with missing task_id or question: {item}")
            results_log.append({"Task ID": task_id if task_id else "MISSING_ID", "Question": question_text if question_text else "MISSING_QUESTION", "Associated File": file_name if file_name else "None", "Submitted Answer": "SKIPPED - Missing data", "From Cache": "N/A"})
            continue
        try:
            submitted_answer_tuple = agent(task_id, question_text, file_name) # Returns (answer, from_cache)
            submitted_answer, from_cache = submitted_answer_tuple
            
            answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
            results_log.append({"Task ID": task_id, "Question": question_text, "Associated File": file_name if file_name else "None", "Submitted Answer": submitted_answer, "From Cache": from_cache})
            successful_answers_count += 1
            if from_cache:
                answers_from_cache_count += 1
                logger.info(f"Agent successfully processed task_id: {task_id} (from cache)")
            else:
                logger.info(f"Agent successfully processed task_id: {task_id} (newly generated)")
        except Exception as e:
            logger.error(f"Error running agent on task {task_id}: {e}", exc_info=True)
            results_log.append({"Task ID": task_id, "Question": question_text, "Associated File": file_name if file_name else "None", "Submitted Answer": f"AGENT ERROR: {e}", "From Cache": False})

    logger.info(f"Agent finished processing. Successfully generated/retrieved answers for {successful_answers_count}/{total_questions_fetched} questions. {answers_from_cache_count} answers were from cache.")

    if not answers_payload:
        logger.warning("Agent did not produce any answers to submit (all attempts might have failed or been skipped).")
        return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)

    submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
    
    summary_line = f"Agent processed {total_questions_fetched} questions. Successfully generated/retrieved {successful_answers_count} answers ({answers_from_cache_count} from cache)."
    logger.info(summary_line)

    # --- TEMPORARILY BYPASS SUBMISSION FOR TESTING ---
    # logger.warning("SUBMISSION TO API IS CURRENTLY BYPASSED FOR TESTING.")
    # bypassed_status_message = (
    #     f"SUBMISSION BYPASSED. {summary_line}\\n"
    #     f"User: '{username}'. Results log is available. Submission data prepared but not sent."
    # )
    # results_df = pd.DataFrame(results_log)
    # return bypassed_status_message, results_df
    # --- END OF TEMPORARY BYPASS ---

    # Call the refactored submission method, passing the global logger instance
    # Note: If re-enabling submission, ensure the summary_line is incorporated into the _submit_answers_to_api or its return message.
    return _submit_answers_to_api(submit_url, submission_data, results_log, logger)


with gr.Blocks() as demo:
    gr.Markdown("# Basic Agent Evaluation Runner")
    gr.Markdown(
        """
        **Instructions:**

        1.  Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
        2.  Log in to your Hugging Face account using the button below. This uses your HF username for submission.
        3.  Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.

        ---
        **Disclaimers:**
        Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
        This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
        """
    )

    gr.LoginButton()

    run_button = gr.Button("Run Evaluation & Submit All Answers")

    status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
    results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)

    run_button.click(
        fn=run_and_submit_all,
        outputs=[status_output, results_table]
    )

if __name__ == "__main__":
    logger.info("App Starting...")
    space_host_startup = os.getenv("SPACE_HOST")
    space_id_startup = os.getenv("SPACE_ID")

    if space_host_startup:
        logger.info(f"SPACE_HOST found: {space_host_startup}")
        logger.info(f"   Runtime URL should be: https://{space_host_startup}.hf.space")
    else:
        logger.info("ℹ️  SPACE_HOST environment variable not found (running locally?).")

    if space_id_startup:
        logger.info(f"SPACE_ID found: {space_id_startup}")
        logger.info(f"   Repo URL: https://huggingface.co/spaces/{space_id_startup}")
        logger.info(f"   Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
    else:
        logger.info("ℹ️  SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")

    logger.info("-"*(60 + len(" App Starting ")) + "\n")

    logger.info("Launching Gradio Interface for Basic Agent Evaluation...")
    demo.launch(debug=True, share=False)