{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "from dotenv import load_dotenv\n",
    "from openai import OpenAI\n",
    "from IPython.display import Markdown"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Refresh dot env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "load_dotenv(override=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "open_api_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "google_api_key = os.getenv(\"GOOGLE_API_KEY\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create initial query to get challange reccomendation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = 'Please come up with a challenging, nuanced question that I can ask a number of LLMs to evaluate their intelligence. '\n",
    "query += 'Answer only with the question, no explanation.'\n",
    "\n",
    "messages = [{'role':'user', 'content':query}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(messages)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Call openai gpt-4o-mini "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "openai = OpenAI()\n",
    "\n",
    "response = openai.chat.completions.create(\n",
    "    messages=messages,\n",
    "    model='gpt-4o-mini'\n",
    ")\n",
    "\n",
    "challange = response.choices[0].message.content\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(challange)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "competitors = []\n",
    "answers = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create messages with the challange query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages = [{'role':'user', 'content':challange}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(messages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ollama pull llama3.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from threading import Thread"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gpt_mini_processor():\n",
    "    modleName = 'gpt-4o-mini'\n",
    "    competitors.append(modleName)\n",
    "    response_gpt = openai.chat.completions.create(\n",
    "        messages=messages,\n",
    "        model=modleName\n",
    "    )\n",
    "    answers.append(response_gpt.choices[0].message.content)\n",
    "\n",
    "def gemini_processor():\n",
    "    gemini = OpenAI(api_key=google_api_key, base_url='https://generativelanguage.googleapis.com/v1beta/openai/')\n",
    "    modleName = 'gemini-2.0-flash'\n",
    "    competitors.append(modleName)\n",
    "    response_gemini = gemini.chat.completions.create(\n",
    "        messages=messages,\n",
    "        model=modleName\n",
    "    )\n",
    "    answers.append(response_gemini.choices[0].message.content)\n",
    "\n",
    "def llama_processor():\n",
    "    ollama = OpenAI(base_url='http://localhost:11434/v1', api_key='ollama')\n",
    "    modleName = 'llama3.2'\n",
    "    competitors.append(modleName)\n",
    "    response_llama = ollama.chat.completions.create(\n",
    "        messages=messages,\n",
    "        model=modleName\n",
    "    )\n",
    "    answers.append(response_llama.choices[0].message.content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Paraller execution of LLM calls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "thread1 = Thread(target=gpt_mini_processor)\n",
    "thread2 = Thread(target=gemini_processor)\n",
    "thread3 = Thread(target=llama_processor)\n",
    "\n",
    "thread1.start()\n",
    "thread2.start()\n",
    "thread3.start()\n",
    "\n",
    "thread1.join()\n",
    "thread2.join()\n",
    "thread3.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(competitors)\n",
    "print(answers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for competitor, answer in zip(competitors, answers):\n",
    "    print(f'Competitor:{competitor}\\n\\n{answer}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "together = ''\n",
    "for index, answer in enumerate(answers):\n",
    "    together += f'# Response from competitor {index + 1}\\n\\n'\n",
    "    together += answer + '\\n\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(together)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prompt to judge the LLM results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_judge = f'''You are judging a competition between {len(competitors)} competitors.\n",
    "Each model has been given this question:\n",
    "\n",
    "{challange}\n",
    "\n",
    "Your job is to evaluate each response for clarity and strength of argument, and rank them in order of best to worst.\n",
    "Respond with JSON, and only JSON, with the following format:\n",
    "{{\"results\": [\"best competitor number\", \"second best competitor number\", \"third best competitor number\", ...]}}\n",
    "\n",
    "Here are the responses from each competitor:\n",
    "\n",
    "{together}\n",
    "\n",
    "Now respond with the JSON with the ranked order of the competitors, nothing else. Do not include markdown formatting or code blocks.\"\"\"\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_judge_message = [{'role':'user', 'content':to_judge}]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Execute o3-mini to analyze the LLM results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "openai = OpenAI()\n",
    "response = openai.chat.completions.create(\n",
    "    messages=to_judge_message,\n",
    "    model='o3-mini'\n",
    ")\n",
    "result = response.choices[0].message.content\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_dict = json.loads(result)\n",
    "ranks = results_dict[\"results\"]\n",
    "for index, result in enumerate(ranks):\n",
    "    competitor = competitors[int(result)-1]\n",
    "    print(f\"Rank {index+1}: {competitor}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}