First version

mini_agents.py

@@ -0,0 +1,45 @@
+from smolagents import CodeAgent, InferenceClientModel
+from tools import sort_list
+import os
+
+MODEL_CHOICES = {
+    "audio": ["whisper-large-v3"], 
+    "vlm": ["Salesforce/blip-image-captioning-base", "smolvlm/vlm-base-patch14-224"], 
+    "code": ["gpt-4o-mini"]}
+
+code_agent = CodeAgent(
+    model=MODEL_CHOICES["code"][0],
+    tools=[sort_list],
+    verbose=True
+)
+
+audio_model = InferenceClientModel(
+    model=MODEL_CHOICES["audio"][0],
+    api_key=os.getenv("HUGGINGFACE_API_KEY"),
+    api_url="https://api.openai.com/v1/audio/transcriptions"
+)
+
+audio_agent = CodeAgent(
+    model=audio_model,
+    tools=[],
+    verbose=True
+)
+
+vlm_model = InferenceClientModel(
+    model=MODEL_CHOICES["vlm"][0],
+    api_key=os.getenv("HUGGINGFACE_API_KEY"),
+    api_url="https://api.openai.com/v1/images/generations"
+)
+
+vlm_agent = CodeAgent(
+    model=vlm_model,
+    tools=[],
+    verbose=True
+)
+
+
+
+
+
+
+

tools.py

@@ -0,0 +1,63 @@
+from langchain_core.tools import tool
+from datetime import datetime
+from typing import Literal, List
+from smolagents import WebSearchTool, DuckDuckGoSearchTool, VisitWebpageTool, WikipediaSearchTool
+
+@tool
+def get_current_time(timezone: str = "America/New_York", format: str = "%Y-%m-%d %H:%M:%S"):
+    """
+    Get the current time
+    Args:
+        timezone: The timezone to get the current time in. Example: "America/New_York"
+        format: The format to return the current time in. Example: "%Y-%m-%d %H:%M:%S"
+    Returns:
+        The current time
+    """
+    return datetime.now(timezone).strftime(format)
+           
+@tool
+def sort_list(my_list: List[int], order: Literal["asc", "desc", "alphabetize", "alphabetize_reverse"]):
+    """
+    Sort a list in ascending or descending order if the list contains numbers. 
+    Sort it in alphabetically or alphabetically in reverse order if the list contains strings or mixed types.
+    
+    Args:
+        my_list: The list to sort
+        order: The order to sort the list in. Must be one of the following:
+            - "asc": Sort the list in ascending order. Only for lists containing numbers.
+            - "desc": Sort the list in descending order. Only for lists containing numbers.
+            - "alphabetize": Sort the list alphabetically. Only for lists containing strings or mixed types.
+            - "alphabetize_reverse": Sort the list alphabetically in reverse order. Only for lists containing strings or mixed types.
+        
+    Returns:
+        The sorted list
+    """
+    if not isinstance(my_list, List):
+        raise ValueError("my_list must be a list")
+    else:
+        if all(isinstance(item, (int, float)) for item in my_list):
+            if order in ["asc", "desc"]:
+                return sorted(my_list, reverse=order == "desc")
+            elif order in ["alphabetize", "alphabetize_reverse"]:
+                how = {
+                    "alphabetize": "asc",
+                    "alphabetize_reverse": "desc"
+                }
+                return sorted(my_list, key=lambda x: str(x), reverse=how[order] == "desc")
+            else:
+                raise ValueError("order must be one of the following: asc, desc, alphabetize, alphabetize_reverse")
+        else:
+            print("This is a mixed list. Converting and sorting alphabetically.")
+            my_list = [str(item) for item in my_list]
+            how = {
+                "alphabetize": "asc",
+                "alphabetize_reverse": "desc"
+            }
+            return sorted(my_list, reverse=how[order] == "desc")
+            
+#smolagents tools
+web_search_tool = WebSearchTool()
+duckduckgo_search_tool = DuckDuckGoSearchTool()
+visit_webpage_tool = VisitWebpageTool()
+wikipedia_search_tool = WikipediaSearchTool()
+

vlm_tools.py

@@ -0,0 +1,54 @@
+from langchain_core.tools import tool
+
+@tool
+def download_image(image_url: str):
+    """
+    Download an image from a url
+    Args:
+        image_url: The url of the image to download
+    Returns:
+        The image as a base64 string
+    """
+    #download the image into a base64 string
+    image = None
+    return image
+
+@tool
+def image_processing(image: str):
+    """
+    Process an image
+    Args:
+        image: The image in base64 format to process
+    Returns:
+        The processed image
+    """
+    processed_image = None
+    return processed_image
+
+@tool
+def object_detection(image: str):
+    """
+    Detect objects in an image
+    Args:
+        image: The image in base64 format to detect objects in
+    Returns:
+        The detected objects
+    """
+    detected_objects = None
+    return detected_objects
+
+@tool
+def ocr_scan(image: str):
+    """
+    Scan an image for text
+    Args:
+        image: The image in base64 format to scan for text
+    Returns:
+        The text in the image
+    """
+    scanned_text = None
+    return scanned_text
+
+
+
+