feat(tool): add audio transcription tool using Whisper AI

ShrewdAgent.py

@@ -18,6 +18,7 @@ from tools.execute_python_code_from_file import execute_python_code_from_file
 from tools.maths import add_integers
 from tools.produce_classifier import produce_classifier
 from tools.sort_words_alphabetically import sort_words_alphabetically
+from tools.transcribe_audio import transcribe_audio
 from tools.web_page_information_extractor import web_page_information_extractor
 from tools.wikipedia_search import wikipedia_search
 from tools.youtube_transcript import youtube_transcript
@@ -52,6 +53,7 @@ class ShrewdAgent:
             excel_to_text,
             execute_python_code_from_file,
             add_integers,
+            transcribe_audio,
         ]
         self.llm = ChatOpenAI(
             model="gpt-4o-mini",

data/99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b218c951c1f888f0bbe6f46c080f57afc7c9348fffc7ba4da35749ff1e2ac40f
+size 179304

requirements.txt

@@ -12,4 +12,5 @@ readability-lxml~=0.8.4.1
 youtube-transcript-api~=1.0.3
 wikipedia~=1.4.0
 langchain_tavily~=0.1.6
-rizaio~=0.11.0
+rizaio~=0.11.0
+openai-whisper~=20240930

tools/transcribe_audio.py

@@ -0,0 +1,42 @@
+import tempfile
+
+import whisper
+from langchain_core.tools import tool
+from loguru import logger
+
+from tools.load_file import load_file
+
+
+@tool("transcribe_audio_tool", parse_docstring=True)
+def transcribe_audio(file_name: str) -> str:
+    """
+    Convert speech from an audio file or URL to text.
+
+    Args:
+        file_name (str): Either a local file path to an audio file or a URL pointing to an audio file.
+
+
+    Returns:
+        str: The transcribed text from the audio file
+    """
+    logger.info(f"use transcribe_audio_tool with param: {file_name}")
+    model_name = "base"
+
+    try:
+        audio_bytes = load_file(file_name)
+        with tempfile.NamedTemporaryFile() as tmp:
+            tmp.write(audio_bytes.getvalue())
+            tmp.flush()
+            model = whisper.load_model(model_name)
+            result = model.transcribe(tmp.name, fp16=False)
+            return result["text"]
+
+    except Exception as e:
+        print(f"Error transcribing audio: {str(e)}")
+        return ""
+
+
+if __name__ == "__main__":
+    print(transcribe_audio.invoke("../data/99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3.mp3"))
+    print(transcribe_audio.invoke(
+        "https://agents-course-unit4-scoring.hf.space/files/99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3"))