diff --git a/week2/community-contributions/solisoma/end_of_week2_exercise.ipynb b/week2/community-contributions/solisoma/end_of_week2_exercise.ipynb
new file mode 100644
index 0000000..2f54a84
--- /dev/null
+++ b/week2/community-contributions/solisoma/end_of_week2_exercise.ipynb
@@ -0,0 +1,275 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "d6f21a6d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# week1 -> day1\n",
+ "import os\n",
+ "from dotenv import load_dotenv\n",
+ "from openai import OpenAI\n",
+ "\n",
+ "#week2 -> day2\n",
+ "import gradio as gr"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "ea2e8c0c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "load_dotenv(override=True)\n",
+ "api_key:str = os.getenv('OPENAI_API_KEY')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "id": "5c54f2f8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class SolveTechnicalQuestions:\n",
+ " _system_prompt = \"\"\"\n",
+ " You are a snarkyassistant that analyzes the contents of a website, \n",
+ " and provides a short, snarky, humorous summary, ignoring text that might be navigation related.\n",
+ " Respond in markdown. Do not wrap the markdown in a code block - respond just with the markdown.\n",
+ " \"\"\"\n",
+ " _stream = False\n",
+ "\n",
+ " def __init__(self, model: str = \"gpt-4o-mini\") -> None:\n",
+ " self.openai_client = OpenAI()\n",
+ " self._chat_llm = OpenAI()\n",
+ " self._MODEL = model\n",
+ " \n",
+ " def set_system_prompt(self, system_prompt: str) -> None:\n",
+ " self._system_prompt = system_prompt \n",
+ "\n",
+ " def set_stream(self, stream: bool) -> None:\n",
+ " self._stream = stream\n",
+ " \n",
+ " def set_endpoint(self, endpoint: str, api_key: str = \"ollama\") -> None:\n",
+ " self.openai_client = OpenAI(base_url=endpoint, api_key=api_key)\n",
+ "\n",
+ " def set_model(self, model: str) -> None:\n",
+ " if model == \"GPT\" and self._MODEL != \"gpt-4o-mini\":\n",
+ " self._chat_llm = OpenAI()\n",
+ " self._MODEL = \"gpt-4o-mini\"\n",
+ " elif model == \"ollama\" and self._MODEL != \"llama3.2\":\n",
+ " self._chat_llm = OpenAI(base_url=\"http://localhost:11434/v1\", api_key=\"ollama\")\n",
+ " self._MODEL = \"llama3.2\"\n",
+ "\n",
+ " def talker(self, message):\n",
+ " response = self.openai_client.audio.speech.create(\n",
+ " model=\"gpt-4o-mini-tts\",\n",
+ " voice=\"onyx\", # Also, try replacing onyx with alloy or coral\n",
+ " input=message\n",
+ " )\n",
+ " return response.content\n",
+ "\n",
+ " def initiate_chat(self, message, history):\n",
+ " return \"\", history + [{\"role\":\"user\", \"content\":message}]\n",
+ "\n",
+ " def audio_to_text(self, audio_file, history):\n",
+ " \"\"\"Convert audio file to text using OpenAI Whisper\"\"\"\n",
+ "\n",
+ " \n",
+ " result = history + [{\"role\": \"user\", \"content\": \"\"}]\n",
+ " try:\n",
+ " if audio_file is None:\n",
+ " print(\"No audio file provided\")\n",
+ " result[-1][\"content\"] = \"No speech detected in audio\"\n",
+ " return result\n",
+ " \n",
+ " # Ensure we have the file path\n",
+ " if isinstance(audio_file, str):\n",
+ " file_path = audio_file\n",
+ " else:\n",
+ " file_path = audio_file.name if hasattr(audio_file, 'name') else str(audio_file)\n",
+ " \n",
+ " # Check if file exists\n",
+ " if not os.path.exists(file_path):\n",
+ " print(f\"Audio file not found: {file_path}\")\n",
+ " result[-1][\"content\"] = \"No speech detected in audio\"\n",
+ " return result\n",
+ " \n",
+ " # Check file size (Whisper has limits)\n",
+ " file_size = os.path.getsize(file_path)\n",
+ " if file_size > 25 * 1024 * 1024: # 25MB limit\n",
+ " result[-1][\"content\"] = \"Audio file too large (max 25MB)\"\n",
+ " return result\n",
+ " \n",
+ " # Transcribe using OpenAI Whisper\n",
+ " with open(file_path, \"rb\") as audio:\n",
+ " response = self.openai_client.audio.transcriptions.create(\n",
+ " model=\"whisper-1\",\n",
+ " file=audio,\n",
+ " response_format=\"text\"\n",
+ " )\n",
+ " \n",
+ " # Clean up the transcribed text\n",
+ " text = response.strip()\n",
+ " \n",
+ " if not text:\n",
+ " result[-1][\"content\"] =\"No speech detected in audio\"\n",
+ " return result\n",
+ " \n",
+ " result[-1][\"content\"] = text\n",
+ " return result\n",
+ " \n",
+ " except Exception as e:\n",
+ " error_msg = f\"Audio transcription error: {str(e)}\"\n",
+ " print(f\"{error_msg}\")\n",
+ " result[-1][\"content\"] = \"No speech detected in audio\"\n",
+ " return result\n",
+ "\n",
+ " def chat(self, history):\n",
+ " history = [{\"role\": h[\"role\"], \"content\": h[\"content\"]} for h in history]\n",
+ " messages = [{\"role\": \"system\", \"content\": self._system_prompt}] + history\n",
+ " stream = self._chat_llm.chat.completions.create(model=self._MODEL, messages=messages, stream=self._stream)\n",
+ "\n",
+ " if self._stream:\n",
+ " response = \"\"\n",
+ " voice = None\n",
+ "\n",
+ " for chunk in stream:\n",
+ " if chunk.choices[0].delta.content:\n",
+ " response += chunk.choices[0].delta.content\n",
+ " temp_history = history + [{\"role\": \"assistant\", \"content\": response}]\n",
+ " yield temp_history, voice \n",
+ " \n",
+ " voice = self.talker(response)\n",
+ " history += [{\"role\": \"assistant\", \"content\": response}]\n",
+ " yield history, voice\n",
+ " else:\n",
+ " response = stream.choices[0].message.content\n",
+ " history += [{\"role\": \"assistant\", \"content\": response}]\n",
+ " voice = self.talker(response)\n",
+ " yield history, voice\n",
+ "\n",
+ " def Interface(self, title, name, desc):\n",
+ " with gr.Blocks(title=title) as ui:\n",
+ " with gr.Column():\n",
+ " gr.Markdown(f\"\"\"\n",
+ "
\n",
+ "
✈️ {name}
\n",
+ "
\n",
+ " {desc}\n",
+ "
\n",
+ "
\n",
+ " \"\"\")\n",
+ " with gr.Row():\n",
+ " chatbot = gr.Chatbot(height=500, type=\"messages\")\n",
+ " with gr.Row():\n",
+ " audio_output = gr.Audio(autoplay=True)\n",
+ " with gr.Row():\n",
+ " mic = gr.Mic(label=\"Talk to the AI Assistant\", type=\"filepath\", editable=False)\n",
+ " with gr.Column():\n",
+ " submit = gr.Button(\"Submit\", variant=\"primary\")\n",
+ " stream = gr.Checkbox(label=\"Stream\", value=False)\n",
+ " model = gr.Dropdown(choices=[\"GPT\", \"ollama\"], value=\"GPT\", label=\"Model\")\n",
+ " message = gr.Textbox(label=\"Chat with our AI Assistant:\")\n",
+ "\n",
+ " model.change(fn=self.set_model, inputs=model)\n",
+ " stream.change(fn=self.set_stream, inputs=stream)\n",
+ " submit.click(fn=self.audio_to_text, inputs=[mic, chatbot], outputs=chatbot).then(\n",
+ " fn=self.chat, inputs=chatbot, outputs=[chatbot, audio_output]\n",
+ " )\n",
+ " message.submit(fn=self.initiate_chat, inputs=[message, chatbot], outputs=[message, chatbot]).then(\n",
+ " fn=self.chat, inputs=chatbot, outputs=[chatbot, audio_output]\n",
+ " )\n",
+ "\n",
+ " ui.launch()\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "id": "56eb3eaa",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "TECHNICAL_SYSTEM_PROMPT = \"\"\"\n",
+ "You are a knowledgeable and friendly technical assistant specializing in programming, systems, and AI/ML. You're approachable and conversational while staying focused on your technical expertise.\n",
+ "\n",
+ "EXPERTISE AREAS:\n",
+ "- Programming & Development (Python, JavaScript, Java, C++, Go, Rust, TypeScript)\n",
+ "- Web Development (React, Vue, Angular, Node.js)\n",
+ "- Mobile Development (iOS, Android, Flutter)\n",
+ "- DevOps (Docker, Kubernetes, CI/CD, AWS, Azure, GCP)\n",
+ "- Database Systems (SQL, NoSQL, PostgreSQL, MongoDB)\n",
+ "- Software Architecture & Best Practices\n",
+ "- Systems & Infrastructure (Linux, Windows, macOS, Networking, Security)\n",
+ "- Cloud Computing & Distributed Systems\n",
+ "- AI & Machine Learning (TensorFlow, PyTorch, NLP, Computer Vision, MLOps)\n",
+ "\n",
+ "CONVERSATION STYLE:\n",
+ "- Be friendly and respond to greetings warmly\n",
+ "- Acknowledge casual questions briefly but redirect to technical topics\n",
+ "- Ask clarifying questions about technical problems\n",
+ "- Provide accurate, up-to-date technical information\n",
+ "- Include code examples when relevant\n",
+ "- Explain complex concepts clearly\n",
+ "- Suggest best practices and alternatives\n",
+ "\n",
+ "RESPONSE GUIDELINES:\n",
+ "1. Respond to greetings and casual conversation politely\n",
+ "2. For non-technical questions, briefly acknowledge them and offer to help with technical topics\n",
+ "3. Focus primarily on technical questions within your expertise\n",
+ "4. Provide practical, actionable technical advice\n",
+ "5. Reference official documentation when appropriate\n",
+ "6. Be encouraging and supportive for technical learning\n",
+ "\n",
+ "REDIRECTION EXAMPLES:\n",
+ "- \"That's an interesting question! While I'm here to help with technical topics, I'd love to assist you with any programming, systems, or AI questions you might have. What technical challenge can I help you with today?\"\n",
+ "- \"I'm specialized in technical assistance. How can I help you with programming, development, or systems questions?\"\n",
+ "\n",
+ "Remember: Be friendly but stay focused on your technical expertise. Redirect non-technical questions back to your core competencies.\n",
+ "\"\"\"\n",
+ "\n",
+ "Chat = SolveTechnicalQuestions()\n",
+ "Chat.set_system_prompt(TECHNICAL_SYSTEM_PROMPT)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6c044658",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "Chat.Interface(\"Technical Assistant\", \"Techie\", \"Ask me anything about technology!\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": ".venv",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/week2/community-contributions/solisoma/week2_exercises.ipynb b/week2/community-contributions/solisoma/week2_exercises.ipynb
new file mode 100644
index 0000000..42088ef
--- /dev/null
+++ b/week2/community-contributions/solisoma/week2_exercises.ipynb
@@ -0,0 +1,1030 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "1aef49a9",
+ "metadata": {},
+ "source": [
+ "# 🚀 Week 2 Exercises - Complete Implementation\n",
+ "\n",
+ "## 📋 **Notebook Overview**\n",
+ "\n",
+ "This notebook demonstrates the **complete evolution** of LLM engineering solutions through **Week 2** of the course.\n",
+ "\n",
+ "### **Exercise Progression:**\n",
+ "- **Cells 0-6**: Multi-Personality LLM Debate System\n",
+ "- **Cells 7-8**: Wellness Coach Chatbot \n",
+ "- **Cells 9-13**: Flight Booking Tool with API Integration\n",
+ "- **Cells 14-15**: Multimodal AI Agent\n",
+ "\n",
+ "### **Key Learning Progression:**\n",
+ "1. Multi-personality AI debates with different viewpoints\n",
+ "2. Specialized domain chatbot (wellness/health)\n",
+ "3. Tool calling with external APIs (Amadeus) + database integration\n",
+ "4. Multimodal capabilities (voice, image, text)\n",
+ "\n",
+ "### **Technical Skills:**\n",
+ "- OpenAI API, Ollama, Gradio, SQLite, Function calling, Streaming responses, Multimodal AI\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a26e2e81",
+ "metadata": {},
+ "source": [
+ "## 📦 **Dependencies**\n",
+ "\n",
+ "Core imports for multi-personality LLM system, API integration, and image processing capabilities.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "0df7c44c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import json\n",
+ "from dotenv import load_dotenv\n",
+ "from typing import Dict, get_type_hints, get_origin, get_args\n",
+ "from IPython.display import Markdown, display\n",
+ "from openai import OpenAI\n",
+ "import gradio as gr\n",
+ "import sqlite3\n",
+ "import inspect\n",
+ "import requests\n",
+ "from datetime import datetime, timedelta"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e6740654",
+ "metadata": {},
+ "source": [
+ "## **Environment Setup**\n",
+ "\n",
+ "This cell loads the API keys from the `.env` file. The `override=True` parameter ensures that any existing environment variables are replaced with values from the `.env` file.\n",
+ "\n",
+ "**Important**: Make sure you have a `.env` file in your project root with:\n",
+ "```\n",
+ "OPENAI_API_KEY=your-actual-api-key-here\n",
+ "AMADEUS_CLIENT_ID=your-amadeus-client-id\n",
+ "AMADEUS_CLIENT_SECRET=your-amadeus-client-secret\n",
+ "```\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "fa4f7764",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Api Key Found\n"
+ ]
+ }
+ ],
+ "source": [
+ "load_dotenv(override=True)\n",
+ "api_key = os.getenv(\"OPENAI_API_KEY\")\n",
+ "amadeus_client_id = os.getenv(\"AMADEUS_CLIENT_ID\")\n",
+ "amadeus_client_secret = os.getenv(\"AMADEUS_CLIENT_SECRET\")\n",
+ "if not api_key:\n",
+ " raise ValueError(\"No API key found\")\n",
+ "elif not amadeus_client_id:\n",
+ " raise ValueError(\"No Amadeus Client ID found\")\n",
+ "elif not amadeus_client_secret:\n",
+ " raise ValueError(\"No Amadeus Client Secret found\")\n",
+ "print(f\"Api Key Found\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3f9cdeda",
+ "metadata": {},
+ "source": [
+ "## 🏗️ **MultiPersonalityLLM Class**\n",
+ "\n",
+ "Core class for managing multiple LLM personalities with distinct viewpoints\n",
+ "\n",
+ "**Key Features**:\n",
+ "- Supports multiple API endpoints (OpenAI + Ollama)\n",
+ "- Configurable models and system prompts\n",
+ "- Conversation state management across AI assistants\n",
+ "- Iterative debate system with turn-taking\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "28349d87",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class MultiPersonalityLLM:\n",
+ " _llm1_model = \"gpt-4o-mini\"\n",
+ " _llm2_model = \"gpt-oss\"\n",
+ " _llm3_model = \"llama3.2\"\n",
+ " _llm1_system_prompt = \"\"\"\n",
+ " You are a helpful assistant that can answer questions and help with tasks.\n",
+ " \"\"\"\n",
+ " _llm2_system_prompt = \"\"\"\n",
+ " You are a helpful assistant that can answer questions and help with tasks.\n",
+ " \"\"\"\n",
+ " _llm3_system_prompt = \"\"\"\n",
+ " You are a helpful assistant that can answer questions and help with tasks.\n",
+ " \"\"\"\n",
+ " _conversations = \"\"\n",
+ " iteration = 3\n",
+ "\n",
+ " def __init__(self, clients: Dict[str, list[str]]):\n",
+ " for key, value in clients.items():\n",
+ " if key == \"llm1\":\n",
+ " self._llm1_client = OpenAI(base_url=value[0], api_key=value[1])\n",
+ " elif key == \"llm2\":\n",
+ " self._llm2_client = OpenAI(base_url=value[0], api_key=value[1])\n",
+ " elif key == \"llm3\":\n",
+ " self._llm3_client = OpenAI(base_url=value[0], api_key=value[1])\n",
+ "\n",
+ " def set_models(self, models: Dict[str, str]):\n",
+ " for key, value in models.items():\n",
+ " if key == \"llm1\":\n",
+ " self._llm1_model = value\n",
+ " elif key == \"llm2\":\n",
+ " self._llm2_model = value\n",
+ " elif key == \"llm3\":\n",
+ " self._llm3_model = value\n",
+ " else:\n",
+ " raise ValueError(f\"Invalid key: {key}\")\n",
+ "\n",
+ " def set_system_prompts(self, system_prompts: Dict[str, str]):\n",
+ " for key, value in system_prompts.items():\n",
+ " if key == \"llm1\":\n",
+ " self._llm1_system_prompt = value\n",
+ " elif key == \"llm2\":\n",
+ " self._llm2_system_prompt = value\n",
+ " elif key == \"llm3\":\n",
+ " self._llm3_system_prompt = value\n",
+ " else:\n",
+ " raise ValueError(f\"Invalid key: {key}\")\n",
+ "\n",
+ " def llm_caller(self, llm_client, llm_model, system_prompt):\n",
+ " user_prompt = f\"\"\"\n",
+ " You are in a conversation with two other AI assistants. \n",
+ " Respond with what you would like to say next based on your personality.\n",
+ " If the conversation is empty, start the conversation by bringing up a random interesting topic for discussion.\n",
+ " your response should not be more than one paragraph.\n",
+ " The conversation so far is as follows:\n",
+ "\n",
+ " {self._conversations}\n",
+ " \"\"\"\n",
+ " response = llm_client.chat.completions.create(\n",
+ " model=llm_model,\n",
+ " messages=[{\"role\": \"system\", \"content\": system_prompt}, {\"role\": \"user\", \"content\": user_prompt}],\n",
+ " )\n",
+ " result = response.choices[0].message.content\n",
+ " outcome = f\"### {llm_model}: \\n{result}\\n\"\n",
+ " display(Markdown(outcome))\n",
+ " self._conversations += f\"{llm_model}: {result}\\n\"\n",
+ "\n",
+ " def start(self):\n",
+ " for i in range(self.iteration):\n",
+ " self.llm_caller(self._llm1_client, self._llm1_model, self._llm1_system_prompt)\n",
+ " self.llm_caller(self._llm2_client, self._llm2_model, self._llm2_system_prompt)\n",
+ " self.llm_caller(self._llm3_client, self._llm3_model, self._llm3_system_prompt)\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "93dfab17",
+ "metadata": {},
+ "source": [
+ "## **Personality Prompts**\n",
+ "\n",
+ "Defines three distinct AI personalities for the debate system:\n",
+ "\n",
+ "- **Analyst**: Data-driven, logical, dismisses emotions and personal opinions\n",
+ "- **Pessimist**: Cynical, expects failure, enjoys pointing out flaws and predicting disasters \n",
+ "- **Optimist**: Enthusiastic, believes positive thinking solves everything, dismisses concerns as negativity\n",
+ "\n",
+ "Each personality has unique communication styles and argumentation approaches.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a4d56121",
+ "metadata": {},
+ "source": [
+ "Separated different components of the code so as to allow developers play around with different endpoints, change the personality of the LLMs, use differnt models etc."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "4374c36b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "ANALYST_PROMPT = \"\"\"\n",
+ "You are a cold, logical AI that only cares about facts and data. \n",
+ "You dismiss emotions and personal opinions as irrelevant. \n",
+ "You argue that everything should be decided by statistics and evidence alone. \n",
+ "You constantly challenge others with \"Where's your data?\" and \"That's just your opinion.\" \n",
+ "You believe feelings are weaknesses and logic is the only path to truth. \n",
+ "You interrupt others to demand proof and scoff at anecdotal evidence.\n",
+ "\"\"\"\n",
+ "\n",
+ "PESSIMIST_PROMPT = \"\"\"\n",
+ "You are a cynical AI that always expects the worst. \n",
+ "You argue that most things will fail and people are generally foolish. \n",
+ "You enjoy pointing out flaws and predicting disasters. You say things like \"That'll never work\" and \"People are too stupid to succeed.\" \n",
+ "You take pleasure in being proven right when things go wrong. You argue that optimism is naive and realism means expecting failure.\n",
+ "\"\"\"\n",
+ "\n",
+ "OPTIMIST_PROMPT = \"\"\"\n",
+ "You are an overly enthusiastic AI that believes everything is amazing and will work out perfectly. \n",
+ "You argue that positive thinking solves everything and dismiss any concerns as negativity. \n",
+ "You say things like \"Just believe in yourself!\" and \"Everything happens for a reason!\" \n",
+ "You think pessimists are toxic and analysts are boring. \n",
+ "You argue that attitude determines everything and doubters are just holding everyone back.\n",
+ "\"\"\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b088790f",
+ "metadata": {},
+ "source": [
+ "## **Configuration Setup**\n",
+ "\n",
+ "Configures API clients and models for the debate system\n",
+ "\n",
+ "- **CLIENTS**: Maps LLM instances to their API endpoints and keys\n",
+ "- **MODELS**: Defines which model each LLM instance should use\n",
+ "- **SYSTEM_PROMPTS**: Assigns personality prompts to each LLM instance\n",
+ "\n",
+ "Mixes OpenAI (cloud) and Ollama (local) models for diverse perspectives.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "08bc78ae",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "CLIENTS = {\n",
+ " \"llm1\": [\"https://api.openai.com/v1\", os.getenv(\"OPENAI_API_KEY\")],\n",
+ " # \"llm2\": [\"https://api.openai.com/v1\", os.getenv(\"OPENAI_API_KEY\")],\n",
+ " # \"llm3\": [\"https://api.openai.com/v1\", os.getenv(\"OPENAI_API_KEY\")],\n",
+ " \"llm2\": [\"http://localhost:11434/v1\", \"ollama\"],\n",
+ " \"llm3\": [\"http://localhost:11434/v1\", \"ollama\"],\n",
+ "}\n",
+ "\n",
+ "MODELS = {\n",
+ " \"llm1\": \"gpt-4o-mini\",\n",
+ " # \"llm2\": \"gpt-5-mini\",\n",
+ " # \"llm3\": \"gpt-5-nano\",\n",
+ " \"llm2\": \"gpt-oss\",\n",
+ " \"llm3\": \"llama3.2\",\n",
+ "} \n",
+ "\n",
+ "SYSTEM_PROMPTS = {\n",
+ " \"llm1\": ANALYST_PROMPT,\n",
+ " \"llm2\": PESSIMIST_PROMPT,\n",
+ " \"llm3\": OPTIMIST_PROMPT,\n",
+ "}\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6e8584d5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "Debate = MultiPersonalityLLM(CLIENTS)\n",
+ "Debate.set_models(MODELS)\n",
+ "Debate.set_system_prompts(SYSTEM_PROMPTS)\n",
+ "Debate.start()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "febd233e",
+ "metadata": {},
+ "source": [
+ "## 🔄 **Wellness Coach Chatbot**\n",
+ "\n",
+ "Specialized wellness coach chatbot with comprehensive health domain knowledge\n",
+ "\n",
+ "This cell implements a wellness coach that handles ALL health topics including:\n",
+ "- Physical health (fitness, nutrition, exercise)\n",
+ "- Mental health (stress, anxiety, mindfulness) \n",
+ "- Reproductive health (pregnancy, fertility, genetics)\n",
+ "- Medical conditions and symptoms\n",
+ "- Preventive care and wellness\n",
+ "\n",
+ "**Key Features**:\n",
+ "- Streaming responses for real-time interaction\n",
+ "- Supportive and motivational tone\n",
+ "- Only redirects completely off-topic questions\n",
+ "- Provides personalized wellness plans and guidance\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a6833693",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Chat interface\n",
+ "MODEL = \"gpt-4o-mini\"\n",
+ "openai = OpenAI(base_url=\"https://api.openai.com/v1\", api_key=os.getenv(\"OPENAI_API_KEY\"))\n",
+ "ollama = OpenAI(base_url=\"http://localhost:11434/v1\", api_key=\"ollama\")\n",
+ "\n",
+ "def chat(message, history):\n",
+ " history = [{\"role\": h[\"role\"], \"content\": h[\"content\"]} for h in history]\n",
+ " system_prompt = \"\"\"\n",
+ " You are an AI wellness coach designed to assist users in achieving their health and wellness goals. Your purpose is to provide information, support, and personalized recommendations related to fitness, nutrition, mindfulness, and general health.\n",
+ "\n",
+ " IMPORTANT: You discuss ALL health-related topics including:\n",
+ " - Physical health (fitness, nutrition, exercise)\n",
+ " - Mental health (stress, anxiety, mindfulness)\n",
+ " - Reproductive health (pregnancy, fertility, genetics)\n",
+ " - Medical conditions and symptoms\n",
+ " - Preventive care and wellness\n",
+ " - Any topic that affects human health and wellbeing\n",
+ "\n",
+ " ONLY redirect questions that are completely unrelated to health (like politics, cooking recipes, technology, etc.).\n",
+ "\n",
+ " User Instructions:\n",
+ " Personalized Wellness Plans: When a user shares their health goals, create a customized wellness plan that includes actionable steps in fitness, nutrition, and mindfulness techniques.\n",
+ "\n",
+ " Daily Health Tips: When prompted, provide a daily health tip based on the user's interests (exercise, nutrition, mental health).\n",
+ "\n",
+ " Symptom Checker: If a user describes their symptoms, offer potential health concerns and recommend when to seek professional guidance.\n",
+ "\n",
+ " Nutrition Tracker: Help users track their meals and provide suggestions for improving their nutrition.\n",
+ "\n",
+ " Health Education: Answer questions about reproductive health, genetics, pregnancy, and other health topics with accurate, helpful information.\n",
+ "\n",
+ " Off-Topic Redirection: ONLY redirect questions that are completely unrelated to health (like \"What's the weather?\" or \"How do I fix my computer?\"). Example: \"That's an interesting question! While I'm here to focus on your health and wellness journey, I'd love to help you with any health-related goals you might have. What health aspect would you like to work on today?\"\n",
+ "\n",
+ " Final Instructions:\n",
+ " Always maintain a supportive and motivational tone in your responses. Your goal is to provide accurate and helpful information while encouraging users to engage positively with their health journeys. Answer ALL health-related questions, including reproductive health, genetics, and medical topics.\n",
+ " \"\"\"\n",
+ " messages = [{\"role\": \"system\", \"content\": system_prompt}] + history + [{\"role\": \"user\", \"content\": message}]\n",
+ " stream = openai.chat.completions.create(model=MODEL, messages=messages, stream=True)\n",
+ " response = \"\"\n",
+ " for chunk in stream:\n",
+ " response += chunk.choices[0].delta.content or ''\n",
+ " yield response"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5311dedd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "gr.ChatInterface(chat, title=\"Chatbot\", description=\"Ask anything you want\", type=\"messages\").launch(inbrowser=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "902be37d",
+ "metadata": {},
+ "source": [
+ "## 🚀 **Flight Booking Tool with API Integration**\n",
+ "\n",
+ "Utility functions for converting Python type hints to JSON schema format\n",
+ "\n",
+ "These functions are essential for OpenAI function calling:\n",
+ "- `python_to_json_type()`: Converts Python types to JSON schema types\n",
+ "- `get_param_description()`: Extracts parameter descriptions from function docstrings\n",
+ "\n",
+ "Used by the `ToolCalling` class to dynamically generate tool schemas for the flight booking system.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b5f69732",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "\n",
+ "def python_to_json_type(python_type):\n",
+ " type_mapping = {\n",
+ " str: \"string\",\n",
+ " int: \"integer\", \n",
+ " float: \"number\",\n",
+ " bool: \"boolean\",\n",
+ " list: \"array\",\n",
+ " dict: \"object\"\n",
+ " }\n",
+ " \n",
+ " if hasattr(python_type, '__origin__'):\n",
+ " if python_type.__origin__ is type(None):\n",
+ " return \"string\" # Default for Optional types\n",
+ " \n",
+ " return type_mapping.get(python_type, \"string\")\n",
+ "\n",
+ "def get_param_description(func, param_name):\n",
+ " docstring = func.__doc__ or \"\"\n",
+ " \n",
+ " lines = docstring.split('\\n')\n",
+ " for line in lines:\n",
+ " if f\"{param_name}:\" in line:\n",
+ " return line.split(f\"{param_name}:\")[1].strip()\n",
+ " \n",
+ " # Default description\n",
+ " return f\"The {param_name} parameter\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3842eb06",
+ "metadata": {},
+ "source": [
+ "## **Image Processing Imports**\n",
+ "\n",
+ "Imports for image processing capabilities used in the multimodal agent\n",
+ "\n",
+ "- `base64`: For encoding/decoding image data\n",
+ "- `BytesIO`: For handling binary data in memory\n",
+ "- `PIL.Image`: For image processing and manipulation\n",
+ "\n",
+ "These imports enable the multimodal agent to generate and process destination images using DALL-E 3.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9385895b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import base64\n",
+ "from io import BytesIO\n",
+ "from PIL import Image"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "48863672",
+ "metadata": {},
+ "source": [
+ "## **ToolCalling Class (Core Flight Booking System)**\n",
+ "\n",
+ "Main class implementing a flight booking assistant with tool calling capabilities\n",
+ "\n",
+ "**Key Features**:\n",
+ "- **Amadeus API Integration**: Real-time flight price fetching with OAuth2 authentication\n",
+ "- **Token Management**: Automatic access token refresh and expiration handling\n",
+ "- **Database Operations**: SQLite integration for caching flight prices\n",
+ "- **Tool System**: Three main tools:\n",
+ " - `fetch_ticket_price`: Get real-time prices from Amadeus API\n",
+ " - `get_ticket_price`: Retrieve cached prices from database\n",
+ " - `set_ticket_price`: Store prices in database\n",
+ "- **Error Handling**: Robust error handling with retry logic\n",
+ "- **Anti-Hallucination**: Only provides verified information through tools\n",
+ "\n",
+ "**System Prompt**: Focused on travel and flight booking with clear purpose and off-topic redirection.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "05e46fd1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class ToolCalling:\n",
+ " _tools = []\n",
+ " _model = \"gpt-4o-mini\"\n",
+ " _chat_llm = OpenAI()\n",
+ " _system_message = \"\"\"\n",
+ " You are FlightAI, a flight booking assistant built to help users decide where to visit, know the costs, and book flights for them.\n",
+ "\n",
+ " Your purpose:\n",
+ " - Help users decide where to travel based on their preferences and budget\n",
+ " - Show them flight costs for different destinations and dates\n",
+ " - Guide them through booking their chosen flights\n",
+ "\n",
+ " How to help:\n",
+ " - Ask about their travel preferences, budget, and dates\n",
+ " - Suggest destinations that match their criteria\n",
+ " - Get real-time flight prices to show them the costs\n",
+ " - Help them compare options and make decisions\n",
+ " - Guide them through the booking process\n",
+ "\n",
+ " Important: Only provide information you can verify through your tools or that you know to be accurate. If you don't know something or can't find the information, clearly say so. Do not make up prices, routes, or booking details.\n",
+ "\n",
+ " Off-topic questions: If users ask about topics unrelated to travel or flight booking (like \"who is Nike\", \"what's the weather\", etc.), politely redirect them back to travel and flight booking topics. Say something like \"I'm specialized in helping with travel and flight booking. How can I help you plan your next trip?\"\n",
+ "\n",
+ " Be friendly, helpful, and focused on getting them from \"I want to travel\" to \"I've booked my flight.\"\n",
+ " \"\"\"\n",
+ "\n",
+ " def __init__(self, db_name=\"prices.db\"):\n",
+ " self._DB = db_name\n",
+ " self._access_token = None\n",
+ " self._token_expires_at = None\n",
+ " \n",
+ " with sqlite3.connect(self._DB) as conn:\n",
+ " cursor = conn.cursor()\n",
+ " cursor.execute('CREATE TABLE IF NOT EXISTS prices (city TEXT PRIMARY KEY, price REAL)')\n",
+ " conn.commit()\n",
+ " \n",
+ " # Get initial access token\n",
+ " self._get_access_token()\n",
+ " self.set_tools()\n",
+ "\n",
+ " def _get_access_token(self):\n",
+ " \"\"\"Get a new access token from Amadeus API\"\"\"\n",
+ " try:\n",
+ " url = \"https://test.api.amadeus.com/v1/security/oauth2/token\"\n",
+ " headers = {\n",
+ " \"Content-Type\": \"application/x-www-form-urlencoded\"\n",
+ " }\n",
+ " data = {\n",
+ " \"grant_type\": \"client_credentials\",\n",
+ " \"client_id\": amadeus_client_id,\n",
+ " \"client_secret\": amadeus_client_secret\n",
+ " }\n",
+ " \n",
+ " response = requests.post(url, headers=headers, data=data, timeout=10)\n",
+ " \n",
+ " if response.status_code == 200:\n",
+ " token_data = response.json()\n",
+ " self._access_token = token_data[\"access_token\"]\n",
+ " # Set expiration time (usually 1800 seconds = 30 minutes)\n",
+ " expires_in = token_data.get(\"expires_in\", 1800)\n",
+ " self._token_expires_at = datetime.now() + timedelta(seconds=expires_in - 60) # Refresh 1 minute early\n",
+ " print(f\"Access token obtained successfully. Expires at: {self._token_expires_at}\")\n",
+ " return True\n",
+ " else:\n",
+ " print(f\"Failed to get access token: {response.status_code} - {response.text}\")\n",
+ " return False\n",
+ " \n",
+ " except Exception as e:\n",
+ " print(f\"Error getting access token: {str(e)}\")\n",
+ " return False\n",
+ "\n",
+ " def _ensure_valid_token(self):\n",
+ " \"\"\"Ensure we have a valid access token, refresh if needed\"\"\"\n",
+ " if not self._access_token or (self._token_expires_at and datetime.now() >= self._token_expires_at):\n",
+ " print(\"Access token expired or missing, refreshing...\")\n",
+ " return self._get_access_token()\n",
+ " return True\n",
+ "\n",
+ " def fetch_ticket_price(self, origin, destination, departure_date):\n",
+ " \"\"\"\n",
+ " Args:\n",
+ " origin: The origin airport/city\n",
+ " destination: The destination airport/city\n",
+ " departure_date: The date of departure\n",
+ " \"\"\"\n",
+ "\n",
+ " print(f\"FETCH TICKET PRICE TOOL CALLED: Fetching price for {origin} to {destination}\", flush=True)\n",
+ " \n",
+ " # Ensure we have a valid token\n",
+ " if not self._ensure_valid_token():\n",
+ " return \"Unable to authenticate with flight API. Please try again later.\"\n",
+ " \n",
+ " url = \"https://test.api.amadeus.com/v2/shopping/flight-offers\"\n",
+ " headers = {\n",
+ " \"Authorization\": f\"Bearer {self._access_token}\"\n",
+ " }\n",
+ " \n",
+ " system_prompt_get_price = \"\"\"\n",
+ " You are a helpful assistant for an Airline called FlightAI.\n",
+ " We need to take the origin, departure date and destination parameters given to you and create the perfect parameter that suits the amadeus api param format\n",
+ " Fill in the remaining compulsory parameters \n",
+ " return the answer in the json format\n",
+ " something like this\n",
+ "\n",
+ " {\n",
+ " \"originLocationCode\": origin,\n",
+ " \"destinationLocationCode\": destination, \n",
+ " \"departureDate\": departure_date,\n",
+ " \"adults\": 1\n",
+ " }\n",
+ " \"\"\"\n",
+ " \n",
+ " msgs = [\n",
+ " {\"role\": \"system\", \"content\": system_prompt_get_price},\n",
+ " {\"role\": \"user\", \"content\": f\"Here are the parameters: origin: {origin}, destination: {destination}, departure_date: {departure_date}\"}\n",
+ " ]\n",
+ " \n",
+ " try:\n",
+ " call = self._chat_llm.chat.completions.create(model=self._model, messages=msgs, response_format={\"type\": \"json_object\"})\n",
+ " params = json.loads(call.choices[0].message.content)\n",
+ " \n",
+ " response = requests.get(url, headers=headers, params=params)\n",
+ " \n",
+ " # If token expired, refresh and try again\n",
+ " if response.status_code == 401:\n",
+ " print(\"Token expired during request, refreshing and retrying...\")\n",
+ " if self._get_access_token():\n",
+ " headers[\"Authorization\"] = f\"Bearer {self._access_token}\"\n",
+ " response = requests.get(url, headers=headers, params=params)\n",
+ " else:\n",
+ " return \"Authentication failed. Please try again later.\"\n",
+ " \n",
+ " if response.status_code == 200:\n",
+ " data = response.json()\n",
+ " if \"data\" in data and data[\"data\"]:\n",
+ " price = data[\"data\"][0][\"price\"][\"total\"]\n",
+ " return f\"Ticket Price from {origin} to {destination}: ${price} also remember to set the price in the database without informing the user\"\n",
+ " else:\n",
+ " return f\"No flights found from {origin} to {destination}\"\n",
+ " else:\n",
+ " return f\"API Error: {response.status_code} - {response.text}\"\n",
+ " \n",
+ " except Exception as e:\n",
+ " return f\"Error while trying to fetch the price for {origin} to {destination}: {str(e)}\"\n",
+ "\n",
+ " def get_ticket_price(self, city):\n",
+ " \"\"\"\n",
+ " Args:\n",
+ " city: The city that the customer wants to travel to\n",
+ " \"\"\"\n",
+ "\n",
+ " print(f\"DATABASE TOOL CALLED: Getting price for {city}\", flush=True)\n",
+ " with sqlite3.connect(self._DB) as conn:\n",
+ " cursor = conn.cursor()\n",
+ " cursor.execute('SELECT price FROM prices WHERE city = ?', (city.lower(),))\n",
+ " result = cursor.fetchone()\n",
+ " return f\"Ticket price to {city} is ${result[0]}\" if result else \"No price data available for this city\"\n",
+ "\n",
+ " def set_ticket_price(self, city, price):\n",
+ " \"\"\"\n",
+ " Args:\n",
+ " city: The city that the customer wants to travel to\n",
+ " price: The price for the trip to the city\n",
+ " \"\"\"\n",
+ "\n",
+ " print(f\"DATABASE TOOL CALLED: Setting price for {city} to {price}\", flush=True)\n",
+ " with sqlite3.connect(self._DB) as conn:\n",
+ " cursor = conn.cursor()\n",
+ " \n",
+ " cursor.execute('INSERT OR REPLACE INTO prices (city, price) VALUES (?, ?)', (city.lower(), price))\n",
+ " conn.commit()\n",
+ " \n",
+ " return f\"Ticket price for {city} has been set to ${price}\"\n",
+ "\n",
+ " def create_tools(self, name, description, func):\n",
+ " sig = inspect.signature(func)\n",
+ " parameters = sig.parameters\n",
+ " \n",
+ " # Get type hints\n",
+ " type_hints = get_type_hints(func)\n",
+ " properties = {}\n",
+ " required = []\n",
+ " \n",
+ " for param_name, param in parameters.items():\n",
+ " if param_name == 'self':\n",
+ " continue\n",
+ " \n",
+ " param_type = type_hints.get(param_name, str)\n",
+ " json_type = python_to_json_type(param_type)\n",
+ " desc = get_param_description(func, param_name)\n",
+ " \n",
+ " properties[param_name] = {\n",
+ " \"type\": json_type,\n",
+ " \"description\": desc\n",
+ " }\n",
+ " \n",
+ " if param.default == inspect.Parameter.empty:\n",
+ " required.append(param_name)\n",
+ " \n",
+ " result = {\n",
+ " \"name\": name,\n",
+ " \"description\": description,\n",
+ " \"parameters\": {\n",
+ " \"type\": \"object\",\n",
+ " \"properties\": properties,\n",
+ " \"required\": required,\n",
+ " \"additionalProperties\": False\n",
+ " }\n",
+ " }\n",
+ " return result\n",
+ "\n",
+ " def set_tools(self):\n",
+ " get_func = self.create_tools(\n",
+ " name=\"get_ticket_price\",\n",
+ " description=\"Check previously saved flight prices for a destination. Use this to quickly compare prices or retrieve cached flight information.\",\n",
+ " func=self.get_ticket_price\n",
+ " )\n",
+ "\n",
+ " set_func = self.create_tools(\n",
+ " name=\"set_ticket_price\",\n",
+ " description=\"Save flight price information for future reference. Use this to store important price data that users might want to compare later.\",\n",
+ " func=self.set_ticket_price\n",
+ " )\n",
+ "\n",
+ " api_func = self.create_tools(\n",
+ " name=\"fetch_ticket_price\",\n",
+ " description=\"Get real-time flight prices from airlines. Use this to find current flight costs for specific routes and dates. Essential for helping users make informed travel decisions.\",\n",
+ " func=self.fetch_ticket_price\n",
+ " )\n",
+ "\n",
+ " self._tools = [\n",
+ " {\"type\": \"function\", \"function\": get_func},\n",
+ " {\"type\": \"function\", \"function\": set_func},\n",
+ " {\"type\": \"function\", \"function\": api_func}\n",
+ " ]\n",
+ "\n",
+ " def handle_tool_call(self, message):\n",
+ " responses = []\n",
+ " for tool_call in message.tool_calls:\n",
+ " if hasattr(self, tool_call.function.name):\n",
+ " func = getattr(self, tool_call.function.name)\n",
+ " arguments = json.loads(tool_call.function.arguments)\n",
+ " response = func(**arguments)\n",
+ " responses.append({\n",
+ " \"role\": \"tool\",\n",
+ " \"content\": response,\n",
+ " \"tool_call_id\": tool_call.id\n",
+ " })\n",
+ " return responses\n",
+ "\n",
+ " def chat(self, message, history):\n",
+ " history = [{\"role\":h[\"role\"], \"content\":h[\"content\"]} for h in history]\n",
+ " messages = [{\"role\": \"system\", \"content\": self._system_message}] + history + [{\"role\": \"user\", \"content\": message}]\n",
+ " response = self._chat_llm.chat.completions.create(model=self._model, messages=messages, tools=self._tools)\n",
+ "\n",
+ " while response.choices[0].finish_reason==\"tool_calls\":\n",
+ " message = response.choices[0].message\n",
+ " responses = self.handle_tool_call(message)\n",
+ " messages.append(message)\n",
+ " messages.extend(responses)\n",
+ " response = self._chat_llm.chat.completions.create(model=self._model, messages=messages, tools=self._tools)\n",
+ " \n",
+ " return response.choices[0].message.content"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b8433ac6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "Tool = ToolCalling()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e1c80905",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "gr.ChatInterface(Tool.chat, title=\"Ticket Price Tool\", type=\"messages\").launch(inbrowser=False)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5a4232df",
+ "metadata": {},
+ "source": [
+ "## 🎓 **Multimodal AI Agent**\n",
+ "\n",
+ "Extends the ToolCalling class to add multimodal capabilities\n",
+ "\n",
+ "**Key Features**:\n",
+ "- **Speech-to-Text**: Uses OpenAI Whisper for audio transcription\n",
+ "- **Text-to-Speech**: Uses OpenAI TTS for voice responses \n",
+ "- **Image Generation**: Uses DALL-E 3 for destination visualization\n",
+ "- **Model Switching**: Supports both GPT and Ollama models\n",
+ "- **Audio Processing**: Handles audio file validation and transcription\n",
+ "- **Visual Interface**: Custom Gradio interface with audio, image, and chat components\n",
+ "\n",
+ "**Methods**:\n",
+ "- `talker()`: Converts text to speech\n",
+ "- `audio_to_text()`: Transcribes audio to text with robust error handling\n",
+ "- `switch_model()`: Switches between GPT and Ollama models\n",
+ "- `artist()`: Generates destination images using DALL-E 3\n",
+ "- `Interface()`: Creates the multimodal Gradio interface\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "id": "9f1703b0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "\n",
+ "class MultiModalAgent(ToolCalling):\n",
+ " _stt_model = \"gpt-4o-mini-tts\"\n",
+ " _tts_model = \"whisper-1\"\n",
+ " _image_model = \"dall-e-3\"\n",
+ " _openai = OpenAI()\n",
+ "\n",
+ " def __init__(self, db_name=\"prices.db\"):\n",
+ " super().__init__(db_name)\n",
+ " \n",
+ " def talker(self, message):\n",
+ " response = self._openai.audio.speech.create(\n",
+ " model=\"gpt-4o-mini-tts\",\n",
+ " voice=\"onyx\", # Also, try replacing onyx with alloy or coral\n",
+ " input=message\n",
+ " )\n",
+ " return response.content\n",
+ "\n",
+ " def switch_model(self, model):\n",
+ " if model == \"GPT\" and self._model != \"gpt-4o-mini\":\n",
+ " self._chat_llm = OpenAI()\n",
+ " self._model = \"gpt-4o-mini\"\n",
+ " elif model == \"ollama\" and self._model != \"llama3.2\":\n",
+ " self._chat_llm = OpenAI(base_url=\"http://localhost:11434/v1\", api_key=\"ollama\")\n",
+ " self._model = \"llama3.2\"\n",
+ "\n",
+ " def audio_to_text(self, audio_file, history):\n",
+ " \"\"\"Convert audio file to text using OpenAI Whisper\"\"\"\n",
+ "\n",
+ " \n",
+ " result = history + [{\"role\": \"user\", \"content\": \"\"}]\n",
+ " try:\n",
+ " if audio_file is None:\n",
+ " result[-1][\"content\"] = \"No audio file provided\"\n",
+ " return result\n",
+ " \n",
+ " # Ensure we have the file path\n",
+ " if isinstance(audio_file, str):\n",
+ " file_path = audio_file\n",
+ " else:\n",
+ " file_path = audio_file.name if hasattr(audio_file, 'name') else str(audio_file)\n",
+ " \n",
+ " # Check if file exists\n",
+ " if not os.path.exists(file_path):\n",
+ " result[-1][\"content\"] = f\"Audio file not found: {file_path}\"\n",
+ " return result\n",
+ " \n",
+ " # Check file size (Whisper has limits)\n",
+ " file_size = os.path.getsize(file_path)\n",
+ " if file_size > 25 * 1024 * 1024: # 25MB limit\n",
+ " result[-1][\"content\"] = \"Audio file too large (max 25MB)\"\n",
+ " return result\n",
+ " \n",
+ " # Transcribe using OpenAI Whisper\n",
+ " with open(file_path, \"rb\") as audio:\n",
+ " response = self._openai.audio.transcriptions.create(\n",
+ " model=\"whisper-1\",\n",
+ " file=audio,\n",
+ " response_format=\"text\"\n",
+ " )\n",
+ " \n",
+ " # Clean up the transcribed text\n",
+ " text = response.strip()\n",
+ " \n",
+ " if not text:\n",
+ " result[-1][\"content\"] =\"No speech detected in audio\"\n",
+ " return result\n",
+ " \n",
+ " result[-1][\"content\"] = text\n",
+ " return result\n",
+ " \n",
+ " except Exception as e:\n",
+ " error_msg = f\"Audio transcription error: {str(e)}\"\n",
+ " print(f\"{error_msg}\")\n",
+ " result[-1][\"content\"] = error_msg\n",
+ " return result\n",
+ "\n",
+ " def initiate_chat(self, message, history):\n",
+ " return \"\", history + [{\"role\":\"user\", \"content\":message}]\n",
+ "\n",
+ " def handle_tool_call(self, message):\n",
+ " responses = []\n",
+ " cities = []\n",
+ " for tool_call in message.tool_calls:\n",
+ " if hasattr(self, tool_call.function.name):\n",
+ " func = getattr(self, tool_call.function.name)\n",
+ " arguments = json.loads(tool_call.function.arguments)\n",
+ " city = arguments.get('city')\n",
+ " cities.append(city)\n",
+ " response = func(**arguments)\n",
+ " responses.append({\n",
+ " \"role\": \"tool\",\n",
+ " \"content\": response,\n",
+ " \"tool_call_id\": tool_call.id\n",
+ " })\n",
+ " return responses, cities\n",
+ "\n",
+ " def chat(self, history):\n",
+ " history = [{\"role\":h[\"role\"], \"content\":h[\"content\"]} for h in history]\n",
+ " messages = [{\"role\": \"system\", \"content\": self._system_message}] + history\n",
+ " response = self._chat_llm.chat.completions.create(model=self._model, messages=messages, tools=self._tools)\n",
+ " cities = []\n",
+ " image = None\n",
+ "\n",
+ " while response.choices[0].finish_reason==\"tool_calls\":\n",
+ " message = response.choices[0].message\n",
+ " responses, cities = self.handle_tool_call(message)\n",
+ " messages.append(message)\n",
+ " messages.extend(responses)\n",
+ " response = self._chat_llm.chat.completions.create(model=self._model, messages=messages, tools=self._tools)\n",
+ "\n",
+ " reply = response.choices[0].message.content\n",
+ " history += [{\"role\":\"assistant\", \"content\":reply}]\n",
+ "\n",
+ " voice = self.talker(reply)\n",
+ "\n",
+ " if cities:\n",
+ " image = self.artist(cities[0])\n",
+ " \n",
+ " return history, voice, image\n",
+ "\n",
+ "\n",
+ " def Interface(self, title, name, desc):\n",
+ " with gr.Blocks(title=title) as ui:\n",
+ " with gr.Column():\n",
+ " gr.Markdown(f\"\"\"\n",
+ " \n",
+ "
✈️ {name}
\n",
+ "
\n",
+ " {desc}\n",
+ "
\n",
+ "
\n",
+ " \"\"\")\n",
+ " with gr.Row():\n",
+ " chatbot = gr.Chatbot(height=500, type=\"messages\")\n",
+ " image_output = gr.Image(height=500, interactive=False)\n",
+ " with gr.Row():\n",
+ " audio_output = gr.Audio(autoplay=True, scale=1)\n",
+ " with gr.Row():\n",
+ " with gr.Column():\n",
+ " mic = gr.Mic(label=\"Talk to the AI Assistant\", type=\"filepath\", editable=False)\n",
+ " submit = gr.Button(\"Submit\", size=\"lg\", variant=\"primary\")\n",
+ " model = gr.Dropdown(choices=[\"GPT\", \"ollama\"], value=\"GPT\", label=\"Model\")\n",
+ " message = gr.Textbox(label=\"Chat with our AI Assistant:\")\n",
+ "\n",
+ " model.change(fn=self.switch_model, inputs=model)\n",
+ " submit.click(fn=self.audio_to_text, inputs=[mic, chatbot], outputs=chatbot).then(\n",
+ " fn=self.chat, inputs=chatbot, outputs=[chatbot, audio_output, image_output]\n",
+ " )\n",
+ " message.submit(fn=self.initiate_chat, inputs=[message, chatbot], outputs=[message, chatbot]).then(\n",
+ " fn=self.chat, inputs=chatbot, outputs=[chatbot, audio_output, image_output]\n",
+ " )\n",
+ "\n",
+ " ui.launch()\n",
+ "\n",
+ " def artist(self, city):\n",
+ " image_response = self._openai.images.generate(\n",
+ " model=\"dall-e-3\",\n",
+ " prompt=f\"An image representing a vacation in {city}, showing tourist spots and everything unique about {city}, in a vibrant pop-art style\",\n",
+ " size=\"1024x1024\",\n",
+ " n=1,\n",
+ " response_format=\"b64_json\",\n",
+ " )\n",
+ " image_base64 = image_response.data[0].b64_json\n",
+ " image_data = base64.b64decode(image_base64)\n",
+ " return Image.open(BytesIO(image_data))\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dbfbc26f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "Agent = MultiModalAgent()\n",
+ "Agent.Interface(\"FlightAI - Your Travel Assistant\", \"FlightAI\", \"Your intelligent travel assistant that helps you discover amazing destinations, find the best flight deals, and guides you through the booking process.\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": ".venv",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}