{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "dfe37963-1af6-44fc-a841-8e462443f5e6",
   "metadata": {},
   "source": [
    "## Expert Knowledge Worker\n",
    "\n",
    "### A question answering agent that is an expert knowledge worker\n",
    "### To be used by employees of Insurellm, an Insurance Tech company\n",
    "### The agent needs to be accurate and the solution should be low cost.\n",
    "\n",
    "This project will use RAG (Retrieval Augmented Generation) to ensure our question/answering assistant has high accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba2779af-84ef-4227-9e9e-6eaf0df87e77",
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "\n",
    "import os\n",
    "import glob\n",
    "from dotenv import load_dotenv\n",
    "import gradio as gr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "802137aa-8a74-45e0-a487-d1974927d7ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports for langchain\n",
    "\n",
    "from langchain.document_loaders import DirectoryLoader, TextLoader\n",
    "from langchain.text_splitter import CharacterTextSplitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58c85082-e417-4708-9efe-81a5d55d1424",
   "metadata": {},
   "outputs": [],
   "source": [
    "# price is a factor for our company, so we're going to use a low cost model\n",
    "\n",
    "MODEL = \"gpt-4o-mini\"\n",
    "db_name = \"vector_db\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee78efcb-60fe-449e-a944-40bab26261af",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load environment variables in a file called .env\n",
    "\n",
    "load_dotenv()\n",
    "os.environ['OPENAI_API_KEY'] = os.getenv('OPENAI_API_KEY', 'your-key-if-not-using-env')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "730711a9-6ffe-4eee-8f48-d6cfb7314905",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in documents using LangChain's loaders\n",
    "# Take everything in all the sub-folders of our knowledgebase\n",
    "\n",
    "folders = glob.glob(\"knowledge-base/*\")\n",
    "\n",
    "# With thanks to CG and Jon R, students on the course, for this fix needed for some users \n",
    "text_loader_kwargs = {'encoding': 'utf-8'}\n",
    "# If that doesn't work, some Windows users might need to uncomment the next line instead\n",
    "# text_loader_kwargs={'autodetect_encoding': True}\n",
    "\n",
    "documents = []\n",
    "for folder in folders:\n",
    "    doc_type = os.path.basename(folder)\n",
    "    loader = DirectoryLoader(folder, glob=\"**/*.md\", loader_cls=TextLoader, loader_kwargs=text_loader_kwargs)\n",
    "    folder_docs = loader.load()\n",
    "    for doc in folder_docs:\n",
    "        doc.metadata[\"doc_type\"] = doc_type\n",
    "        documents.append(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "252f17e9-3529-4e81-996c-cfa9f08e75a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e8decb0-d9b0-4d51-8402-7a6174d22159",
   "metadata": {},
   "outputs": [],
   "source": [
    "documents[24]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7310c9c8-03c1-4efc-a104-5e89aec6db1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
    "chunks = text_splitter.split_documents(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd06e02f-6d9b-44cc-a43d-e1faa8acc7bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2562754-9052-4aae-92c1-37236435ea06",
   "metadata": {},
   "outputs": [],
   "source": [
    "chunks[6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c54b4b6-06da-463d-bee7-4dd456c2b887",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_types = set(chunk.metadata['doc_type'] for chunk in chunks)\n",
    "print(f\"Document types found: {', '.join(doc_types)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "128c73f7-f149-4904-a554-8140941fce0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "for chunk in chunks:\n",
    "    if 'CEO' in chunk.page_content:\n",
    "        print(chunk)\n",
    "        print(\"_________\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6965971c-fb97-482c-a497-4e81a0ac83df",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
