brad/LLM_Engineering_OLD
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

small fixes

Browse Source
This commit is contained in:
Dmitry Kisselev
2025-10-19 14:59:38 -07:00
parent 1268763737
commit f0718b6512
1 changed files with 1895 additions and 1827 deletions
Download Patch File Download Diff File Expand all files Collapse all files

376
week3/community-contributions/dkisselev-zz/Week3_Excercise_Synthetic_Dataset_PGx.ipynb
View File

@@ -3,7 +3,9 @@
{
"cell_type": "markdown",
"id": "57eebd05",
"metadata": {},
"metadata": {
"id": "57eebd05"
},
"source": [
"# Synthetic Dataset Generator with Quality Scoring\n",
"\n",
@@ -29,21 +31,45 @@
"cell_type": "code",
"execution_count": null,
"id": "a1673e5a",
"metadata": {},
"metadata": {
"id": "a1673e5a"
},
"outputs": [],
"source": [
"# Install dependencies\n",
"%pip install -q --upgrade torch==2.5.1+cu124 torchvision==0.20.1+cu124 torchaudio==2.5.1+cu124 --index-url https://download.pytorch.org/whl/cu124\n",
"%pip install -q requests bitsandbytes==0.46.0 transformers==4.48.3 accelerate==1.3.0\n",
"%pip install -q anthropic openai gradio nltk pandas pyarrow\n",
"%pip install -q google-generativeai deepseek-ai\n"
"%pip install -q --upgrade torch==2.6.0+cu124 --index-url https://download.pytorch.org/whl/cu124\n",
"%pip install -q requests bitsandbytes==0.48.1 transformers==4.57.1 accelerate==1.10.1\n",
"%pip install -q openai gradio nltk pandas\n"
]
},
{
"cell_type": "code",
"source": [
"gpu_info = !nvidia-smi\n",
"gpu_info = '\\n'.join(gpu_info)\n",
"if gpu_info.find('failed') >= 0:\n",
" print('Not connected to a GPU')\n",
"else:\n",
" print(gpu_info)\n",
" if gpu_info.find('Tesla T4') >= 0:\n",
" print(\"Success - Connected to a T4\")\n",
" else:\n",
" print(\"NOT CONNECTED TO A T4\")"
],
"metadata": {
"id": "m-yhYlN4OQEC"
},
"id": "m-yhYlN4OQEC",
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "5ab3109c",
"metadata": {},
"metadata": {
"id": "5ab3109c"
},
"outputs": [],
"source": [
"# Imports and Setup\n",
@@ -91,7 +117,9 @@
"cell_type": "code",
"execution_count": null,
"id": "a206f9d4",
"metadata": {},
"metadata": {
"id": "a206f9d4"
},
"outputs": [],
"source": [
"# API Key Setup - Support both Colab and Local environments\n",
@@ -146,7 +174,7 @@
" # clients['anthropic'] = anthropic.Anthropic(api_key=api_keys['anthropic'])\n",
" if api_keys['google']:\n",
" # genai.configure(api_key=api_keys['google'])\n",
" clients['gemini'] = OpenAI(api_key=api_keys['google'], base_url=gemini_url)\n",
" clients['google'] = OpenAI(api_key=api_keys['google'], base_url=gemini_url)\n",
" if api_keys['deepseek']:\n",
" clients['deepseek'] = OpenAI(api_key=api_keys['deepseek'], base_url=deepseek_url)\n",
" # clients['deepseek'] = DeepSeek(api_key=api_keys['deepseek'])\n",
@@ -165,7 +193,9 @@
"cell_type": "code",
"execution_count": null,
"id": "5a791f39",
"metadata": {},
"metadata": {
"id": "5a791f39"
},
"outputs": [],
"source": [
"# Model Configuration\n",
@@ -273,11 +303,25 @@
"print(f\"🌐 Available Commercial models: {len(COMMERCIAL_MODELS)}\")\n"
]
},
{
"cell_type": "code",
"source": [
"schema_manager.generate_schema_with_llm(\"realstate dataset for residential houses\",'Gemini 2.5 Flash', 0.7)"
],
"metadata": {
"id": "dFYWA5y0ZmJr"
},
"id": "dFYWA5y0ZmJr",
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "5d2f459a",
"metadata": {},
"metadata": {
"id": "5d2f459a"
},
"outputs": [],
"source": [
"# Schema Management Module\n",
@@ -298,7 +342,7 @@
" Use realistic field names and appropriate data types (TEXT, INT, FLOAT, BOOLEAN, ARRAY).\n",
" Provide clear descriptions and realistic examples.\"\"\"\n",
"\n",
" user_prompt = f\"\"\"Business case: {business_case}\n",
" user_prompt = f\"\"\"\\n\\nBusiness case: {business_case}\n",
"\n",
" Generate a dataset schema for this business case. Include fields that would be relevant for analysis and decision-making.\"\"\"\n",
"\n",
@@ -319,7 +363,7 @@
"\n",
" Return the enhanced schema in the same format as the original.\"\"\"\n",
"\n",
" user_prompt = f\"\"\"Business case: {business_case}\n",
" user_prompt = f\"\"\"\\n\\nBusiness case: {business_case}\n",
"\n",
" Current partial schema:\n",
" {partial_schema}\n",
@@ -396,6 +440,7 @@
" provider = model_info[\"provider\"]\n",
" model_id = model_info[\"model_id\"]\n",
"\n",
"\n",
" try:\n",
" response = clients[provider].chat.completions.create(\n",
" model=model_id,\n",
@@ -419,7 +464,9 @@
"cell_type": "code",
"execution_count": null,
"id": "dd37ee66",
"metadata": {},
"metadata": {
"id": "dd37ee66"
},
"outputs": [],
"source": [
"# Dataset Generation Module\n",
@@ -627,136 +674,13 @@
"print(\"✅ Dataset Generation Module loaded!\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "756883cd",
"metadata": {},
"outputs": [],
"source": [
"# Synonym Permutation Module\n",
"class SynonymPermutator:\n",
" \"\"\"Handles synonym replacement to increase dataset diversity\"\"\"\n",
" \n",
" def __init__(self):\n",
" self.synonym_cache = {} # Cache for synonyms to avoid repeated lookups\n",
" \n",
" def get_synonyms(self, word: str) -> List[str]:\n",
" \"\"\"Get synonyms for a word using NLTK WordNet\"\"\"\n",
" if word.lower() in self.synonym_cache:\n",
" return self.synonym_cache[word.lower()]\n",
" \n",
" synonyms = set()\n",
" try:\n",
" for syn in wordnet.synsets(word.lower()):\n",
" for lemma in syn.lemmas():\n",
" synonym = lemma.name().replace('_', ' ').lower()\n",
" if synonym != word.lower() and len(synonym) > 2:\n",
" synonyms.add(synonym)\n",
" except:\n",
" pass\n",
" \n",
" # Filter out very similar words and keep only relevant ones\n",
" filtered_synonyms = []\n",
" for syn in synonyms:\n",
" if (len(syn) >= 3 and \n",
" syn != word.lower() and \n",
" not syn.endswith('ing') or word.endswith('ing') and\n",
" not syn.endswith('ed') or word.endswith('ed')):\n",
" filtered_synonyms.append(syn)\n",
" \n",
" # Limit to 5 synonyms max\n",
" filtered_synonyms = filtered_synonyms[:5]\n",
" self.synonym_cache[word.lower()] = filtered_synonyms\n",
" return filtered_synonyms\n",
" \n",
" def identify_text_fields(self, dataset: List[Dict]) -> List[str]:\n",
" \"\"\"Auto-detect text fields suitable for synonym permutation\"\"\"\n",
" if not dataset:\n",
" return []\n",
" \n",
" text_fields = []\n",
" for key, value in dataset[0].items():\n",
" if isinstance(value, str) and len(value) > 3:\n",
" # Check if field contains meaningful text (not just IDs or codes)\n",
" if not re.match(r'^[A-Z0-9_\\-]+$', value) and not value.isdigit():\n",
" text_fields.append(key)\n",
" \n",
" return text_fields\n",
" \n",
" def permute_with_synonyms(self, dataset: List[Dict], fields_to_permute: List[str], \n",
" permutation_rate: float = 0.3) -> Tuple[List[Dict], Dict[str, int]]:\n",
" \"\"\"Replace words with synonyms in specified fields\"\"\"\n",
" if not dataset or not fields_to_permute:\n",
" return dataset, {}\n",
" \n",
" permuted_dataset = []\n",
" replacement_stats = {field: 0 for field in fields_to_permute}\n",
" \n",
" for record in dataset:\n",
" permuted_record = record.copy()\n",
" \n",
" for field in fields_to_permute:\n",
" if field in record and isinstance(record[field], str):\n",
" original_text = record[field]\n",
" permuted_text = self._permute_text(original_text, permutation_rate)\n",
" permuted_record[field] = permuted_text\n",
" \n",
" # Count replacements\n",
" if original_text != permuted_text:\n",
" replacement_stats[field] += 1\n",
" \n",
" permuted_dataset.append(permuted_record)\n",
" \n",
" return permuted_dataset, replacement_stats\n",
" \n",
" def _permute_text(self, text: str, permutation_rate: float) -> str:\n",
" \"\"\"Permute words in text with synonyms\"\"\"\n",
" words = text.split()\n",
" if len(words) < 2: # Skip very short texts\n",
" return text\n",
" \n",
" num_replacements = max(1, int(len(words) * permutation_rate))\n",
" words_to_replace = random.sample(range(len(words)), min(num_replacements, len(words)))\n",
" \n",
" permuted_words = words.copy()\n",
" for word_idx in words_to_replace:\n",
" word = words[word_idx]\n",
" # Clean word for synonym lookup\n",
" clean_word = re.sub(r'[^\\w]', '', word.lower())\n",
" \n",
" if len(clean_word) > 3: # Only replace meaningful words\n",
" synonyms = self.get_synonyms(clean_word)\n",
" if synonyms:\n",
" chosen_synonym = random.choice(synonyms)\n",
" # Preserve original capitalization and punctuation\n",
" if word.isupper():\n",
" chosen_synonym = chosen_synonym.upper()\n",
" elif word.istitle():\n",
" chosen_synonym = chosen_synonym.title()\n",
" \n",
" permuted_words[word_idx] = word.replace(clean_word, chosen_synonym)\n",
" \n",
" return ' '.join(permuted_words)\n",
" \n",
" def get_permutation_preview(self, text: str, permutation_rate: float = 0.3) -> str:\n",
" \"\"\"Get a preview of how text would look after permutation\"\"\"\n",
" return self._permute_text(text, permutation_rate)\n",
" \n",
" def clear_cache(self):\n",
" \"\"\"Clear the synonym cache to free memory\"\"\"\n",
" self.synonym_cache.clear()\n",
"\n",
"# Initialize synonym permutator\n",
"synonym_permutator = SynonymPermutator()\n",
"print(\"✅ Synonym Permutation Module loaded!\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "350a1468",
"metadata": {},
"metadata": {
"id": "350a1468"
},
"outputs": [],
"source": [
"# Quality Scoring Module\n",
@@ -979,11 +903,140 @@
"print(\"✅ Quality Scoring Module loaded!\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "756883cd",
"metadata": {
"id": "756883cd"
},
"outputs": [],
"source": [
"# Synonym Permutation Module\n",
"class SynonymPermutator:\n",
" \"\"\"Handles synonym replacement to increase dataset diversity\"\"\"\n",
"\n",
" def __init__(self):\n",
" self.synonym_cache = {} # Cache for synonyms to avoid repeated lookups\n",
"\n",
" def get_synonyms(self, word: str) -> List[str]:\n",
" \"\"\"Get synonyms for a word using NLTK WordNet\"\"\"\n",
" if word.lower() in self.synonym_cache:\n",
" return self.synonym_cache[word.lower()]\n",
"\n",
" synonyms = set()\n",
" try:\n",
" for syn in wordnet.synsets(word.lower()):\n",
" for lemma in syn.lemmas():\n",
" synonym = lemma.name().replace('_', ' ').lower()\n",
" if synonym != word.lower() and len(synonym) > 2:\n",
" synonyms.add(synonym)\n",
" except:\n",
" pass\n",
"\n",
" # Filter out very similar words and keep only relevant ones\n",
" filtered_synonyms = []\n",
" for syn in synonyms:\n",
" if (len(syn) >= 3 and\n",
" syn != word.lower() and\n",
" not syn.endswith('ing') or word.endswith('ing') and\n",
" not syn.endswith('ed') or word.endswith('ed')):\n",
" filtered_synonyms.append(syn)\n",
"\n",
" # Limit to 5 synonyms max\n",
" filtered_synonyms = filtered_synonyms[:5]\n",
" self.synonym_cache[word.lower()] = filtered_synonyms\n",
" return filtered_synonyms\n",
"\n",
" def identify_text_fields(self, dataset: List[Dict]) -> List[str]:\n",
" \"\"\"Auto-detect text fields suitable for synonym permutation\"\"\"\n",
" if not dataset:\n",
" return []\n",
"\n",
" text_fields = []\n",
" for key, value in dataset[0].items():\n",
" if isinstance(value, str) and len(value) > 3:\n",
" # Check if field contains meaningful text (not just IDs or codes)\n",
" if not re.match(r'^[A-Z0-9_\\-]+$', value) and not value.isdigit():\n",
" text_fields.append(key)\n",
"\n",
" return text_fields\n",
"\n",
" def permute_with_synonyms(self, dataset: List[Dict], fields_to_permute: List[str],\n",
" permutation_rate: float = 0.3) -> Tuple[List[Dict], Dict[str, int]]:\n",
" \"\"\"Replace words with synonyms in specified fields\"\"\"\n",
" if not dataset or not fields_to_permute:\n",
" return dataset, {}\n",
"\n",
" permuted_dataset = []\n",
" replacement_stats = {field: 0 for field in fields_to_permute}\n",
"\n",
" for record in dataset:\n",
" permuted_record = record.copy()\n",
"\n",
" for field in fields_to_permute:\n",
" if field in record and isinstance(record[field], str):\n",
" original_text = record[field]\n",
" permuted_text = self._permute_text(original_text, permutation_rate)\n",
" permuted_record[field] = permuted_text\n",
"\n",
" # Count replacements\n",
" if original_text != permuted_text:\n",
" replacement_stats[field] += 1\n",
"\n",
" permuted_dataset.append(permuted_record)\n",
"\n",
" return permuted_dataset, replacement_stats\n",
"\n",
" def _permute_text(self, text: str, permutation_rate: float) -> str:\n",
" \"\"\"Permute words in text with synonyms\"\"\"\n",
" words = text.split()\n",
" if len(words) < 2: # Skip very short texts\n",
" return text\n",
"\n",
" num_replacements = max(1, int(len(words) * permutation_rate))\n",
" words_to_replace = random.sample(range(len(words)), min(num_replacements, len(words)))\n",
"\n",
" permuted_words = words.copy()\n",
" for word_idx in words_to_replace:\n",
" word = words[word_idx]\n",
" # Clean word for synonym lookup\n",
" clean_word = re.sub(r'[^\\w]', '', word.lower())\n",
"\n",
" if len(clean_word) > 3: # Only replace meaningful words\n",
" synonyms = self.get_synonyms(clean_word)\n",
" if synonyms:\n",
" chosen_synonym = random.choice(synonyms)\n",
" # Preserve original capitalization and punctuation\n",
" if word.isupper():\n",
" chosen_synonym = chosen_synonym.upper()\n",
" elif word.istitle():\n",
" chosen_synonym = chosen_synonym.title()\n",
"\n",
" permuted_words[word_idx] = word.replace(clean_word, chosen_synonym)\n",
"\n",
" return ' '.join(permuted_words)\n",
"\n",
" def get_permutation_preview(self, text: str, permutation_rate: float = 0.3) -> str:\n",
" \"\"\"Get a preview of how text would look after permutation\"\"\"\n",
" return self._permute_text(text, permutation_rate)\n",
"\n",
" def clear_cache(self):\n",
" \"\"\"Clear the synonym cache to free memory\"\"\"\n",
" self.synonym_cache.clear()\n",
"\n",
"# Initialize synonym permutator\n",
"synonym_permutator = SynonymPermutator()\n",
"print(\"✅ Synonym Permutation Module loaded!\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cda75e7c",
"metadata": {},
"metadata": {
"id": "cda75e7c"
},
"outputs": [],
"source": [
"# Output & Export Module\n",
@@ -1123,7 +1176,9 @@
"cell_type": "code",
"execution_count": null,
"id": "2a85481e",
"metadata": {},
"metadata": {
"id": "2a85481e"
},
"outputs": [],
"source": [
"# Global state variables\n",
@@ -1233,7 +1288,9 @@
"cell_type": "code",
"execution_count": null,
"id": "ccc985a6",
"metadata": {},
"metadata": {
"id": "ccc985a6"
},
"outputs": [],
"source": [
"# Create Gradio Interface\n",
@@ -1459,8 +1516,7 @@
" )\n",
"\n",
" quality_report = gr.JSON(\n",
" label=\"Quality Report\",\n",
" interactive=False\n",
" label=\"Quality Report\"\n",
" )\n",
"\n",
" # Tab 5: Export\n",
@@ -1551,7 +1607,9 @@
"cell_type": "code",
"execution_count": null,
"id": "70d39131",
"metadata": {},
"metadata": {
"id": "70d39131"
},
"outputs": [],
"source": [
"# Launch the Gradio Interface\n",
@@ -1563,7 +1621,9 @@
{
"cell_type": "markdown",
"id": "212aa78a",
"metadata": {},
"metadata": {
"id": "212aa78a"
},
"source": [
"## Example Workflow: Pharmacogenomics Dataset\n",
"\n",
@@ -1611,7 +1671,9 @@
"cell_type": "code",
"execution_count": null,
"id": "9789613e",
"metadata": {},
"metadata": {
"id": "9789613e"
},
"outputs": [],
"source": [
"# Testing and Validation Functions\n",
@@ -1633,7 +1695,7 @@
" result = schema_manager.generate_schema_with_llm(\n",
" \"Generate a dataset for e-commerce customer analysis\",\n",
" \"GPT-5 Mini\",\n",
" 0.7\n",
" 1\n",
" )\n",
" print(f\"✅ OpenAI schema generation: {len(result)} characters\")\n",
"\n",
@@ -1654,7 +1716,7 @@
" if \"openai\" in clients:\n",
" print(\"🔄 Testing OpenAI dataset generation...\")\n",
" status, records = dataset_generator.generate_dataset(\n",
" test_schema, business_case, \"GPT-4o Mini\", 0.7, 5, \"\"\n",
" test_schema, business_case, \"GPT-5 Mini\", 1, 5, \"\"\n",
" )\n",
" print(f\"✅ OpenAI generation: {status}\")\n",
" if records:\n",
@@ -1731,7 +1793,9 @@
{
"cell_type": "markdown",
"id": "6577036b",
"metadata": {},
"metadata": {
"id": "6577036b"
},
"source": [
"## 🎯 Key Features Summary\n",
"\n",
@@ -1807,8 +1871,7 @@
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
@@ -1822,7 +1885,12 @@
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.10"
}
},
"colab": {
"provenance": [],
"gpuType": "T4"
},
"accelerator": "GPU"
},
"nbformat": 4,
"nbformat_minor": 5