brad/LLM_Engineering_OLD
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added timeouts to give Modal time

Browse Source
This commit is contained in:
Edward Donner
2024-11-13 23:38:39 +00:00
parent 8f9c0905ee
commit 3189509cfa
4 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
week8/llama.py
View File

@@ -7,11 +7,11 @@ app = modal.App("llama")
image = Image.debian_slim().pip_install("torch", "transformers", "bitsandbytes", "accelerate")
secrets = [modal.Secret.from_name("hf-secret")]
GPU = "T4"
MODEL_NAME = "meta-llama/Meta-Llama-3.1-8B"
MODEL_NAME = "meta-llama/Meta-Llama-3.1-8B" # "google/gemma-2-2b"
@app.function(image=image, secrets=secrets, gpu=GPU)
@app.function(image=image, secrets=secrets, gpu=GPU, timeout=1800)
def generate(prompt: str) -> str:
import os
import torch