tcsenpai/agenticSeek
1
0
Fork 0
mirror of https://github.com/tcsenpai/agenticSeek.git synced 2025-06-06 19:15:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix : server problem with llamacpp

Browse Source
This commit is contained in:
martin legrand 2025-03-29 13:31:13 +01:00
parent 4676b817e9
commit 44e0508ae5
1 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
server/sources/llamacpp.py
View File

@ -9,13 +9,15 @@ class LlamacppLLM(GeneratorLLM):
Handle generation using llama.cpp Handle generation using llama.cpp
""" """
super().__init__() super().__init__()
self.llm = Llama.from_pretrained( self.llm = None
repo_id=self.model,
filename="*q8_0.gguf",
verbose=True
)
def generate(self, history): def generate(self, history):
if self.model is None:
self.llm = Llama.from_pretrained(
repo_id=self.model,
filename="*q8_0.gguf",
verbose=True
)
self.logger.info(f"Using {self.model} for generation with Llama.cpp") self.logger.info(f"Using {self.model} for generation with Llama.cpp")
self.llm.create_chat_completion( self.llm.create_chat_completion(
messages = history messages = history