From 44e0508ae5661505b938fa7d8b0ab2761a637088 Mon Sep 17 00:00:00 2001 From: martin legrand Date: Sat, 29 Mar 2025 13:31:13 +0100 Subject: [PATCH] fix : server problem with llamacpp --- server/sources/llamacpp.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/server/sources/llamacpp.py b/server/sources/llamacpp.py index 09fd465..76252ec 100644 --- a/server/sources/llamacpp.py +++ b/server/sources/llamacpp.py @@ -9,13 +9,15 @@ class LlamacppLLM(GeneratorLLM): Handle generation using llama.cpp """ super().__init__() - self.llm = Llama.from_pretrained( - repo_id=self.model, - filename="*q8_0.gguf", - verbose=True - ) + self.llm = None def generate(self, history): + if self.model is None: + self.llm = Llama.from_pretrained( + repo_id=self.model, + filename="*q8_0.gguf", + verbose=True + ) self.logger.info(f"Using {self.model} for generation with Llama.cpp") self.llm.create_chat_completion( messages = history