feat : increase ctx size

2025-06-06 11:05:26 +00:00 · 2025-03-29 19:54:45 +01:00 · 2025-03-29 19:54:45 +01:00 · 875ab3bd8e
commit 875ab3bd8e
parent 1dd8e0a016
1 changed files with 1 additions and 0 deletions
--- a/server/sources/llamacpp_handler.py
+++ b/server/sources/llamacpp_handler.py
@ -17,6 +17,7 @@ class LlamacppLLM(GeneratorLLM):
            self.llm = Llama.from_pretrained(
                repo_id=self.model,
                filename="*Q8_0.gguf",
+                n_ctx=4096,
                verbose=True
            )
        self.logger.info(f"Using {self.model} for generation with Llama.cpp")