From 7aa08a77caf0121b9c9be8df1922b5fb8679d377 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Mon, 29 Apr 2024 10:07:30 -0400 Subject: [PATCH] llm: dont cap context window limit to training context window (#3988) --- llm/server.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llm/server.go b/llm/server.go index 84babe46..231100a0 100644 --- a/llm/server.go +++ b/llm/server.go @@ -73,8 +73,7 @@ func LoadModel(model string) (*GGML, error) { func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, projectors []string, opts api.Options) (LlamaServer, error) { var err error if opts.NumCtx > int(ggml.KV().ContextLength()) { - slog.Warn("requested context length is greater than model max context length", "requested", opts.NumCtx, "model", ggml.KV().ContextLength()) - opts.NumCtx = int(ggml.KV().ContextLength()) + slog.Warn("requested context length is greater than the model's training context window size", "requested", opts.NumCtx, "training size", ggml.KV().ContextLength()) } if opts.NumCtx < 4 {