From ce0dc33cb809405fda18a8077da4058d1f7a5374 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Thu, 6 Jun 2024 23:14:33 -0700 Subject: [PATCH] llm: patch to fix qwen 2 temporarily on nvidia (#4897) --- llm/patches/06-qwen2.diff | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 llm/patches/06-qwen2.diff diff --git a/llm/patches/06-qwen2.diff b/llm/patches/06-qwen2.diff new file mode 100644 index 00000000..d7b0c155 --- /dev/null +++ b/llm/patches/06-qwen2.diff @@ -0,0 +1,13 @@ +diff --git a/llama.cpp b/llama.cpp +index 40d2ec2c..f34eb79a 100644 +--- a/llama.cpp ++++ b/llama.cpp +@@ -6943,7 +6943,7 @@ static struct ggml_tensor * llm_build_kqv( + struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q); + cb(kq, "kq", il); + +- if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX) { ++ if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_QWEN2) { + // for this arch, we need to perform the KQ multiplication with F32 precision, otherwise we get NaNs + // ref: https://github.com/ggerganov/llama.cpp/pull/4490#issuecomment-1859055847 + ggml_mul_mat_set_prec(kq, GGML_PREC_F32);