tcsenpai/ollama
1
0
Fork 0
mirror of https://github.com/tcsenpai/ollama.git synced 2025-06-08 12:15:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

working causal attention

Browse Source
This commit is contained in:
Josh Yan 2024-08-27 11:34:32 -07:00
parent 80eef7c7b1
commit a6d30ecefe
1 changed files with 17 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
llm/patches/12-paligemma.diff
View File

@ -1,26 +1,25 @@
diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp
index 54aa822c..67a02c4c 100644 index 9c0d351e..019a147c 100644
--- a/examples/llava/clip.cpp --- a/examples/llava/clip.cpp
+++ b/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp
@@ -764,11 +764,12 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 @@ -718,10 +718,12 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
if (ctx->proj_type == PROJECTOR_TYPE_MLP) {
embeddings = ggml_mul_mat(ctx0, model.mm_0_w, embeddings); embeddings = ggml_mul_mat(ctx0, model.mm_0_w, embeddings);
embeddings = ggml_add(ctx0, embeddings, model.mm_0_b); embeddings = ggml_add(ctx0, embeddings, model.mm_0_b);
-
- embeddings = ggml_gelu(ctx0, embeddings); - embeddings = ggml_gelu(ctx0, embeddings);
- embeddings = ggml_mul_mat(ctx0, model.mm_2_w, embeddings); - embeddings = ggml_mul_mat(ctx0, model.mm_2_w, embeddings);
- embeddings = ggml_add(ctx0, embeddings, model.mm_2_b); - embeddings = ggml_add(ctx0, embeddings, model.mm_2_b);
- -
+ if (model.mm_2_w) + if (model.mm_2_w)
+ { + {
+ embeddings = ggml_gelu(ctx0, embeddings); + embeddings = ggml_gelu(ctx0, embeddings);
+ embeddings = ggml_mul_mat(ctx0, model.mm_2_w, embeddings); + embeddings = ggml_mul_mat(ctx0, model.mm_2_w, embeddings);
+ embeddings = ggml_add(ctx0, embeddings, model.mm_2_b); + embeddings = ggml_add(ctx0, embeddings, model.mm_2_b);
+ } + }
} else if (ctx->proj_type == PROJECTOR_TYPE_MLP_NORM) { } else if (ctx->proj_type == PROJECTOR_TYPE_MLP_NORM) {
embeddings = ggml_mul_mat(ctx0, model.mm_0_w, embeddings); embeddings = ggml_mul_mat(ctx0, model.mm_0_w, embeddings);
embeddings = ggml_add(ctx0, embeddings, model.mm_0_b); embeddings = ggml_add(ctx0, embeddings, model.mm_0_b);
@@ -2542,6 +2543,10 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { @@ -2102,6 +2104,10 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->vision_model.mm_model_peg_0_b->ne[0]; return ctx->vision_model.mm_model_peg_0_b->ne[0];
} }
if (ctx->proj_type == PROJECTOR_TYPE_MLP) { if (ctx->proj_type == PROJECTOR_TYPE_MLP) {
@ -32,7 +31,7 @@ index 54aa822c..67a02c4c 100644
} }
if (ctx->proj_type == PROJECTOR_TYPE_MLP_NORM) { if (ctx->proj_type == PROJECTOR_TYPE_MLP_NORM) {
diff --git a/include/llama.h b/include/llama.h diff --git a/include/llama.h b/include/llama.h
index ce07f4fa..07b09c9a 100644 index 6072e76e..4c572a74 100644
--- a/include/llama.h --- a/include/llama.h
+++ b/include/llama.h +++ b/include/llama.h
@@ -444,6 +444,12 @@ extern "C" { @@ -444,6 +444,12 @@ extern "C" {
@ -49,10 +48,10 @@ index ce07f4fa..07b09c9a 100644
LLAMA_API size_t llama_max_devices(void); LLAMA_API size_t llama_max_devices(void);
diff --git a/src/llama.cpp b/src/llama.cpp diff --git a/src/llama.cpp b/src/llama.cpp
index 7f2f0003..754d3d5f 100644 index d883ed19..322b4b59 100644
--- a/src/llama.cpp --- a/src/llama.cpp
+++ b/src/llama.cpp +++ b/src/llama.cpp
@@ -2719,6 +2719,8 @@ struct llama_context { @@ -2710,6 +2710,8 @@ struct llama_context {
bool logits_all = false; bool logits_all = false;
@ -61,7 +60,7 @@ index 7f2f0003..754d3d5f 100644
// embeddings output (2-dimensional array: [n_outputs][n_embd]) // embeddings output (2-dimensional array: [n_outputs][n_embd])
// populated only when pooling_type == LLAMA_POOLING_TYPE_NONE // populated only when pooling_type == LLAMA_POOLING_TYPE_NONE
size_t embd_size = 0; // capacity (of floats) for embeddings size_t embd_size = 0; // capacity (of floats) for embeddings
@@ -11660,6 +11662,15 @@ struct llm_build_context { @@ -11591,6 +11593,15 @@ struct llm_build_context {
inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
@ -77,7 +76,7 @@ index 7f2f0003..754d3d5f 100644
inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd)); inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd));
cb(inpL, "inp_scaled", -1); cb(inpL, "inp_scaled", -1);
@@ -14565,6 +14576,7 @@ static int llama_decode_internal( @@ -14468,6 +14479,7 @@ static int llama_decode_internal(
const int64_t n_embd = hparams.n_embd; const int64_t n_embd = hparams.n_embd;
const int64_t n_vocab = hparams.n_vocab; const int64_t n_vocab = hparams.n_vocab;
@ -85,16 +84,17 @@ index 7f2f0003..754d3d5f 100644
uint32_t n_outputs = 0; uint32_t n_outputs = 0;
uint32_t n_outputs_prev = 0; uint32_t n_outputs_prev = 0;
@@ -14678,7 +14690,7 @@ static int llama_decode_internal( @@ -14581,7 +14593,8 @@ static int llama_decode_internal(
} }
// non-causal masks do not use the KV cache // non-causal masks do not use the KV cache
- if (hparams.causal_attn) { - if (hparams.causal_attn) {
+ if (hparams.causal_attn || lctx.image_embeds) { + if (hparams.causal_attn || lctx.image_embeds)
+ {
llama_kv_cache_update(&lctx); llama_kv_cache_update(&lctx);
// if we have enough unused cells before the current head -> // if we have enough unused cells before the current head ->
@@ -16589,6 +16601,16 @@ void llama_free_model(struct llama_model * model) { @@ -16455,6 +16468,16 @@ void llama_free_model(struct llama_model * model) {
delete model; delete model;
} }