tcsenpai/ollama
1
0
Fork 0
mirror of https://github.com/tcsenpai/ollama.git synced 2025-06-08 20:25:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

runner.go: Update TODOs

Browse Source
This commit is contained in:
Jesse Gross 2024-08-27 17:05:23 -07:00 committed by jmorganca
parent 46a7c682f2
commit c8a1741d9b
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
llama/runner/runner.go
View File

@ -176,6 +176,8 @@ func (s *Server) shiftContext(seqIndex int) {
slog.Debug("context limit hit - shifting", "limit", s.numCtx, "nPast", seq.nPast, slog.Debug("context limit hit - shifting", "limit", s.numCtx, "nPast", seq.nPast,
"numKeep", seq.numKeep, "numLeft", numLeft, "numDiscard", numDiscard) "numKeep", seq.numKeep, "numLeft", numLeft, "numDiscard", numDiscard)
// TODO (jessegross): KV cache removal can fail for certain types of models
// server.cpp doesn't handle this, though we can be more graceful
s.lc.KvCacheSeqRm(seqIndex, seq.numKeep, seq.numKeep+numDiscard) s.lc.KvCacheSeqRm(seqIndex, seq.numKeep, seq.numKeep+numDiscard)
s.lc.KvCacheSeqAdd(seqIndex, seq.numKeep+numDiscard, seq.nPast, -numDiscard) s.lc.KvCacheSeqAdd(seqIndex, seq.numKeep+numDiscard, seq.nPast, -numDiscard)
@ -327,13 +329,11 @@ func (s *Server) processBatch() {
slog.Debug("sampled", "piece", piece) slog.Debug("sampled", "piece", piece)
// if it's an end of sequence token, break // if it's an end of sequence token, break
// TODO: just end this sequence
if s.model.TokenIsEog(token) { if s.model.TokenIsEog(token) {
// TODO (jmorganca): we should send this back // TODO (jmorganca): we should send this back
// as it's important for the /api/generate context // as it's important for the /api/generate context
// seq.responses <- piece // seq.responses <- piece
// TODO: end the sequence instead of quitting the pool
s.removeSequence(i, "stop") s.removeSequence(i, "stop")
continue continue
} }