tcsenpai/ollama
1
0
Fork 0
mirror of https://github.com/tcsenpai/ollama.git synced 2025-06-08 12:15:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

save whisper port

Browse Source
This commit is contained in:
Roy Han 2024-08-05 13:19:01 -07:00
parent 97d9dffa80
commit 17f9dc6d08
2 changed files with 37 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
server/routes.go
View File

@ -109,7 +109,15 @@ func (s *Server) scheduleRunner(ctx context.Context, name string, caps []Capabil
return runner.llama, model, &opts, nil return runner.llama, model, &opts, nil
} }
func runWhisperServer(c *gin.Context, portCh chan int) { func (s *Server) runWhisperServer(c *gin.Context, portCh chan int) {
s.sched.whisperMu.Lock()
if s.sched.whisperPort != nil {
slog.Info("whisper server already running", "port", *s.sched.whisperPort)
portCh <- *s.sched.whisperPort
s.sched.whisperMu.Unlock()
return
}
whisperServer := "/Users/royhan-ollama/ollama/llm/whisper.cpp/server" whisperServer := "/Users/royhan-ollama/ollama/llm/whisper.cpp/server"
// Find an available port for whisper // Find an available port for whisper
@ -126,7 +134,7 @@ func runWhisperServer(c *gin.Context, portCh chan int) {
slog.Debug("ResolveTCPAddr failed") slog.Debug("ResolveTCPAddr failed")
port = rand.Intn(65535-49152) + 49152 // get a random port in the ephemeral range port = rand.Intn(65535-49152) + 49152 // get a random port in the ephemeral range
} }
finalParams := append(params, "--port", strconv.Itoa(port), "--model", "/Users/royhan-ollama/ollama/llm/whisper.cpp/models/ggml-base.en.bin") finalParams := append(params, "--port", strconv.Itoa(port), "--model", "/Users/royhan-ollama/.ollama/whisper/ggml-base.en.bin")
cmd := exec.Command(whisperServer, finalParams...) cmd := exec.Command(whisperServer, finalParams...)
slog.Info("starting whisper server", "cmd", cmd.String()) slog.Info("starting whisper server", "cmd", cmd.String())
@ -138,18 +146,32 @@ func runWhisperServer(c *gin.Context, portCh chan int) {
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "failed to start whisper server"}) c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "failed to start whisper server"})
} }
// wait for server to start retries := 10
time.Sleep(250 * time.Millisecond) for range retries {
time.Sleep(25 * time.Millisecond)
conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), time.Second)
if err == nil {
conn.Close()
break
}
}
if err != nil {
slog.Error("failed to connect to whisper server", "error", err)
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "failed to connect to whisper server"})
}
portCh <- port portCh <- port
s.sched.whisperPort = &port
s.sched.whisperMu.Unlock()
// Wait for the whisper server to exit // Wait for the whisper server to exit
defer func() {
err = cmd.Wait() err = cmd.Wait()
if err != nil { if err != nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "whisper server exited"}) c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "whisper server exited"})
} }
defer func() {
err := cmd.Process.Kill() err := cmd.Process.Kill()
if err != nil { if err != nil {
slog.Error("failed to kill whisper server", "error", err) slog.Error("failed to kill whisper server", "error", err)
@ -232,7 +254,6 @@ func whisperInference(c *gin.Context, filePath string, port int) (*api.WhisperCo
} }
func (s *Server) GenerateHandler(c *gin.Context) { func (s *Server) GenerateHandler(c *gin.Context) {
slog.Info("generate request", "method", c.Request.Method, "url", c.Request.URL.String())
checkpointStart := time.Now() checkpointStart := time.Now()
var req api.GenerateRequest var req api.GenerateRequest
if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) { if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
@ -258,7 +279,7 @@ func (s *Server) GenerateHandler(c *gin.Context) {
if req.Audio != "" { if req.Audio != "" {
port := make(chan int, 1) port := make(chan int, 1)
go runWhisperServer(c, port) go s.runWhisperServer(c, port)
w, err := whisperInference(c, req.Audio, <-port) w, err := whisperInference(c, req.Audio, <-port)
if err != nil { if err != nil {

3
server/sched.go
View File

@ -46,6 +46,9 @@ type Scheduler struct {
getGpuFn func() gpu.GpuInfoList getGpuFn func() gpu.GpuInfoList
getCpuFn func() gpu.GpuInfoList getCpuFn func() gpu.GpuInfoList
reschedDelay time.Duration reschedDelay time.Duration
whisperPort *int
whisperMu sync.Mutex
} }
// Default automatic value for number of models we allow per GPU // Default automatic value for number of models we allow per GPU