mirror of
https://github.com/ollama/ollama.git
synced 2026-11-19 02:41:27 -06:00
* mlx: batch requests through a shared scheduler Use one request queue and scheduler for generation, decisions, embeddings and speech, including parallelism one. Share compatible model operations while keeping request caches, sampling state and response streams independent. Interleave prompt chunks with decoding and retain speculative decoding when generation runs alone. Cover isolation, cancellation, slow readers, cache reuse and mixed requests with unit and integration tests. Remove the stale llama-server architecture restriction on parallel requests. * mlx: prevent cache snapshots from retaining batch peers Copy each request’s initial batched K/V row so a sliding-cache snapshot cannot retain the whole batch allocation while accounting for only one row. Exclude grammar compilation waits from prompt timing and reuse row-selected attention masks across layers. Return errors for requests rejected before admission and strengthen numerical and compiled-speech regression checks. * address comments
303 lines
8.4 KiB
Go
303 lines
8.4 KiB
Go
package mlxrunner
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/ollama/ollama/api"
|
|
"github.com/ollama/ollama/envconfig"
|
|
"github.com/ollama/ollama/logutil"
|
|
"github.com/ollama/ollama/mlx"
|
|
"github.com/ollama/ollama/mlx/mlxthread"
|
|
"github.com/ollama/ollama/mlxrunner/sample"
|
|
)
|
|
|
|
func Execute(args []string) error {
|
|
slog.SetDefault(logutil.NewLogger(os.Stderr, envconfig.LogLevel()))
|
|
|
|
var (
|
|
modelName string
|
|
port int
|
|
)
|
|
|
|
flagSet := flag.NewFlagSet("mlxrunner", flag.ExitOnError)
|
|
flagSet.StringVar(&modelName, "model", "", "Model name")
|
|
flagSet.IntVar(&port, "port", 0, "Port to listen on")
|
|
_ = flagSet.Bool("verbose", false, "Enable debug logging")
|
|
flagSet.Parse(args)
|
|
|
|
worker, err := mlxthread.Start("mlxrunner", func() error {
|
|
if err := mlx.CheckInit(); err != nil {
|
|
return fmt.Errorf("MLX not available: %w", err)
|
|
}
|
|
|
|
if mlx.GPUIsAvailable() {
|
|
if mlx.CUDAIsAvailable() {
|
|
if err := mlx.CheckCUDADeviceSupport(); err != nil {
|
|
return fmt.Errorf("MLX CUDA initialization failed: %w", err)
|
|
}
|
|
}
|
|
mlx.SetDefaultDeviceGPU()
|
|
slog.Info("MLX engine initialized", "MLX version", mlx.Version(), "device", "gpu")
|
|
} else {
|
|
slog.Info("MLX engine initialized", "MLX version", mlx.Version(), "device", "cpu")
|
|
}
|
|
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer worker.Stop(context.Background(), mlx.ClearCache)
|
|
runnerCtx, cancelRunner := context.WithCancel(context.Background())
|
|
defer cancelRunner()
|
|
|
|
runner := Runner{
|
|
parallel: max(1, int(envconfig.NumParallel())),
|
|
Requests: make(chan queuedRequest),
|
|
mlxThread: worker,
|
|
}
|
|
|
|
if err := worker.Do(context.Background(), func() error {
|
|
return runner.Load(modelName)
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
defer runner.Close()
|
|
|
|
readMemory := func() (uint64, error) {
|
|
return uint64(mlx.ActiveMemory() + mlx.CacheMemory()), nil
|
|
}
|
|
initialMemory, err := mlxthread.Call(context.Background(), worker, readMemory)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
memoryCache := newStatusMemoryCache(
|
|
runnerCtx,
|
|
initialMemory,
|
|
time.Now(),
|
|
statusMemoryRefreshWait,
|
|
func() (uint64, error) {
|
|
return mlxthread.Call(runnerCtx, worker, readMemory)
|
|
},
|
|
)
|
|
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("POST /v1/score", runner.scoreHandler)
|
|
mux.HandleFunc("GET /v1/status", func(w http.ResponseWriter, r *http.Request) {
|
|
resp := statusResponse{
|
|
Status: 0,
|
|
Progress: 100,
|
|
ContextLength: runner.contextLength,
|
|
Memory: memoryCache.Memory(),
|
|
}
|
|
if em, ok := runner.Model.(interface{ EmbeddingDimensions() []int }); ok {
|
|
resp.EmbeddingDimensions = em.EmbeddingDimensions()
|
|
}
|
|
if err := json.NewEncoder(w).Encode(resp); err != nil {
|
|
slog.Error("Failed to encode response", "error", err)
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
})
|
|
|
|
mux.HandleFunc("/v1/models", func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.Method {
|
|
case "POST":
|
|
fallthrough
|
|
case "GET":
|
|
if err := json.NewEncoder(w).Encode(map[string]any{
|
|
"Success": true,
|
|
}); err != nil {
|
|
slog.Error("Failed to encode response", "error", err)
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
case "DELETE":
|
|
// TODO: cleanup model and cache
|
|
}
|
|
})
|
|
|
|
admission := make(chan struct{}, runner.parallel+int(envconfig.MaxQueue()))
|
|
mux.HandleFunc("POST /v1/completions", func(w http.ResponseWriter, r *http.Request) {
|
|
if runner.Sampler == nil {
|
|
http.Error(w, "this model does not support text generation", http.StatusBadRequest)
|
|
return
|
|
}
|
|
select {
|
|
case admission <- struct{}{}:
|
|
defer func() { <-admission }()
|
|
default:
|
|
http.Error(w, "runner queue is full", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
request := request{Responses: make(chan CompletionResponse, 8)}
|
|
|
|
if err := json.NewDecoder(r.Body).Decode(&request.CompletionRequest); err != nil {
|
|
slog.Error("Failed to decode request", "error", err)
|
|
http.Error(w, "Bad Request", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
request.SamplerOpts = sample.Options{
|
|
Temperature: request.Options.Temperature,
|
|
TopP: request.Options.TopP,
|
|
MinP: request.Options.MinP,
|
|
TopK: request.Options.TopK,
|
|
RepeatLastN: request.Options.RepeatLastN,
|
|
RepeatPenalty: request.Options.RepeatPenalty,
|
|
PresencePenalty: request.Options.PresencePenalty,
|
|
FrequencyPenalty: request.Options.FrequencyPenalty,
|
|
Seed: request.Options.Seed,
|
|
UseSeed: request.Options.Seed >= 0,
|
|
Logprobs: request.Logprobs,
|
|
TopLogprobs: request.TopLogprobs,
|
|
}
|
|
|
|
if err := runner.Prepare(&request); err != nil {
|
|
var statusErr api.StatusError
|
|
if errors.As(err, &statusErr) {
|
|
http.Error(w, statusErr.ErrorMessage, statusErr.StatusCode)
|
|
} else {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
}
|
|
return
|
|
}
|
|
|
|
var cancel context.CancelFunc
|
|
request.Ctx, cancel = context.WithCancel(r.Context())
|
|
defer cancel()
|
|
|
|
select {
|
|
case <-r.Context().Done():
|
|
// Never queued, so the runner will not close the grammar.
|
|
request.Grammar.close()
|
|
return
|
|
case runner.Requests <- request:
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/jsonl")
|
|
w.WriteHeader(http.StatusOK)
|
|
enc := json.NewEncoder(w)
|
|
for {
|
|
select {
|
|
case <-r.Context().Done():
|
|
return
|
|
case response, ok := <-request.Responses:
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
if err := enc.Encode(response); err != nil {
|
|
slog.Error("Failed to encode response", "error", err)
|
|
return
|
|
}
|
|
|
|
if f, ok := w.(http.Flusher); ok {
|
|
f.Flush()
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
mux.HandleFunc("POST /v1/tokenize", func(w http.ResponseWriter, r *http.Request) {
|
|
if runner.Tokenizer == nil {
|
|
http.Error(w, "this model does not support text tokenization", http.StatusBadRequest)
|
|
return
|
|
}
|
|
var b bytes.Buffer
|
|
if _, err := io.Copy(&b, r.Body); err != nil {
|
|
slog.Error("Failed to read request body", "error", err)
|
|
http.Error(w, "Bad Request", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
tokens := runner.Tokenizer.Encode(b.String(), runner.Tokenizer.AddBOS())
|
|
|
|
if err := json.NewEncoder(w).Encode(tokens); err != nil {
|
|
slog.Error("Failed to encode response", "error", err)
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
})
|
|
|
|
mux.HandleFunc("POST /v1/detokenize", func(w http.ResponseWriter, r *http.Request) {
|
|
if runner.Tokenizer == nil {
|
|
http.Error(w, "this model does not support text tokenization", http.StatusBadRequest)
|
|
return
|
|
}
|
|
var tokens []int32
|
|
if err := json.NewDecoder(r.Body).Decode(&tokens); err != nil {
|
|
http.Error(w, "Bad Request", http.StatusBadRequest)
|
|
return
|
|
}
|
|
if err := json.NewEncoder(w).Encode(runner.Tokenizer.Decode(tokens)); err != nil {
|
|
slog.Error("Failed to encode response", "error", err)
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
}
|
|
})
|
|
|
|
mux.HandleFunc("POST /v1/embeddings", runner.handleEmbed)
|
|
mux.HandleFunc("POST /v1/audio/speech", runner.speechHandler)
|
|
|
|
for source, target := range map[string]string{
|
|
"GET /health": "/v1/status",
|
|
"POST /load": "/v1/models",
|
|
"POST /completion": "/v1/completions",
|
|
} {
|
|
mux.Handle(source, http.RedirectHandler(target, http.StatusPermanentRedirect))
|
|
}
|
|
|
|
return runner.Run("127.0.0.1", strconv.Itoa(port), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
recorder := &statusRecorder{ResponseWriter: w, code: http.StatusOK}
|
|
t := time.Now()
|
|
mux.ServeHTTP(recorder, r)
|
|
|
|
var level slog.Level
|
|
switch {
|
|
case recorder.code >= 500:
|
|
level = slog.LevelError
|
|
case recorder.code >= 400:
|
|
level = slog.LevelWarn
|
|
case recorder.code >= 300:
|
|
return
|
|
}
|
|
|
|
slog.Log(r.Context(), level, "ServeHTTP", "method", r.Method, "path", r.URL.Path, "took", time.Since(t), "status", recorder.Status())
|
|
}))
|
|
}
|
|
|
|
type statusRecorder struct {
|
|
http.ResponseWriter
|
|
code int
|
|
}
|
|
|
|
func (w *statusRecorder) WriteHeader(code int) {
|
|
w.code = code
|
|
w.ResponseWriter.WriteHeader(code)
|
|
}
|
|
|
|
func (w *statusRecorder) Status() string {
|
|
return strconv.Itoa(w.code) + " " + http.StatusText(w.code)
|
|
}
|
|
|
|
// Unwrap lets net/http's ResponseController reach deadlines and other controls.
|
|
func (w *statusRecorder) Unwrap() http.ResponseWriter { return w.ResponseWriter }
|
|
|
|
func (w *statusRecorder) FlushError() error {
|
|
return http.NewResponseController(w.ResponseWriter).Flush()
|
|
}
|
|
|
|
func (w *statusRecorder) Flush() { _ = w.FlushError() }
|