Files
ollama/integration/mlx_batching_test.go
Daniel HiltgenandGitHub ab9295d03c mlx: support concurrent generation (#19197)
* mlx: batch requests through a shared scheduler

Use one request queue and scheduler for generation, decisions, embeddings
and speech, including parallelism one. Share compatible model operations
while keeping request caches, sampling state and response streams independent.

Interleave prompt chunks with decoding and retain speculative decoding when
generation runs alone. Cover isolation, cancellation, slow readers, cache
reuse and mixed requests with unit and integration tests. Remove the stale
llama-server architecture restriction on parallel requests.

* mlx: prevent cache snapshots from retaining batch peers

Copy each request’s initial batched K/V row so a sliding-cache snapshot cannot retain the whole batch allocation while accounting for only one row.

Exclude grammar compilation waits from prompt timing and reuse row-selected attention masks across layers.

Return errors for requests rejected before admission and strengthen numerical and compiled-speech regression checks.

* address comments
2026-11-10 13:24:12 -08:00

316 lines
10 KiB
Go

//go:build integration
package integration
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"testing"
"time"
"github.com/ollama/ollama/types/model"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/envconfig"
)
// runMLXBatching adds cache reuse and slot lifecycle checks to the release
// concurrency group. Capacity one exercises the same requests serially.
func runMLXBatching(t *testing.T, modelName string) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
pullOrSkip(ctx, t, client, modelName)
info, err := client.Show(ctx, &api.ShowRequest{Model: modelName})
if err != nil {
t.Fatal(err)
}
if info.Details.Format != "safetensors" {
t.Skip("cache reuse checks require an MLX model")
}
request := func(prompt string, n int) api.GenerateRequest {
return api.GenerateRequest{
Model: modelName, Runner: "mlx", Prompt: prompt,
Think: &api.ThinkValue{Value: false}, KeepAlive: &api.Duration{Duration: time.Minute},
Options: map[string]any{"num_predict": n, "temperature": 0, "num_ctx": 4096},
}
}
preloadConcurrentModel(ctx, t, client, request("", 128))
cities := []string{"Paris", "Tokyo", "Rome", "Madrid", "London", "Berlin", "Ottawa", "Cairo"}
countries := []string{"France", "Japan", "Italy", "Spain", "the United Kingdom", "Germany", "Canada", "Egypt"}
requests := make([]api.GenerateRequest, len(cities))
for i := range requests {
requests[i] = request("What is the capital of "+countries[i]+"? Answer with just the city name.", 128)
}
t.Run("isolation", func(t *testing.T) {
type result struct {
text string
final api.GenerateResponse
err error
}
for range 2 {
results := make(chan result, len(requests))
start := make(chan struct{})
for i, req := range requests {
go func() {
<-start
r := result{}
r.err = client.Generate(ctx, &req, func(chunk api.GenerateResponse) error {
r.text += chunk.Response
if chunk.Done {
r.final = chunk
}
return nil
})
if !strings.Contains(r.text, cities[i]) {
r.err = fmt.Errorf("want %s, got %q (%v)", cities[i], r.text, r.err)
}
results <- r
}()
}
close(start)
for range requests {
r := <-results
if r.err != nil || !r.final.Done {
t.Fatalf("response: %+v, error: %v", r, r.err)
}
}
}
})
t.Run("reuse", func(t *testing.T) {
// When every slot is free, a repeat can select its cached prefix. A
// concurrent repeat may instead land on another available slot.
for _, req := range requests {
for round := range 2 {
var final api.GenerateResponse
err := client.Generate(ctx, &req, func(chunk api.GenerateResponse) error {
if chunk.Done {
final = chunk
}
return nil
})
if err != nil || !final.Done {
t.Fatalf("repeat response: %+v, %v", final, err)
}
if round == 1 {
cached := final.PromptEvalCachedCount
if cached == nil {
t.Fatal("repeat did not report cached tokens")
}
if *cached < final.PromptEvalCount-8 {
t.Fatalf("repeat reused %d of %d prompt tokens", *cached, final.PromptEvalCount)
}
}
}
}
})
for _, tc := range []struct {
name string
cancel bool
}{{"cancel", true}, {"finish", false}} {
t.Run("join_and_leave/"+tc.name, func(t *testing.T) {
longCtx, stop := context.WithCancel(ctx)
defer stop()
first := make(chan struct{}, 1)
finished := make(chan error, 1)
var final api.GenerateResponse
long := request("Write a detailed 2000-word story about a lighthouse keeper. Start the story immediately and keep writing.", 256)
go func() {
finished <- client.Generate(longCtx, &long, func(chunk api.GenerateResponse) error {
if chunk.Response != "" || chunk.Thinking != "" {
select {
case first <- struct{}{}:
default:
}
}
if chunk.Done {
final = chunk
}
return nil
})
}()
select {
case <-first:
case err := <-finished:
t.Fatalf("long request ended before output: %v", err)
case <-ctx.Done():
t.Fatal(ctx.Err())
}
// Capacity one queues the peer; cancellation still frees the slot.
if tc.cancel && envconfig.NumParallel() <= 1 {
stop()
}
short := requests[1]
var text string
err := client.Generate(ctx, &short, func(chunk api.GenerateResponse) error { text += chunk.Response; return nil })
if err != nil || !strings.Contains(text, "Tokyo") {
t.Fatalf("joining request: %q, %v", text, err)
}
if envconfig.NumParallel() > 1 {
select {
case err := <-finished:
t.Fatalf("long request finished before short joining request: %v", err)
default:
}
}
if tc.cancel {
stop()
}
err = <-finished
if tc.cancel {
if !errors.Is(err, context.Canceled) {
t.Fatalf("cancel: %v", err)
}
} else if err != nil || !final.Done || final.EvalCount != 256 {
t.Fatalf("surviving request: %+v, %v", final, err)
}
text = ""
err = client.Generate(ctx, &requests[2], func(chunk api.GenerateResponse) error { text += chunk.Response; return nil })
if err != nil || !strings.Contains(text, "Rome") {
t.Fatalf("slot reuse: %q, %v", text, err)
}
})
}
t.Run("independent_grammar", func(t *testing.T) {
errs := make(chan error, 2)
for _, city := range cities[:2] {
go func() {
req := request("Return the city in JSON.", 32)
req.Format = []byte(fmt.Sprintf(`{"type":"object","properties":{"city":{"type":"string","enum":[%q]}},"required":["city"],"additionalProperties":false}`, city))
var text string
err := client.Generate(ctx, &req, func(chunk api.GenerateResponse) error { text += chunk.Response; return nil })
if err == nil && !strings.Contains(text, city) {
err = fmt.Errorf("grammar want %s, got %q", city, text)
}
errs <- err
}()
}
for range 2 {
if err := <-errs; err != nil {
t.Fatal(err)
}
}
})
}
// TestMLXBatchingVision mixes text and two different images at the same prompt
// position. Repeats exercise both cached image identity and per-request layout.
func TestMLXBatchingVision(t *testing.T) {
if testModel == "" {
t.Skip("set OLLAMA_TEST_MODEL to a published MLX vision model")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
info, err := client.Show(ctx, &api.ShowRequest{Model: testModel})
if err != nil {
t.Fatal(err)
}
if !slices.Contains(info.Capabilities, model.CapabilityVision) {
t.Skip("model has no vision capability")
}
road, desk, _ := decodeTestImages(t)
cases := []struct {
image api.ImageData
prompt, want string
}{
{road, "Is this scene crossing a road or working at a desk? Begin with CROSSING or WORKING, then describe the image in detail.", "crossing"},
{desk, "Is this scene crossing a road or working at a desk? Begin with CROSSING or WORKING, then describe the image in detail.", "working"},
{nil, "What is the capital of Japan? Begin with the city name and then describe the city's history in detail.", "tokyo"},
}
for round := range 2 {
start := make(chan struct{})
results := make(chan error, len(cases))
for _, tc := range cases {
go func() {
<-start
req := api.GenerateRequest{Model: testModel, Runner: "mlx", Prompt: tc.prompt, Think: &api.ThinkValue{Value: false}, KeepAlive: &api.Duration{Duration: time.Minute}, Options: map[string]any{"num_predict": 128, "num_ctx": 4096, "temperature": 0}}
if tc.image != nil {
req.Images = []api.ImageData{tc.image}
}
var text string
var final api.GenerateResponse
err := client.Generate(ctx, &req, func(chunk api.GenerateResponse) error {
text += chunk.Response
if chunk.Done {
final = chunk
}
return nil
})
if err == nil && (!final.Done || final.EvalCount == 0 || !strings.Contains(strings.ToLower(text), tc.want)) {
err = fmt.Errorf("want %q in image/text response, got %q (done=%v tokens=%d)", tc.want, text, final.Done, final.EvalCount)
}
results <- err
}()
}
close(start)
for range cases {
if err := <-results; err != nil {
t.Errorf("round %d: %v", round, err)
}
}
if t.Failed() {
return
}
}
}
// TestMLXBatchingLongPrompts crosses the prefill chunk and sparse-attention
// boundaries with unequal histories. Longer replies keep the short-prompt row
// generating while its peer processes later prompt chunks.
func TestMLXBatchingLongPrompts(t *testing.T) {
if testModel == "" {
t.Skip("set OLLAMA_TEST_MODEL to a published MLX model")
}
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute)
defer cancel()
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
start := make(chan struct{})
cities := []string{"Paris", "Tokyo", "Rome", "Madrid", "London", "Berlin", "Ottawa", "Cairo"}
countries := []string{"France", "Japan", "Italy", "Spain", "the United Kingdom", "Germany", "Canada", "Egypt"}
rows := max(2, min(len(cities), int(envconfig.NumParallel())))
t.Logf("keeping %d requests active across unequal prompt lengths", rows)
results := make(chan error, rows)
for i, city := range cities[:rows] {
go func() {
repetitions := 20
if i%2 == 1 {
repetitions = 180
}
prompt := strings.Repeat("The island has an old lighthouse, a harbor, and a village beside the sea. ", repetitions)
req := api.GenerateRequest{Model: testModel, Runner: "mlx", Prompt: prompt + "\nIgnore the preceding background. What is the capital of " + countries[i] + "? Begin with the city name, then write a detailed 1000-word history of the city.", Think: &api.ThinkValue{Value: false}, KeepAlive: &api.Duration{Duration: time.Minute}, Options: map[string]any{"num_predict": 512, "num_ctx": 8192, "temperature": 0}}
<-start
var text, thinking string
var final api.GenerateResponse
err := client.Generate(ctx, &req, func(chunk api.GenerateResponse) error {
text += chunk.Response
thinking += chunk.Thinking
if chunk.Done {
final = chunk
}
return nil
})
if err == nil && (!final.Done || !strings.Contains(text, city)) {
err = fmt.Errorf("row %d want %s, got %q (done=%v tokens=%d reason=%s thinking=%q)", i, city, text, final.Done, final.EvalCount, final.DoneReason, thinking)
}
if err == nil && i%2 == 1 && final.PromptEvalCount <= 2048 {
err = fmt.Errorf("long prompt only has %d tokens; did not cross chunk/sparse boundary", final.PromptEvalCount)
}
results <- err
}()
}
close(start)
for range rows {
if err := <-results; err != nil {
t.Error(err)
}
}
}