Files
ollama/mlxrunner/cache/batch_test.go
T
Daniel HiltgenandGitHub ab9295d03c mlx: support concurrent generation (#19197)
* mlx: batch requests through a shared scheduler

Use one request queue and scheduler for generation, decisions, embeddings
and speech, including parallelism one. Share compatible model operations
while keeping request caches, sampling state and response streams independent.

Interleave prompt chunks with decoding and retain speculative decoding when
generation runs alone. Cover isolation, cancellation, slow readers, cache
reuse and mixed requests with unit and integration tests. Remove the stale
llama-server architecture restriction on parallel requests.

* mlx: prevent cache snapshots from retaining batch peers

Copy each request’s initial batched K/V row so a sliding-cache snapshot cannot retain the whole batch allocation while accounting for only one row.

Exclude grammar compilation waits from prompt timing and reuse row-selected attention masks across layers.

Return errors for requests rejected before admission and strengthen numerical and compiled-speech regression checks.

* address comments
2026-11-10 13:24:12 -08:00

404 lines
14 KiB
Go

package cache
import (
"fmt"
"math"
"slices"
"testing"
"github.com/ollama/ollama/mlx"
"github.com/ollama/ollama/mlx/mlxtest"
"github.com/ollama/ollama/mlxrunner/batch"
"github.com/ollama/ollama/mlxrunner/nn"
)
func compareRowArrays(got, want *mlx.Array) error {
if !slices.Equal(got.Dims(), want.Dims()) {
return fmt.Errorf("shape = %v, want %v", got.Dims(), want.Dims())
}
mlx.Eval(got, want)
g, w := got.Floats(), want.Floats()
if len(g) == 0 || len(g) != len(w) {
return fmt.Errorf("value count = %d, want %d", len(g), len(w))
}
for i := range w {
if delta := math.Abs(float64(g[i] - w[i])); math.IsNaN(delta) || delta > 1e-5 {
return fmt.Errorf("element %d = %g, want %g", i, g[i], w[i])
}
}
return nil
}
func TestCompareRowArraysRejectsInvalidComparisons(t *testing.T) {
for _, tt := range []struct {
name string
got []float32
want []float32
wantErr bool
}{
{name: "valid", got: []float32{1, 2}, want: []float32{1, 2}},
{name: "actual nan", got: []float32{float32(math.NaN())}, want: []float32{0}, wantErr: true},
{name: "expected nan", got: []float32{0}, want: []float32{float32(math.NaN())}, wantErr: true},
{name: "matching infinity", got: []float32{float32(math.Inf(1))}, want: []float32{float32(math.Inf(1))}, wantErr: true},
{name: "empty", wantErr: true},
{name: "shape mismatch", got: []float32{1}, want: []float32{1, 2}, wantErr: true},
} {
mlxtest.RunSubtest(t, tt.name, func(t *mlxtest.T) {
array := func(values []float32) *mlx.Array {
if len(values) == 0 {
return mlx.Zeros(mlx.DTypeFloat32, 0)
}
return mlx.FromValues(values, len(values))
}
err := compareRowArrays(array(tt.got), array(tt.want))
if (err != nil) != tt.wantErr {
t.Fatalf("comparison error = %v, want error %t", err, tt.wantErr)
}
})
}
}
func TestBatchAttentionRows(t *testing.T) {
for _, kind := range []string{"full", "rotating"} {
mlxtest.RunSubtest(t, kind, func(t *mlxtest.T) {
newCache := func() Cache {
if kind == "rotating" {
return NewRotatingKVCache(4)
}
return NewKVCache()
}
owners, refs := make([]Cache, 3), make([]Cache, 3)
for i := range owners {
owners[i], refs[i] = newCache(), newCache()
defer owners[i].Free()
defer refs[i].Free()
}
kv := func(row, pos int) (*mlx.Array, *mlx.Array) {
values := []float32{float32(row+1) * .2, float32(pos+1) * .1, -.4, .7}
k := mlx.FromValues(values, 1, 1, 1, 4)
return k, mlx.MulScalar(k, float32(row+2))
}
// Different history lengths and ring positions before requests meet.
for row, n := range []int{1, 7, 3} {
for pos := range n {
k, v := kv(row, pos)
for _, c := range []Cache{owners[row], refs[row]} {
c.(Attention).Update(newKVBatch(pos, 1), k, v)
}
}
}
for _, active := range [][]int{{0, 1}, {1, 0}, {2, 1}, {0}, {2, 0}} {
rows := make([][]Cache, len(active))
b := &batch.Batch{
InputIDs: mlx.Zeros(mlx.DTypeInt32, len(active), 1),
}
var ks, vs, qs []*mlx.Array
for i, row := range active {
rows[i] = []Cache{owners[row]}
pos := owners[row].Offset()
b.SeqOffsets = append(b.SeqOffsets, int32(pos))
b.SeqQueryLens = append(b.SeqQueryLens, 1)
k, v := kv(row, pos)
ks, vs = append(ks, k), append(vs, v)
qs = append(qs, mlx.AddScalar(k, .3))
}
layers, err := Batch(rows)
if err != nil {
t.Fatal(err)
}
c := layers[0].(Attention)
h := c.Update(b, mlx.Concatenate(ks, 0), mlx.Concatenate(vs, 0))
q := mlx.Concatenate(qs, 0)
got := nn.ScaledDotProductAttention(b, q, .5, nn.WithKVHistory(h), nn.WithMask(nn.CausalMask()))
// View is also the read-only shared-history path used by donors.
view := nn.ScaledDotProductAttention(b, q, .5, nn.WithKVHistory(c.View(b)), nn.WithMask(nn.CausalMask()))
var want []*mlx.Array
for i, row := range active {
rb := newKVBatch(refs[row].Offset(), 1)
rh := refs[row].(Attention).Update(rb, ks[i], vs[i])
want = append(want, nn.ScaledDotProductAttention(rb, qs[i], .5, nn.WithKVHistory(rh), nn.WithMask(nn.CausalMask())))
if owners[row].Offset() != refs[row].Offset() {
t.Fatalf("row %d offset = %d, want %d", row, owners[row].Offset(), refs[row].Offset())
}
}
expected := mlx.Concatenate(want, 0)
if err := compareRowArrays(got, expected); err != nil {
t.Fatal(err)
}
if err := compareRowArrays(view, expected); err != nil {
t.Fatal(err)
}
for row := range owners {
for i, state := range owners[row].State() {
if err := compareRowArrays(state, refs[row].State()[i]); err != nil {
t.Fatalf("row %d state %d: %v", row, i, err)
}
}
}
}
})
}
}
func TestBatchRecurrentOwnersAndSnapshots(t *testing.T) {
mlxtest.Run(t, func(t *mlxtest.T) {
a, b := NewRecurrentCache(2, 3, 1, 2, 2), NewRecurrentCache(2, 3, 1, 2, 2)
defer a.Free()
defer b.Free()
for i, c := range []*RecurrentCache{a, b} {
rb := newKVBatch(0, i+1)
h := c.Get(rb, mlx.DTypeFloat32)
c.Put(rb, []*mlx.Array{mlx.AddScalar(h.ConvState(), float32(i+1))},
[]*mlx.Array{mlx.AddScalar(h.DeltaState(), float32(10+i))})
c.PrepareSnapshots([]int{c.Offset() + 1})
}
layers, err := Batch([][]Cache{{b}, {a}})
if err != nil {
t.Fatal(err)
}
rb := &batch.Batch{
InputIDs: mlx.Zeros(mlx.DTypeInt32, 2, 1), SeqOffsets: []int32{2, 1}, SeqQueryLens: []int32{1, 1},
}
c := layers[0].(Recurrent)
h := c.Get(rb, mlx.DTypeFloat32)
if err := compareRowArrays(h.ConvState(), mlx.Concatenate([]*mlx.Array{b.State()[0], a.State()[0]}, 0)); err != nil {
t.Fatal(err)
}
c.Put(rb, []*mlx.Array{mlx.AddScalar(h.ConvState(), 3)}, []*mlx.Array{mlx.AddScalar(h.DeltaState(), 7)})
if a.Offset() != 2 || b.Offset() != 3 {
t.Fatalf("offsets = %d, %d, want 2, 3", a.Offset(), b.Offset())
}
mlx.Eval(c.State()...)
sa, sb := a.TakeSnapshots()[0], b.TakeSnapshots()[0]
defer sa.Close()
defer sb.Close()
// Keep b parked while a advances and restores its own boundary.
before := b.State()[1]
ah := a.Get(newKVBatch(2, 1), mlx.DTypeFloat32)
a.Put(newKVBatch(2, 1), []*mlx.Array{mlx.AddScalar(ah.ConvState(), 100)},
[]*mlx.Array{mlx.AddScalar(ah.DeltaState(), 100)})
if !a.Restore(sa, 2) {
t.Fatal("cannot restore row a's captured boundary")
}
if err := compareRowArrays(b.State()[1], before); err != nil {
t.Fatal(err)
}
if err := compareRowArrays(a.State()[0], mlx.AddScalar(mlx.Zeros(mlx.DTypeFloat32, 1, 2, 3), 4)); err != nil {
t.Fatal(err)
}
if err := compareRowArrays(a.State()[1], mlx.AddScalar(mlx.Zeros(mlx.DTypeFloat32, 1, 1, 2, 2), 17)); err != nil {
t.Fatal(err)
}
if b.Offset() != 3 {
t.Fatal("restoring a changed b's offset")
}
})
}
func TestBatchRejectsIncompatibleCaches(t *testing.T) {
if Layers(nil) != nil {
t.Fatal("no-cache forwards must retain a nil cache bundle")
}
mlxtest.Run(t, func(t *mlxtest.T) {
a, b := NewKVCache(), NewRotatingKVCache(4)
defer a.Free()
defer b.Free()
for _, rows := range [][][]Cache{nil, {{a}, {}}, {{a}, {b}}, {{a}, {a}}, {{nil}, {a}}} {
if _, err := Batch(rows); err == nil {
t.Fatalf("accepted incompatible rows %v", rows)
}
}
layers, err := Batch([][]Cache{{a}})
if err != nil || len(layers) != 1 || layers[0] != a {
t.Fatalf("single row changed its original owner: %v, %v", layers, err)
}
})
}
func TestBatchPrefillAttention(t *testing.T) {
for _, tt := range []struct {
name string
rotating bool
seeded bool
}{
{name: "full/empty"},
{name: "full/populated", seeded: true},
{name: "rotating/empty", rotating: true},
{name: "rotating/populated", rotating: true, seeded: true},
} {
mlxtest.RunSubtest(t, tt.name, func(t *mlxtest.T) {
var owners, refs [2]Cache
for i := range owners {
if tt.rotating {
owners[i], refs[i] = NewRotatingKVCache(4), NewRotatingKVCache(4)
} else {
owners[i], refs[i] = NewKVCache(), NewKVCache()
}
defer owners[i].Free()
defer refs[i].Free()
if tt.seeded {
seed := mlx.AddScalar(mlx.Zeros(mlx.DTypeFloat32, 1, 1, i+2, 2), float32(i+1))
for _, c := range []Cache{owners[i], refs[i]} {
c.(Attention).Update(newKVBatch(0, i+2), seed, seed)
}
}
}
for _, n := range []int{3, 1, 5} {
b := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 2, n), SeqQueryLens: []int32{int32(n), int32(n)}}
var keys, wants []*mlx.Array
for i := range owners {
pos := owners[i].Offset()
b.SeqOffsets = append(b.SeqOffsets, int32(pos))
k := mlx.AddScalar(mlx.Arange(0, float64(n*2), 1, mlx.DTypeFloat32).Reshape(1, 1, n, 2), float32(i+1))
keys = append(keys, k)
rb := newKVBatch(pos, n)
h := refs[i].(Attention).Update(rb, k, k)
wants = append(wants, nn.ScaledDotProductAttention(rb, k, .5, nn.WithKVHistory(h), nn.WithMask(nn.CausalMask())))
}
layers, err := Batch([][]Cache{{owners[0]}, {owners[1]}})
if err != nil {
t.Fatal(err)
}
k := mlx.Concatenate(keys, 0)
h := layers[0].(Attention).Update(b, k, k)
got := nn.ScaledDotProductAttention(b, k, .5, nn.WithKVHistory(h), nn.WithMask(nn.CausalMask()))
if err := compareRowArrays(got, mlx.Concatenate(wants, 0)); err != nil {
t.Fatalf("n=%d: %v", n, err)
}
for i := range owners {
if owners[i].Offset() != refs[i].Offset() {
t.Fatal("cache position mismatch")
}
}
}
})
}
}
func TestBatchRecurrentIndependentSplits(t *testing.T) {
mlxtest.Run(t, func(t *mlxtest.T) {
a, b := NewRecurrentCache(1, 1, 1, 1, 1), NewRecurrentCache(1, 1, 1, 1, 1)
defer a.Free()
defer b.Free()
a.Get(newKVBatch(0, 1), mlx.DTypeFloat32)
b.Get(newKVBatch(0, 1), mlx.DTypeFloat32)
a.PrepareSnapshots([]int{1, 4})
b.PrepareSnapshots([]int{2, 3, 4})
layers, err := Batch([][]Cache{{a}, {b}})
if err != nil {
t.Fatal(err)
}
c := layers[0].(Recurrent)
if got := c.SnapshotSplits(4); !slices.Equal(got, []int{1, 2, 3}) {
t.Fatalf("splits %v", got)
}
var conv, delta []*mlx.Array
for _, boundary := range []float32{1, 2, 3, 4} {
conv = append(conv, mlx.FromValues([]float32{boundary, 10 + boundary}, 2, 1, 1))
delta = append(delta, mlx.FromValues([]float32{100 + boundary, 200 + boundary}, 2, 1, 1, 1))
}
c.Put(&batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 2, 4), SeqOffsets: []int32{0, 0}, SeqQueryLens: []int32{4, 4}}, conv, delta)
mlx.Eval(c.State()...)
for row, owner := range []*RecurrentCache{a, b} {
boundaries := [][]int{{1, 4}, {2, 3, 4}}[row]
snaps := owner.TakeSnapshots()
if len(snaps) != len(boundaries) {
t.Fatalf("row %d snapshots %d", row, len(snaps))
}
for i, snap := range snaps {
if !owner.Restore(snap, boundaries[i]) {
t.Fatal("restore failed")
}
want := float32(100*(row+1) + boundaries[i])
if got := owner.State()[1].Floats()[0]; got != want {
t.Fatalf("row %d boundary %d: %g, want %g", row, boundaries[i], got, want)
}
snap.Close()
}
}
})
}
// A snapshot of one request must not retain the allocations for its batch peers.
func TestBatchAttentionSnapshotOwnsRow(t *testing.T) {
for _, kind := range []string{"full", "rotating"} {
for _, scheduled := range []bool{false, true} {
mlxtest.RunSubtest(t, fmt.Sprintf("%s/scheduled=%t", kind, scheduled), func(t *mlxtest.T) {
const rows, heads, tokens, dims, keep = 8, 4, 256, 64, 3
const rowElements = heads * tokens * dims
const rowBytes = 2 * rowElements * 4
newCache := func() Cache {
if kind == "rotating" {
return NewRotatingKVCache(512)
}
return NewKVCache()
}
mlx.DefaultStream().Synchronize()
baseline := settledActiveMemory()
var snapshot Snapshot
defer func() {
if snapshot != nil {
snapshot.Close()
}
}()
mlx.Scoped(func() {
owners := make([][]Cache, rows)
b := &batch.Batch{
InputIDs: mlx.Zeros(mlx.DTypeInt32, rows, tokens),
SeqOffsets: make([]int32, rows), SeqQueryLens: make([]int32, rows),
}
data := make([]float32, rows*rowElements)
for i := range rows {
owners[i] = []Cache{newCache()}
defer owners[i][0].Free()
b.SeqQueryLens[i] = tokens
for j := range rowElements {
data[i*rowElements+j] = float32(i + 1)
}
}
if scheduled {
owners[keep][0].PrepareSnapshots([]int{tokens})
}
layers, err := Batch(owners)
if err != nil {
t.Fatal(err)
}
keys := mlx.FromValues(data, rows, heads, tokens, dims)
values := mlx.Neg(keys)
layers[0].(Attention).Update(b, keys, values)
mlx.Eval(layers[0].State()...)
if scheduled {
snapshot = owners[keep][0].TakeSnapshots()[0]
} else {
snapshot = owners[keep][0].Snapshot(0)
}
})
mlx.DefaultStream().Synchronize()
if got := snapshot.Size(); got != rowBytes {
t.Fatalf("snapshot accounts for %d bytes, want one row (%d)", got, rowBytes)
}
// Allow allocator rounding, but not another row's allocation.
if retained := settledActiveMemory() - baseline; retained > rowBytes+64*1024 {
t.Fatalf("one %d-byte snapshot retains %d bytes after freeing all batch owners", rowBytes, retained)
}
restored := newCache()
defer restored.Free()
if !restored.Restore(snapshot, tokens) {
t.Fatal("could not restore the surviving row")
}
for i, state := range restored.State() {
mlx.Eval(state)
want := float32(keep + 1)
if i == 1 {
want = -want
}
matches := state.Equal(mlx.FromValue(want)).AsType(mlx.DTypeInt32).Reshape(-1)
if got := mlx.Sum(matches, 0, false).Int(); got != int32(state.Size()) {
t.Fatalf("restored state %d has %d matching elements, want %d", i, got, state.Size())
}
}
})
}
}
}