Files
ollama/mlxrunner/cache/batch.go
T
Daniel HiltgenandGitHub ab9295d03c mlx: support concurrent generation (#19197)
* mlx: batch requests through a shared scheduler

Use one request queue and scheduler for generation, decisions, embeddings
and speech, including parallelism one. Share compatible model operations
while keeping request caches, sampling state and response streams independent.

Interleave prompt chunks with decoding and retain speculative decoding when
generation runs alone. Cover isolation, cancellation, slow readers, cache
reuse and mixed requests with unit and integration tests. Remove the stale
llama-server architecture restriction on parallel requests.

* mlx: prevent cache snapshots from retaining batch peers

Copy each request’s initial batched K/V row so a sliding-cache snapshot cannot retain the whole batch allocation while accounting for only one row.

Exclude grammar compilation waits from prompt timing and reuse row-selected attention masks across layers.

Return errors for requests rejected before admission and strengthen numerical and compiled-speech regression checks.

* address comments
2026-11-10 13:24:12 -08:00

195 lines
6.3 KiB
Go

package cache
import (
"fmt"
"slices"
"github.com/ollama/ollama/mlx"
"github.com/ollama/ollama/mlxrunner/batch"
"github.com/ollama/ollama/mlxrunner/nn"
)
// Batcher adapts model-specific cache state to a shared forward. Batch
// must validate every row before returning, without mutating any owner. The
// returned Layer borrows the owners and does not manage their lifetime.
type Batcher interface {
Batch([]Cache) (Layer, error)
}
// Batch borrows one cache bundle per request for an unpadded forward with
// equal query lengths. It neither owns nor rearranges the original caches.
// Keep every owner alive until the forward and its state evaluate.
// Rebuild the views when the participating requests or their order changes.
// A single row returns its original layers.
//
// Standard attention and recurrent caches are adapted here. Model-specific
// caches implement Batcher. This validates cache compatibility, not whether a
// model's computation supports multiple rows.
func Batch(rows [][]Cache) ([]Layer, error) {
if len(rows) == 0 {
return nil, fmt.Errorf("cache batch: no rows")
}
if len(rows) == 1 {
return Layers(rows[0]), nil
}
for i, row := range rows {
if len(row) != len(rows[0]) {
return nil, fmt.Errorf("cache batch: row %d has %d layers, want %d", i, len(row), len(rows[0]))
}
}
layers := make([]Layer, len(rows[0]))
seen := make(map[Cache]bool)
for l, first := range rows[0] {
view := batchedLayer{rows: make([]Cache, len(rows))}
for i, row := range rows {
c := row[l]
var compatible bool
switch a := first.(type) {
case nil:
compatible = c == nil
case *KVCache:
_, compatible = c.(*KVCache)
case *RotatingKVCache:
b, ok := c.(*RotatingKVCache)
compatible = ok && a.maxSize == b.maxSize
case *RecurrentCache:
b, ok := c.(*RecurrentCache)
compatible = ok && a.convTail == b.convTail && a.convDim == b.convDim &&
a.numVHeads == b.numVHeads && a.headVDim == b.headVDim && a.headKDim == b.headKDim
case Batcher:
compatible = true // The model validates its rows below.
default:
return nil, fmt.Errorf("cache batch: unsupported layer %d cache %T", l, first)
}
if !compatible {
return nil, fmt.Errorf("cache batch: incompatible row %d layer %d cache %T (first %T)", i, l, c, first)
}
if c != nil {
if seen[c] {
return nil, fmt.Errorf("cache batch: reused cache owner in row %d layer %d", i, l)
}
seen[c] = true
}
view.rows[i] = c
}
switch first := first.(type) {
case *KVCache, *RotatingKVCache:
layers[l] = &attentionBatch{view}
case *RecurrentCache:
layers[l] = &recurrentBatch{view}
case Batcher:
var err error
layers[l], err = first.Batch(view.rows)
if err != nil {
return nil, fmt.Errorf("cache batch: layer %d: %w", l, err)
}
}
}
return layers, nil
}
type batchedLayer struct {
rows []Cache
}
func (c *batchedLayer) State() []*mlx.Array {
var state []*mlx.Array
for _, row := range c.rows {
state = append(state, row.State()...)
}
return state
}
func (c *batchedLayer) check(b *batch.Batch) {
if b.InputIDs.Dim(0) != len(c.rows) || b.InputIDs.Dim(1) < 1 ||
len(b.SeqOffsets) != len(c.rows) || len(b.SeqQueryLens) != len(c.rows) {
panic("cache batch: expected equal-length query rows")
}
for _, n := range b.SeqQueryLens {
if int(n) != b.InputIDs.Dim(1) {
panic("cache batch: padded cache rows are not supported")
}
}
}
type attentionBatch struct{ batchedLayer }
func (c *attentionBatch) Update(b *batch.Batch, keys, values *mlx.Array) *nn.KVHistory {
c.check(b)
if keys.Dim(0) != len(c.rows) || values.Dim(0) != len(c.rows) || keys.Dim(2) != b.InputIDs.Dim(1) || values.Dim(2) != b.InputIDs.Dim(1) {
panic("cache batch: key/value shape does not match query rows")
}
histories := make([]*nn.KVHistory, len(c.rows))
for i, row := range c.rows {
var k, v *mlx.Array
if row.Offset() == 0 && keys.Dim(2) > 1 {
// An empty cache may retain its first prompt directly. Give it
// its own row so a snapshot cannot pin the entire batch.
index := mlx.FromValues([]int32{int32(i)}, 1)
k, v = keys.TakeAxis(index, 0), values.TakeAxis(index, 0)
} else {
k = keys.Slice(mlx.Slice(i, i+1), mlx.Slice(), mlx.Slice(), mlx.Slice())
v = values.Slice(mlx.Slice(i, i+1), mlx.Slice(), mlx.Slice(), mlx.Slice())
}
histories[i] = row.(Attention).Update(b.Row(i), k, v)
}
return nn.NewRowKVHistory(histories)
}
func (c *attentionBatch) View(b *batch.Batch) *nn.KVHistory {
c.check(b)
histories := make([]*nn.KVHistory, len(c.rows))
for i, row := range c.rows {
histories[i] = row.(Attention).View(b.Row(i))
}
return nn.NewRowKVHistory(histories)
}
type recurrentBatch struct{ batchedLayer }
func (c *recurrentBatch) Get(b *batch.Batch, dtype mlx.DType) *nn.RecurrentHistory {
c.check(b)
conv, delta := make([]*mlx.Array, len(c.rows)), make([]*mlx.Array, len(c.rows))
for i, row := range c.rows {
h := row.(Recurrent).Get(b.Row(i), dtype)
conv[i], delta[i] = h.ConvState(), h.DeltaState()
}
return nn.NewRecurrentHistory(mlx.Concatenate(conv, 0), mlx.Concatenate(delta, 0))
}
func (c *recurrentBatch) SnapshotSplits(forwardLen int) []int {
var splits []int
for _, row := range c.rows {
splits = append(splits, row.(Recurrent).SnapshotSplits(forwardLen)...)
}
slices.Sort(splits)
return slices.Compact(splits)
}
func (c *recurrentBatch) Put(b *batch.Batch, convStates, deltaStates []*mlx.Array) {
c.check(b)
n := b.InputIDs.Dim(1)
boundaries := append(c.SnapshotSplits(n), n)
if len(convStates) != len(boundaries) || len(deltaStates) != len(boundaries) {
panic("cache batch: recurrent states do not match snapshot boundaries")
}
for i := range boundaries {
if convStates[i].Dim(0) != len(c.rows) || deltaStates[i].Dim(0) != len(c.rows) {
panic("cache batch: recurrent state row count does not match caches")
}
}
for i, row := range c.rows {
owner := row.(Recurrent)
wanted := append(owner.SnapshotSplits(n), n)
conv, delta := make([]*mlx.Array, len(wanted)), make([]*mlx.Array, len(wanted))
index := mlx.FromValues([]int32{int32(i)}, 1)
for j, boundary := range wanted {
at, _ := slices.BinarySearch(boundaries, boundary)
// Keep each owner compact rather than retaining its peers' states.
conv[j] = convStates[at].TakeAxis(index, 0)
delta[j] = deltaStates[at].TakeAxis(index, 0)
}
owner.Put(b.Row(i), conv, delta)
}
}