mirror of
https://github.com/ollama/ollama.git
synced 2026-11-18 09:51:49 -06:00
* mlx: batch requests through a shared scheduler Use one request queue and scheduler for generation, decisions, embeddings and speech, including parallelism one. Share compatible model operations while keeping request caches, sampling state and response streams independent. Interleave prompt chunks with decoding and retain speculative decoding when generation runs alone. Cover isolation, cancellation, slow readers, cache reuse and mixed requests with unit and integration tests. Remove the stale llama-server architecture restriction on parallel requests. * mlx: prevent cache snapshots from retaining batch peers Copy each request’s initial batched K/V row so a sliding-cache snapshot cannot retain the whole batch allocation while accounting for only one row. Exclude grammar compilation waits from prompt timing and reuse row-selected attention masks across layers. Return errors for requests rejected before admission and strengthen numerical and compiled-speech regression checks. * address comments
170 lines
6.7 KiB
Go
170 lines
6.7 KiB
Go
package cache
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"github.com/ollama/ollama/mlx"
|
|
)
|
|
|
|
// Layer is one cache slot used during a model forward. Attention and Recurrent
|
|
// provide its read/write operations. It may be a request's Cache or a temporary
|
|
// view of the corresponding caches from several requests. The view does not own
|
|
// those caches and cannot manage their snapshots or lifetime.
|
|
type Layer interface {
|
|
// State returns state roots that should be kept/evaluated.
|
|
State() []*mlx.Array
|
|
}
|
|
|
|
// Cache owns one cache slot's state and snapshots for a single request.
|
|
// A model usually needs one slot per layer, returned together by NewCaches.
|
|
type Cache interface {
|
|
Layer
|
|
Free()
|
|
Offset() int
|
|
|
|
// Snapshot retains state from fromOffset to the current offset without
|
|
// changing the active cache. Storage may be copied, shared immutably, or
|
|
// copied lazily before the active cache overwrites it.
|
|
Snapshot(fromOffset int) Snapshot
|
|
|
|
// PrepareSnapshots schedules the cache to capture a snapshot as its
|
|
// storage offset reaches each listed offset during subsequent writes.
|
|
// Offsets are storage offsets, must not already be passed (current
|
|
// offset <= offset), and must be sorted ascending and unique. Scheduled
|
|
// offsets persist across multiple writes until TakeSnapshots is called;
|
|
// captures accumulate. The previous schedule must be drained first.
|
|
//
|
|
// Unlike Snapshot, capture happens at the moment a write crosses the
|
|
// scheduled offset, so it can record states interior to a batched write
|
|
// without the caller breaking the write into pieces.
|
|
PrepareSnapshots(offsets []int)
|
|
|
|
// TakeSnapshots returns the snapshots captured since PrepareSnapshots,
|
|
// one per scheduled offset in the caller's order, and clears the
|
|
// schedule. An entry may be nil for an empty range or an offset that
|
|
// writes never reached, for example after cancellation.
|
|
TakeSnapshots() []Snapshot
|
|
|
|
// Restore brings the cache to target. If snapshot is nil, rewinds
|
|
// using the cache's own live state. Returns false if the requested state
|
|
// is unavailable or target is negative.
|
|
Restore(snapshot Snapshot, target int) bool
|
|
|
|
// Merge combines two sequential snapshots [a,b) and [b,c) into [a,c).
|
|
// Takes ownership of both inputs.
|
|
Merge(parent, child Snapshot) Snapshot
|
|
|
|
// Split divides a snapshot [a,c) at offset b into [a,b) and [b,c).
|
|
// Takes ownership of the input. Cache types that cannot split
|
|
// (e.g. recurrent) return (nil, snapshot).
|
|
Split(snapshot Snapshot, at int) (parent, child Snapshot)
|
|
}
|
|
|
|
// Layers exposes a request's cache slots through the forward-only interface.
|
|
// It converts the slice without copying any cache state or transferring ownership.
|
|
func Layers(caches []Cache) []Layer {
|
|
if caches == nil {
|
|
return nil
|
|
}
|
|
layers := make([]Layer, len(caches))
|
|
for i, c := range caches {
|
|
layers[i] = c
|
|
}
|
|
return layers
|
|
}
|
|
|
|
// Snapshot is paged-out cache state that can be restored later.
|
|
type Snapshot interface {
|
|
// Size returns the retained VRAM bytes charged to the snapshot. Immutable
|
|
// views may conservatively charge shared backing storage more than once. A lazy
|
|
// snapshot that still indexes a live cache buffer returns 0 — it owns
|
|
// no extra memory yet. Once materialized (the cache copies the range
|
|
// out before overwriting its slots), Size returns the owned bytes.
|
|
Size() int
|
|
|
|
// SetMaterializeHook installs a callback fired once when a lazy
|
|
// snapshot materializes (allocates its owned arrays). delta is the
|
|
// newly-allocated byte count. Pass nil to detach. Snapshots that are
|
|
// never lazy may treat this as a no-op.
|
|
SetMaterializeHook(func(delta int))
|
|
|
|
// Close releases the snapshot's ownership of retained arrays.
|
|
Close()
|
|
}
|
|
|
|
// pendingSnapshots holds the scheduled snapshot capture state shared by all
|
|
// cache kinds. The owning cache calls captureReached from its write path
|
|
// to retain state at each scheduled offset the write reaches.
|
|
type pendingSnapshots struct {
|
|
offsets []int // scheduled storage offsets, in caller order
|
|
captured []Snapshot // captured[i] corresponds to offsets[i]; nil until reached
|
|
base int // running capture cursor: the from-offset of the next capture
|
|
}
|
|
|
|
// prepare schedules the listed storage offsets. See Cache.PrepareSnapshots.
|
|
func (p *pendingSnapshots) prepare(currentOffset int, offsets []int) {
|
|
if p.captured != nil {
|
|
panic("PrepareSnapshots: previous schedule not drained; TakeSnapshots first or its captures leak")
|
|
}
|
|
for i, o := range offsets {
|
|
if o < currentOffset {
|
|
panic(fmt.Sprintf("PrepareSnapshots: offset %d already passed (current %d)", o, currentOffset))
|
|
}
|
|
// captureReached and scheduledIn walk offsets in storage order and rely
|
|
// on it being ascending and unique.
|
|
if i > 0 && o <= offsets[i-1] {
|
|
panic(fmt.Sprintf("PrepareSnapshots: offsets must be sorted and unique, got %v", offsets))
|
|
}
|
|
}
|
|
p.offsets = append([]int(nil), offsets...)
|
|
p.captured = make([]Snapshot, len(offsets))
|
|
p.base = currentOffset
|
|
}
|
|
|
|
// take returns the captured snapshots and clears the schedule. See
|
|
// Cache.TakeSnapshots. Captures fire in ascending offset order; offsets beyond
|
|
// the last completed write remain nil.
|
|
func (p *pendingSnapshots) take() []Snapshot {
|
|
out := p.captured
|
|
p.offsets, p.captured = nil, nil
|
|
return out
|
|
}
|
|
|
|
// captureReached captures a snapshot for every scheduled offset that equals
|
|
// reached and hasn't been captured yet, using snap to produce the rollback
|
|
// state. The capture cursor base holds the previous scheduled boundary: snap
|
|
// reads it (yielding a [base, reached) range for position-sliceable caches),
|
|
// then base advances to reached so the next capture starts there. base only
|
|
// advances when a capture actually fires — write boundaries between scheduled
|
|
// offsets must not move it, or a later capture would record a fromOffset past
|
|
// the previous scheduled offset and break the alignment between each capture's
|
|
// range and the trie edge the caller attaches it to.
|
|
func (p *pendingSnapshots) captureReached(reached int, snap func(offset int) Snapshot) {
|
|
captured := false
|
|
for i, o := range p.offsets {
|
|
if p.captured[i] == nil && o == reached {
|
|
p.captured[i] = snap(o)
|
|
captured = true
|
|
}
|
|
}
|
|
if captured {
|
|
p.base = reached
|
|
}
|
|
}
|
|
|
|
// scheduledIn returns the scheduled offsets in [start, end], ascending. A write
|
|
// spanning [start, end) drives captureReached at each of these so any capture
|
|
// due there fires; for position-sliceable caches, the resulting snapshots cover
|
|
// [previous-scheduled-offset, this-offset) via captureReached's running cursor.
|
|
// p.offsets is ascending and unique (prepare enforces it), so the range filter
|
|
// preserves both.
|
|
func (p *pendingSnapshots) scheduledIn(start, end int) []int {
|
|
var boundaries []int
|
|
for _, o := range p.offsets {
|
|
if o >= start && o <= end {
|
|
boundaries = append(boundaries, o)
|
|
}
|
|
}
|
|
return boundaries
|
|
}
|