ollama/mlxrunner/model/model.go

package model

import (
	"encoding/json"
	"fmt"
	"log/slog"
	"sync"

	"github.com/ollama/ollama/mlx"
	"github.com/ollama/ollama/mlxrunner/batch"
	"github.com/ollama/ollama/mlxrunner/cache"
	"github.com/ollama/ollama/mlxrunner/tokenizer"
)

// Model is the interface that model implementations must satisfy.
type Model interface {
	// LoadWeights receives all tensors loaded from the manifest and assigns
	// them to model fields. Model-specific logic (MLA absorption, expert
	// stacking, quantized layer creation) happens here.
	LoadWeights(tensors map[string]*mlx.Array) error

	// NewCaches builds the cache slots this model's layers need.
	NewCaches() []cache.Cache

	// Forward returns the hidden state to unembed and the state a draft
	// model conditions on; plain models return the final hidden for both.
	Forward(b *batch.Batch, cache []cache.Cache) (hidden, auxHidden *mlx.Array)
	Unembed(x *mlx.Array) *mlx.Array

	Tokenizer() *tokenizer.Tokenizer
	MaxContextLength() int
}

// SkippedTensorsModel identifies checkpoint tensors to exclude from MLX loading.
// Names use the original safetensors namespace, including any quantization
// companions. The model is responsible for accessing any required skipped tensors.
type SkippedTensorsModel interface {
	SkippedTensorNames() []string
}

// DraftModel is an auxiliary model alongside a target that proposes speculative
// tokens.
type DraftModel interface {
	// LoadWeights assigns manifest tensors to the draft model's fields. An
	// inline head has nothing to do here; its weights load with the target's.
	LoadWeights(tensors map[string]*mlx.Array) error

	// NewCaches builds the cache slots this draft model writes, or nil
	// when it keeps no KV.
	NewCaches() []cache.Cache

	// Forward consumes b.Hidden (the draft-conditioning state) and returns
	// its hidden plus the aux hidden that seeds the next step. targetCaches
	// is read-only, for drafts that attend over the target's history.
	Forward(b *batch.Batch, targetCaches, draftCaches []cache.Cache) (hidden, auxHidden *mlx.Array)

	// Unembed projects a hidden state to vocabulary logits.
	Unembed(x *mlx.Array) *mlx.Array
}

// BlockDraft is a DraftModel that drafts a whole block per forward (block
// diffusion), conditioned on features tapped from target layers rather than
// the final hidden state.
type BlockDraft interface {
	DraftModel

	// BlockParams returns the trained block length and the mask token
	// standing in for undrafted positions.
	BlockParams() (blockSize int, maskToken int32)
}

// SelfDraft is implemented by models whose draft head ships inline with the
// target weights; it returns the head, or nil when the checkpoint shipped none.
type SelfDraft interface {
	SelfDraft() DraftModel
}

// BlockSelector is a BlockDraft with a learned selector that chooses each
// proposal conditional on its predecessor. The runner owns sampling policy
// and RNG; the model owns candidate scores.
type BlockSelector interface {
	BlockDraft

	// Select proposes a block from the draft's hidden rows [B,L,H], their
	// logits [B,L,V] and the anchor token [B] preceding the block. choose
	// samples one of each row's unique candidate IDs, taking IDs and scores
	// [N,K] and returning tokens [N] and probabilities [N,K]. Select returns
	// tokens [B,L] and each position's candidate IDs and probabilities [B,L,K],
	// which verification uses to correct the proposal.
	Select(hidden, logits, anchor *mlx.Array, choose func(ids, scores *mlx.Array) (tokens, probs *mlx.Array)) (tokens, ids, probs *mlx.Array)
}

var (
	mu            sync.Mutex
	registry      = make(map[string]func(root *Root) (Model, error))
	draftRegistry = make(map[string]func(root *Root, target Model) (DraftModel, error))
)

// Register registers a model constructor by architecture name.
// Called from init() in model packages. Panics on duplicate registration.
func Register(arch string, fn func(root *Root) (Model, error)) {
	mu.Lock()
	defer mu.Unlock()

	if _, exists := registry[arch]; exists {
		panic(fmt.Sprintf("model architecture %q already registered", arch))
	}
	registry[arch] = fn
}

// RegisterDraft registers a draft model constructor by architecture name.
func RegisterDraft(arch string, fn func(root *Root, target Model) (DraftModel, error)) {
	mu.Lock()
	defer mu.Unlock()

	if _, exists := draftRegistry[arch]; exists {
		panic(fmt.Sprintf("draft model architecture %q already registered", arch))
	}
	draftRegistry[arch] = fn
}

// SupportsArchitecture reports whether a target model constructor is registered.
func SupportsArchitecture(arch string) bool {
	mu.Lock()
	defer mu.Unlock()
	_, ok := registry[arch]
	return ok
}

// SupportsDraftArchitecture reports whether a draft model constructor is registered.
func SupportsDraftArchitecture(arch string) bool {
	mu.Lock()
	defer mu.Unlock()
	_, ok := draftRegistry[arch]
	return ok
}

// New reads config.json from the manifest, detects the architecture, looks up
// the registered constructor, and calls it to create the model (with config
// parsed and struct created, but weights not yet loaded).
func New(root *Root) (Model, error) {
	configData, err := root.Manifest.ReadConfig("config.json")
	if err != nil {
		return nil, fmt.Errorf("failed to read config.json: %w", err)
	}

	var archConfig struct {
		Architectures []string `json:"architectures"`
	}
	if err := json.Unmarshal(configData, &archConfig); err != nil {
		return nil, fmt.Errorf("failed to parse config.json: %w", err)
	}

	if len(archConfig.Architectures) == 0 {
		return nil, fmt.Errorf("no architectures found in config.json")
	}

	arch := archConfig.Architectures[0]
	slog.Info("Model architecture", "arch", arch)

	mu.Lock()
	fn, ok := registry[arch]
	mu.Unlock()

	if !ok {
		return nil, fmt.Errorf("unsupported architecture: %s", arch)
	}

	return fn(root)
}

// NewDraft constructs the draft model described by the manifest config, if any.
func NewDraft(root *Root, target Model) (DraftModel, error) {
	if root == nil || root.Draft == nil {
		return nil, nil
	}

	configPath := root.Draft.Config
	if configPath == "" {
		configPath = "draft/config.json"
	}
	configData, err := root.Manifest.ReadConfig(configPath)
	if err != nil {
		return nil, fmt.Errorf("failed to read %s: %w", configPath, err)
	}

	var archConfig struct {
		Architectures []string `json:"architectures"`
		ModelType     string   `json:"model_type"`
	}
	if err := json.Unmarshal(configData, &archConfig); err != nil {
		return nil, fmt.Errorf("failed to parse %s: %w", configPath, err)
	}

	arch := root.Draft.Architecture
	if arch == "" && len(archConfig.Architectures) > 0 {
		arch = archConfig.Architectures[0]
	}
	if arch == "" {
		arch = archConfig.ModelType
	}
	if arch == "" {
		return nil, fmt.Errorf("no draft architecture found in %s", configPath)
	}
	slog.Info("Draft model architecture", "arch", arch)

	mu.Lock()
	fn, ok := draftRegistry[arch]
	mu.Unlock()
	if !ok {
		return nil, fmt.Errorf("unsupported draft architecture: %s", arch)
	}

	return fn(root, target)
}