mirror of
https://github.com/ollama/ollama.git
synced 2026-11-10 15:02:13 -06:00
Daniel HiltgenandGitHub
3fbc609be4
Fixes for running LFM3 GGUFs on the llama-server runner, plus GGUF text-to-speech that turns on once llama.cpp is bumped past b11901. - BOS: honor an explicit tokenizer.ggml.add_bos_token. LFM3 GGUFs (and lfm2:24b) set it to false and expect the template's BOS, which the lfm2 default was stripping. The lfm2 default still applies when the key is absent. On lfm2:24b, WikiText PPL goes from 81.3 to 72.0. - repeat_last_n -1 now maps to the context size; llama-server rejects negative values. - Audio-only projectors (LFM3-Audio) no longer advertise vision, so clients don't send images the model can't take. - /v1/audio/speech on GGUF models calls llama-server /tts chunk by chunk. Renderers opt in through a new SpeechPrompter (voices, voice parsing, chunking), so other speech models can add their own. The LFM3 voice and chunking helpers move from the MLX runner into model/renderers and are shared by both runners.
71 lines
2.0 KiB
Go
71 lines
2.0 KiB
Go
package create
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/ollama/ollama/model/renderers"
|
|
"github.com/ollama/ollama/types/model"
|
|
)
|
|
|
|
// lfm3ImportTransform quantizes only the language backbone of the multimodal
|
|
// LFM3 checkpoints; vision, audio encoder, projectors, and the speech decoder
|
|
// stay at source precision.
|
|
type lfm3ImportTransform struct {
|
|
backbone string
|
|
// Audio nests its language model config under "lfm", where shared
|
|
// config inference doesn't look; the manifest records it instead.
|
|
contextLen, embedLen int
|
|
// speech models' voices are prompt descriptions, not voice tensors, so
|
|
// the manifest names them.
|
|
speech bool
|
|
}
|
|
|
|
func newLFM3ImportTransform(json.RawMessage) (quantizePolicy, error) {
|
|
return lfm3ImportTransform{backbone: "model.language_model."}, nil
|
|
}
|
|
|
|
func newLFM3OmniImportTransform(json.RawMessage) (quantizePolicy, error) {
|
|
return lfm3ImportTransform{backbone: "model.language_model.", speech: true}, nil
|
|
}
|
|
|
|
func newLFM3AudioImportTransform(raw json.RawMessage) (quantizePolicy, error) {
|
|
var cfg struct {
|
|
LFM struct {
|
|
HiddenSize int `json:"hidden_size"`
|
|
MaxPositionEmbeddings int `json:"max_position_embeddings"`
|
|
} `json:"lfm"`
|
|
}
|
|
if err := json.Unmarshal(raw, &cfg); err != nil {
|
|
return nil, err
|
|
}
|
|
if cfg.LFM.HiddenSize <= 0 || cfg.LFM.MaxPositionEmbeddings <= 0 {
|
|
return nil, fmt.Errorf("missing lfm.hidden_size or lfm.max_position_embeddings")
|
|
}
|
|
return lfm3ImportTransform{
|
|
backbone: "lfm.model.",
|
|
contextLen: cfg.LFM.MaxPositionEmbeddings,
|
|
embedLen: cfg.LFM.HiddenSize,
|
|
speech: true,
|
|
}, nil
|
|
}
|
|
|
|
func (t lfm3ImportTransform) augmentConfig(cfg *model.ConfigV2) error {
|
|
if t.contextLen > 0 {
|
|
cfg.ContextLen = t.contextLen
|
|
cfg.EmbedLen = t.embedLen
|
|
}
|
|
if t.speech {
|
|
cfg.Voices = renderers.LFM3SpeechVoices()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (t lfm3ImportTransform) quantizationType(name string, shape []int32, quantize string) string {
|
|
if !strings.HasPrefix(name, t.backbone) {
|
|
return ""
|
|
}
|
|
return GetTensorQuantization(name, shape, quantize)
|
|
}
|