mirror of
https://github.com/ollama/ollama.git
synced 2026-11-15 10:08:04 -06:00
* mlx: batch requests through a shared scheduler Use one request queue and scheduler for generation, decisions, embeddings and speech, including parallelism one. Share compatible model operations while keeping request caches, sampling state and response streams independent. Interleave prompt chunks with decoding and retain speculative decoding when generation runs alone. Cover isolation, cancellation, slow readers, cache reuse and mixed requests with unit and integration tests. Remove the stale llama-server architecture restriction on parallel requests. * mlx: prevent cache snapshots from retaining batch peers Copy each request’s initial batched K/V row so a sliding-cache snapshot cannot retain the whole batch allocation while accounting for only one row. Exclude grammar compilation waits from prompt timing and reuse row-selected attention masks across layers. Return errors for requests rejected before admission and strengthen numerical and compiled-speech regression checks. * address comments
166 lines
5.5 KiB
Go
166 lines
5.5 KiB
Go
//go:build integration && release
|
|
|
|
package integration
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"math"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/ollama/ollama/api"
|
|
"github.com/ollama/ollama/types/model"
|
|
)
|
|
|
|
func runConcurrentEmbed(t *testing.T, modelName string) {
|
|
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute)
|
|
defer cancel()
|
|
client, _, cleanup := InitServerConnection(ctx, t)
|
|
defer cleanup()
|
|
requireCapability(ctx, t, client, modelName, "embedding")
|
|
skipIfModelTooLargeForVRAM(ctx, t, client, modelName)
|
|
info, err := client.Show(ctx, &api.ShowRequest{Model: modelName})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
inputs := []any{"A red bicycle.", "The library closes at six on Saturdays.", "A recipe for bread made with flour, water, salt, and yeast."}
|
|
if slices.Contains(info.Capabilities, model.CapabilityVision) {
|
|
image, _, _ := decodeTestImages(t)
|
|
inputs = append(inputs, map[string]string{"text": "A street scene.", "image": base64.StdEncoding.EncodeToString(image)})
|
|
}
|
|
if slices.Contains(info.Capabilities, model.CapabilityAudio) {
|
|
inputs = append(inputs, map[string]string{"audio": base64.StdEncoding.EncodeToString(decodeTestAudio(t))})
|
|
}
|
|
want := make([][]float32, len(inputs))
|
|
for i, input := range inputs {
|
|
response, err := client.Embed(ctx, &api.EmbedRequest{Model: modelName, Input: input})
|
|
if err != nil {
|
|
skipIfMLXUnsupported(t, err)
|
|
t.Fatal(err)
|
|
}
|
|
if len(response.Embeddings) != 1 || len(response.Embeddings[0]) == 0 {
|
|
t.Fatal("empty baseline embedding")
|
|
}
|
|
want[i] = response.Embeddings[0]
|
|
}
|
|
t.Run("concurrent", func(t *testing.T) {
|
|
for i, input := range inputs {
|
|
t.Run(fmt.Sprint(i), func(t *testing.T) {
|
|
t.Parallel()
|
|
got, err := client.Embed(ctx, &api.EmbedRequest{Model: modelName, Input: input})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got.Embeddings) != 1 || len(got.Embeddings[0]) != len(want[i]) {
|
|
t.Fatal("embedding dimensions changed")
|
|
}
|
|
for j, value := range got.Embeddings[0] {
|
|
if math.IsNaN(float64(value)) || math.IsInf(float64(value), 0) {
|
|
t.Fatalf("embedding %d value %d is not finite: %g", i, j, value)
|
|
}
|
|
}
|
|
// Quantized kernels can round differently for a wider batch.
|
|
// Use the same whole-vector parity check as the embed suite.
|
|
requireSimilarEmbedding(t, want[i], got.Embeddings[0])
|
|
})
|
|
}
|
|
})
|
|
}
|
|
|
|
func runConcurrentSpeech(t *testing.T, modelName string) {
|
|
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute)
|
|
defer cancel()
|
|
client, endpoint, cleanup := InitServerConnection(ctx, t)
|
|
defer cleanup()
|
|
requireCapability(ctx, t, client, modelName, "speech")
|
|
skipIfModelTooLargeForVRAM(ctx, t, client, modelName)
|
|
info, err := client.Show(ctx, &api.ShowRequest{Model: modelName})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
speech := func(ctx context.Context, text string) (*http.Response, error) {
|
|
body, err := json.Marshal(map[string]string{"model": modelName, "input": text, "response_format": "pcm"})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://"+endpoint+"/v1/audio/speech", bytes.NewReader(body))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
return http.DefaultClient.Do(req)
|
|
}
|
|
check := func(t *testing.T, ctx context.Context, text string) {
|
|
t.Helper()
|
|
res, err := speech(ctx, text)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer res.Body.Close()
|
|
pcm, err := io.ReadAll(io.LimitReader(res.Body, 8<<20))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if res.StatusCode != http.StatusOK {
|
|
err := fmt.Errorf("speech HTTP %d: %s", res.StatusCode, pcm)
|
|
skipIfMLXUnsupported(t, err)
|
|
t.Fatal(err)
|
|
}
|
|
if len(pcm) < 4800 || len(pcm)%2 != 0 || len(pcm) == 8<<20 || bytes.Equal(pcm, make([]byte, len(pcm))) {
|
|
t.Fatalf("invalid or silent PCM stream (%d bytes)", len(pcm))
|
|
}
|
|
}
|
|
// Load on the parent goroutine so an unsupported platform skips the case.
|
|
check(t, ctx, "Hello there.")
|
|
for round := range 2 {
|
|
t.Run(fmt.Sprint(round), func(t *testing.T) {
|
|
if slices.Contains(info.Capabilities, model.CapabilityCompletion) {
|
|
t.Run("chat", func(t *testing.T) {
|
|
t.Parallel()
|
|
var text strings.Builder
|
|
err := client.Chat(ctx, &api.ChatRequest{
|
|
Model: modelName, Messages: []api.Message{{Role: "user", Content: "The box is red. What color is the box? Reply with just its color."}},
|
|
Think: &api.ThinkValue{Value: false}, Options: map[string]any{"temperature": 0, "num_predict": 32},
|
|
}, func(r api.ChatResponse) error { text.WriteString(r.Message.Content); return nil })
|
|
if err != nil || !strings.Contains(strings.ToLower(text.String()), "red") {
|
|
t.Fatalf("chat alongside speech = %q, error %v", text.String(), err)
|
|
}
|
|
})
|
|
}
|
|
for i, text := range []string{"The red bicycle is waiting outside.", "Please bring the books back to the library before six.", "This request is canceled while its peers continue speaking."} {
|
|
t.Run(fmt.Sprint(i), func(t *testing.T) {
|
|
t.Parallel()
|
|
if i < 2 {
|
|
check(t, ctx, text)
|
|
return
|
|
}
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
res, err := speech(ctx, text)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer res.Body.Close()
|
|
if res.StatusCode != http.StatusOK {
|
|
t.Fatalf("cancel request HTTP %d", res.StatusCode)
|
|
}
|
|
var sample [2]byte
|
|
if _, err := io.ReadFull(res.Body, sample[:]); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cancel()
|
|
})
|
|
}
|
|
})
|
|
}
|
|
check(t, ctx, "All streams have finished.")
|
|
}
|