mirror of
https://github.com/ollama/ollama.git
synced 2026-11-12 03:46:14 -06:00
a00d083bb5
* mlxrunner: restore System One scoring
Carry the MLX implementation and its tests from the original System One
commit. The API/compiler and llama-server implementation already exist on
the release branch, so retain those versions. Adapt the optional model test
to the renamed decision package and its explicit rendering/context setup.
(cherry picked from commit 26e546ddb3)
* Enable MLX System One and add regression coverage
* mlxrunner: add Laya support for System One
Import Laya checkpoints and implement bidirectional decision scoring.
* mlxrunner: release cached buffers after System One scoring
* decision: match Tev1 and Nimble publisher prompt formats
Select Tev1 prompts using renderer metadata and fix JSON formatting, with regression coverage for GGUF and MLX.
* mlx: support Clef and Clef Flash decision models
Add model import, quantization safeguards, and text/image scoring with the Clef joint decision head.
* support vision capabilities for decision models
* Make Clef scoring cancellable between prefill chunks
Preserve joint-head inputs and test cancellation, cleanup, and recovery.
* mlx: separate decision preparation and readout from forward
Let serial and batched scoring share model-owned preparation and readout
without duplicating architecture-specific logic in the runner.
* mlx: add Strands Decider support
* implement caching
---------
Co-authored-by: ParthSareen <parth.sareen@ollama.com>
47 lines
1.1 KiB
Go
47 lines
1.1 KiB
Go
package mlxrunner
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"github.com/ollama/ollama/api"
|
|
"github.com/ollama/ollama/llm"
|
|
)
|
|
|
|
var _ llm.Scorer = (*Client)(nil)
|
|
|
|
func (c *Client) Score(ctx context.Context, input llm.ScoreRequest) (llm.ScoreResponse, error) {
|
|
var result llm.ScoreResponse
|
|
data, err := json.Marshal(input)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/v1/score", c.port), bytes.NewReader(data))
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
res, err := c.client.Do(req)
|
|
if err != nil {
|
|
if errMsg := c.status.LastError(); errMsg != "" {
|
|
return result, fmt.Errorf("mlx runner failed: %s", errMsg)
|
|
}
|
|
return result, err
|
|
}
|
|
defer res.Body.Close()
|
|
if res.StatusCode != http.StatusOK {
|
|
body, err := io.ReadAll(res.Body)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
return result, api.StatusError{StatusCode: res.StatusCode, ErrorMessage: strings.TrimSpace(string(body))}
|
|
}
|
|
err = json.NewDecoder(res.Body).Decode(&result)
|
|
return result, err
|
|
}
|