github-starred/ollama-ollama
2
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2025-12-05 18:46:22 -06:00
Code Issues 2.2k Packages Projects Releases 100 Wiki Activity

added batch fix

Browse Source
This commit is contained in:
nicole pardal
2025-11-21 16:29:56 -08:00
parent 47e272c35a
commit cee4922649
1 changed files with 5 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
llm/server.go
View File

@@ -170,6 +170,11 @@ func NewLlamaServer(systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, modelPath st
opts.NumBatch = min(opts.NumBatch, opts.NumCtx)
if f.KV().Architecture() == "nomic-bert" {
opts.NumBatch = opts.NumCtx
slog.Debug("nomic-bert model detected, setting batch size equal to context length", "num_batch", opts.NumBatch, "num_ctx", opts.NumCtx)
}
loadRequest := LoadRequest{LoraPath: adapters, KvSize: opts.NumCtx * numParallel, BatchSize: opts.NumBatch, Parallel: numParallel, MultiUserCache: envconfig.MultiUserCache()}
defaultThreads := systemInfo.ThreadCount