mirror of
https://github.com/ollama/ollama.git
synced 2026-11-10 15:02:13 -06:00
Daniel HiltgenandGitHub
195f4cdca1
* mlx: match publisher tokenizer semantics Honor pretokenizer stage order, split behavior, Unicode boundaries, added-token normalization, and ranked BPE merges. Handle empty added tokens and empty Metaspace input consistently. Add shared Go/Python reference cases using published tokenizers, pulling missing models directly and failing on errors, plus focused regressions for configuration precedence, byte fallback, and parallel encoding. * address comments
42 lines
1.1 KiB
YAML
42 lines
1.1 KiB
YAML
name: test-tokenizer
|
|
|
|
on:
|
|
pull_request:
|
|
paths:
|
|
- 'mlxrunner/tokenizer/**'
|
|
- '.github/workflows/test-tokenizer.yaml'
|
|
workflow_dispatch:
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
reference:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
env:
|
|
FETCH_TOKENIZERS: '1'
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-go@v5
|
|
with:
|
|
go-version-file: go.mod
|
|
- name: Cache tokenizer data
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: .cache/tokenizers
|
|
key: tokenizers-${{ hashFiles('mlxrunner/tokenizer/tokenizer_reference_test.go') }}
|
|
restore-keys: tokenizers-
|
|
- name: Test published tokenizers
|
|
run: |
|
|
set -euo pipefail
|
|
go test -race -count=1 -timeout=10m -v ./mlxrunner/tokenizer/... | tee "${RUNNER_TEMP}/tokenizer-test.log"
|
|
if grep -q -- '--- SKIP' "${RUNNER_TEMP}/tokenizer-test.log"; then
|
|
echo "Tokenizer CI must run every case without skips" >&2
|
|
exit 1
|
|
fi
|