From d7f52eb9918dad2e888f6cb0b2e3f662e593f948 Mon Sep 17 00:00:00 2001 From: vad Date: Mon, 28 Sep 2026 15:37:36 +0000 Subject: [PATCH] feat(inference): Go hf2gguf converter + open-1b local run, byte-identical GGUF (#4) --- .gitignore | 3 + README.md | 2 +- reports/open1b-local-inference.md | 332 +++++++++++++++++++++++++ tools/hf2gguf/README.md | 82 +++++++ tools/hf2gguf/convert.go | 396 ++++++++++++++++++++++++++++++ tools/hf2gguf/convert_test.go | 161 ++++++++++++ tools/hf2gguf/go.mod | 5 + tools/hf2gguf/go.sum | 2 + tools/hf2gguf/hfconfig.go | 223 +++++++++++++++++ tools/hf2gguf/main.go | 284 +++++++++++++++++++++ tools/hf2gguf/rawkv.go | 353 ++++++++++++++++++++++++++ tools/hf2gguf/safetensors.go | 154 ++++++++++++ tools/hf2gguf/verify.go | 253 +++++++++++++++++++ 13 files changed, 2249 insertions(+), 1 deletion(-) create mode 100644 reports/open1b-local-inference.md create mode 100644 tools/hf2gguf/README.md create mode 100644 tools/hf2gguf/convert.go create mode 100644 tools/hf2gguf/convert_test.go create mode 100644 tools/hf2gguf/go.mod create mode 100644 tools/hf2gguf/go.sum create mode 100644 tools/hf2gguf/hfconfig.go create mode 100644 tools/hf2gguf/main.go create mode 100644 tools/hf2gguf/rawkv.go create mode 100644 tools/hf2gguf/safetensors.go create mode 100644 tools/hf2gguf/verify.go diff --git a/.gitignore b/.gitignore index 6e8f1a7..4bd2757 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ gitea/ *.whl *.gguf __pycache__/ + +# compiled tool binaries +/tools/hf2gguf/hf2gguf diff --git a/README.md b/README.md index 94d43fd..d25b9ba 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ |--------|--------| | Аудит init-unit open-1b (CPU) | ✅ **MATCH** — `state_hash 16554a11…`; см. [`reports/audit-open1b-init.md`](reports/audit-open1b-init.md) | | Аудит тренировочного интервала | ⛔ заблокирован (нет опубликованного interval-unit + RAM/диск) | -| Локальный inference на Arc B580 | 🔄 в работе | +| Локальный inference на Arc B580 | ✅ **работает** — Go-конвертер, GGUF байт-идентичен; **~114 tok/s** decode, ~892 prefill, ~3.6 GiB VRAM; см. [`reports/open1b-local-inference.md`](reports/open1b-local-inference.md) | ## Ссылки diff --git a/reports/open1b-local-inference.md b/reports/open1b-local-inference.md new file mode 100644 index 0000000..96c4d30 --- /dev/null +++ b/reports/open1b-local-inference.md @@ -0,0 +1,332 @@ +# Gensyn open-1b (SFT) — local bring-up & proof of run + +Epic: `vado/1b#1` / task #4 (SE). Workdir: `/root/projects/gensyn/inference/`. +Engine: existing shared llama.cpp **Vulkan** build (`/root/projects/inference/build-vulkan/bin/llama-server`). +Language: **Go 1.27 only** (see "Process note" re one accidental `python3` call). + +## TL;DR + +- Built a pure-Go tool `hf2gguf` (module `hf2gguf`) that **dumps** the reference GGUF + (go-gguf + a minimal raw-KV walker for arrays), **converts** `model.safetensors` + → `models/open-1b-sft-f16-go.gguf`, and **verifies** the two files. +- **Validation: PASS — 39/39 metadata KV byte-identical, 220/220 tensors byte-identical.** + Whole files are byte-for-byte identical (`cmp` + SHA-256 match). +- **Served** on one Arc (Vulkan2 / `07:00.0`) with the existing Vulkan build; coherent + output, **~114 tok/s decode**, **~892 tok/s prefill**, ~**3.6 GiB VRAM**. +- No commit made (all under gitignored `inference/`). + +Reference SHA-256 (and our output — identical): +`a18530417c732bb115a3501a7e1388ea7a54bd2a96c70be571d1249aa210fbae` + +--- + +## 1. Inputs & topology + +| Item | Path | Notes | +|------|------|-------| +| HF checkpoint (fp32) | `models/open-1b-sft/model.safetensors` | 6,432,081,672 B; dtype **F32**; header 34560 B | +| Reference GGUF (f16) | `models/open-1b-sft-f16.gguf` | 3,220,153,856 B; vendor converter; **read-only oracle** | +| HF config | `models/open-1b-sft/config.json` | `model_type=open1b`, 24 layers, hidden 2048, heads 16/4, head_dim 128, ffn 5632, vocab 128256, rope_theta 5e5, rms_eps 1e-5, swa 512 every 5, qk_norm gain-free, embedding_norm, untied | +| Vendor Python | `models/open-1b-sft/modeling_open1b.py` | read only, never executed | +| Engine | `/root/projects/inference/build-vulkan/bin/llama-server` | symlink → `/mnt/2TB/projects/build-vulkan`; shared prod tree, untouched | + +GPU devices as reported by the engine (`--list-devices`): + +``` +Vulkan0: AMD Ryzen 7 7700X (RADV RAPHAEL_MENDOCINO) (15835 MiB) +Vulkan1: Intel(R) Arc(tm) B580 Graphics (BMG G21) (12216 MiB) +Vulkan2: Intel(R) Arc(tm) B580 Graphics (BMG G21) (12216 MiB) +Vulkan3: Intel(R) Arc(tm) B580 Graphics (BMG G21) (12216 MiB) +``` + +`Vulkan2` was confirmed to map to PCI `0000:07:00.0` (Arc #2) via +`/sys/class/drm/renderD129/device`. + +--- + +## 2. Tool: `hf2gguf` (Go 1.27, `github.com/cymertek/go-gguf`) + +``` +inference/hf2gguf/ + go.mod, go.sum + main.go # subcommand dispatch + JSON manifest types + rawkv.go # minimal read-only GGUF KV walker (fills go-gguf's array gap) + safetensors.go # safetensors header + tensor byte access + hfconfig.go # config.json / tokenizer_config.json / tokenizer.json validation + convert.go # metadata reproduction + HF→GGUF tensor mapping + F32→F16 + verify.go # byte-level metadata & tensor comparison + convert_test.go +``` + +### Why a raw KV walker + +`go-gguf`'s `MetadataEntry.Value()` intentionally returns an empty value for +`BTypeArray` (see `readLazyValue` in the library). The tokenizer arrays +(`tokens`, `scores`→`token_type`, `merges`) are exactly the values we must copy +verbatim, so `rawkv.go` reads the KV section directly and preserves each array's +`elem_type`, element count, and **raw element bytes**. Those raw bytes are fed +straight into go-gguf's writer `Value{Raw: ...}`, so serialization is still done +by the library (nothing hand-rolled on the write path). + +### Subcommands + +```bash +# 2a. dump — emit the reproduction contract as JSON +go run . dump -gguf models/open-1b-sft-f16.gguf -o models/open-1b-sft-f16.manifest.json + +# 2b/c. convert — reproduce metadata + map safetensors→GGUF (fp32→f16) +go run . convert \ + -ref models/open-1b-sft-f16.gguf \ + -safetensors models/open-1b-sft/model.safetensors \ + -config models/open-1b-sft/config.json \ + -tokenizer-config models/open-1b-sft/tokenizer_config.json \ + -tokenizer models/open-1b-sft/tokenizer.json \ + -o models/open-1b-sft-f16-go.gguf + +# 3. verify — byte-level equality against the reference +go run . verify -ref models/open-1b-sft-f16.gguf -got models/open-1b-sft-f16-go.gguf +``` + +`dump` output (the contract) is at +`models/open-1b-sft-f16.manifest.json` (39 metadata entries + 220 tensors, +with per-array `raw_sha256`). + +### Tensor mapping (reference names are the contract) + +Shapes are the reference's; ggml `ne0` is the innermost (= HF input) dim, so HF +bytes are copied unpermuted with the dims reversed. + +| Reference tensor | HF source | Shape | Type | Count | +|---|---|---|---|---| +| `token_embd.weight` | `embedding.tok_embeddings.weight` | [2048,128256] | F16 | 1 | +| `token_embd_norm.weight` | `emb_norm.weight` | [2048] | F32 | 1 | +| `output.weight` | `embedding.output.weight` | [2048,128256] | F16 | 1 | +| `output_norm.weight` | `norm_out.weight` | [2048] | F32 | 1 | +| `blk.{i}.attn_q.weight` | `blocks.{i}.attn.wq.weight` | [2048,2048] | F16 | 24 | +| `blk.{i}.attn_k.weight` | `blocks.{i}.attn.wk.weight` | [2048,512] | F16 | 24 | +| `blk.{i}.attn_v.weight` | `blocks.{i}.attn.wv.weight` | [2048,512] | F16 | 24 | +| `blk.{i}.attn_output.weight` | `blocks.{i}.attn.wo.weight` | [2048,2048] | F16 | 24 | +| `blk.{i}.ffn_gate.weight` | `blocks.{i}.ffn.w_gate_up.weight` (first half) | [2048,5632] | F16 | 24 | +| `blk.{i}.ffn_up.weight` | `blocks.{i}.ffn.w_gate_up.weight` (second half) | [2048,5632] | F16 | 24 | +| `blk.{i}.ffn_down.weight` | `blocks.{i}.ffn.w_down.weight` | [5632,2048] | F16 | 24 | +| `blk.{i}.attn_norm.weight` | `blocks.{i}.norm1.weight` | [2048] | F32 | 24 | +| `blk.{i}.ffn_norm.weight` | `blocks.{i}.norm2.weight` | [2048] | F32 | 24 | + +- The fused `w_gate_up` is split at the row boundary (`chunk(2,-1)` → first half + `gate`, second half `up`), matching the reference. +- Quantized per-channel `*.weight_scale` tensors in the checkpoint are **dropped** + (the reference has none). +- F32→F16 uses IEEE round-to-nearest, ties-to-even (`f32bitsToF16`), matching + numpy/torch casts; the reference tensors are byte-identical to ours, which + empirically proves the rounding matched. +- HF config/tokenizer are **parsed and validated** against the reference, but the + reference remains authoritative for every emitted value. + +--- + +## 3. Validation + +`go run . verify` streams both files and compares on raw wire bytes: + +``` +metadata: 39 checked, 0 mismatches +tensors: 220 checked, 220 byte-identical, 0 mismatches +verification PASSED: 39 metadata keys and 220 tensors byte-identical +``` + +Whole-file check: + +``` +$ cmp models/open-1b-sft-f16.gguf models/open-1b-sft-f16-go.gguf && echo IDENTICAL +IDENTICAL +$ sha256sum models/open-1b-sft-f16.gguf models/open-1b-sft-f16-go.gguf +a18530417c732bb115a3501a7e1388ea7a54bd2a96c70be571d1249aa210fbae open-1b-sft-f16.gguf +a18530417c732bb115a3501a7e1388ea7a54bd2a96c70be571d1249aa210fbae open-1b-sft-f16-go.gguf +``` + +The per-tensor diff is empty (0 mismatches) — every tensor's payload is +byte-identical, in the same order, with the same shape and ggml type. + +### Metadata contract (all 39 keys, all byte-identical) + +| Key | Type | Value | +|---|---|---| +| `general.architecture` | STRING | `open1b` | +| `general.type` | STRING | `model` | +| `general.name` | STRING | `Open 1b Sft` | +| `general.finetune` | STRING | `sft` | +| `general.basename` | STRING | `open` | +| `general.size_label` | STRING | `1B` | +| `general.license` | STRING | `apache-2.0` | +| `general.base_model.count` | UINT32 | 1 | +| `general.base_model.0.name` | STRING | `Open 1b Midtrained 93B` | +| `general.base_model.0.organization` | STRING | `Gensyn` | +| `general.base_model.0.repo_url` | STRING | `https://huggingface.co/Gensyn/open-1b-midtrained-93B` | +| `general.dataset.count` | UINT32 | 1 | +| `general.dataset.0.name` | STRING | `Tulu 3 Sft Olmo 2 Mixture 0225` | +| `general.dataset.0.version` | STRING | `0225` | +| `general.dataset.0.organization` | STRING | `Allenai` | +| `general.dataset.0.repo_url` | STRING | `https://huggingface.co/allenai/tulu-3-sft-olmo-2-mixture-0225` | +| `general.tags` | ARRAY[STRING] | 5 (`gensyn`, `verifiable-training`, `reproducible-training`, `repops`, `text-generation`) | +| `general.languages` | ARRAY[STRING] | 1 (`en`) | +| `open1b.block_count` | UINT32 | 24 | +| `open1b.context_length` | UINT32 | 4096 | +| `open1b.embedding_length` | UINT32 | 2048 | +| `open1b.feed_forward_length` | UINT32 | 5632 | +| `open1b.attention.head_count` | UINT32 | 16 | +| `open1b.attention.head_count_kv` | UINT32 | 4 | +| `open1b.rope.freq_base` | FLOAT32 | 500000 | +| `open1b.attention.layer_norm_rms_epsilon` | FLOAT32 | 1e-05 | +| `open1b.attention.key_length` | UINT32 | 128 | +| `open1b.attention.value_length` | UINT32 | 128 | +| `general.file_type` | UINT32 | 1 (F16) | +| `open1b.attention.sliding_window` | UINT32 | 512 | +| `open1b.attention.sliding_window_pattern` | UINT32 | 5 | +| `general.quantization_version` | UINT32 | 2 | +| `tokenizer.ggml.model` | STRING | `gpt2` | +| `tokenizer.ggml.pre` | STRING | `gpt-2` | +| `tokenizer.ggml.tokens` | ARRAY[STRING] | 128256 (sha256 `30bab437…`) | +| `tokenizer.ggml.token_type` | ARRAY[INT32] | 128256 (sha256 `5ca8c567…`) | +| `tokenizer.ggml.merges` | ARRAY[STRING] | 127994 (sha256 `9e3a7dd4…`) | +| `tokenizer.ggml.eos_token_id` | UINT32 | 4 | +| `tokenizer.chat_template` | STRING | (501-byte Jinja template) | + +Array `raw_sha256` values are in `models/open-1b-sft-f16.manifest.json`. + +### HF cross-checks performed by `convert` + +``` +config.json OK: arch=open1b layers=24 hidden=2048 heads=16/4 head_dim=128 ffn=5632 vocab=128256 swa=512/5 tie=false qk_norm=true(gain=false) emb_norm=true +tokenizer_config.json OK: eos=<|eot|> chat_template=501 bytes +tokenizer.json OK: type=BPE vocab=128256 +added=6 -> 128256 tokens, 127994 merges +``` + +### Go build / vet / test + +``` +$ go build ./... # OK +$ go vet ./... # OK +$ go test -race -v ./... +--- PASS: TestF32BitsToF16 (0.00s) +--- PASS: TestF32ToF16Bytes (0.00s) +--- PASS: TestRawKVRoundTrip (0.00s) +--- PASS: TestVerifyRealFiles (6.59s) +PASS +ok hf2gguf 7.602s +``` + +`TestVerifyRealFiles` re-runs the full 3.2 GB byte comparison (skipped on +`-short` or when the artifacts are absent). + +--- + +## 4. Serve on one Arc (Vulkan2) + +``` +setsid --fork /root/projects/inference/build-vulkan/bin/llama-server \ + -m /root/projects/gensyn/inference/models/open-1b-sft-f16-go.gguf \ + --device Vulkan2 -ngl 99 -c 32768 -fa off --jinja \ + --spec-type ngram-map-k,ngram-cache -t 16 --parallel 1 --alias open-1b \ + --host 127.0.0.1 --port 8085 >/tmp/open1b.log 2>&1 n_ctx_train (4096) -- possible training context overflow +load_model: the slot context (32768) exceeds the training context of the model (4096) - capping +load_model: initializing, n_slots = 1, n_ctx_slot = 4096, kv_unified = 'false' +model loaded +listening on http://127.0.0.1:8085 +``` + +Device confirmation: PID `1378706` maps `/dev/dri/renderD129` with large +`rw-s` regions; `renderD129 → /sys/devices/.../0000:07:00.0` = Arc #2 = Vulkan2. +`/props` reports `"speculative.types":"none,ngram-map-k,ngram-cache"` and +`"model_ftype":"F16"`. + +### Performance (single Arc, 1 slot, `-fa off`, ngram spec) + +| Metric | Value | +|---|---| +| Decode | **~111.7–117.2 tok/s** (e.g. 256 tok @ 111.65; 96 tok @ 117.20) | +| Prefill | **~892 tok/s** (961 prompt tok in 1077 ms) | +| VRAM on Vulkan2 | free 10971 → 7258 MiB ⇒ **≈ 3.6 GiB** used; fully released after `kill` | + +### Sample outputs + +``` +$ curl ... /completion {"prompt":"The three primary colors are", greedy} + red, blue, and yellow. + +$ curl ... /completion {"prompt":"Q: What is the capital of France?\nA:", greedy, 96} + Paris + +$ curl ... /completion (seeded chat turn, see caveat) 113.9 tok/s +" Venus and Mars. Venus is the hottest planet in our Solar System, with + temperatures reaching 465°C (1,470°F). Mars, on the other hand, is a + terrestrial planet, meaning it has a solid surface and liquid water at its poles." +``` + +### Stopping + +``` +$ ss -ltnp | grep :8085 # -> pid=1378706 +$ kill 1378706 # never pkill -f +``` + +Both the test server (:8085) and a diagnostic server (:8086) were stopped; +`--list-devices` returned to 10971 MiB free on Vulkan2/3. + +--- + +## 5. Caveats + +1. **Chat template newline quirk (model-side, not conversion-side).** With the + exact vendor template tail `<|end_header|>\n\n` for the assistant turn, this + checkpoint emits `<|eot|>` as the first sampled token → empty reply / instant + EOS (reproduced on both :8085 with ngram spec and a :8086 run with + `--spec-type none`, so it is **not** the speculative decoder). A single `\n`, + `\n\n\n`, or a seed token makes it answer coherently: + + | tail after `assistant<|end_header|>` | result | + |---|---| + | `\n\n` (vendor template) | immediate EOS | + | `\n` | `\n1. **Earth**: …` (coherent) | + | `\n\n\n` | `\n1. **Earth**: …` (coherent) | + | ` ` | `\nSure, here are two names…` (coherent) | + + Because our GGUF is byte-identical to the vendor reference, this is a + property of the released checkpoint/GGUF (likely SFT formatting), not of our + conversion. Workarounds for consumers: use `ignore_eos` + trim, seed the + assistant turn, or fix the template's trailing newline. +2. **Context capped to 4096.** `open1b.context_length = 4096`, so the requested + `-c 32768` is capped to `n_ctx_slot = 4096`. KV is f16 (`-fa off` mandatory on + Arc per `brainstation-gpu`). +3. **`special_eos_id is not in special_eog_ids`** warning on load — present for + the reference file too (byte-identical), so not introduced by us. +4. **1B greedy degeneration.** Long greedy generation without repetition penalty + loops on this small model; use sampling / `repeat_penalty` for long outputs. +5. **Process note (disclosure).** During an early diagnostic I piped one + `/completion` response through `python3 -c` to pretty-print JSON — a breach of + the "no Python" rule (the tool itself is 100% Go, and nothing in the pipeline + depends on Python). All subsequent parsing was done with Go/raw JSON. No `.py` + file or venv was created; the only Python files in the tree are the vendor's + read-only `models/open-1b-sft/*.py`. + +--- + +## 6. Artifacts + +| Path | What | +|---|---| +| `hf2gguf/` | Go module (`dump` / `convert` / `verify`) + tests | +| `hf2gguf/README.md` | tool usage | +| `models/open-1b-sft-f16-go.gguf` | our output (byte-identical to reference) | +| `models/open-1b-sft-f16.manifest.json` | dumped contract (39 KV + 220 tensors) | +| `models/open-1b-sft-f16.gguf` | reference (untouched) | +| `/tmp/open1b.log` | server log | + +No changes were committed; everything lives under gitignored `inference/`. diff --git a/tools/hf2gguf/README.md b/tools/hf2gguf/README.md new file mode 100644 index 0000000..54aacd5 --- /dev/null +++ b/tools/hf2gguf/README.md @@ -0,0 +1,82 @@ +# hf2gguf + +Pure-Go (Go 1.27) tool to convert a HuggingFace `open-1b` safetensors checkpoint +into GGUF **byte-identically** to a supplied reference GGUF, plus a byte-level +verifier. Built for epic `vado/1b#1` (Gensyn open-1b local bring-up). + +Library: [`github.com/cymertek/go-gguf`](https://github.com/cymertek/go-gguf) +(reader + writer). No Python anywhere in the tool. + +## Build + +```bash +go build ./... +go vet ./... +go test -race ./... # full test includes the 3.2 GB byte comparison +``` + +## Subcommands + +### `dump` — the reproduction contract + +Reads the reference GGUF and emits a JSON manifest of **all** metadata KV pairs +(key, type, value / array elem-type + count + `raw_sha256`) and every tensor +(name, shape, ggml type, nbytes). + +```bash +go run . dump -gguf models/open-1b-sft-f16.gguf -o models/open-1b-sft-f16.manifest.json +``` + +`go-gguf` exposes scalar/string metadata and tensors; its `MetadataEntry.Value()` +returns an empty value for `BTypeArray`, so `rawkv.go` walks the KV section +directly to preserve tokenizer arrays (`tokens`, `merges`, `token_type`) as raw +bytes. Writing still goes through `go-gguf`'s writer — nothing hand-rolls GGUF +serialization. + +### `convert` — emit our GGUF + +```bash +go run . convert \ + -ref models/open-1b-sft-f16.gguf \ + -safetensors models/open-1b-sft/model.safetensors \ + -config models/open-1b-sft/config.json \ + -tokenizer-config models/open-1b-sft/tokenizer_config.json \ + -tokenizer models/open-1b-sft/tokenizer.json \ + -o models/open-1b-sft-f16-go.gguf +``` + +- Metadata is copied verbatim from the reference KV section. +- Tensors are mapped HF FQN → GGUF name (see the module docs in `convert.go`), + with `blocks.{i}.ffn.w_gate_up.weight` split into `ffn_gate`/`ffn_up`, and + `*.weight_scale` dropped (the reference has none). +- Payloads stream from safetensors and are converted F32→F16 with IEEE + round-to-nearest-even on the fly (O(1) memory per tensor). +- `-config` / `-tokenizer-config` / `-tokenizer` are optional; when given they + are parsed and **validated** against the reference (mismatch is fatal). + +### `verify` — byte-level equality + +```bash +go run . verify -ref models/open-1b-sft-f16.gguf -got models/open-1b-sft-f16-go.gguf +``` + +Compares metadata on raw wire bytes and tensors by shape, type and full payload +equality (streaming, first-diff offset reported on failure). + +``` +metadata: 39 checked, 0 mismatches +tensors: 220 checked, 220 byte-identical, 0 mismatches +verification PASSED: 39 metadata keys and 220 tensors byte-identical +``` + +## Layout + +| File | Purpose | +|---|---| +| `main.go` | dispatch, manifest types, `dump` | +| `rawkv.go` | read-only GGUF KV walker (array support) | +| `safetensors.go` | safetensors header/byte access | +| `hfconfig.go` | HF config/tokenizer parsing + validation | +| `convert.go` | metadata reproduction, tensor mapping, F32→F16 | +| `verify.go` | `verify` command + comparison engine | +| `convert_test.go` | F16 rounding, KV round-trip, full-file verification | diff --git a/tools/hf2gguf/convert.go b/tools/hf2gguf/convert.go new file mode 100644 index 0000000..16dfc87 --- /dev/null +++ b/tools/hf2gguf/convert.go @@ -0,0 +1,396 @@ +package main + +// convert.go — emit models/open-1b-sft-f16-go.gguf from the HF safetensors, +// reproducing the reference GGUF's metadata (verbatim) and tensor set/order. +// +// Tensor mapping (HF FQN -> GGUF name), shapes are the reference's (ggml ne0 is +// the innermost/HF input dim, so the HF bytes are used unpermuted): +// +// embedding.tok_embeddings.weight -> token_embd.weight +// emb_norm.weight -> token_embd_norm.weight +// embedding.output.weight -> output.weight +// norm_out.weight -> output_norm.weight +// blocks.{i}.attn.wq.weight -> blk.{i}.attn_q.weight +// blocks.{i}.attn.wk.weight -> blk.{i}.attn_k.weight +// blocks.{i}.attn.wv.weight -> blk.{i}.attn_v.weight +// blocks.{i}.attn.wo.weight -> blk.{i}.attn_output.weight +// blocks.{i}.ffn.w_gate_up.weight -> blk.{i}.ffn_gate.weight (first half) +// -> blk.{i}.ffn_up.weight (second half) +// blocks.{i}.ffn.w_down.weight -> blk.{i}.ffn_down.weight +// blocks.{i}.norm1.weight -> blk.{i}.attn_norm.weight +// blocks.{i}.norm2.weight -> blk.{i}.ffn_norm.weight +// +// The quantized per-channel weight_scale tensors in the checkpoint are dropped, +// matching the reference. + +import ( + "flag" + "fmt" + "io" + "os" + "regexp" + + gguf "github.com/cymertek/go-gguf" +) + +// hfRef resolves a GGUF tensor name to a HF safetensors tensor plus an optional +// element range [start,end) (for the fused gate+up split). end=-1 means "all". +type hfRef struct { + Name string + Start int64 + End int64 +} + +func resolveHF(refName string) (hfRef, error) { + switch refName { + case "token_embd.weight": + return hfRef{"embedding.tok_embeddings.weight", 0, -1}, nil + case "token_embd_norm.weight": + return hfRef{"emb_norm.weight", 0, -1}, nil + case "output.weight": + return hfRef{"embedding.output.weight", 0, -1}, nil + case "output_norm.weight": + return hfRef{"norm_out.weight", 0, -1}, nil + } + m := blkRe.FindStringSubmatch(refName) + if m == nil { + return hfRef{}, fmt.Errorf("unmapped tensor %q", refName) + } + i := m[1] + leaf := m[2] + switch leaf { + case "attn_q": + return hfRef{"blocks." + i + ".attn.wq.weight", 0, -1}, nil + case "attn_k": + return hfRef{"blocks." + i + ".attn.wk.weight", 0, -1}, nil + case "attn_v": + return hfRef{"blocks." + i + ".attn.wv.weight", 0, -1}, nil + case "attn_output": + return hfRef{"blocks." + i + ".attn.wo.weight", 0, -1}, nil + case "ffn_gate": + // fused w_gate_up is [2*ffn, hidden]; chunk(2) => gate = first half + return hfRef{"blocks." + i + ".ffn.w_gate_up.weight", 0, 0}, nil // end filled later + case "ffn_up": + return hfRef{"blocks." + i + ".ffn.w_gate_up.weight", 0, 0}, nil + case "ffn_down": + return hfRef{"blocks." + i + ".ffn.w_down.weight", 0, -1}, nil + case "attn_norm": + return hfRef{"blocks." + i + ".norm1.weight", 0, -1}, nil + case "ffn_norm": + return hfRef{"blocks." + i + ".norm2.weight", 0, -1}, nil + } + return hfRef{}, fmt.Errorf("unmapped block tensor %q", refName) +} + +var blkRe = regexp.MustCompile(`^blk\.(\d+)\.(.+)\.weight$`) + +func cmdConvert(args []string) error { + fs := flag.NewFlagSet("convert", flag.ExitOnError) + refGGUF := fs.String("ref", "", "reference GGUF (metadata/tensor contract)") + stPath := fs.String("safetensors", "", "HF model.safetensors") + hfConfig := fs.String("config", "", "HF config.json (validated against ref)") + hfTokConfig := fs.String("tokenizer-config", "", "HF tokenizer_config.json (validated against ref)") + hfTokenizer := fs.String("tokenizer", "", "HF tokenizer.json (validated against ref)") + outPath := fs.String("o", "", "output GGUF path") + fs.Parse(args) + if *refGGUF == "" || *stPath == "" || *outPath == "" { + return fmt.Errorf("convert: -ref, -safetensors and -o are required") + } + + hdr, kvs, err := readRawKV(*refGGUF) + if err != nil { + return fmt.Errorf("read reference metadata: %w", err) + } + ref, err := gguf.Open(*refGGUF) + if err != nil { + return fmt.Errorf("open reference: %w", err) + } + refTensors, err := ref.Tensors() + if err != nil { + return fmt.Errorf("reference tensors: %w", err) + } + + st, err := openSafetensors(*stPath) + if err != nil { + return fmt.Errorf("open safetensors: %w", err) + } + defer st.Close() + + // Validate HF config against the reference contract (no values are taken from + // it — the reference is authoritative — but mismatches are fatal). + if *hfConfig != "" { + if err := validateConfig(*hfConfig, kvs); err != nil { + return fmt.Errorf("config validation: %w", err) + } + } + if *hfTokConfig != "" { + if err := validateTokenizerConfig(*hfTokConfig, kvs); err != nil { + return fmt.Errorf("tokenizer config validation: %w", err) + } + } + if *hfTokenizer != "" { + if err := validateTokenizerJSON(*hfTokenizer, kvs); err != nil { + return fmt.Errorf("tokenizer.json validation: %w", err) + } + } + + w, err := gguf.OpenForWrite(*outPath) + if err != nil { + return fmt.Errorf("create output: %w", err) + } + // Metadata first: copy the reference KV section verbatim. + for _, kv := range kvs { + v, err := kvToValue(kv) + if err != nil { + w.Close() + return fmt.Errorf("metadata %q: %w", kv.Key, err) + } + if err := w.SetKV(kv.Key, v); err != nil { + w.Close() + return fmt.Errorf("set %q: %w", kv.Key, err) + } + } + + // Tensors in reference order so the data layout matches. + byName := map[string]gguf.TensorInfo{} + for _, t := range refTensors { + byName[t.Info().Name] = t.Info() + } + for _, t := range refTensors { + info := t.Info() + r, err := resolveHF(info.Name) + if err != nil { + w.Close() + return err + } + hfT, ok := st.Tensors[r.Name] + if !ok { + w.Close() + return fmt.Errorf("tensor %s: HF source %q missing", info.Name, r.Name) + } + elemBytes, err := stDtypeBytes(hfT.Dtype) + if err != nil { + w.Close() + return fmt.Errorf("tensor %s: %w", info.Name, err) + } + total := numElems(hfT.Shape) + if hfT.Dtype != "F32" && hfT.Dtype != "F16" { + w.Close() + return fmt.Errorf("tensor %s: source dtype %s unsupported", info.Name, hfT.Dtype) + } + // Resolve the fused gate/up split now that we know the element count. + start, end := r.Start, r.End + if end == -1 { + end = total + } + if isFusedSplit(info.Name) { + half := total / 2 + switch { + case hasSuffix(info.Name, "ffn_gate.weight"): + start, end = 0, half + case hasSuffix(info.Name, "ffn_up.weight"): + start, end = half, total + } + } + if end > total || start < 0 || start >= end { + w.Close() + return fmt.Errorf("tensor %s: bad element range [%d,%d) of %d", info.Name, start, end, total) + } + wantBytes := uint64(end-start) * uint64(info.GgmlType.BlockBytes()) / uint64(info.GgmlType.ElementsPerBlock()) + if info.NBytes != wantBytes { + w.Close() + return fmt.Errorf("tensor %s: ref nbytes %d != expected %d", info.Name, info.NBytes, wantBytes) + } + + // Shape check: reference shape must be the HF shape reversed (ggml ne0 + // innermost) and consistent with the slice. + if !shapeMatches(info.Shape, hfT.Shape, start, end, total) { + w.Close() + return fmt.Errorf("tensor %s: ref shape %v vs HF %v (range %d:%d)", info.Name, info.Shape, hfT.Shape, start, end) + } + + dataOff := st.dataStart + hfT.Start + start*int64(elemBytes) + rem := (end - start) * int64(elemBytes) + + idx := w.AddTensor(info.Name, info.Shape, info.GgmlType) + var rd io.Reader + switch info.GgmlType { + case gguf.GgmlF16: + rd = &f16StreamReader{f: st.f, pos: dataOff, rem: rem} + case gguf.GgmlF32: + rd = &f32StreamReader{f: st.f, pos: dataOff, rem: rem} + default: + w.Close() + return fmt.Errorf("tensor %s: unsupported target type %s", info.Name, info.GgmlType.GgmlName()) + } + if err := w.WriteTensorData(idx, rd); err != nil { + w.Close() + return fmt.Errorf("tensor %s: %w", info.Name, err) + } + } + + n, err := w.Close() + if err != nil { + return fmt.Errorf("finalize: %w", err) + } + fmt.Printf("wrote %s: %d bytes, %d metadata keys, %d tensors (ref nKV=%d nTensors=%d)\n", + *outPath, n, len(kvs), len(refTensors), hdr.NKV, hdr.NTensors) + return nil +} + +func isFusedSplit(name string) bool { + return hasSuffix(name, "ffn_gate.weight") || hasSuffix(name, "ffn_up.weight") +} + +func hasSuffix(s, suffix string) bool { + return len(s) >= len(suffix) && s[len(s)-len(suffix):] == suffix +} + +// shapeMatches verifies ref ggml shape against the HF shape for a slice. +// Whole tensors: ref == reverse(hf). A gate/up half: ref == [hf[1], hf[0]/2]. +func shapeMatches(ref []uint64, hf []int64, start, end, total int64) bool { + if len(ref) != len(hf) { + return false + } + if end-start != total { + // fused split: both ref and hf are 2-D; ref[0]==hf[1], + // ref[1]==hf[0]/2 and the split is on the outer (row) dim. + if len(ref) != 2 { + return false + } + return ref[0] == uint64(hf[1]) && ref[1] == uint64(hf[0]/2) && hf[0]%2 == 0 + } + for i := range ref { + if ref[i] != uint64(hf[len(hf)-1-i]) { + return false + } + } + return true +} + +// --------------------------------------------------------------------------- +// streaming readers that convert F32 -> F16 on the fly (RNE) +// --------------------------------------------------------------------------- + +const streamChunk = 1 << 20 // 1 MiB of source bytes per read + +type f16StreamReader struct { + f *os.File + pos int64 + rem int64 + pending []byte +} + +func (r *f16StreamReader) Read(p []byte) (int, error) { + for len(r.pending) == 0 { + if r.rem <= 0 { + return 0, io.EOF + } + n := r.rem + if n > streamChunk { + n = streamChunk + } + n -= n % 4 // whole f32 elements + buf := make([]byte, n) + if _, err := r.f.ReadAt(buf, r.pos); err != nil { + return 0, err + } + r.pos += n + r.rem -= n + r.pending = f32ToF16Bytes(buf) + } + n := copy(p, r.pending) + r.pending = r.pending[n:] + return n, nil +} + +type f32StreamReader struct { + f *os.File + pos int64 + rem int64 + pending []byte +} + +func (r *f32StreamReader) Read(p []byte) (int, error) { + for len(r.pending) == 0 { + if r.rem <= 0 { + return 0, io.EOF + } + n := r.rem + if n > streamChunk { + n = streamChunk + } + buf := make([]byte, n) + if _, err := r.f.ReadAt(buf, r.pos); err != nil { + return 0, err + } + r.pos += n + r.rem -= n + r.pending = buf + } + n := copy(p, r.pending) + r.pending = r.pending[n:] + return n, nil +} + +// --------------------------------------------------------------------------- +// FP32 -> FP16 (IEEE-754 round-to-nearest-even), matching numpy/torch casts. +// --------------------------------------------------------------------------- + +func f32ToF16Bytes(src []byte) []byte { + out := make([]byte, len(src)/4*2) + for i := 0; i+4 <= len(src); i += 4 { + bits := uint32(src[i]) | uint32(src[i+1])<<8 | uint32(src[i+2])<<16 | uint32(src[i+3])<<24 + half := f32bitsToF16(bits) + j := i / 4 * 2 + out[j] = byte(half) + out[j+1] = byte(half >> 8) + } + return out +} + +// f32bitsToF16 converts the raw bits of an IEEE-754 binary32 to binary16 using +// round-to-nearest, ties-to-even. +func f32bitsToF16(b uint32) uint16 { + sign := uint16((b >> 16) & 0x8000) + exp := int32((b >> 23) & 0xff) + mant := b & 0x7fffff + + if exp == 0xff { // Inf or NaN + if mant != 0 { + return sign | 0x7e00 // quiet NaN + } + return sign | 0x7c00 // Inf + } + + e := exp - 127 // unbiased + + if e > 15 { // overflow -> Inf + return sign | 0x7c00 + } + if e >= -14 { // normal + hm := uint16(e+15)<<10 | uint16(mant>>13) + rem := mant & 0x1fff + if rem > 0x1000 || (rem == 0x1000 && hm&1 == 1) { + hm++ + } + return sign | hm + } + if e < -25 { // too small -> signed zero + return sign + } + // subnormal: implicit leading 1, shift into place + m := mant | 0x800000 + shift := uint32(-e - 14) // 1..11 + hm := uint16(m >> (13 + shift)) + rem := m & ((1 << (13 + shift)) - 1) + halfBit := uint32(1) << (12 + shift) + if rem > halfBit || (rem == halfBit && hm&1 == 1) { + hm++ + } + return sign | hm +} + +// --------------------------------------------------------------------------- +// HF config / tokenizer config validation (reference is authoritative) +// --------------------------------------------------------------------------- diff --git a/tools/hf2gguf/convert_test.go b/tools/hf2gguf/convert_test.go new file mode 100644 index 0000000..bb257c7 --- /dev/null +++ b/tools/hf2gguf/convert_test.go @@ -0,0 +1,161 @@ +package main + +import ( + "math" + "os" + "path/filepath" + "testing" + + gguf "github.com/cymertek/go-gguf" +) + +func TestF32BitsToF16(t *testing.T) { + cases := []struct { + name string + in uint32 + want uint16 + }{ + {"zero", 0x00000000, 0x0000}, + {"negzero", 0x80000000, 0x8000}, + {"one", 0x3f800000, 0x3c00}, + {"neg2", 0xc0000000, 0xc000}, + {"half", 0x3f000000, 0x3800}, + {"max_f16", 0x477fe000, 0x7bff}, + {"overflow_to_inf", 0x477ff000, 0x7c00}, + {"inf", 0x7f800000, 0x7c00}, + {"neginf", 0xff800000, 0xfc00}, + {"qnan", 0x7fc00000, 0x7e00}, + {"snan", 0x7f800001, 0x7e00}, + {"min_normal", 0x38800000, 0x0400}, // 2^-14 + {"min_subnormal", 0x33800000, 0x0001}, // 2^-24 + {"tie_to_even_zero", 0x33000000, 0x0000}, // 2^-25 (half of min subnormal) + {"tie_to_even_two", 0x33c00000, 0x0002}, // 3 * 2^-25 + {"one_tenth", 0x3dcccccd, 0x2e66}, + } + for _, c := range cases { + if got := f32bitsToF16(c.in); got != c.want { + t.Errorf("%s: f32=%08x got %04x want %04x", c.name, c.in, got, c.want) + } + } +} + +func TestF32ToF16Bytes(t *testing.T) { + in := make([]byte, 8) + putF32 := func(off int, f float32) { + b := math.Float32bits(f) + in[off] = byte(b) + in[off+1] = byte(b >> 8) + in[off+2] = byte(b >> 16) + in[off+3] = byte(b >> 24) + } + putF32(0, 1.0) + putF32(4, -2.0) + out := f32ToF16Bytes(in) + want := []byte{0x00, 0x3c, 0x00, 0xc0} // 1.0 -> 0x3c00 LE, -2.0 -> 0xc000 LE + for i := range want { + if out[i] != want[i] { + t.Fatalf("byte %d: got %02x want %02x", i, out[i], want[i]) + } + } +} + +// TestRawKVRoundTrip creates a small GGUF with go-gguf, reads it with our raw +// walker, and re-writes the KVs through kvToValue to confirm arrays (including +// string arrays) survive byte-exactly. +func TestRawKVRoundTrip(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "a.gguf") + dst := filepath.Join(dir, "b.gguf") + + w, err := gguf.OpenForWrite(src) + if err != nil { + t.Fatal(err) + } + strs := []byte{} + for _, s := range []string{"alpha", "béta", ""} { + strs = append(strs, leU64(uint64(len(s)))...) + strs = append(strs, s...) + } + kvs := []gguf.KVEntry{ + {Key: "general.architecture", Value: gguf.Value{BType: gguf.BTypeString, Str: "open1b"}}, + {Key: "x.count", Value: gguf.Value{BType: gguf.BTypeUint32, Int: 7}}, + {Key: "x.f", Value: gguf.Value{BType: gguf.BTypeFloat32, Float: float64(float32(1e-5))}}, + {Key: "x.b", Value: gguf.Value{BType: gguf.BTypeBool, Int: 1}}, + {Key: "x.tags", Value: gguf.Value{BType: gguf.BTypeArray, ElemType: gguf.BTypeString, Int: 3, Raw: strs}}, + } + for _, kv := range kvs { + if err := w.SetKV(kv.Key, kv.Value); err != nil { + t.Fatal(err) + } + } + if _, err := w.Close(); err != nil { + t.Fatal(err) + } + + _, got, err := readRawKV(src) + if err != nil { + t.Fatal(err) + } + if len(got) != len(kvs) { + t.Fatalf("kv count %d want %d", len(got), len(kvs)) + } + + w2, err := gguf.OpenForWrite(dst) + if err != nil { + t.Fatal(err) + } + for _, kv := range got { + v, err := kvToValue(kv) + if err != nil { + t.Fatal(err) + } + if err := w2.SetKV(kv.Key, v); err != nil { + t.Fatal(err) + } + } + if _, err := w2.Close(); err != nil { + t.Fatal(err) + } + + _, round, err := readRawKV(dst) + if err != nil { + t.Fatal(err) + } + res, err := compareGGUF(src, dst) + if err != nil { + t.Fatal(err) + } + if !res.OK() { + t.Fatalf("roundtrip not identical: meta=%v tensor=%v", res.MetaMismatches, res.TensorMismatch) + } + if len(round) != len(got) { + t.Fatalf("rewritten kv count %d want %d", len(round), len(got)) + } +} + +// TestVerifyRealFiles runs the full byte-level verification against the built +// artifacts when they are present in the workdir. Skipped when absent (fresh +// checkout) or in -short mode. +func TestVerifyRealFiles(t *testing.T) { + if testing.Short() { + t.Skip("skipping multi-GB comparison in -short mode") + } + ref := "../models/open-1b-sft-f16.gguf" + got := "../models/open-1b-sft-f16-go.gguf" + if _, err := os.Stat(ref); err != nil { + t.Skipf("reference GGUF not present: %v", err) + } + if _, err := os.Stat(got); err != nil { + t.Skipf("candidate GGUF not present (run convert first): %v", err) + } + res, err := compareGGUF(ref, got) + if err != nil { + t.Fatal(err) + } + if !res.OK() { + t.Fatalf("verification failed:\n metadata=%v\n tensors=%v", res.MetaMismatches, res.TensorMismatch) + } + if res.ByteEqual != res.TensorsChecked || res.TensorsChecked == 0 { + t.Fatalf("expected all tensors byte-identical: %d/%d", res.ByteEqual, res.TensorsChecked) + } +} diff --git a/tools/hf2gguf/go.mod b/tools/hf2gguf/go.mod new file mode 100644 index 0000000..6044c99 --- /dev/null +++ b/tools/hf2gguf/go.mod @@ -0,0 +1,5 @@ +module hf2gguf + +go 1.27.0 + +require github.com/cymertek/go-gguf v0.0.0-20260818124528-20850052d9e2 diff --git a/tools/hf2gguf/go.sum b/tools/hf2gguf/go.sum new file mode 100644 index 0000000..b998491 --- /dev/null +++ b/tools/hf2gguf/go.sum @@ -0,0 +1,2 @@ +github.com/cymertek/go-gguf v0.0.0-20260818124528-20850052d9e2 h1:7WUU9/eb/AcBNCyMXIrPuaUaeLJXlYzSxQfNmEGwNV4= +github.com/cymertek/go-gguf v0.0.0-20260818124528-20850052d9e2/go.mod h1:F8ePnvWZX6ZD/sc6NAg3Zjm7T2b7f+kDWnCkKzyNVUw= diff --git a/tools/hf2gguf/hfconfig.go b/tools/hf2gguf/hfconfig.go new file mode 100644 index 0000000..81cf19e --- /dev/null +++ b/tools/hf2gguf/hfconfig.go @@ -0,0 +1,223 @@ +package main + +// hfconfig.go — parse and validate the HF config/tokenizer against the +// reference GGUF metadata. The reference remains authoritative for values; this +// only proves the reference is consistent with the published HF files and that +// we parsed them (per the task), it does not source any GGUF value from them. + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "os" + "sort" +) + +type hfConfigFile struct { + ModelType string `json:"model_type"` + HiddenSize int64 `json:"hidden_size"` + NumHiddenLayers int64 `json:"num_hidden_layers"` + NumAttentionHeads int64 `json:"num_attention_heads"` + NumKeyValueHeads int64 `json:"num_key_value_heads"` + HeadDim int64 `json:"head_dim"` + IntermediateSize int64 `json:"intermediate_size"` + VocabSize int64 `json:"vocab_size"` + MaxPositionEmbeddings int64 `json:"max_position_embeddings"` + RopeTheta float64 `json:"rope_theta"` + RMSNormEps float64 `json:"rms_norm_eps"` + TieWordEmbeddings bool `json:"tie_word_embeddings"` + QKNorm bool `json:"qk_norm"` + QKNormGain bool `json:"qk_norm_gain"` + EmbeddingNorm bool `json:"embedding_norm"` + SlidingWindow int64 `json:"sliding_window"` + SWAFullEvery int64 `json:"swa_full_every"` +} + +func findKV(kvs []rawKV, key string) (rawKV, bool) { + for _, kv := range kvs { + if kv.Key == key { + return kv, true + } + } + return rawKV{}, false +} + +func validateConfig(path string, kvs []rawKV) error { + b, err := os.ReadFile(path) + if err != nil { + return err + } + var c hfConfigFile + if err := json.Unmarshal(b, &c); err != nil { + return err + } + + intChecks := []struct { + name string + got int64 + }{ + {"open1b.embedding_length", c.HiddenSize}, + {"open1b.block_count", c.NumHiddenLayers}, + {"open1b.attention.head_count", c.NumAttentionHeads}, + {"open1b.attention.head_count_kv", c.NumKeyValueHeads}, + {"open1b.attention.key_length", c.HeadDim}, + {"open1b.attention.value_length", c.HeadDim}, + {"open1b.feed_forward_length", c.IntermediateSize}, + {"open1b.context_length", c.MaxPositionEmbeddings}, + {"open1b.attention.sliding_window", c.SlidingWindow}, + {"open1b.attention.sliding_window_pattern", c.SWAFullEvery}, + } + for _, chk := range intChecks { + kv, ok := findKV(kvs, chk.name) + if !ok { + return fmt.Errorf("reference missing %s", chk.name) + } + if kv.Int != chk.got { + return fmt.Errorf("%s: HF=%d ref=%d", chk.name, chk.got, kv.Int) + } + } + floatChecks := []struct { + name string + got float64 + }{ + {"open1b.rope.freq_base", c.RopeTheta}, + {"open1b.attention.layer_norm_rms_epsilon", c.RMSNormEps}, + } + for _, chk := range floatChecks { + kv, ok := findKV(kvs, chk.name) + if !ok { + return fmt.Errorf("reference missing %s", chk.name) + } + if float32(kv.Float) != float32(chk.got) { + return fmt.Errorf("%s: HF=%v ref=%v", chk.name, chk.got, kv.Float) + } + } + fmt.Printf("config.json OK: arch=%s layers=%d hidden=%d heads=%d/%d head_dim=%d ffn=%d vocab=%d swa=%d/%d tie=%v qk_norm=%v(gain=%v) emb_norm=%v\n", + c.ModelType, c.NumHiddenLayers, c.HiddenSize, c.NumAttentionHeads, c.NumKeyValueHeads, + c.HeadDim, c.IntermediateSize, c.VocabSize, c.SlidingWindow, c.SWAFullEvery, + c.TieWordEmbeddings, c.QKNorm, c.QKNormGain, c.EmbeddingNorm) + return nil +} + +type tokConfigFile struct { + EOSToken string `json:"eos_token"` + ModelMaxLen int64 `json:"model_max_length"` + ChatTemplate string `json:"chat_template"` + AddedDecoders map[string]struct { + Content string `json:"content"` + } `json:"added_tokens_decoder"` +} + +func validateTokenizerConfig(path string, kvs []rawKV) error { + b, err := os.ReadFile(path) + if err != nil { + return err + } + var tc tokConfigFile + if err := json.Unmarshal(b, &tc); err != nil { + return err + } + // chat_template must match the reference verbatim. + if kv, ok := findKV(kvs, "tokenizer.chat_template"); ok { + if kv.Str != tc.ChatTemplate { + return fmt.Errorf("chat_template differs (HF %d bytes, ref %d bytes)", len(tc.ChatTemplate), len(kv.Str)) + } + } else { + return fmt.Errorf("reference missing tokenizer.chat_template") + } + // eos token id from added_tokens_decoder. + if kv, ok := findKV(kvs, "tokenizer.ggml.eos_token_id"); ok { + var eosID int64 = -1 + keys := make([]string, 0, len(tc.AddedDecoders)) + for k := range tc.AddedDecoders { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if tc.AddedDecoders[k].Content == tc.EOSToken { + var id int64 + fmt.Sscanf(k, "%d", &id) + eosID = id + } + } + if eosID != kv.Int { + return fmt.Errorf("eos_token %q: HF id=%d ref=%d", tc.EOSToken, eosID, kv.Int) + } + } + // context length consistency + if kv, ok := findKV(kvs, "open1b.context_length"); ok && tc.ModelMaxLen != 0 && tc.ModelMaxLen != kv.Int { + return fmt.Errorf("model_max_length: HF=%d ref=%d", tc.ModelMaxLen, kv.Int) + } + fmt.Printf("tokenizer_config.json OK: eos=%s chat_template=%d bytes\n", tc.EOSToken, len(tc.ChatTemplate)) + return nil +} + +type tokenizerJSON struct { + Model struct { + Type string `json:"type"` + Vocab map[string]int `json:"vocab"` + Merges []json.RawMessage `json:"merges"` + } `json:"model"` + AddedTokens []struct { + ID int64 `json:"id"` + Content string `json:"content"` + } `json:"added_tokens"` +} + +// validateTokenizerJSON proves the tokenizer.json we parsed is the source of the +// reference tokenizer arrays: same BPE type, same vocab+added token count, same +// merge count, and the added special tokens are the reference's leading tokens. +func validateTokenizerJSON(path string, kvs []rawKV) error { + b, err := os.ReadFile(path) + if err != nil { + return err + } + var tj tokenizerJSON + if err := json.Unmarshal(b, &tj); err != nil { + return err + } + if kv, ok := findKV(kvs, "tokenizer.ggml.model"); ok && kv.Str != "gpt2" { + return fmt.Errorf("ref tokenizer model %q, expected gpt2", kv.Str) + } + if tj.Model.Type != "BPE" { + return fmt.Errorf("tokenizer.json model.type=%q, expected BPE", tj.Model.Type) + } + tokensKV, ok := findKV(kvs, "tokenizer.ggml.tokens") + if !ok { + return fmt.Errorf("reference missing tokenizer.ggml.tokens") + } + // tokenizer.json's vocab sometimes already contains the added specials and + // sometimes not; accept either interpretation as long as the count matches. + vocabN := uint64(len(tj.Model.Vocab)) + got := vocabN + if got != tokensKV.Count { + got = vocabN + uint64(len(tj.AddedTokens)) + } + if got != tokensKV.Count { + return fmt.Errorf("vocab=%d (+added=%d =%d) vs ref tokens=%d", vocabN, len(tj.AddedTokens), got, tokensKV.Count) + } + mergesKV, ok := findKV(kvs, "tokenizer.ggml.merges") + if !ok { + return fmt.Errorf("reference missing tokenizer.ggml.merges") + } + if uint64(len(tj.Model.Merges)) != mergesKV.Count { + return fmt.Errorf("tokenizer.json merges=%d vs ref=%d", len(tj.Model.Merges), mergesKV.Count) + } + // Leading tokens (added specials, sorted by id) must match the ref array head. + sort.Slice(tj.AddedTokens, func(i, j int) bool { return tj.AddedTokens[i].ID < tj.AddedTokens[j].ID }) + refHead := decodeStringArray(tokensKV.Raw, uint64(len(tj.AddedTokens))) + for i := range tj.AddedTokens { + if i >= len(refHead) { + return fmt.Errorf("ref tokens shorter than added tokens") + } + if refHead[i] != tj.AddedTokens[i].Content { + return fmt.Errorf("token[%d]: HF %q ref %q", i, tj.AddedTokens[i].Content, refHead[i]) + } + } + fmt.Printf("tokenizer.json OK: type=BPE vocab=%d +added=%d -> %d tokens, %d merges\n", + vocabN, len(tj.AddedTokens), got, len(tj.Model.Merges)) + return nil +} + +// keep the binary import used (helps future raw comparisons). +var _ = binary.LittleEndian diff --git a/tools/hf2gguf/main.go b/tools/hf2gguf/main.go new file mode 100644 index 0000000..f0db669 --- /dev/null +++ b/tools/hf2gguf/main.go @@ -0,0 +1,284 @@ +package main + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/json" + "flag" + "fmt" + "math" + "os" + + gguf "github.com/cymertek/go-gguf" +) + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "usage: hf2gguf [flags]") + os.Exit(2) + } + var err error + switch os.Args[1] { + case "dump": + err = cmdDump(os.Args[2:]) + case "convert": + err = cmdConvert(os.Args[2:]) + case "verify": + err = cmdVerify(os.Args[2:]) + default: + err = fmt.Errorf("unknown subcommand %q", os.Args[1]) + } + if err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} + +// --------------------------------------------------------------------------- +// Manifest types — the reproduction contract. +// --------------------------------------------------------------------------- + +type manifest struct { + Source string `json:"source"` + Version uint32 `json:"version"` + NTensors uint64 `json:"n_tensors"` + NKV uint64 `json:"n_kv"` + Metadata []metaKV `json:"metadata"` + Tensors []tensorRow `json:"tensors"` +} + +type metaKV struct { + Key string `json:"key"` + Type string `json:"type"` + Value any `json:"value,omitempty"` + ElemType string `json:"elem_type,omitempty"` + Count uint64 `json:"count,omitempty"` + RawBytes int `json:"raw_bytes,omitempty"` + RawSHA256 string `json:"raw_sha256,omitempty"` + Preview []string `json:"preview,omitempty"` + Numbers []float64 `json:"numbers,omitempty"` +} + +type tensorRow struct { + Name string `json:"name"` + Shape []uint64 `json:"shape"` + Type string `json:"type"` + TypeID uint32 `json:"type_id"` + NBytes uint64 `json:"nbytes"` +} + +func buildManifest(ggufPath string) (*manifest, error) { + hdr, kvs, err := readRawKV(ggufPath) + if err != nil { + return nil, err + } + m := &manifest{ + Source: ggufPath, + Version: hdr.Version, + NTensors: hdr.NTensors, + NKV: hdr.NKV, + } + for _, kv := range kvs { + m.Metadata = append(m.Metadata, kvToManifest(kv)) + } + + g, err := gguf.Open(ggufPath) + if err != nil { + return nil, fmt.Errorf("go-gguf open: %w", err) + } + defer g.Close() + tensors, err := g.Tensors() + if err != nil { + return nil, fmt.Errorf("go-gguf tensors: %w", err) + } + for _, t := range tensors { + info := t.Info() + m.Tensors = append(m.Tensors, tensorRow{ + Name: info.Name, + Shape: info.Shape, + Type: info.GgmlType.GgmlName(), + TypeID: uint32(info.GgmlType), + NBytes: info.NBytes, + }) + } + if uint64(len(m.Tensors)) != hdr.NTensors { + return nil, fmt.Errorf("tensor count mismatch: header %d, walked %d", hdr.NTensors, len(m.Tensors)) + } + if uint64(len(m.Metadata)) != hdr.NKV { + return nil, fmt.Errorf("kv count mismatch: header %d, walked %d", hdr.NKV, len(m.Metadata)) + } + return m, nil +} + +func kvToManifest(kv rawKV) metaKV { + out := metaKV{Key: kv.Key, Type: kv.BType.name()} + switch kv.BType { + case btBool: + out.Value = kv.Int != 0 + case btUint8, btInt8, btUint16, btInt16, btUint32, btInt32, btUint64, btInt64: + out.Value = kv.Int + case btFloat32: + out.Value = float32(kv.Float) + case btFloat64: + out.Value = kv.Float + case btString: + out.Value = kv.Str + case btArray: + out.ElemType = kv.ElemType.name() + out.Count = kv.Count + out.RawBytes = len(kv.Raw) + sum := sha256.Sum256(kv.Raw) + out.RawSHA256 = fmt.Sprintf("%x", sum) + decodePreview(kv, &out) + } + return out +} + +// decodePreview fills Preview/Numbers for small arrays so the manifest stays readable. +func decodePreview(kv rawKV, out *metaKV) { + if kv.Count > 64 { + // give a tiny human preview even for huge token arrays + if kv.ElemType == btString { + out.Preview = decodeStringArray(kv.Raw, 3) + } + return + } + switch kv.ElemType { + case btString: + out.Preview = decodeStringArray(kv.Raw, kv.Count) + case btFloat32: + n := int(kv.Count) + for i := 0; i < n; i++ { + bits := binary.LittleEndian.Uint32(kv.Raw[i*4:]) + out.Numbers = append(out.Numbers, float64(math.Float32frombits(bits))) + } + case btInt32: + n := int(kv.Count) + for i := 0; i < n; i++ { + out.Numbers = append(out.Numbers, float64(int32(binary.LittleEndian.Uint32(kv.Raw[i*4:])))) + } + case btUint32: + n := int(kv.Count) + for i := 0; i < n; i++ { + out.Numbers = append(out.Numbers, float64(binary.LittleEndian.Uint32(kv.Raw[i*4:]))) + } + } +} + +func decodeStringArray(raw []byte, limit uint64) []string { + var out []string + off := 0 + for i := uint64(0); i < limit; i++ { + if off+8 > len(raw) { + break + } + n := binary.LittleEndian.Uint64(raw[off:]) + off += 8 + if off+int(n) > len(raw) { + break + } + out = append(out, string(raw[off:off+int(n)])) + off += int(n) + } + return out +} + +// --------------------------------------------------------------------------- +// dump +// --------------------------------------------------------------------------- + +func cmdDump(args []string) error { + fs := flag.NewFlagSet("dump", flag.ExitOnError) + ggufPath := fs.String("gguf", "", "reference GGUF path") + outPath := fs.String("o", "", "output JSON manifest path (default stdout)") + fs.Parse(args) + if *ggufPath == "" { + return fmt.Errorf("dump: -gguf is required") + } + m, err := buildManifest(*ggufPath) + if err != nil { + return err + } + b, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + b = append(b, '\n') + if *outPath == "" { + _, err = os.Stdout.Write(b) + return err + } + return os.WriteFile(*outPath, b, 0o644) +} + +// --------------------------------------------------------------------------- +// shared helpers for convert/verify +// --------------------------------------------------------------------------- + +// btypeFromName maps the manifest type name back to the raw enum. +func btypeFromName(s string) (ggufBType, error) { + for _, t := range []ggufBType{btUint8, btInt8, btUint16, btInt16, btUint32, + btInt32, btFloat32, btBool, btString, btArray, btUint64, btInt64, btFloat64} { + if t.name() == s { + return t, nil + } + } + return 0, fmt.Errorf("unknown btype %q", s) +} + +// kvToValue converts a rawKV into a go-gguf write Value, preserving the wire +// representation exactly (arrays keep ElemType + count + raw element bytes). +func kvToValue(kv rawKV) (gguf.Value, error) { + switch kv.BType { + case btBool: + return gguf.Value{BType: gguf.BTypeBool, Int: kv.Int}, nil + case btUint8: + return gguf.Value{BType: gguf.BTypeUint8, Int: kv.Int}, nil + case btInt8: + return gguf.Value{BType: gguf.BTypeInt8, Int: kv.Int}, nil + case btUint16: + return gguf.Value{BType: gguf.BTypeUint16, Int: kv.Int}, nil + case btInt16: + return gguf.Value{BType: gguf.BTypeInt16, Int: kv.Int}, nil + case btUint32: + return gguf.Value{BType: gguf.BTypeUint32, Int: kv.Int}, nil + case btInt32: + return gguf.Value{BType: gguf.BTypeInt32, Int: kv.Int}, nil + case btUint64: + return gguf.Value{BType: gguf.BTypeUint64, Int: kv.Int}, nil + case btInt64: + return gguf.Value{BType: gguf.BTypeInt64, Int: kv.Int}, nil + case btFloat32: + return gguf.Value{BType: gguf.BTypeFloat32, Float: kv.Float}, nil + case btFloat64: + return gguf.Value{BType: gguf.BTypeFloat64, Float: kv.Float}, nil + case btString: + return gguf.Value{BType: gguf.BTypeString, Str: kv.Str}, nil + case btArray: + return gguf.Value{ + BType: gguf.BTypeArray, + ElemType: btypeToGguf(kv.ElemType), + Int: int64(kv.Count), + Raw: kv.Raw, + }, nil + default: + return gguf.Value{}, fmt.Errorf("unsupported btype %d for key %q", kv.BType, kv.Key) + } +} + +func btypeToGguf(t ggufBType) gguf.BType { + return gguf.BType(t) +} + +func sha256Hex(b []byte) string { + s := sha256.Sum256(b) + return fmt.Sprintf("%x", s) +} + +func shortHash(b []byte) string { + s := sha256Hex(b) + if len(s) > 12 { + return s[:12] + } + return s +} diff --git a/tools/hf2gguf/rawkv.go b/tools/hf2gguf/rawkv.go new file mode 100644 index 0000000..f373221 --- /dev/null +++ b/tools/hf2gguf/rawkv.go @@ -0,0 +1,353 @@ +package main + +// rawkv.go implements a minimal read-only GGUF metadata (KV) section walker. +// +// Why not go-gguf for this: the library's MetadataEntry.Value() deliberately +// returns an empty Value for BTypeArray (see readLazyValue in lazy.go), which is +// fine for scalar/string consumers but loses the tokenizer arrays +// (tokenizer.ggml.tokens / scores / token_type / merges). To reproduce the +// reference file's metadata byte-for-byte we need the raw array element bytes. +// go-gguf is still used for all tensor reading/writing and for scalar metadata; +// this walker only fills the array gap. It never writes GGUF. + +import ( + "encoding/binary" + "fmt" + "hash/crc32" + "io" + "math" + "os" +) + +// ggufBType wire enum (mirrors gguf.BType). +type ggufBType uint32 + +const ( + btUint8 ggufBType = 0 + btInt8 ggufBType = 1 + btUint16 ggufBType = 2 + btInt16 ggufBType = 3 + btUint32 ggufBType = 4 + btInt32 ggufBType = 5 + btFloat32 ggufBType = 6 + btBool ggufBType = 7 + btString ggufBType = 8 + btArray ggufBType = 9 + btUint64 ggufBType = 10 + btInt64 ggufBType = 11 + btFloat64 ggufBType = 12 +) + +func (t ggufBType) name() string { + switch t { + case btUint8: + return "UINT8" + case btInt8: + return "INT8" + case btUint16: + return "UINT16" + case btInt16: + return "INT16" + case btUint32: + return "UINT32" + case btInt32: + return "INT32" + case btFloat32: + return "FLOAT32" + case btBool: + return "BOOL" + case btString: + return "STRING" + case btArray: + return "ARRAY" + case btUint64: + return "UINT64" + case btInt64: + return "INT64" + case btFloat64: + return "FLOAT64" + default: + return fmt.Sprintf("BTYPE(%d)", uint32(t)) + } +} + +func (t ggufBType) size() int { + switch t { + case btUint8, btInt8, btBool: + return 1 + case btUint16, btInt16: + return 2 + case btUint32, btInt32, btFloat32: + return 4 + case btUint64, btInt64, btFloat64: + return 8 + default: + return 0 + } +} + +// rawKV is one metadata entry with its raw wire value preserved. +type rawKV struct { + Key string + BType ggufBType + Int int64 // integer/bool scalars + Float float64 // float scalars + Str string // string scalars + ElemType ggufBType // arrays: element type + Count uint64 // arrays: element count + Raw []byte // arrays: concatenated element bytes (excludes elem_type+count) +} + +// rawKVHeader holds the GGUF header fields. +type rawKVHeader struct { + Version uint32 + NTensors uint64 + NKV uint64 +} + +// readRawKV opens path and returns the header plus every KV entry in file order. +func readRawKV(path string) (rawKVHeader, []rawKV, error) { + f, err := os.Open(path) + if err != nil { + return rawKVHeader{}, nil, err + } + defer f.Close() + + var hdr rawKVHeader + var b [24]byte + if _, err := io.ReadFull(f, b[:]); err != nil { + return hdr, nil, fmt.Errorf("read header: %w", err) + } + if string(b[0:4]) != "GGUF" { + return hdr, nil, fmt.Errorf("bad magic %q", b[0:4]) + } + hdr.Version = binary.LittleEndian.Uint32(b[4:8]) + hdr.NTensors = binary.LittleEndian.Uint64(b[8:16]) + hdr.NKV = binary.LittleEndian.Uint64(b[16:24]) + + br := newBufReader(f) + out := make([]rawKV, 0, hdr.NKV) + for i := uint64(0); i < hdr.NKV; i++ { + kv, err := readOneKV(br) + if err != nil { + return hdr, nil, fmt.Errorf("kv[%d]: %w", i, err) + } + out = append(out, kv) + } + return hdr, out, nil +} + +func readOneKV(br *bufReader) (rawKV, error) { + var kv rawKV + key, err := br.str() + if err != nil { + return kv, fmt.Errorf("key: %w", err) + } + kv.Key = key + bt, err := br.u32() + if err != nil { + return kv, fmt.Errorf("btype: %w", err) + } + kv.BType = ggufBType(bt) + + switch kv.BType { + case btBool, btUint8, btInt8: + v, err := br.u8() + if err != nil { + return kv, err + } + kv.Int = int64(v) + case btUint16, btInt16: + v, err := br.u16() + if err != nil { + return kv, err + } + if kv.BType == btInt16 { + kv.Int = int64(int16(v)) + } else { + kv.Int = int64(v) + } + case btUint32, btInt32: + v, err := br.u32() + if err != nil { + return kv, err + } + if kv.BType == btInt32 { + kv.Int = int64(int32(v)) + } else { + kv.Int = int64(v) + } + case btFloat32: + v, err := br.u32() + if err != nil { + return kv, err + } + kv.Float = float64(math.Float32frombits(v)) + case btUint64, btInt64: + v, err := br.u64() + if err != nil { + return kv, err + } + kv.Int = int64(v) + case btFloat64: + v, err := br.u64() + if err != nil { + return kv, err + } + kv.Float = math.Float64frombits(v) + case btString: + s, err := br.str() + if err != nil { + return kv, err + } + kv.Str = s + case btArray: + et, err := br.u32() + if err != nil { + return kv, fmt.Errorf("array elem_type: %w", err) + } + kv.ElemType = ggufBType(et) + cnt, err := br.u64() + if err != nil { + return kv, fmt.Errorf("array count: %w", err) + } + kv.Count = cnt + raw, err := readArrayElems(br, kv.ElemType, cnt) + if err != nil { + return kv, fmt.Errorf("array elems: %w", err) + } + kv.Raw = raw + default: + return kv, fmt.Errorf("unsupported btype %d", kv.BType) + } + return kv, nil +} + +// readArrayElems reads `count` elements of elemType, returning their concatenated +// wire bytes (exactly the bytes go-gguf's writer expects in Value.Raw). +func readArrayElems(br *bufReader, elemType ggufBType, count uint64) ([]byte, error) { + if elemType == btArray || elemType == btString { + // strings (and nested arrays, unsupported) are variable length + if elemType != btString { + return nil, fmt.Errorf("nested array element type unsupported") + } + var out []byte + for i := uint64(0); i < count; i++ { + slen, err := br.u64() + if err != nil { + return nil, err + } + out = append(out, leU64(slen)...) + buf := make([]byte, slen) + if _, err := io.ReadFull(br.r, buf); err != nil { + return nil, err + } + out = append(out, buf...) + } + return out, nil + } + sz := elemType.size() + if sz == 0 { + return nil, fmt.Errorf("unknown array element type %d", elemType) + } + total := int(count) * sz + // sanity cap: 256 MiB + if uint64(total) > 256<<20 { + return nil, fmt.Errorf("array too large: %d bytes", total) + } + buf := make([]byte, total) + if _, err := io.ReadFull(br.r, buf); err != nil { + return nil, err + } + return buf, nil +} + +// rawKVHash is a stable fingerprint of a KV entry's full wire value. +func (kv rawKV) rawKVHash() string { + h := crc32.NewIEEE() + h.Write([]byte(kv.Key)) + var b [8]byte + binary.LittleEndian.PutUint32(b[:4], uint32(kv.BType)) + h.Write(b[:4]) + switch kv.BType { + case btBool, btUint8, btInt8, btUint16, btInt16, btUint32, btInt32, btUint64, btInt64: + binary.LittleEndian.PutUint64(b[:], uint64(kv.Int)) + h.Write(b[:]) + case btFloat32: + binary.LittleEndian.PutUint32(b[:], math.Float32bits(float32(kv.Float))) + h.Write(b[:4]) + case btFloat64: + binary.LittleEndian.PutUint64(b[:], math.Float64bits(kv.Float)) + h.Write(b[:]) + case btString: + h.Write([]byte(kv.Str)) + case btArray: + binary.LittleEndian.PutUint32(b[:4], uint32(kv.ElemType)) + h.Write(b[:4]) + binary.LittleEndian.PutUint64(b[:], kv.Count) + h.Write(b[:]) + h.Write(kv.Raw) + } + return fmt.Sprintf("%08x", h.Sum32()) +} + +// --- tiny buffered reader helpers --- + +type bufReader struct { + r io.Reader +} + +func newBufReader(r io.Reader) *bufReader { return &bufReader{r: r} } + +func (b *bufReader) u8() (uint8, error) { + var x [1]byte + if _, err := io.ReadFull(b.r, x[:]); err != nil { + return 0, err + } + return x[0], nil +} + +func (b *bufReader) u16() (uint16, error) { + var x [2]byte + if _, err := io.ReadFull(b.r, x[:]); err != nil { + return 0, err + } + return binary.LittleEndian.Uint16(x[:]), nil +} + +func (b *bufReader) u32() (uint32, error) { + var x [4]byte + if _, err := io.ReadFull(b.r, x[:]); err != nil { + return 0, err + } + return binary.LittleEndian.Uint32(x[:]), nil +} + +func (b *bufReader) u64() (uint64, error) { + var x [8]byte + if _, err := io.ReadFull(b.r, x[:]); err != nil { + return 0, err + } + return binary.LittleEndian.Uint64(x[:]), nil +} + +func (b *bufReader) str() (string, error) { + n, err := b.u64() + if err != nil { + return "", err + } + if n > 1<<28 { + return "", fmt.Errorf("string too long: %d", n) + } + buf := make([]byte, n) + if _, err := io.ReadFull(b.r, buf); err != nil { + return "", err + } + return string(buf), nil +} + +func leU64(v uint64) []byte { + var b [8]byte + binary.LittleEndian.PutUint64(b[:], v) + return b[:] +} diff --git a/tools/hf2gguf/safetensors.go b/tools/hf2gguf/safetensors.go new file mode 100644 index 0000000..4317565 --- /dev/null +++ b/tools/hf2gguf/safetensors.go @@ -0,0 +1,154 @@ +package main + +// safetensors.go — minimal read-only HuggingFace safetensors index. +// +// Format: [u64 LE header_len][header_len bytes of JSON][tensor data ...]. +// The JSON header maps tensor name -> {dtype, shape, data_offsets:[start,end]} +// with data_offsets relative to the start of the data section. + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "io" + "os" +) + +type stTensor struct { + Dtype string `json:"dtype"` + Shape []int64 `json:"shape"` + Start int64 `json:"-"` + End int64 `json:"-"` +} + +type stHeader struct { + Dtype string `json:"dtype"` + Shape []int64 `json:"shape"` + DataOffsets [2]int64 `json:"data_offsets"` + Extra json.RawMessage `json:"-"` +} + +type safetensors struct { + path string + f *os.File + dataStart int64 + Meta map[string]string + Order []string + Tensors map[string]stTensor +} + +func openSafetensors(path string) (*safetensors, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + var lenBuf [8]byte + if _, err := io.ReadFull(f, lenBuf[:]); err != nil { + f.Close() + return nil, fmt.Errorf("read header length: %w", err) + } + hlen := binary.LittleEndian.Uint64(lenBuf[:]) + if hlen == 0 || hlen > 1<<30 { + f.Close() + return nil, fmt.Errorf("implausible safetensors header length %d", hlen) + } + raw := make([]byte, hlen) + if _, err := io.ReadFull(f, raw); err != nil { + f.Close() + return nil, fmt.Errorf("read header json: %w", err) + } + + // Top-level: either "__metadata__" (object of strings) or a tensor descriptor. + var top map[string]json.RawMessage + if err := json.Unmarshal(raw, &top); err != nil { + f.Close() + return nil, fmt.Errorf("parse header json: %w", err) + } + + st := &safetensors{ + path: path, + f: f, + dataStart: int64(8 + hlen), + Meta: map[string]string{}, + Tensors: map[string]stTensor{}, + } + if metaRaw, ok := top["__metadata__"]; ok { + var m map[string]string + if err := json.Unmarshal(metaRaw, &m); err == nil { + st.Meta = m + } + delete(top, "__metadata__") + } + for name, rawDesc := range top { + var h stHeader + if err := json.Unmarshal(rawDesc, &h); err != nil { + f.Close() + return nil, fmt.Errorf("tensor %q: %w", name, err) + } + st.Tensors[name] = stTensor{ + Dtype: h.Dtype, + Shape: h.Shape, + Start: h.DataOffsets[0], + End: h.DataOffsets[1], + } + st.Order = append(st.Order, name) + } + return st, nil +} + +func (s *safetensors) Close() error { + if s.f != nil { + return s.f.Close() + } + return nil +} + +// readTensor returns the raw little-endian bytes of a tensor. +func (s *safetensors) readTensor(name string) ([]byte, stTensor, error) { + t, ok := s.Tensors[name] + if !ok { + return nil, stTensor{}, fmt.Errorf("no tensor %q", name) + } + n := t.End - t.Start + if n < 0 { + return nil, t, fmt.Errorf("tensor %q: negative size", name) + } + buf := make([]byte, n) + if _, err := s.f.ReadAt(buf, s.dataStart+t.Start); err != nil { + return nil, t, fmt.Errorf("read %q: %w", name, err) + } + return buf, t, nil +} + +// numElems returns the product of shape dimensions. +func numElems(shape []int64) int64 { + n := int64(1) + for _, d := range shape { + n *= d + } + return n +} + +// stDtypeBytes returns bytes per element for the supported safetensors dtypes. +func stDtypeBytes(dtype string) (int, error) { + switch dtype { + case "F32": + return 4, nil + case "F16": + return 2, nil + case "BF16": + return 2, nil + case "I64": + return 8, nil + case "I32": + return 4, nil + case "I8": + return 1, nil + case "U8": + return 1, nil + case "F64": + return 8, nil + default: + return 0, fmt.Errorf("unsupported safetensors dtype %q", dtype) + } +} diff --git a/tools/hf2gguf/verify.go b/tools/hf2gguf/verify.go new file mode 100644 index 0000000..b8046bb --- /dev/null +++ b/tools/hf2gguf/verify.go @@ -0,0 +1,253 @@ +package main + +// verify.go — prove the emitted GGUF equals the reference: identical metadata +// (keys, types, values; verified on raw wire bytes) and identical tensor set, +// shapes, ggml types and per-tensor payload bytes. + +import ( + "bytes" + "flag" + "fmt" + "io" + + gguf "github.com/cymertek/go-gguf" +) + +type verifyResult struct { + MetadataChecked int + TensorsChecked int + ByteEqual int + MetaMismatches []string + TensorMismatch []string + OrderDiffers bool +} + +func (r *verifyResult) OK() bool { + return len(r.MetaMismatches) == 0 && len(r.TensorMismatch) == 0 && !r.OrderDiffers +} + +func compareGGUF(refPath, gotPath string) (*verifyResult, error) { + res := &verifyResult{} + + // --- metadata on raw wire bytes --- + _, refKVs, err := readRawKV(refPath) + if err != nil { + return nil, fmt.Errorf("read ref kv: %w", err) + } + _, gotKVs, err := readRawKV(gotPath) + if err != nil { + return nil, fmt.Errorf("read got kv: %w", err) + } + refMap := map[string]rawKV{} + for _, kv := range refKVs { + refMap[kv.Key] = kv + } + gotMap := map[string]rawKV{} + for _, kv := range gotKVs { + gotMap[kv.Key] = kv + } + for i, rk := range refKVs { + gk, ok := gotMap[rk.Key] + if !ok { + res.MetaMismatches = append(res.MetaMismatches, fmt.Sprintf("missing key %q", rk.Key)) + continue + } + if i < len(gotKVs) && gotKVs[i].Key != rk.Key { + res.OrderDiffers = true + } + switch { + case gk.BType != rk.BType: + res.MetaMismatches = append(res.MetaMismatches, fmt.Sprintf("%s: type %s != %s", rk.Key, gk.BType.name(), rk.BType.name())) + case gk.rawKVHash() != rk.rawKVHash(): + res.MetaMismatches = append(res.MetaMismatches, fmt.Sprintf("%s: value bytes differ", rk.Key)) + } + res.MetadataChecked++ + } + for _, gk := range gotKVs { + if _, ok := refMap[gk.Key]; !ok { + res.MetaMismatches = append(res.MetaMismatches, fmt.Sprintf("extra key %q", gk.Key)) + } + } + + // --- tensor metadata --- + refT, err := loadTensorSet(refPath) + if err != nil { + return nil, err + } + gotT, err := loadTensorSet(gotPath) + if err != nil { + return nil, err + } + if len(refT.order) != len(gotT.order) { + res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("tensor count %d != %d", len(gotT.order), len(refT.order))) + } + for i, name := range refT.order { + if i < len(gotT.order) && gotT.order[i] != name { + res.OrderDiffers = true + } + ri := refT.byName[name] + gi, ok := gotT.byName[name] + if !ok { + res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("missing tensor %s", name)) + continue + } + if !shapesEqual(ri.Shape, gi.Shape) { + res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("%s: shape %v != %v", name, gi.Shape, ri.Shape)) + continue + } + if ri.GgmlType != gi.GgmlType { + res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("%s: type %s != %s", name, gi.GgmlType.GgmlName(), ri.GgmlType.GgmlName())) + continue + } + res.TensorsChecked++ + } + for _, name := range gotT.order { + if _, ok := refT.byName[name]; !ok { + res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("extra tensor %s", name)) + } + } + + // --- per-tensor payload bytes --- + refG, err := gguf.Open(refPath) + if err != nil { + return nil, err + } + defer refG.Close() + gotG, err := gguf.Open(gotPath) + if err != nil { + return nil, err + } + defer gotG.Close() + + refHandles := map[string]*gguf.Tensor{} + for _, t := range mustTensors(refG) { + refHandles[t.Info().Name] = t + } + gotHandles := map[string]*gguf.Tensor{} + for _, t := range mustTensors(gotG) { + gotHandles[t.Info().Name] = t + } + for _, name := range refT.order { + rh, ok := refHandles[name] + if !ok { + continue + } + gh, ok := gotHandles[name] + if !ok { + continue + } + eq, diffOff, err := streamsEqual(rh.Reader(), gh.Reader()) + if err != nil { + return nil, fmt.Errorf("compare %s: %w", name, err) + } + if !eq { + res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("%s: payload differs at byte %d", name, diffOff)) + continue + } + res.ByteEqual++ + } + return res, nil +} + +// tensorSet is an ordered view of a GGUF's tensor metadata. +type tensorSet struct { + order []string + byName map[string]gguf.TensorInfo +} + +func loadTensorSet(path string) (*tensorSet, error) { + g, err := gguf.Open(path) + if err != nil { + return nil, err + } + defer g.Close() + ts, err := g.Tensors() + if err != nil { + return nil, err + } + out := &tensorSet{byName: map[string]gguf.TensorInfo{}} + for _, t := range ts { + info := t.Info() + out.byName[info.Name] = info + out.order = append(out.order, info.Name) + } + return out, nil +} + +func mustTensors(g *gguf.GGUF) []*gguf.Tensor { + ts, err := g.Tensors() + if err != nil { + panic(err) + } + return ts +} + +func shapesEqual(a, b []uint64) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// streamsEqual compares two readers byte-by-byte, returning the offset of the +// first difference. +func streamsEqual(a, b io.Reader) (bool, int64, error) { + const chunk = 1 << 20 + ba := make([]byte, chunk) + bb := make([]byte, chunk) + var off int64 + for { + na, ea := io.ReadFull(a, ba) + nb, eb := io.ReadFull(b, bb) + if na != nb { + return false, off + int64(min(na, nb)), nil + } + if !bytes.Equal(ba[:na], bb[:nb]) { + for i := 0; i < na; i++ { + if ba[i] != bb[i] { + return false, off + int64(i), nil + } + } + } + off += int64(na) + if ea == io.EOF || ea == io.ErrUnexpectedEOF { + endB := eb == io.EOF || eb == io.ErrUnexpectedEOF + return endB, off, nil + } + } +} + +func cmdVerify(args []string) error { + fs := flag.NewFlagSet("verify", flag.ExitOnError) + ref := fs.String("ref", "", "reference GGUF") + got := fs.String("got", "", "candidate GGUF") + fs.Parse(args) + if *ref == "" || *got == "" { + return fmt.Errorf("verify: -ref and -got are required") + } + res, err := compareGGUF(*ref, *got) + if err != nil { + return err + } + fmt.Printf("metadata: %d checked, %d mismatches\n", res.MetadataChecked, len(res.MetaMismatches)) + fmt.Printf("tensors: %d checked, %d byte-identical, %d mismatches\n", res.TensorsChecked, res.ByteEqual, len(res.TensorMismatch)) + if res.OrderDiffers { + fmt.Println("WARNING: metadata or tensor order differs from reference") + } + for _, m := range res.MetaMismatches { + fmt.Println(" META ", m) + } + for _, m := range res.TensorMismatch { + fmt.Println(" TENSOR", m) + } + if !res.OK() { + return fmt.Errorf("verification FAILED") + } + fmt.Printf("verification PASSED: %d metadata keys and %d tensors byte-identical\n", res.MetadataChecked, res.ByteEqual) + return nil +}