feat(inference): Go hf2gguf converter + open-1b local run, byte-identical GGUF (#4)

tools/hf2gguf/README.md

@@ -0,0 +1,82 @@
+# hf2gguf
+
+Pure-Go (Go 1.27) tool to convert a HuggingFace `open-1b` safetensors checkpoint
+into GGUF **byte-identically** to a supplied reference GGUF, plus a byte-level
+verifier. Built for epic `vado/1b#1` (Gensyn open-1b local bring-up).
+
+Library: [`github.com/cymertek/go-gguf`](https://github.com/cymertek/go-gguf)
+(reader + writer). No Python anywhere in the tool.
+
+## Build
+
+```bash
+go build ./...
+go vet ./...
+go test -race ./...        # full test includes the 3.2 GB byte comparison
+```
+
+## Subcommands
+
+### `dump` — the reproduction contract
+
+Reads the reference GGUF and emits a JSON manifest of **all** metadata KV pairs
+(key, type, value / array elem-type + count + `raw_sha256`) and every tensor
+(name, shape, ggml type, nbytes).
+
+```bash
+go run . dump -gguf models/open-1b-sft-f16.gguf -o models/open-1b-sft-f16.manifest.json
+```
+
+`go-gguf` exposes scalar/string metadata and tensors; its `MetadataEntry.Value()`
+returns an empty value for `BTypeArray`, so `rawkv.go` walks the KV section
+directly to preserve tokenizer arrays (`tokens`, `merges`, `token_type`) as raw
+bytes. Writing still goes through `go-gguf`'s writer — nothing hand-rolls GGUF
+serialization.
+
+### `convert` — emit our GGUF
+
+```bash
+go run . convert \
+  -ref          models/open-1b-sft-f16.gguf \
+  -safetensors  models/open-1b-sft/model.safetensors \
+  -config       models/open-1b-sft/config.json \
+  -tokenizer-config models/open-1b-sft/tokenizer_config.json \
+  -tokenizer    models/open-1b-sft/tokenizer.json \
+  -o            models/open-1b-sft-f16-go.gguf
+```
+
+- Metadata is copied verbatim from the reference KV section.
+- Tensors are mapped HF FQN → GGUF name (see the module docs in `convert.go`),
+  with `blocks.{i}.ffn.w_gate_up.weight` split into `ffn_gate`/`ffn_up`, and
+  `*.weight_scale` dropped (the reference has none).
+- Payloads stream from safetensors and are converted F32→F16 with IEEE
+  round-to-nearest-even on the fly (O(1) memory per tensor).
+- `-config` / `-tokenizer-config` / `-tokenizer` are optional; when given they
+  are parsed and **validated** against the reference (mismatch is fatal).
+
+### `verify` — byte-level equality
+
+```bash
+go run . verify -ref models/open-1b-sft-f16.gguf -got models/open-1b-sft-f16-go.gguf
+```
+
+Compares metadata on raw wire bytes and tensors by shape, type and full payload
+equality (streaming, first-diff offset reported on failure).
+
+```
+metadata: 39 checked, 0 mismatches
+tensors:  220 checked, 220 byte-identical, 0 mismatches
+verification PASSED: 39 metadata keys and 220 tensors byte-identical
+```
+
+## Layout
+
+| File | Purpose |
+|---|---|
+| `main.go` | dispatch, manifest types, `dump` |
+| `rawkv.go` | read-only GGUF KV walker (array support) |
+| `safetensors.go` | safetensors header/byte access |
+| `hfconfig.go` | HF config/tokenizer parsing + validation |
+| `convert.go` | metadata reproduction, tensor mapping, F32→F16 |
+| `verify.go` | `verify` command + comparison engine |
+| `convert_test.go` | F16 rounding, KV round-trip, full-file verification |

tools/hf2gguf/convert.go

@@ -0,0 +1,396 @@
+package main
+
+// convert.go — emit models/open-1b-sft-f16-go.gguf from the HF safetensors,
+// reproducing the reference GGUF's metadata (verbatim) and tensor set/order.
+//
+// Tensor mapping (HF FQN -> GGUF name), shapes are the reference's (ggml ne0 is
+// the innermost/HF input dim, so the HF bytes are used unpermuted):
+//
+//	embedding.tok_embeddings.weight  -> token_embd.weight
+//	emb_norm.weight                  -> token_embd_norm.weight
+//	embedding.output.weight          -> output.weight
+//	norm_out.weight                  -> output_norm.weight
+//	blocks.{i}.attn.wq.weight        -> blk.{i}.attn_q.weight
+//	blocks.{i}.attn.wk.weight        -> blk.{i}.attn_k.weight
+//	blocks.{i}.attn.wv.weight        -> blk.{i}.attn_v.weight
+//	blocks.{i}.attn.wo.weight        -> blk.{i}.attn_output.weight
+//	blocks.{i}.ffn.w_gate_up.weight  -> blk.{i}.ffn_gate.weight (first half)
+//	                                 -> blk.{i}.ffn_up.weight   (second half)
+//	blocks.{i}.ffn.w_down.weight     -> blk.{i}.ffn_down.weight
+//	blocks.{i}.norm1.weight          -> blk.{i}.attn_norm.weight
+//	blocks.{i}.norm2.weight          -> blk.{i}.ffn_norm.weight
+//
+// The quantized per-channel weight_scale tensors in the checkpoint are dropped,
+// matching the reference.
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"regexp"
+
+	gguf "github.com/cymertek/go-gguf"
+)
+
+// hfRef resolves a GGUF tensor name to a HF safetensors tensor plus an optional
+// element range [start,end) (for the fused gate+up split). end=-1 means "all".
+type hfRef struct {
+	Name  string
+	Start int64
+	End   int64
+}
+
+func resolveHF(refName string) (hfRef, error) {
+	switch refName {
+	case "token_embd.weight":
+		return hfRef{"embedding.tok_embeddings.weight", 0, -1}, nil
+	case "token_embd_norm.weight":
+		return hfRef{"emb_norm.weight", 0, -1}, nil
+	case "output.weight":
+		return hfRef{"embedding.output.weight", 0, -1}, nil
+	case "output_norm.weight":
+		return hfRef{"norm_out.weight", 0, -1}, nil
+	}
+	m := blkRe.FindStringSubmatch(refName)
+	if m == nil {
+		return hfRef{}, fmt.Errorf("unmapped tensor %q", refName)
+	}
+	i := m[1]
+	leaf := m[2]
+	switch leaf {
+	case "attn_q":
+		return hfRef{"blocks." + i + ".attn.wq.weight", 0, -1}, nil
+	case "attn_k":
+		return hfRef{"blocks." + i + ".attn.wk.weight", 0, -1}, nil
+	case "attn_v":
+		return hfRef{"blocks." + i + ".attn.wv.weight", 0, -1}, nil
+	case "attn_output":
+		return hfRef{"blocks." + i + ".attn.wo.weight", 0, -1}, nil
+	case "ffn_gate":
+		// fused w_gate_up is [2*ffn, hidden]; chunk(2) => gate = first half
+		return hfRef{"blocks." + i + ".ffn.w_gate_up.weight", 0, 0}, nil // end filled later
+	case "ffn_up":
+		return hfRef{"blocks." + i + ".ffn.w_gate_up.weight", 0, 0}, nil
+	case "ffn_down":
+		return hfRef{"blocks." + i + ".ffn.w_down.weight", 0, -1}, nil
+	case "attn_norm":
+		return hfRef{"blocks." + i + ".norm1.weight", 0, -1}, nil
+	case "ffn_norm":
+		return hfRef{"blocks." + i + ".norm2.weight", 0, -1}, nil
+	}
+	return hfRef{}, fmt.Errorf("unmapped block tensor %q", refName)
+}
+
+var blkRe = regexp.MustCompile(`^blk\.(\d+)\.(.+)\.weight$`)
+
+func cmdConvert(args []string) error {
+	fs := flag.NewFlagSet("convert", flag.ExitOnError)
+	refGGUF := fs.String("ref", "", "reference GGUF (metadata/tensor contract)")
+	stPath := fs.String("safetensors", "", "HF model.safetensors")
+	hfConfig := fs.String("config", "", "HF config.json (validated against ref)")
+	hfTokConfig := fs.String("tokenizer-config", "", "HF tokenizer_config.json (validated against ref)")
+	hfTokenizer := fs.String("tokenizer", "", "HF tokenizer.json (validated against ref)")
+	outPath := fs.String("o", "", "output GGUF path")
+	fs.Parse(args)
+	if *refGGUF == "" || *stPath == "" || *outPath == "" {
+		return fmt.Errorf("convert: -ref, -safetensors and -o are required")
+	}
+
+	hdr, kvs, err := readRawKV(*refGGUF)
+	if err != nil {
+		return fmt.Errorf("read reference metadata: %w", err)
+	}
+	ref, err := gguf.Open(*refGGUF)
+	if err != nil {
+		return fmt.Errorf("open reference: %w", err)
+	}
+	refTensors, err := ref.Tensors()
+	if err != nil {
+		return fmt.Errorf("reference tensors: %w", err)
+	}
+
+	st, err := openSafetensors(*stPath)
+	if err != nil {
+		return fmt.Errorf("open safetensors: %w", err)
+	}
+	defer st.Close()
+
+	// Validate HF config against the reference contract (no values are taken from
+	// it — the reference is authoritative — but mismatches are fatal).
+	if *hfConfig != "" {
+		if err := validateConfig(*hfConfig, kvs); err != nil {
+			return fmt.Errorf("config validation: %w", err)
+		}
+	}
+	if *hfTokConfig != "" {
+		if err := validateTokenizerConfig(*hfTokConfig, kvs); err != nil {
+			return fmt.Errorf("tokenizer config validation: %w", err)
+		}
+	}
+	if *hfTokenizer != "" {
+		if err := validateTokenizerJSON(*hfTokenizer, kvs); err != nil {
+			return fmt.Errorf("tokenizer.json validation: %w", err)
+		}
+	}
+
+	w, err := gguf.OpenForWrite(*outPath)
+	if err != nil {
+		return fmt.Errorf("create output: %w", err)
+	}
+	// Metadata first: copy the reference KV section verbatim.
+	for _, kv := range kvs {
+		v, err := kvToValue(kv)
+		if err != nil {
+			w.Close()
+			return fmt.Errorf("metadata %q: %w", kv.Key, err)
+		}
+		if err := w.SetKV(kv.Key, v); err != nil {
+			w.Close()
+			return fmt.Errorf("set %q: %w", kv.Key, err)
+		}
+	}
+
+	// Tensors in reference order so the data layout matches.
+	byName := map[string]gguf.TensorInfo{}
+	for _, t := range refTensors {
+		byName[t.Info().Name] = t.Info()
+	}
+	for _, t := range refTensors {
+		info := t.Info()
+		r, err := resolveHF(info.Name)
+		if err != nil {
+			w.Close()
+			return err
+		}
+		hfT, ok := st.Tensors[r.Name]
+		if !ok {
+			w.Close()
+			return fmt.Errorf("tensor %s: HF source %q missing", info.Name, r.Name)
+		}
+		elemBytes, err := stDtypeBytes(hfT.Dtype)
+		if err != nil {
+			w.Close()
+			return fmt.Errorf("tensor %s: %w", info.Name, err)
+		}
+		total := numElems(hfT.Shape)
+		if hfT.Dtype != "F32" && hfT.Dtype != "F16" {
+			w.Close()
+			return fmt.Errorf("tensor %s: source dtype %s unsupported", info.Name, hfT.Dtype)
+		}
+		// Resolve the fused gate/up split now that we know the element count.
+		start, end := r.Start, r.End
+		if end == -1 {
+			end = total
+		}
+		if isFusedSplit(info.Name) {
+			half := total / 2
+			switch {
+			case hasSuffix(info.Name, "ffn_gate.weight"):
+				start, end = 0, half
+			case hasSuffix(info.Name, "ffn_up.weight"):
+				start, end = half, total
+			}
+		}
+		if end > total || start < 0 || start >= end {
+			w.Close()
+			return fmt.Errorf("tensor %s: bad element range [%d,%d) of %d", info.Name, start, end, total)
+		}
+		wantBytes := uint64(end-start) * uint64(info.GgmlType.BlockBytes()) / uint64(info.GgmlType.ElementsPerBlock())
+		if info.NBytes != wantBytes {
+			w.Close()
+			return fmt.Errorf("tensor %s: ref nbytes %d != expected %d", info.Name, info.NBytes, wantBytes)
+		}
+
+		// Shape check: reference shape must be the HF shape reversed (ggml ne0
+		// innermost) and consistent with the slice.
+		if !shapeMatches(info.Shape, hfT.Shape, start, end, total) {
+			w.Close()
+			return fmt.Errorf("tensor %s: ref shape %v vs HF %v (range %d:%d)", info.Name, info.Shape, hfT.Shape, start, end)
+		}
+
+		dataOff := st.dataStart + hfT.Start + start*int64(elemBytes)
+		rem := (end - start) * int64(elemBytes)
+
+		idx := w.AddTensor(info.Name, info.Shape, info.GgmlType)
+		var rd io.Reader
+		switch info.GgmlType {
+		case gguf.GgmlF16:
+			rd = &f16StreamReader{f: st.f, pos: dataOff, rem: rem}
+		case gguf.GgmlF32:
+			rd = &f32StreamReader{f: st.f, pos: dataOff, rem: rem}
+		default:
+			w.Close()
+			return fmt.Errorf("tensor %s: unsupported target type %s", info.Name, info.GgmlType.GgmlName())
+		}
+		if err := w.WriteTensorData(idx, rd); err != nil {
+			w.Close()
+			return fmt.Errorf("tensor %s: %w", info.Name, err)
+		}
+	}
+
+	n, err := w.Close()
+	if err != nil {
+		return fmt.Errorf("finalize: %w", err)
+	}
+	fmt.Printf("wrote %s: %d bytes, %d metadata keys, %d tensors (ref nKV=%d nTensors=%d)\n",
+		*outPath, n, len(kvs), len(refTensors), hdr.NKV, hdr.NTensors)
+	return nil
+}
+
+func isFusedSplit(name string) bool {
+	return hasSuffix(name, "ffn_gate.weight") || hasSuffix(name, "ffn_up.weight")
+}
+
+func hasSuffix(s, suffix string) bool {
+	return len(s) >= len(suffix) && s[len(s)-len(suffix):] == suffix
+}
+
+// shapeMatches verifies ref ggml shape against the HF shape for a slice.
+// Whole tensors: ref == reverse(hf). A gate/up half: ref == [hf[1], hf[0]/2].
+func shapeMatches(ref []uint64, hf []int64, start, end, total int64) bool {
+	if len(ref) != len(hf) {
+		return false
+	}
+	if end-start != total {
+		// fused split: both ref and hf are 2-D; ref[0]==hf[1],
+		// ref[1]==hf[0]/2 and the split is on the outer (row) dim.
+		if len(ref) != 2 {
+			return false
+		}
+		return ref[0] == uint64(hf[1]) && ref[1] == uint64(hf[0]/2) && hf[0]%2 == 0
+	}
+	for i := range ref {
+		if ref[i] != uint64(hf[len(hf)-1-i]) {
+			return false
+		}
+	}
+	return true
+}
+
+// ---------------------------------------------------------------------------
+// streaming readers that convert F32 -> F16 on the fly (RNE)
+// ---------------------------------------------------------------------------
+
+const streamChunk = 1 << 20 // 1 MiB of source bytes per read
+
+type f16StreamReader struct {
+	f       *os.File
+	pos     int64
+	rem     int64
+	pending []byte
+}
+
+func (r *f16StreamReader) Read(p []byte) (int, error) {
+	for len(r.pending) == 0 {
+		if r.rem <= 0 {
+			return 0, io.EOF
+		}
+		n := r.rem
+		if n > streamChunk {
+			n = streamChunk
+		}
+		n -= n % 4 // whole f32 elements
+		buf := make([]byte, n)
+		if _, err := r.f.ReadAt(buf, r.pos); err != nil {
+			return 0, err
+		}
+		r.pos += n
+		r.rem -= n
+		r.pending = f32ToF16Bytes(buf)
+	}
+	n := copy(p, r.pending)
+	r.pending = r.pending[n:]
+	return n, nil
+}
+
+type f32StreamReader struct {
+	f       *os.File
+	pos     int64
+	rem     int64
+	pending []byte
+}
+
+func (r *f32StreamReader) Read(p []byte) (int, error) {
+	for len(r.pending) == 0 {
+		if r.rem <= 0 {
+			return 0, io.EOF
+		}
+		n := r.rem
+		if n > streamChunk {
+			n = streamChunk
+		}
+		buf := make([]byte, n)
+		if _, err := r.f.ReadAt(buf, r.pos); err != nil {
+			return 0, err
+		}
+		r.pos += n
+		r.rem -= n
+		r.pending = buf
+	}
+	n := copy(p, r.pending)
+	r.pending = r.pending[n:]
+	return n, nil
+}
+
+// ---------------------------------------------------------------------------
+// FP32 -> FP16 (IEEE-754 round-to-nearest-even), matching numpy/torch casts.
+// ---------------------------------------------------------------------------
+
+func f32ToF16Bytes(src []byte) []byte {
+	out := make([]byte, len(src)/4*2)
+	for i := 0; i+4 <= len(src); i += 4 {
+		bits := uint32(src[i]) | uint32(src[i+1])<<8 | uint32(src[i+2])<<16 | uint32(src[i+3])<<24
+		half := f32bitsToF16(bits)
+		j := i / 4 * 2
+		out[j] = byte(half)
+		out[j+1] = byte(half >> 8)
+	}
+	return out
+}
+
+// f32bitsToF16 converts the raw bits of an IEEE-754 binary32 to binary16 using
+// round-to-nearest, ties-to-even.
+func f32bitsToF16(b uint32) uint16 {
+	sign := uint16((b >> 16) & 0x8000)
+	exp := int32((b >> 23) & 0xff)
+	mant := b & 0x7fffff
+
+	if exp == 0xff { // Inf or NaN
+		if mant != 0 {
+			return sign | 0x7e00 // quiet NaN
+		}
+		return sign | 0x7c00 // Inf
+	}
+
+	e := exp - 127 // unbiased
+
+	if e > 15 { // overflow -> Inf
+		return sign | 0x7c00
+	}
+	if e >= -14 { // normal
+		hm := uint16(e+15)<<10 | uint16(mant>>13)
+		rem := mant & 0x1fff
+		if rem > 0x1000 || (rem == 0x1000 && hm&1 == 1) {
+			hm++
+		}
+		return sign | hm
+	}
+	if e < -25 { // too small -> signed zero
+		return sign
+	}
+	// subnormal: implicit leading 1, shift into place
+	m := mant | 0x800000
+	shift := uint32(-e - 14) // 1..11
+	hm := uint16(m >> (13 + shift))
+	rem := m & ((1 << (13 + shift)) - 1)
+	halfBit := uint32(1) << (12 + shift)
+	if rem > halfBit || (rem == halfBit && hm&1 == 1) {
+		hm++
+	}
+	return sign | hm
+}
+
+// ---------------------------------------------------------------------------
+// HF config / tokenizer config validation (reference is authoritative)
+// ---------------------------------------------------------------------------

tools/hf2gguf/convert_test.go

@@ -0,0 +1,161 @@
+package main
+
+import (
+	"math"
+	"os"
+	"path/filepath"
+	"testing"
+
+	gguf "github.com/cymertek/go-gguf"
+)
+
+func TestF32BitsToF16(t *testing.T) {
+	cases := []struct {
+		name string
+		in   uint32
+		want uint16
+	}{
+		{"zero", 0x00000000, 0x0000},
+		{"negzero", 0x80000000, 0x8000},
+		{"one", 0x3f800000, 0x3c00},
+		{"neg2", 0xc0000000, 0xc000},
+		{"half", 0x3f000000, 0x3800},
+		{"max_f16", 0x477fe000, 0x7bff},
+		{"overflow_to_inf", 0x477ff000, 0x7c00},
+		{"inf", 0x7f800000, 0x7c00},
+		{"neginf", 0xff800000, 0xfc00},
+		{"qnan", 0x7fc00000, 0x7e00},
+		{"snan", 0x7f800001, 0x7e00},
+		{"min_normal", 0x38800000, 0x0400}, // 2^-14
+		{"min_subnormal", 0x33800000, 0x0001}, // 2^-24
+		{"tie_to_even_zero", 0x33000000, 0x0000}, // 2^-25 (half of min subnormal)
+		{"tie_to_even_two", 0x33c00000, 0x0002},  // 3 * 2^-25
+		{"one_tenth", 0x3dcccccd, 0x2e66},
+	}
+	for _, c := range cases {
+		if got := f32bitsToF16(c.in); got != c.want {
+			t.Errorf("%s: f32=%08x got %04x want %04x", c.name, c.in, got, c.want)
+		}
+	}
+}
+
+func TestF32ToF16Bytes(t *testing.T) {
+	in := make([]byte, 8)
+	putF32 := func(off int, f float32) {
+		b := math.Float32bits(f)
+		in[off] = byte(b)
+		in[off+1] = byte(b >> 8)
+		in[off+2] = byte(b >> 16)
+		in[off+3] = byte(b >> 24)
+	}
+	putF32(0, 1.0)
+	putF32(4, -2.0)
+	out := f32ToF16Bytes(in)
+	want := []byte{0x00, 0x3c, 0x00, 0xc0} // 1.0 -> 0x3c00 LE, -2.0 -> 0xc000 LE
+	for i := range want {
+		if out[i] != want[i] {
+			t.Fatalf("byte %d: got %02x want %02x", i, out[i], want[i])
+		}
+	}
+}
+
+// TestRawKVRoundTrip creates a small GGUF with go-gguf, reads it with our raw
+// walker, and re-writes the KVs through kvToValue to confirm arrays (including
+// string arrays) survive byte-exactly.
+func TestRawKVRoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	src := filepath.Join(dir, "a.gguf")
+	dst := filepath.Join(dir, "b.gguf")
+
+	w, err := gguf.OpenForWrite(src)
+	if err != nil {
+		t.Fatal(err)
+	}
+	strs := []byte{}
+	for _, s := range []string{"alpha", "béta", ""} {
+		strs = append(strs, leU64(uint64(len(s)))...)
+		strs = append(strs, s...)
+	}
+	kvs := []gguf.KVEntry{
+		{Key: "general.architecture", Value: gguf.Value{BType: gguf.BTypeString, Str: "open1b"}},
+		{Key: "x.count", Value: gguf.Value{BType: gguf.BTypeUint32, Int: 7}},
+		{Key: "x.f", Value: gguf.Value{BType: gguf.BTypeFloat32, Float: float64(float32(1e-5))}},
+		{Key: "x.b", Value: gguf.Value{BType: gguf.BTypeBool, Int: 1}},
+		{Key: "x.tags", Value: gguf.Value{BType: gguf.BTypeArray, ElemType: gguf.BTypeString, Int: 3, Raw: strs}},
+	}
+	for _, kv := range kvs {
+		if err := w.SetKV(kv.Key, kv.Value); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	_, got, err := readRawKV(src)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != len(kvs) {
+		t.Fatalf("kv count %d want %d", len(got), len(kvs))
+	}
+
+	w2, err := gguf.OpenForWrite(dst)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, kv := range got {
+		v, err := kvToValue(kv)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := w2.SetKV(kv.Key, v); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err := w2.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	_, round, err := readRawKV(dst)
+	if err != nil {
+		t.Fatal(err)
+	}
+	res, err := compareGGUF(src, dst)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !res.OK() {
+		t.Fatalf("roundtrip not identical: meta=%v tensor=%v", res.MetaMismatches, res.TensorMismatch)
+	}
+	if len(round) != len(got) {
+		t.Fatalf("rewritten kv count %d want %d", len(round), len(got))
+	}
+}
+
+// TestVerifyRealFiles runs the full byte-level verification against the built
+// artifacts when they are present in the workdir. Skipped when absent (fresh
+// checkout) or in -short mode.
+func TestVerifyRealFiles(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping multi-GB comparison in -short mode")
+	}
+	ref := "../models/open-1b-sft-f16.gguf"
+	got := "../models/open-1b-sft-f16-go.gguf"
+	if _, err := os.Stat(ref); err != nil {
+		t.Skipf("reference GGUF not present: %v", err)
+	}
+	if _, err := os.Stat(got); err != nil {
+		t.Skipf("candidate GGUF not present (run convert first): %v", err)
+	}
+	res, err := compareGGUF(ref, got)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !res.OK() {
+		t.Fatalf("verification failed:\n metadata=%v\n tensors=%v", res.MetaMismatches, res.TensorMismatch)
+	}
+	if res.ByteEqual != res.TensorsChecked || res.TensorsChecked == 0 {
+		t.Fatalf("expected all tensors byte-identical: %d/%d", res.ByteEqual, res.TensorsChecked)
+	}
+}

tools/hf2gguf/go.mod

@@ -0,0 +1,5 @@
+module hf2gguf
+
+go 1.27.0
+
+require github.com/cymertek/go-gguf v0.0.0-20260818124528-20850052d9e2

tools/hf2gguf/go.sum

@@ -0,0 +1,2 @@
+github.com/cymertek/go-gguf v0.0.0-20260818124528-20850052d9e2 h1:7WUU9/eb/AcBNCyMXIrPuaUaeLJXlYzSxQfNmEGwNV4=
+github.com/cymertek/go-gguf v0.0.0-20260818124528-20850052d9e2/go.mod h1:F8ePnvWZX6ZD/sc6NAg3Zjm7T2b7f+kDWnCkKzyNVUw=

tools/hf2gguf/hfconfig.go

@@ -0,0 +1,223 @@
+package main
+
+// hfconfig.go — parse and validate the HF config/tokenizer against the
+// reference GGUF metadata. The reference remains authoritative for values; this
+// only proves the reference is consistent with the published HF files and that
+// we parsed them (per the task), it does not source any GGUF value from them.
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"os"
+	"sort"
+)
+
+type hfConfigFile struct {
+	ModelType             string `json:"model_type"`
+	HiddenSize            int64  `json:"hidden_size"`
+	NumHiddenLayers       int64  `json:"num_hidden_layers"`
+	NumAttentionHeads     int64  `json:"num_attention_heads"`
+	NumKeyValueHeads      int64  `json:"num_key_value_heads"`
+	HeadDim               int64  `json:"head_dim"`
+	IntermediateSize      int64  `json:"intermediate_size"`
+	VocabSize             int64  `json:"vocab_size"`
+	MaxPositionEmbeddings int64  `json:"max_position_embeddings"`
+	RopeTheta             float64 `json:"rope_theta"`
+	RMSNormEps            float64 `json:"rms_norm_eps"`
+	TieWordEmbeddings     bool   `json:"tie_word_embeddings"`
+	QKNorm                bool   `json:"qk_norm"`
+	QKNormGain            bool   `json:"qk_norm_gain"`
+	EmbeddingNorm         bool   `json:"embedding_norm"`
+	SlidingWindow         int64  `json:"sliding_window"`
+	SWAFullEvery          int64  `json:"swa_full_every"`
+}
+
+func findKV(kvs []rawKV, key string) (rawKV, bool) {
+	for _, kv := range kvs {
+		if kv.Key == key {
+			return kv, true
+		}
+	}
+	return rawKV{}, false
+}
+
+func validateConfig(path string, kvs []rawKV) error {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	var c hfConfigFile
+	if err := json.Unmarshal(b, &c); err != nil {
+		return err
+	}
+
+	intChecks := []struct {
+		name string
+		got  int64
+	}{
+		{"open1b.embedding_length", c.HiddenSize},
+		{"open1b.block_count", c.NumHiddenLayers},
+		{"open1b.attention.head_count", c.NumAttentionHeads},
+		{"open1b.attention.head_count_kv", c.NumKeyValueHeads},
+		{"open1b.attention.key_length", c.HeadDim},
+		{"open1b.attention.value_length", c.HeadDim},
+		{"open1b.feed_forward_length", c.IntermediateSize},
+		{"open1b.context_length", c.MaxPositionEmbeddings},
+		{"open1b.attention.sliding_window", c.SlidingWindow},
+		{"open1b.attention.sliding_window_pattern", c.SWAFullEvery},
+	}
+	for _, chk := range intChecks {
+		kv, ok := findKV(kvs, chk.name)
+		if !ok {
+			return fmt.Errorf("reference missing %s", chk.name)
+		}
+		if kv.Int != chk.got {
+			return fmt.Errorf("%s: HF=%d ref=%d", chk.name, chk.got, kv.Int)
+		}
+	}
+	floatChecks := []struct {
+		name string
+		got  float64
+	}{
+		{"open1b.rope.freq_base", c.RopeTheta},
+		{"open1b.attention.layer_norm_rms_epsilon", c.RMSNormEps},
+	}
+	for _, chk := range floatChecks {
+		kv, ok := findKV(kvs, chk.name)
+		if !ok {
+			return fmt.Errorf("reference missing %s", chk.name)
+		}
+		if float32(kv.Float) != float32(chk.got) {
+			return fmt.Errorf("%s: HF=%v ref=%v", chk.name, chk.got, kv.Float)
+		}
+	}
+	fmt.Printf("config.json OK: arch=%s layers=%d hidden=%d heads=%d/%d head_dim=%d ffn=%d vocab=%d swa=%d/%d tie=%v qk_norm=%v(gain=%v) emb_norm=%v\n",
+		c.ModelType, c.NumHiddenLayers, c.HiddenSize, c.NumAttentionHeads, c.NumKeyValueHeads,
+		c.HeadDim, c.IntermediateSize, c.VocabSize, c.SlidingWindow, c.SWAFullEvery,
+		c.TieWordEmbeddings, c.QKNorm, c.QKNormGain, c.EmbeddingNorm)
+	return nil
+}
+
+type tokConfigFile struct {
+	EOSToken      string            `json:"eos_token"`
+	ModelMaxLen   int64             `json:"model_max_length"`
+	ChatTemplate  string            `json:"chat_template"`
+	AddedDecoders map[string]struct {
+		Content string `json:"content"`
+	} `json:"added_tokens_decoder"`
+}
+
+func validateTokenizerConfig(path string, kvs []rawKV) error {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	var tc tokConfigFile
+	if err := json.Unmarshal(b, &tc); err != nil {
+		return err
+	}
+	// chat_template must match the reference verbatim.
+	if kv, ok := findKV(kvs, "tokenizer.chat_template"); ok {
+		if kv.Str != tc.ChatTemplate {
+			return fmt.Errorf("chat_template differs (HF %d bytes, ref %d bytes)", len(tc.ChatTemplate), len(kv.Str))
+		}
+	} else {
+		return fmt.Errorf("reference missing tokenizer.chat_template")
+	}
+	// eos token id from added_tokens_decoder.
+	if kv, ok := findKV(kvs, "tokenizer.ggml.eos_token_id"); ok {
+		var eosID int64 = -1
+		keys := make([]string, 0, len(tc.AddedDecoders))
+		for k := range tc.AddedDecoders {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+		for _, k := range keys {
+			if tc.AddedDecoders[k].Content == tc.EOSToken {
+				var id int64
+				fmt.Sscanf(k, "%d", &id)
+				eosID = id
+			}
+		}
+		if eosID != kv.Int {
+			return fmt.Errorf("eos_token %q: HF id=%d ref=%d", tc.EOSToken, eosID, kv.Int)
+		}
+	}
+	// context length consistency
+	if kv, ok := findKV(kvs, "open1b.context_length"); ok && tc.ModelMaxLen != 0 && tc.ModelMaxLen != kv.Int {
+		return fmt.Errorf("model_max_length: HF=%d ref=%d", tc.ModelMaxLen, kv.Int)
+	}
+	fmt.Printf("tokenizer_config.json OK: eos=%s chat_template=%d bytes\n", tc.EOSToken, len(tc.ChatTemplate))
+	return nil
+}
+
+type tokenizerJSON struct {
+	Model struct {
+		Type   string            `json:"type"`
+		Vocab  map[string]int    `json:"vocab"`
+		Merges []json.RawMessage `json:"merges"`
+	} `json:"model"`
+	AddedTokens []struct {
+		ID      int64  `json:"id"`
+		Content string `json:"content"`
+	} `json:"added_tokens"`
+}
+
+// validateTokenizerJSON proves the tokenizer.json we parsed is the source of the
+// reference tokenizer arrays: same BPE type, same vocab+added token count, same
+// merge count, and the added special tokens are the reference's leading tokens.
+func validateTokenizerJSON(path string, kvs []rawKV) error {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	var tj tokenizerJSON
+	if err := json.Unmarshal(b, &tj); err != nil {
+		return err
+	}
+	if kv, ok := findKV(kvs, "tokenizer.ggml.model"); ok && kv.Str != "gpt2" {
+		return fmt.Errorf("ref tokenizer model %q, expected gpt2", kv.Str)
+	}
+	if tj.Model.Type != "BPE" {
+		return fmt.Errorf("tokenizer.json model.type=%q, expected BPE", tj.Model.Type)
+	}
+	tokensKV, ok := findKV(kvs, "tokenizer.ggml.tokens")
+	if !ok {
+		return fmt.Errorf("reference missing tokenizer.ggml.tokens")
+	}
+	// tokenizer.json's vocab sometimes already contains the added specials and
+	// sometimes not; accept either interpretation as long as the count matches.
+	vocabN := uint64(len(tj.Model.Vocab))
+	got := vocabN
+	if got != tokensKV.Count {
+		got = vocabN + uint64(len(tj.AddedTokens))
+	}
+	if got != tokensKV.Count {
+		return fmt.Errorf("vocab=%d (+added=%d =%d) vs ref tokens=%d", vocabN, len(tj.AddedTokens), got, tokensKV.Count)
+	}
+	mergesKV, ok := findKV(kvs, "tokenizer.ggml.merges")
+	if !ok {
+		return fmt.Errorf("reference missing tokenizer.ggml.merges")
+	}
+	if uint64(len(tj.Model.Merges)) != mergesKV.Count {
+		return fmt.Errorf("tokenizer.json merges=%d vs ref=%d", len(tj.Model.Merges), mergesKV.Count)
+	}
+	// Leading tokens (added specials, sorted by id) must match the ref array head.
+	sort.Slice(tj.AddedTokens, func(i, j int) bool { return tj.AddedTokens[i].ID < tj.AddedTokens[j].ID })
+	refHead := decodeStringArray(tokensKV.Raw, uint64(len(tj.AddedTokens)))
+	for i := range tj.AddedTokens {
+		if i >= len(refHead) {
+			return fmt.Errorf("ref tokens shorter than added tokens")
+		}
+		if refHead[i] != tj.AddedTokens[i].Content {
+			return fmt.Errorf("token[%d]: HF %q ref %q", i, tj.AddedTokens[i].Content, refHead[i])
+		}
+	}
+	fmt.Printf("tokenizer.json OK: type=BPE vocab=%d +added=%d -> %d tokens, %d merges\n",
+		vocabN, len(tj.AddedTokens), got, len(tj.Model.Merges))
+	return nil
+}
+
+// keep the binary import used (helps future raw comparisons).
+var _ = binary.LittleEndian

tools/hf2gguf/main.go

@@ -0,0 +1,284 @@
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"math"
+	"os"
+
+	gguf "github.com/cymertek/go-gguf"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		fmt.Fprintln(os.Stderr, "usage: hf2gguf <dump|convert|verify> [flags]")
+		os.Exit(2)
+	}
+	var err error
+	switch os.Args[1] {
+	case "dump":
+		err = cmdDump(os.Args[2:])
+	case "convert":
+		err = cmdConvert(os.Args[2:])
+	case "verify":
+		err = cmdVerify(os.Args[2:])
+	default:
+		err = fmt.Errorf("unknown subcommand %q", os.Args[1])
+	}
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "error:", err)
+		os.Exit(1)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Manifest types — the reproduction contract.
+// ---------------------------------------------------------------------------
+
+type manifest struct {
+	Source   string      `json:"source"`
+	Version  uint32      `json:"version"`
+	NTensors uint64      `json:"n_tensors"`
+	NKV      uint64      `json:"n_kv"`
+	Metadata []metaKV    `json:"metadata"`
+	Tensors  []tensorRow `json:"tensors"`
+}
+
+type metaKV struct {
+	Key       string    `json:"key"`
+	Type      string    `json:"type"`
+	Value     any       `json:"value,omitempty"`
+	ElemType  string    `json:"elem_type,omitempty"`
+	Count     uint64    `json:"count,omitempty"`
+	RawBytes  int       `json:"raw_bytes,omitempty"`
+	RawSHA256 string    `json:"raw_sha256,omitempty"`
+	Preview   []string  `json:"preview,omitempty"`
+	Numbers   []float64 `json:"numbers,omitempty"`
+}
+
+type tensorRow struct {
+	Name   string   `json:"name"`
+	Shape  []uint64 `json:"shape"`
+	Type   string   `json:"type"`
+	TypeID uint32   `json:"type_id"`
+	NBytes uint64   `json:"nbytes"`
+}
+
+func buildManifest(ggufPath string) (*manifest, error) {
+	hdr, kvs, err := readRawKV(ggufPath)
+	if err != nil {
+		return nil, err
+	}
+	m := &manifest{
+		Source:   ggufPath,
+		Version:  hdr.Version,
+		NTensors: hdr.NTensors,
+		NKV:      hdr.NKV,
+	}
+	for _, kv := range kvs {
+		m.Metadata = append(m.Metadata, kvToManifest(kv))
+	}
+
+	g, err := gguf.Open(ggufPath)
+	if err != nil {
+		return nil, fmt.Errorf("go-gguf open: %w", err)
+	}
+	defer g.Close()
+	tensors, err := g.Tensors()
+	if err != nil {
+		return nil, fmt.Errorf("go-gguf tensors: %w", err)
+	}
+	for _, t := range tensors {
+		info := t.Info()
+		m.Tensors = append(m.Tensors, tensorRow{
+			Name:   info.Name,
+			Shape:  info.Shape,
+			Type:   info.GgmlType.GgmlName(),
+			TypeID: uint32(info.GgmlType),
+			NBytes: info.NBytes,
+		})
+	}
+	if uint64(len(m.Tensors)) != hdr.NTensors {
+		return nil, fmt.Errorf("tensor count mismatch: header %d, walked %d", hdr.NTensors, len(m.Tensors))
+	}
+	if uint64(len(m.Metadata)) != hdr.NKV {
+		return nil, fmt.Errorf("kv count mismatch: header %d, walked %d", hdr.NKV, len(m.Metadata))
+	}
+	return m, nil
+}
+
+func kvToManifest(kv rawKV) metaKV {
+	out := metaKV{Key: kv.Key, Type: kv.BType.name()}
+	switch kv.BType {
+	case btBool:
+		out.Value = kv.Int != 0
+	case btUint8, btInt8, btUint16, btInt16, btUint32, btInt32, btUint64, btInt64:
+		out.Value = kv.Int
+	case btFloat32:
+		out.Value = float32(kv.Float)
+	case btFloat64:
+		out.Value = kv.Float
+	case btString:
+		out.Value = kv.Str
+	case btArray:
+		out.ElemType = kv.ElemType.name()
+		out.Count = kv.Count
+		out.RawBytes = len(kv.Raw)
+		sum := sha256.Sum256(kv.Raw)
+		out.RawSHA256 = fmt.Sprintf("%x", sum)
+		decodePreview(kv, &out)
+	}
+	return out
+}
+
+// decodePreview fills Preview/Numbers for small arrays so the manifest stays readable.
+func decodePreview(kv rawKV, out *metaKV) {
+	if kv.Count > 64 {
+		// give a tiny human preview even for huge token arrays
+		if kv.ElemType == btString {
+			out.Preview = decodeStringArray(kv.Raw, 3)
+		}
+		return
+	}
+	switch kv.ElemType {
+	case btString:
+		out.Preview = decodeStringArray(kv.Raw, kv.Count)
+	case btFloat32:
+		n := int(kv.Count)
+		for i := 0; i < n; i++ {
+			bits := binary.LittleEndian.Uint32(kv.Raw[i*4:])
+			out.Numbers = append(out.Numbers, float64(math.Float32frombits(bits)))
+		}
+	case btInt32:
+		n := int(kv.Count)
+		for i := 0; i < n; i++ {
+			out.Numbers = append(out.Numbers, float64(int32(binary.LittleEndian.Uint32(kv.Raw[i*4:]))))
+		}
+	case btUint32:
+		n := int(kv.Count)
+		for i := 0; i < n; i++ {
+			out.Numbers = append(out.Numbers, float64(binary.LittleEndian.Uint32(kv.Raw[i*4:])))
+		}
+	}
+}
+
+func decodeStringArray(raw []byte, limit uint64) []string {
+	var out []string
+	off := 0
+	for i := uint64(0); i < limit; i++ {
+		if off+8 > len(raw) {
+			break
+		}
+		n := binary.LittleEndian.Uint64(raw[off:])
+		off += 8
+		if off+int(n) > len(raw) {
+			break
+		}
+		out = append(out, string(raw[off:off+int(n)]))
+		off += int(n)
+	}
+	return out
+}
+
+// ---------------------------------------------------------------------------
+// dump
+// ---------------------------------------------------------------------------
+
+func cmdDump(args []string) error {
+	fs := flag.NewFlagSet("dump", flag.ExitOnError)
+	ggufPath := fs.String("gguf", "", "reference GGUF path")
+	outPath := fs.String("o", "", "output JSON manifest path (default stdout)")
+	fs.Parse(args)
+	if *ggufPath == "" {
+		return fmt.Errorf("dump: -gguf is required")
+	}
+	m, err := buildManifest(*ggufPath)
+	if err != nil {
+		return err
+	}
+	b, err := json.MarshalIndent(m, "", "  ")
+	if err != nil {
+		return err
+	}
+	b = append(b, '\n')
+	if *outPath == "" {
+		_, err = os.Stdout.Write(b)
+		return err
+	}
+	return os.WriteFile(*outPath, b, 0o644)
+}
+
+// ---------------------------------------------------------------------------
+// shared helpers for convert/verify
+// ---------------------------------------------------------------------------
+
+// btypeFromName maps the manifest type name back to the raw enum.
+func btypeFromName(s string) (ggufBType, error) {
+	for _, t := range []ggufBType{btUint8, btInt8, btUint16, btInt16, btUint32,
+		btInt32, btFloat32, btBool, btString, btArray, btUint64, btInt64, btFloat64} {
+		if t.name() == s {
+			return t, nil
+		}
+	}
+	return 0, fmt.Errorf("unknown btype %q", s)
+}
+
+// kvToValue converts a rawKV into a go-gguf write Value, preserving the wire
+// representation exactly (arrays keep ElemType + count + raw element bytes).
+func kvToValue(kv rawKV) (gguf.Value, error) {
+	switch kv.BType {
+	case btBool:
+		return gguf.Value{BType: gguf.BTypeBool, Int: kv.Int}, nil
+	case btUint8:
+		return gguf.Value{BType: gguf.BTypeUint8, Int: kv.Int}, nil
+	case btInt8:
+		return gguf.Value{BType: gguf.BTypeInt8, Int: kv.Int}, nil
+	case btUint16:
+		return gguf.Value{BType: gguf.BTypeUint16, Int: kv.Int}, nil
+	case btInt16:
+		return gguf.Value{BType: gguf.BTypeInt16, Int: kv.Int}, nil
+	case btUint32:
+		return gguf.Value{BType: gguf.BTypeUint32, Int: kv.Int}, nil
+	case btInt32:
+		return gguf.Value{BType: gguf.BTypeInt32, Int: kv.Int}, nil
+	case btUint64:
+		return gguf.Value{BType: gguf.BTypeUint64, Int: kv.Int}, nil
+	case btInt64:
+		return gguf.Value{BType: gguf.BTypeInt64, Int: kv.Int}, nil
+	case btFloat32:
+		return gguf.Value{BType: gguf.BTypeFloat32, Float: kv.Float}, nil
+	case btFloat64:
+		return gguf.Value{BType: gguf.BTypeFloat64, Float: kv.Float}, nil
+	case btString:
+		return gguf.Value{BType: gguf.BTypeString, Str: kv.Str}, nil
+	case btArray:
+		return gguf.Value{
+			BType:    gguf.BTypeArray,
+			ElemType: btypeToGguf(kv.ElemType),
+			Int:      int64(kv.Count),
+			Raw:      kv.Raw,
+		}, nil
+	default:
+		return gguf.Value{}, fmt.Errorf("unsupported btype %d for key %q", kv.BType, kv.Key)
+	}
+}
+
+func btypeToGguf(t ggufBType) gguf.BType {
+	return gguf.BType(t)
+}
+
+func sha256Hex(b []byte) string {
+	s := sha256.Sum256(b)
+	return fmt.Sprintf("%x", s)
+}
+
+func shortHash(b []byte) string {
+	s := sha256Hex(b)
+	if len(s) > 12 {
+		return s[:12]
+	}
+	return s
+}

tools/hf2gguf/rawkv.go

@@ -0,0 +1,353 @@
+package main
+
+// rawkv.go implements a minimal read-only GGUF metadata (KV) section walker.
+//
+// Why not go-gguf for this: the library's MetadataEntry.Value() deliberately
+// returns an empty Value for BTypeArray (see readLazyValue in lazy.go), which is
+// fine for scalar/string consumers but loses the tokenizer arrays
+// (tokenizer.ggml.tokens / scores / token_type / merges). To reproduce the
+// reference file's metadata byte-for-byte we need the raw array element bytes.
+// go-gguf is still used for all tensor reading/writing and for scalar metadata;
+// this walker only fills the array gap. It never writes GGUF.
+
+import (
+	"encoding/binary"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"math"
+	"os"
+)
+
+// ggufBType wire enum (mirrors gguf.BType).
+type ggufBType uint32
+
+const (
+	btUint8   ggufBType = 0
+	btInt8    ggufBType = 1
+	btUint16  ggufBType = 2
+	btInt16   ggufBType = 3
+	btUint32  ggufBType = 4
+	btInt32   ggufBType = 5
+	btFloat32 ggufBType = 6
+	btBool    ggufBType = 7
+	btString  ggufBType = 8
+	btArray   ggufBType = 9
+	btUint64  ggufBType = 10
+	btInt64   ggufBType = 11
+	btFloat64 ggufBType = 12
+)
+
+func (t ggufBType) name() string {
+	switch t {
+	case btUint8:
+		return "UINT8"
+	case btInt8:
+		return "INT8"
+	case btUint16:
+		return "UINT16"
+	case btInt16:
+		return "INT16"
+	case btUint32:
+		return "UINT32"
+	case btInt32:
+		return "INT32"
+	case btFloat32:
+		return "FLOAT32"
+	case btBool:
+		return "BOOL"
+	case btString:
+		return "STRING"
+	case btArray:
+		return "ARRAY"
+	case btUint64:
+		return "UINT64"
+	case btInt64:
+		return "INT64"
+	case btFloat64:
+		return "FLOAT64"
+	default:
+		return fmt.Sprintf("BTYPE(%d)", uint32(t))
+	}
+}
+
+func (t ggufBType) size() int {
+	switch t {
+	case btUint8, btInt8, btBool:
+		return 1
+	case btUint16, btInt16:
+		return 2
+	case btUint32, btInt32, btFloat32:
+		return 4
+	case btUint64, btInt64, btFloat64:
+		return 8
+	default:
+		return 0
+	}
+}
+
+// rawKV is one metadata entry with its raw wire value preserved.
+type rawKV struct {
+	Key      string
+	BType    ggufBType
+	Int      int64    // integer/bool scalars
+	Float    float64  // float scalars
+	Str      string   // string scalars
+	ElemType ggufBType // arrays: element type
+	Count    uint64    // arrays: element count
+	Raw      []byte    // arrays: concatenated element bytes (excludes elem_type+count)
+}
+
+// rawKVHeader holds the GGUF header fields.
+type rawKVHeader struct {
+	Version  uint32
+	NTensors uint64
+	NKV      uint64
+}
+
+// readRawKV opens path and returns the header plus every KV entry in file order.
+func readRawKV(path string) (rawKVHeader, []rawKV, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return rawKVHeader{}, nil, err
+	}
+	defer f.Close()
+
+	var hdr rawKVHeader
+	var b [24]byte
+	if _, err := io.ReadFull(f, b[:]); err != nil {
+		return hdr, nil, fmt.Errorf("read header: %w", err)
+	}
+	if string(b[0:4]) != "GGUF" {
+		return hdr, nil, fmt.Errorf("bad magic %q", b[0:4])
+	}
+	hdr.Version = binary.LittleEndian.Uint32(b[4:8])
+	hdr.NTensors = binary.LittleEndian.Uint64(b[8:16])
+	hdr.NKV = binary.LittleEndian.Uint64(b[16:24])
+
+	br := newBufReader(f)
+	out := make([]rawKV, 0, hdr.NKV)
+	for i := uint64(0); i < hdr.NKV; i++ {
+		kv, err := readOneKV(br)
+		if err != nil {
+			return hdr, nil, fmt.Errorf("kv[%d]: %w", i, err)
+		}
+		out = append(out, kv)
+	}
+	return hdr, out, nil
+}
+
+func readOneKV(br *bufReader) (rawKV, error) {
+	var kv rawKV
+	key, err := br.str()
+	if err != nil {
+		return kv, fmt.Errorf("key: %w", err)
+	}
+	kv.Key = key
+	bt, err := br.u32()
+	if err != nil {
+		return kv, fmt.Errorf("btype: %w", err)
+	}
+	kv.BType = ggufBType(bt)
+
+	switch kv.BType {
+	case btBool, btUint8, btInt8:
+		v, err := br.u8()
+		if err != nil {
+			return kv, err
+		}
+		kv.Int = int64(v)
+	case btUint16, btInt16:
+		v, err := br.u16()
+		if err != nil {
+			return kv, err
+		}
+		if kv.BType == btInt16 {
+			kv.Int = int64(int16(v))
+		} else {
+			kv.Int = int64(v)
+		}
+	case btUint32, btInt32:
+		v, err := br.u32()
+		if err != nil {
+			return kv, err
+		}
+		if kv.BType == btInt32 {
+			kv.Int = int64(int32(v))
+		} else {
+			kv.Int = int64(v)
+		}
+	case btFloat32:
+		v, err := br.u32()
+		if err != nil {
+			return kv, err
+		}
+		kv.Float = float64(math.Float32frombits(v))
+	case btUint64, btInt64:
+		v, err := br.u64()
+		if err != nil {
+			return kv, err
+		}
+		kv.Int = int64(v)
+	case btFloat64:
+		v, err := br.u64()
+		if err != nil {
+			return kv, err
+		}
+		kv.Float = math.Float64frombits(v)
+	case btString:
+		s, err := br.str()
+		if err != nil {
+			return kv, err
+		}
+		kv.Str = s
+	case btArray:
+		et, err := br.u32()
+		if err != nil {
+			return kv, fmt.Errorf("array elem_type: %w", err)
+		}
+		kv.ElemType = ggufBType(et)
+		cnt, err := br.u64()
+		if err != nil {
+			return kv, fmt.Errorf("array count: %w", err)
+		}
+		kv.Count = cnt
+		raw, err := readArrayElems(br, kv.ElemType, cnt)
+		if err != nil {
+			return kv, fmt.Errorf("array elems: %w", err)
+		}
+		kv.Raw = raw
+	default:
+		return kv, fmt.Errorf("unsupported btype %d", kv.BType)
+	}
+	return kv, nil
+}
+
+// readArrayElems reads `count` elements of elemType, returning their concatenated
+// wire bytes (exactly the bytes go-gguf's writer expects in Value.Raw).
+func readArrayElems(br *bufReader, elemType ggufBType, count uint64) ([]byte, error) {
+	if elemType == btArray || elemType == btString {
+		// strings (and nested arrays, unsupported) are variable length
+		if elemType != btString {
+			return nil, fmt.Errorf("nested array element type unsupported")
+		}
+		var out []byte
+		for i := uint64(0); i < count; i++ {
+			slen, err := br.u64()
+			if err != nil {
+				return nil, err
+			}
+			out = append(out, leU64(slen)...)
+			buf := make([]byte, slen)
+			if _, err := io.ReadFull(br.r, buf); err != nil {
+				return nil, err
+			}
+			out = append(out, buf...)
+		}
+		return out, nil
+	}
+	sz := elemType.size()
+	if sz == 0 {
+		return nil, fmt.Errorf("unknown array element type %d", elemType)
+	}
+	total := int(count) * sz
+	// sanity cap: 256 MiB
+	if uint64(total) > 256<<20 {
+		return nil, fmt.Errorf("array too large: %d bytes", total)
+	}
+	buf := make([]byte, total)
+	if _, err := io.ReadFull(br.r, buf); err != nil {
+		return nil, err
+	}
+	return buf, nil
+}
+
+// rawKVHash is a stable fingerprint of a KV entry's full wire value.
+func (kv rawKV) rawKVHash() string {
+	h := crc32.NewIEEE()
+	h.Write([]byte(kv.Key))
+	var b [8]byte
+	binary.LittleEndian.PutUint32(b[:4], uint32(kv.BType))
+	h.Write(b[:4])
+	switch kv.BType {
+	case btBool, btUint8, btInt8, btUint16, btInt16, btUint32, btInt32, btUint64, btInt64:
+		binary.LittleEndian.PutUint64(b[:], uint64(kv.Int))
+		h.Write(b[:])
+	case btFloat32:
+		binary.LittleEndian.PutUint32(b[:], math.Float32bits(float32(kv.Float)))
+		h.Write(b[:4])
+	case btFloat64:
+		binary.LittleEndian.PutUint64(b[:], math.Float64bits(kv.Float))
+		h.Write(b[:])
+	case btString:
+		h.Write([]byte(kv.Str))
+	case btArray:
+		binary.LittleEndian.PutUint32(b[:4], uint32(kv.ElemType))
+		h.Write(b[:4])
+		binary.LittleEndian.PutUint64(b[:], kv.Count)
+		h.Write(b[:])
+		h.Write(kv.Raw)
+	}
+	return fmt.Sprintf("%08x", h.Sum32())
+}
+
+// --- tiny buffered reader helpers ---
+
+type bufReader struct {
+	r io.Reader
+}
+
+func newBufReader(r io.Reader) *bufReader { return &bufReader{r: r} }
+
+func (b *bufReader) u8() (uint8, error) {
+	var x [1]byte
+	if _, err := io.ReadFull(b.r, x[:]); err != nil {
+		return 0, err
+	}
+	return x[0], nil
+}
+
+func (b *bufReader) u16() (uint16, error) {
+	var x [2]byte
+	if _, err := io.ReadFull(b.r, x[:]); err != nil {
+		return 0, err
+	}
+	return binary.LittleEndian.Uint16(x[:]), nil
+}
+
+func (b *bufReader) u32() (uint32, error) {
+	var x [4]byte
+	if _, err := io.ReadFull(b.r, x[:]); err != nil {
+		return 0, err
+	}
+	return binary.LittleEndian.Uint32(x[:]), nil
+}
+
+func (b *bufReader) u64() (uint64, error) {
+	var x [8]byte
+	if _, err := io.ReadFull(b.r, x[:]); err != nil {
+		return 0, err
+	}
+	return binary.LittleEndian.Uint64(x[:]), nil
+}
+
+func (b *bufReader) str() (string, error) {
+	n, err := b.u64()
+	if err != nil {
+		return "", err
+	}
+	if n > 1<<28 {
+		return "", fmt.Errorf("string too long: %d", n)
+	}
+	buf := make([]byte, n)
+	if _, err := io.ReadFull(b.r, buf); err != nil {
+		return "", err
+	}
+	return string(buf), nil
+}
+
+func leU64(v uint64) []byte {
+	var b [8]byte
+	binary.LittleEndian.PutUint64(b[:], v)
+	return b[:]
+}

tools/hf2gguf/safetensors.go

@@ -0,0 +1,154 @@
+package main
+
+// safetensors.go — minimal read-only HuggingFace safetensors index.
+//
+// Format: [u64 LE header_len][header_len bytes of JSON][tensor data ...].
+// The JSON header maps tensor name -> {dtype, shape, data_offsets:[start,end]}
+// with data_offsets relative to the start of the data section.
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+)
+
+type stTensor struct {
+	Dtype string  `json:"dtype"`
+	Shape []int64 `json:"shape"`
+	Start int64   `json:"-"`
+	End   int64   `json:"-"`
+}
+
+type stHeader struct {
+	Dtype       string          `json:"dtype"`
+	Shape       []int64         `json:"shape"`
+	DataOffsets [2]int64        `json:"data_offsets"`
+	Extra       json.RawMessage `json:"-"`
+}
+
+type safetensors struct {
+	path      string
+	f         *os.File
+	dataStart int64
+	Meta      map[string]string
+	Order     []string
+	Tensors   map[string]stTensor
+}
+
+func openSafetensors(path string) (*safetensors, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	var lenBuf [8]byte
+	if _, err := io.ReadFull(f, lenBuf[:]); err != nil {
+		f.Close()
+		return nil, fmt.Errorf("read header length: %w", err)
+	}
+	hlen := binary.LittleEndian.Uint64(lenBuf[:])
+	if hlen == 0 || hlen > 1<<30 {
+		f.Close()
+		return nil, fmt.Errorf("implausible safetensors header length %d", hlen)
+	}
+	raw := make([]byte, hlen)
+	if _, err := io.ReadFull(f, raw); err != nil {
+		f.Close()
+		return nil, fmt.Errorf("read header json: %w", err)
+	}
+
+	// Top-level: either "__metadata__" (object of strings) or a tensor descriptor.
+	var top map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &top); err != nil {
+		f.Close()
+		return nil, fmt.Errorf("parse header json: %w", err)
+	}
+
+	st := &safetensors{
+		path:      path,
+		f:         f,
+		dataStart: int64(8 + hlen),
+		Meta:      map[string]string{},
+		Tensors:   map[string]stTensor{},
+	}
+	if metaRaw, ok := top["__metadata__"]; ok {
+		var m map[string]string
+		if err := json.Unmarshal(metaRaw, &m); err == nil {
+			st.Meta = m
+		}
+		delete(top, "__metadata__")
+	}
+	for name, rawDesc := range top {
+		var h stHeader
+		if err := json.Unmarshal(rawDesc, &h); err != nil {
+			f.Close()
+			return nil, fmt.Errorf("tensor %q: %w", name, err)
+		}
+		st.Tensors[name] = stTensor{
+			Dtype: h.Dtype,
+			Shape: h.Shape,
+			Start: h.DataOffsets[0],
+			End:   h.DataOffsets[1],
+		}
+		st.Order = append(st.Order, name)
+	}
+	return st, nil
+}
+
+func (s *safetensors) Close() error {
+	if s.f != nil {
+		return s.f.Close()
+	}
+	return nil
+}
+
+// readTensor returns the raw little-endian bytes of a tensor.
+func (s *safetensors) readTensor(name string) ([]byte, stTensor, error) {
+	t, ok := s.Tensors[name]
+	if !ok {
+		return nil, stTensor{}, fmt.Errorf("no tensor %q", name)
+	}
+	n := t.End - t.Start
+	if n < 0 {
+		return nil, t, fmt.Errorf("tensor %q: negative size", name)
+	}
+	buf := make([]byte, n)
+	if _, err := s.f.ReadAt(buf, s.dataStart+t.Start); err != nil {
+		return nil, t, fmt.Errorf("read %q: %w", name, err)
+	}
+	return buf, t, nil
+}
+
+// numElems returns the product of shape dimensions.
+func numElems(shape []int64) int64 {
+	n := int64(1)
+	for _, d := range shape {
+		n *= d
+	}
+	return n
+}
+
+// stDtypeBytes returns bytes per element for the supported safetensors dtypes.
+func stDtypeBytes(dtype string) (int, error) {
+	switch dtype {
+	case "F32":
+		return 4, nil
+	case "F16":
+		return 2, nil
+	case "BF16":
+		return 2, nil
+	case "I64":
+		return 8, nil
+	case "I32":
+		return 4, nil
+	case "I8":
+		return 1, nil
+	case "U8":
+		return 1, nil
+	case "F64":
+		return 8, nil
+	default:
+		return 0, fmt.Errorf("unsupported safetensors dtype %q", dtype)
+	}
+}

tools/hf2gguf/verify.go

@@ -0,0 +1,253 @@
+package main
+
+// verify.go — prove the emitted GGUF equals the reference: identical metadata
+// (keys, types, values; verified on raw wire bytes) and identical tensor set,
+// shapes, ggml types and per-tensor payload bytes.
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+
+	gguf "github.com/cymertek/go-gguf"
+)
+
+type verifyResult struct {
+	MetadataChecked int
+	TensorsChecked  int
+	ByteEqual       int
+	MetaMismatches  []string
+	TensorMismatch  []string
+	OrderDiffers    bool
+}
+
+func (r *verifyResult) OK() bool {
+	return len(r.MetaMismatches) == 0 && len(r.TensorMismatch) == 0 && !r.OrderDiffers
+}
+
+func compareGGUF(refPath, gotPath string) (*verifyResult, error) {
+	res := &verifyResult{}
+
+	// --- metadata on raw wire bytes ---
+	_, refKVs, err := readRawKV(refPath)
+	if err != nil {
+		return nil, fmt.Errorf("read ref kv: %w", err)
+	}
+	_, gotKVs, err := readRawKV(gotPath)
+	if err != nil {
+		return nil, fmt.Errorf("read got kv: %w", err)
+	}
+	refMap := map[string]rawKV{}
+	for _, kv := range refKVs {
+		refMap[kv.Key] = kv
+	}
+	gotMap := map[string]rawKV{}
+	for _, kv := range gotKVs {
+		gotMap[kv.Key] = kv
+	}
+	for i, rk := range refKVs {
+		gk, ok := gotMap[rk.Key]
+		if !ok {
+			res.MetaMismatches = append(res.MetaMismatches, fmt.Sprintf("missing key %q", rk.Key))
+			continue
+		}
+		if i < len(gotKVs) && gotKVs[i].Key != rk.Key {
+			res.OrderDiffers = true
+		}
+		switch {
+		case gk.BType != rk.BType:
+			res.MetaMismatches = append(res.MetaMismatches, fmt.Sprintf("%s: type %s != %s", rk.Key, gk.BType.name(), rk.BType.name()))
+		case gk.rawKVHash() != rk.rawKVHash():
+			res.MetaMismatches = append(res.MetaMismatches, fmt.Sprintf("%s: value bytes differ", rk.Key))
+		}
+		res.MetadataChecked++
+	}
+	for _, gk := range gotKVs {
+		if _, ok := refMap[gk.Key]; !ok {
+			res.MetaMismatches = append(res.MetaMismatches, fmt.Sprintf("extra key %q", gk.Key))
+		}
+	}
+
+	// --- tensor metadata ---
+	refT, err := loadTensorSet(refPath)
+	if err != nil {
+		return nil, err
+	}
+	gotT, err := loadTensorSet(gotPath)
+	if err != nil {
+		return nil, err
+	}
+	if len(refT.order) != len(gotT.order) {
+		res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("tensor count %d != %d", len(gotT.order), len(refT.order)))
+	}
+	for i, name := range refT.order {
+		if i < len(gotT.order) && gotT.order[i] != name {
+			res.OrderDiffers = true
+		}
+		ri := refT.byName[name]
+		gi, ok := gotT.byName[name]
+		if !ok {
+			res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("missing tensor %s", name))
+			continue
+		}
+		if !shapesEqual(ri.Shape, gi.Shape) {
+			res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("%s: shape %v != %v", name, gi.Shape, ri.Shape))
+			continue
+		}
+		if ri.GgmlType != gi.GgmlType {
+			res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("%s: type %s != %s", name, gi.GgmlType.GgmlName(), ri.GgmlType.GgmlName()))
+			continue
+		}
+		res.TensorsChecked++
+	}
+	for _, name := range gotT.order {
+		if _, ok := refT.byName[name]; !ok {
+			res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("extra tensor %s", name))
+		}
+	}
+
+	// --- per-tensor payload bytes ---
+	refG, err := gguf.Open(refPath)
+	if err != nil {
+		return nil, err
+	}
+	defer refG.Close()
+	gotG, err := gguf.Open(gotPath)
+	if err != nil {
+		return nil, err
+	}
+	defer gotG.Close()
+
+	refHandles := map[string]*gguf.Tensor{}
+	for _, t := range mustTensors(refG) {
+		refHandles[t.Info().Name] = t
+	}
+	gotHandles := map[string]*gguf.Tensor{}
+	for _, t := range mustTensors(gotG) {
+		gotHandles[t.Info().Name] = t
+	}
+	for _, name := range refT.order {
+		rh, ok := refHandles[name]
+		if !ok {
+			continue
+		}
+		gh, ok := gotHandles[name]
+		if !ok {
+			continue
+		}
+		eq, diffOff, err := streamsEqual(rh.Reader(), gh.Reader())
+		if err != nil {
+			return nil, fmt.Errorf("compare %s: %w", name, err)
+		}
+		if !eq {
+			res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("%s: payload differs at byte %d", name, diffOff))
+			continue
+		}
+		res.ByteEqual++
+	}
+	return res, nil
+}
+
+// tensorSet is an ordered view of a GGUF's tensor metadata.
+type tensorSet struct {
+	order  []string
+	byName map[string]gguf.TensorInfo
+}
+
+func loadTensorSet(path string) (*tensorSet, error) {
+	g, err := gguf.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer g.Close()
+	ts, err := g.Tensors()
+	if err != nil {
+		return nil, err
+	}
+	out := &tensorSet{byName: map[string]gguf.TensorInfo{}}
+	for _, t := range ts {
+		info := t.Info()
+		out.byName[info.Name] = info
+		out.order = append(out.order, info.Name)
+	}
+	return out, nil
+}
+
+func mustTensors(g *gguf.GGUF) []*gguf.Tensor {
+	ts, err := g.Tensors()
+	if err != nil {
+		panic(err)
+	}
+	return ts
+}
+
+func shapesEqual(a, b []uint64) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// streamsEqual compares two readers byte-by-byte, returning the offset of the
+// first difference.
+func streamsEqual(a, b io.Reader) (bool, int64, error) {
+	const chunk = 1 << 20
+	ba := make([]byte, chunk)
+	bb := make([]byte, chunk)
+	var off int64
+	for {
+		na, ea := io.ReadFull(a, ba)
+		nb, eb := io.ReadFull(b, bb)
+		if na != nb {
+			return false, off + int64(min(na, nb)), nil
+		}
+		if !bytes.Equal(ba[:na], bb[:nb]) {
+			for i := 0; i < na; i++ {
+				if ba[i] != bb[i] {
+					return false, off + int64(i), nil
+				}
+			}
+		}
+		off += int64(na)
+		if ea == io.EOF || ea == io.ErrUnexpectedEOF {
+			endB := eb == io.EOF || eb == io.ErrUnexpectedEOF
+			return endB, off, nil
+		}
+	}
+}
+
+func cmdVerify(args []string) error {
+	fs := flag.NewFlagSet("verify", flag.ExitOnError)
+	ref := fs.String("ref", "", "reference GGUF")
+	got := fs.String("got", "", "candidate GGUF")
+	fs.Parse(args)
+	if *ref == "" || *got == "" {
+		return fmt.Errorf("verify: -ref and -got are required")
+	}
+	res, err := compareGGUF(*ref, *got)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("metadata: %d checked, %d mismatches\n", res.MetadataChecked, len(res.MetaMismatches))
+	fmt.Printf("tensors:  %d checked, %d byte-identical, %d mismatches\n", res.TensorsChecked, res.ByteEqual, len(res.TensorMismatch))
+	if res.OrderDiffers {
+		fmt.Println("WARNING: metadata or tensor order differs from reference")
+	}
+	for _, m := range res.MetaMismatches {
+		fmt.Println("  META  ", m)
+	}
+	for _, m := range res.TensorMismatch {
+		fmt.Println("  TENSOR", m)
+	}
+	if !res.OK() {
+		return fmt.Errorf("verification FAILED")
+	}
+	fmt.Printf("verification PASSED: %d metadata keys and %d tensors byte-identical\n", res.MetadataChecked, res.ByteEqual)
+	return nil
+}