open-1b-verify/tools/hf2gguf/verify.go

package main

// verify.go proves the emitted GGUF equals the reference: identical metadata
// (keys, types, values; verified on raw wire bytes) and identical tensor set,
// shapes, ggml types and per-tensor payload bytes.

import (
	"bytes"
	"flag"
	"fmt"
	"io"

	gguf "github.com/cymertek/go-gguf"
)

type verifyResult struct {
	MetadataChecked int
	TensorsChecked  int
	ByteEqual       int
	MetaMismatches  []string
	TensorMismatch  []string
	OrderDiffers    bool
}

func (r *verifyResult) OK() bool {
	return len(r.MetaMismatches) == 0 && len(r.TensorMismatch) == 0 && !r.OrderDiffers
}

func compareGGUF(refPath, gotPath string) (*verifyResult, error) {
	res := &verifyResult{}

	// --- metadata on raw wire bytes ---
	_, refKVs, err := readRawKV(refPath)
	if err != nil {
		return nil, fmt.Errorf("read ref kv: %w", err)
	}
	_, gotKVs, err := readRawKV(gotPath)
	if err != nil {
		return nil, fmt.Errorf("read got kv: %w", err)
	}
	refMap := map[string]rawKV{}
	for _, kv := range refKVs {
		refMap[kv.Key] = kv
	}
	gotMap := map[string]rawKV{}
	for _, kv := range gotKVs {
		gotMap[kv.Key] = kv
	}
	for i, rk := range refKVs {
		gk, ok := gotMap[rk.Key]
		if !ok {
			res.MetaMismatches = append(res.MetaMismatches, fmt.Sprintf("missing key %q", rk.Key))
			continue
		}
		if i < len(gotKVs) && gotKVs[i].Key != rk.Key {
			res.OrderDiffers = true
		}
		switch {
		case gk.BType != rk.BType:
			res.MetaMismatches = append(res.MetaMismatches, fmt.Sprintf("%s: type %s != %s", rk.Key, gk.BType.name(), rk.BType.name()))
		case gk.rawKVHash() != rk.rawKVHash():
			res.MetaMismatches = append(res.MetaMismatches, fmt.Sprintf("%s: value bytes differ", rk.Key))
		}
		res.MetadataChecked++
	}
	for _, gk := range gotKVs {
		if _, ok := refMap[gk.Key]; !ok {
			res.MetaMismatches = append(res.MetaMismatches, fmt.Sprintf("extra key %q", gk.Key))
		}
	}

	// --- tensor metadata ---
	refT, err := loadTensorSet(refPath)
	if err != nil {
		return nil, err
	}
	gotT, err := loadTensorSet(gotPath)
	if err != nil {
		return nil, err
	}
	if len(refT.order) != len(gotT.order) {
		res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("tensor count %d != %d", len(gotT.order), len(refT.order)))
	}
	for i, name := range refT.order {
		if i < len(gotT.order) && gotT.order[i] != name {
			res.OrderDiffers = true
		}
		ri := refT.byName[name]
		gi, ok := gotT.byName[name]
		if !ok {
			res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("missing tensor %s", name))
			continue
		}
		if !shapesEqual(ri.Shape, gi.Shape) {
			res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("%s: shape %v != %v", name, gi.Shape, ri.Shape))
			continue
		}
		if ri.GgmlType != gi.GgmlType {
			res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("%s: type %s != %s", name, gi.GgmlType.GgmlName(), ri.GgmlType.GgmlName()))
			continue
		}
		res.TensorsChecked++
	}
	for _, name := range gotT.order {
		if _, ok := refT.byName[name]; !ok {
			res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("extra tensor %s", name))
		}
	}

	// --- per-tensor payload bytes ---
	refG, err := gguf.Open(refPath)
	if err != nil {
		return nil, err
	}
	defer refG.Close()
	gotG, err := gguf.Open(gotPath)
	if err != nil {
		return nil, err
	}
	defer gotG.Close()

	refHandles := map[string]*gguf.Tensor{}
	for _, t := range mustTensors(refG) {
		refHandles[t.Info().Name] = t
	}
	gotHandles := map[string]*gguf.Tensor{}
	for _, t := range mustTensors(gotG) {
		gotHandles[t.Info().Name] = t
	}
	for _, name := range refT.order {
		rh, ok := refHandles[name]
		if !ok {
			continue
		}
		gh, ok := gotHandles[name]
		if !ok {
			continue
		}
		eq, diffOff, err := streamsEqual(rh.Reader(), gh.Reader())
		if err != nil {
			return nil, fmt.Errorf("compare %s: %w", name, err)
		}
		if !eq {
			res.TensorMismatch = append(res.TensorMismatch, fmt.Sprintf("%s: payload differs at byte %d", name, diffOff))
			continue
		}
		res.ByteEqual++
	}
	return res, nil
}

// tensorSet is an ordered view of a GGUF's tensor metadata.
type tensorSet struct {
	order  []string
	byName map[string]gguf.TensorInfo
}

func loadTensorSet(path string) (*tensorSet, error) {
	g, err := gguf.Open(path)
	if err != nil {
		return nil, err
	}
	defer g.Close()
	ts, err := g.Tensors()
	if err != nil {
		return nil, err
	}
	out := &tensorSet{byName: map[string]gguf.TensorInfo{}}
	for _, t := range ts {
		info := t.Info()
		out.byName[info.Name] = info
		out.order = append(out.order, info.Name)
	}
	return out, nil
}

func mustTensors(g *gguf.GGUF) []*gguf.Tensor {
	ts, err := g.Tensors()
	if err != nil {
		panic(err)
	}
	return ts
}

func shapesEqual(a, b []uint64) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}

// streamsEqual compares two readers byte-by-byte, returning the offset of the
// first difference.
func streamsEqual(a, b io.Reader) (bool, int64, error) {
	const chunk = 1 << 20
	ba := make([]byte, chunk)
	bb := make([]byte, chunk)
	var off int64
	for {
		na, ea := io.ReadFull(a, ba)
		nb, eb := io.ReadFull(b, bb)
		if na != nb {
			return false, off + int64(min(na, nb)), nil
		}
		if !bytes.Equal(ba[:na], bb[:nb]) {
			for i := 0; i < na; i++ {
				if ba[i] != bb[i] {
					return false, off + int64(i), nil
				}
			}
		}
		off += int64(na)
		if ea == io.EOF || ea == io.ErrUnexpectedEOF {
			endB := eb == io.EOF || eb == io.ErrUnexpectedEOF
			return endB, off, nil
		}
	}
}

func cmdVerify(args []string) error {
	fs := flag.NewFlagSet("verify", flag.ExitOnError)
	ref := fs.String("ref", "", "reference GGUF")
	got := fs.String("got", "", "candidate GGUF")
	fs.Parse(args)
	if *ref == "" || *got == "" {
		return fmt.Errorf("verify: -ref and -got are required")
	}
	res, err := compareGGUF(*ref, *got)
	if err != nil {
		return err
	}
	fmt.Printf("metadata: %d checked, %d mismatches\n", res.MetadataChecked, len(res.MetaMismatches))
	fmt.Printf("tensors:  %d checked, %d byte-identical, %d mismatches\n", res.TensorsChecked, res.ByteEqual, len(res.TensorMismatch))
	if res.OrderDiffers {
		fmt.Println("WARNING: metadata or tensor order differs from reference")
	}
	for _, m := range res.MetaMismatches {
		fmt.Println("  META  ", m)
	}
	for _, m := range res.TensorMismatch {
		fmt.Println("  TENSOR", m)
	}
	if !res.OK() {
		return fmt.Errorf("verification FAILED")
	}
	fmt.Printf("verification PASSED: %d metadata keys and %d tensors byte-identical\n", res.MetadataChecked, res.ByteEqual)
	return nil
}