package main import ( "crypto/sha256" "encoding/binary" "encoding/json" "flag" "fmt" "math" "os" gguf "github.com/cymertek/go-gguf" ) func main() { if len(os.Args) < 2 { fmt.Fprintln(os.Stderr, "usage: hf2gguf [flags]") os.Exit(2) } var err error switch os.Args[1] { case "dump": err = cmdDump(os.Args[2:]) case "convert": err = cmdConvert(os.Args[2:]) case "verify": err = cmdVerify(os.Args[2:]) default: err = fmt.Errorf("unknown subcommand %q", os.Args[1]) } if err != nil { fmt.Fprintln(os.Stderr, "error:", err) os.Exit(1) } } // --------------------------------------------------------------------------- // Manifest types — the reproduction contract. // --------------------------------------------------------------------------- type manifest struct { Source string `json:"source"` Version uint32 `json:"version"` NTensors uint64 `json:"n_tensors"` NKV uint64 `json:"n_kv"` Metadata []metaKV `json:"metadata"` Tensors []tensorRow `json:"tensors"` } type metaKV struct { Key string `json:"key"` Type string `json:"type"` Value any `json:"value,omitempty"` ElemType string `json:"elem_type,omitempty"` Count uint64 `json:"count,omitempty"` RawBytes int `json:"raw_bytes,omitempty"` RawSHA256 string `json:"raw_sha256,omitempty"` Preview []string `json:"preview,omitempty"` Numbers []float64 `json:"numbers,omitempty"` } type tensorRow struct { Name string `json:"name"` Shape []uint64 `json:"shape"` Type string `json:"type"` TypeID uint32 `json:"type_id"` NBytes uint64 `json:"nbytes"` } func buildManifest(ggufPath string) (*manifest, error) { hdr, kvs, err := readRawKV(ggufPath) if err != nil { return nil, err } m := &manifest{ Source: ggufPath, Version: hdr.Version, NTensors: hdr.NTensors, NKV: hdr.NKV, } for _, kv := range kvs { m.Metadata = append(m.Metadata, kvToManifest(kv)) } g, err := gguf.Open(ggufPath) if err != nil { return nil, fmt.Errorf("go-gguf open: %w", err) } defer g.Close() tensors, err := g.Tensors() if err != nil { return nil, fmt.Errorf("go-gguf tensors: %w", err) } for _, t := range tensors { info := t.Info() m.Tensors = append(m.Tensors, tensorRow{ Name: info.Name, Shape: info.Shape, Type: info.GgmlType.GgmlName(), TypeID: uint32(info.GgmlType), NBytes: info.NBytes, }) } if uint64(len(m.Tensors)) != hdr.NTensors { return nil, fmt.Errorf("tensor count mismatch: header %d, walked %d", hdr.NTensors, len(m.Tensors)) } if uint64(len(m.Metadata)) != hdr.NKV { return nil, fmt.Errorf("kv count mismatch: header %d, walked %d", hdr.NKV, len(m.Metadata)) } return m, nil } func kvToManifest(kv rawKV) metaKV { out := metaKV{Key: kv.Key, Type: kv.BType.name()} switch kv.BType { case btBool: out.Value = kv.Int != 0 case btUint8, btInt8, btUint16, btInt16, btUint32, btInt32, btUint64, btInt64: out.Value = kv.Int case btFloat32: out.Value = float32(kv.Float) case btFloat64: out.Value = kv.Float case btString: out.Value = kv.Str case btArray: out.ElemType = kv.ElemType.name() out.Count = kv.Count out.RawBytes = len(kv.Raw) sum := sha256.Sum256(kv.Raw) out.RawSHA256 = fmt.Sprintf("%x", sum) decodePreview(kv, &out) } return out } // decodePreview fills Preview/Numbers for small arrays so the manifest stays readable. func decodePreview(kv rawKV, out *metaKV) { if kv.Count > 64 { // give a tiny human preview even for huge token arrays if kv.ElemType == btString { out.Preview = decodeStringArray(kv.Raw, 3) } return } switch kv.ElemType { case btString: out.Preview = decodeStringArray(kv.Raw, kv.Count) case btFloat32: n := int(kv.Count) for i := 0; i < n; i++ { bits := binary.LittleEndian.Uint32(kv.Raw[i*4:]) out.Numbers = append(out.Numbers, float64(math.Float32frombits(bits))) } case btInt32: n := int(kv.Count) for i := 0; i < n; i++ { out.Numbers = append(out.Numbers, float64(int32(binary.LittleEndian.Uint32(kv.Raw[i*4:])))) } case btUint32: n := int(kv.Count) for i := 0; i < n; i++ { out.Numbers = append(out.Numbers, float64(binary.LittleEndian.Uint32(kv.Raw[i*4:]))) } } } func decodeStringArray(raw []byte, limit uint64) []string { var out []string off := 0 for i := uint64(0); i < limit; i++ { if off+8 > len(raw) { break } n := binary.LittleEndian.Uint64(raw[off:]) off += 8 if off+int(n) > len(raw) { break } out = append(out, string(raw[off:off+int(n)])) off += int(n) } return out } // --------------------------------------------------------------------------- // dump // --------------------------------------------------------------------------- func cmdDump(args []string) error { fs := flag.NewFlagSet("dump", flag.ExitOnError) ggufPath := fs.String("gguf", "", "reference GGUF path") outPath := fs.String("o", "", "output JSON manifest path (default stdout)") fs.Parse(args) if *ggufPath == "" { return fmt.Errorf("dump: -gguf is required") } m, err := buildManifest(*ggufPath) if err != nil { return err } b, err := json.MarshalIndent(m, "", " ") if err != nil { return err } b = append(b, '\n') if *outPath == "" { _, err = os.Stdout.Write(b) return err } return os.WriteFile(*outPath, b, 0o644) } // --------------------------------------------------------------------------- // shared helpers for convert/verify // --------------------------------------------------------------------------- // btypeFromName maps the manifest type name back to the raw enum. func btypeFromName(s string) (ggufBType, error) { for _, t := range []ggufBType{btUint8, btInt8, btUint16, btInt16, btUint32, btInt32, btFloat32, btBool, btString, btArray, btUint64, btInt64, btFloat64} { if t.name() == s { return t, nil } } return 0, fmt.Errorf("unknown btype %q", s) } // kvToValue converts a rawKV into a go-gguf write Value, preserving the wire // representation exactly (arrays keep ElemType + count + raw element bytes). func kvToValue(kv rawKV) (gguf.Value, error) { switch kv.BType { case btBool: return gguf.Value{BType: gguf.BTypeBool, Int: kv.Int}, nil case btUint8: return gguf.Value{BType: gguf.BTypeUint8, Int: kv.Int}, nil case btInt8: return gguf.Value{BType: gguf.BTypeInt8, Int: kv.Int}, nil case btUint16: return gguf.Value{BType: gguf.BTypeUint16, Int: kv.Int}, nil case btInt16: return gguf.Value{BType: gguf.BTypeInt16, Int: kv.Int}, nil case btUint32: return gguf.Value{BType: gguf.BTypeUint32, Int: kv.Int}, nil case btInt32: return gguf.Value{BType: gguf.BTypeInt32, Int: kv.Int}, nil case btUint64: return gguf.Value{BType: gguf.BTypeUint64, Int: kv.Int}, nil case btInt64: return gguf.Value{BType: gguf.BTypeInt64, Int: kv.Int}, nil case btFloat32: return gguf.Value{BType: gguf.BTypeFloat32, Float: kv.Float}, nil case btFloat64: return gguf.Value{BType: gguf.BTypeFloat64, Float: kv.Float}, nil case btString: return gguf.Value{BType: gguf.BTypeString, Str: kv.Str}, nil case btArray: return gguf.Value{ BType: gguf.BTypeArray, ElemType: btypeToGguf(kv.ElemType), Int: int64(kv.Count), Raw: kv.Raw, }, nil default: return gguf.Value{}, fmt.Errorf("unsupported btype %d for key %q", kv.BType, kv.Key) } } func btypeToGguf(t ggufBType) gguf.BType { return gguf.BType(t) } func sha256Hex(b []byte) string { s := sha256.Sum256(b) return fmt.Sprintf("%x", s) } func shortHash(b []byte) string { s := sha256Hex(b) if len(s) > 12 { return s[:12] } return s }