nip77: convert to dealing with bytes instead of hex strings.

it was cool but this should be faster and less confusing.

nip77/negentropy/encoding.go

@@ -1,12 +1,13 @@
 package negentropy
 
 import (
+	"bytes"
 	"fmt"
 
 	"fiatjaf.com/nostr"
 )
 
-func (n *Negentropy) readTimestamp(reader *StringHexReader) (nostr.Timestamp, error) {
+func (n *Negentropy) readTimestamp(reader *bytes.Reader) (nostr.Timestamp, error) {
 	delta, err := readVarInt(reader)
 	if err != nil {
 		return 0, err
@@ -31,7 +32,7 @@ func (n *Negentropy) readTimestamp(reader *StringHexReader) (nostr.Timestamp, er
 	return timestamp, nil
 }
 
-func (n *Negentropy) readBound(reader *StringHexReader) (Bound, error) {
+func (n *Negentropy) readBound(reader *bytes.Reader) (Bound, error) {
 	timestamp, err := n.readTimestamp(reader)
 	if err != nil {
 		return Bound{}, fmt.Errorf("failed to decode bound timestamp: %w", err)
@@ -43,14 +44,14 @@ func (n *Negentropy) readBound(reader *StringHexReader) (Bound, error) {
 	}
 
 	pfb := make([]byte, length)
-	if err := reader.ReadHexBytes(pfb); err != nil {
+	if _, err := reader.Read(pfb); err != nil {
 		return Bound{}, fmt.Errorf("failed to read bound id: %w", err)
 	}
 
 	return Bound{timestamp, pfb}, nil
 }
 
-func (n *Negentropy) writeTimestamp(w *StringHexWriter, timestamp nostr.Timestamp) {
+func (n *Negentropy) writeTimestamp(w *bytes.Buffer, timestamp nostr.Timestamp) {
 	if timestamp == maxTimestamp {
 		// zeroes are infinite
 		n.lastTimestampOut = maxTimestamp // cache this (see below)
@@ -69,10 +70,10 @@ func (n *Negentropy) writeTimestamp(w *StringHexWriter, timestamp nostr.Timestam
 	return
 }
 
-func (n *Negentropy) writeBound(w *StringHexWriter, bound Bound) {
+func (n *Negentropy) writeBound(w *bytes.Buffer, bound Bound) {
 	n.writeTimestamp(w, bound.Timestamp)
 	writeVarInt(w, len(bound.IDPrefix))
-	w.WriteBytes(bound.IDPrefix)
+	w.Write(bound.IDPrefix)
 }
 
 func getMinimalBound(prev, curr Item) Bound {
@@ -92,11 +93,11 @@ func getMinimalBound(prev, curr Item) Bound {
 	return Bound{curr.Timestamp, curr.ID[:(sharedPrefixBytes + 1)]}
 }
 
-func readVarInt(reader *StringHexReader) (int, error) {
+func readVarInt(reader *bytes.Reader) (int, error) {
 	var res int = 0
 
 	for {
-		b, err := reader.ReadHexByte()
+		b, err := reader.ReadByte()
 		if err != nil {
 			return 0, err
 		}
@@ -110,13 +111,13 @@ func readVarInt(reader *StringHexReader) (int, error) {
 	return res, nil
 }
 
-func writeVarInt(w *StringHexWriter, n int) {
+func writeVarInt(w *bytes.Buffer, n int) {
 	if n == 0 {
 		w.WriteByte(0)
 		return
 	}
 
-	w.WriteBytes(EncodeVarInt(n))
+	w.Write(EncodeVarInt(n))
 }
 
 func EncodeVarInt(n int) []byte {

nip77/negentropy/hex.go

@@ -1,93 +0,0 @@
-package negentropy
-
-import (
-	"encoding/hex"
-	"io"
-)
-
-func NewStringHexReader(source string) *StringHexReader {
-	return &StringHexReader{source, 0, make([]byte, 1)}
-}
-
-type StringHexReader struct {
-	source string
-	idx    int
-
-	tmp []byte
-}
-
-func (r *StringHexReader) Len() int {
-	return len(r.source) - r.idx
-}
-
-func (r *StringHexReader) ReadHexBytes(buf []byte) error {
-	n := len(buf) * 2
-	r.idx += n
-	if len(r.source) < r.idx {
-		return io.EOF
-	}
-	_, err := hex.Decode(buf, []byte(r.source[r.idx-n:r.idx]))
-	return err
-}
-
-func (r *StringHexReader) ReadHexByte() (byte, error) {
-	err := r.ReadHexBytes(r.tmp)
-	return r.tmp[0], err
-}
-
-func (r *StringHexReader) ReadString(size int) (string, error) {
-	r.idx += size
-	if len(r.source) < r.idx {
-		return "", io.EOF
-	}
-	return r.source[r.idx-size : r.idx], nil
-}
-
-func NewStringHexWriter(buf []byte) *StringHexWriter {
-	return &StringHexWriter{buf, make([]byte, 2)}
-}
-
-type StringHexWriter struct {
-	hexbuf []byte
-
-	tmp []byte
-}
-
-func (r *StringHexWriter) Len() int {
-	return len(r.hexbuf)
-}
-
-func (r *StringHexWriter) Hex() string {
-	return string(r.hexbuf)
-}
-
-func (r *StringHexWriter) Reset() {
-	r.hexbuf = r.hexbuf[:0]
-}
-
-func (r *StringHexWriter) WriteHex(hexString string) {
-	r.hexbuf = append(r.hexbuf, hexString...)
-	return
-}
-
-func (r *StringHexWriter) WriteByte(b byte) error {
-	hex.Encode(r.tmp, []byte{b})
-	r.hexbuf = append(r.hexbuf, r.tmp...)
-	return nil
-}
-
-func (r *StringHexWriter) WriteBytes(in []byte) {
-	r.hexbuf = hex.AppendEncode(r.hexbuf, in)
-
-	// curr := len(r.hexbuf)
-	// next := curr + len(in)*2
-	// for cap(r.hexbuf) < next {
-	// 	r.hexbuf = append(r.hexbuf, in...)
-	// }
-	// r.hexbuf = r.hexbuf[0:next]
-	// dst := r.hexbuf[curr:next]
-
-	// hex.Encode(dst, in)
-
-	return
-}

nip77/negentropy/negentropy.go

@@ -1,10 +1,11 @@
 package negentropy
 
 import (
+	"bytes"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"math"
-	"strings"
 	"unsafe"
 
 	"fiatjaf.com/nostr"
@@ -60,55 +61,60 @@ func (n *Negentropy) Start() string {
 	n.initialized = true
 	n.isClient = true
 
-	output := NewStringHexWriter(make([]byte, 0, 1+n.storage.Size()*64))
+	output := bytes.NewBuffer(make([]byte, 0, 1+n.storage.Size()*64))
 	output.WriteByte(protocolVersion)
 	n.SplitRange(0, n.storage.Size(), InfiniteBound, output)
 
-	return output.Hex()
+	return hex.EncodeToString(output.Bytes())
 }
 
-func (n *Negentropy) Reconcile(msg string) (output string, err error) {
+func (n *Negentropy) Reconcile(msg string) (string, error) {
 	n.initialized = true
-	reader := NewStringHexReader(msg)
+	msgb, err := hex.DecodeString(msg)
-
-	output, err = n.reconcileAux(reader)
 	if err != nil {
 		return "", err
 	}
 
-	if len(output) == 2 && n.isClient {
+	reader := bytes.NewReader(msgb)
+
+	output, err := n.reconcileAux(reader)
+	if err != nil {
+		return "", err
+	}
+
+	if len(output) == 1 && n.isClient {
 		close(n.Haves)
 		close(n.HaveNots)
 		return "", nil
 	}
 
-	return output, nil
+	return hex.EncodeToString(output), nil
 }
 
-func (n *Negentropy) reconcileAux(reader *StringHexReader) (string, error) {
+func (n *Negentropy) reconcileAux(reader *bytes.Reader) ([]byte, error) {
 	n.lastTimestampIn, n.lastTimestampOut = 0, 0 // reset for each message
 
-	fullOutput := NewStringHexWriter(make([]byte, 0, 5000))
+	fullOutput := bytes.NewBuffer(make([]byte, 0, 5000))
 	fullOutput.WriteByte(protocolVersion)
 
-	pv, err := reader.ReadHexByte()
+	pv, err := reader.ReadByte()
 	if err != nil {
-		return "", fmt.Errorf("failed to read pv: %w", err)
+		return nil, fmt.Errorf("failed to read pv: %w", err)
 	}
 	if pv != protocolVersion {
 		if n.isClient {
-			return "", fmt.Errorf("unsupported negentropy protocol version %v", pv)
+			return nil, fmt.Errorf("unsupported negentropy protocol version %v", pv)
 		}
 
 		// if we're a server we just return our protocol version
-		return fullOutput.Hex(), nil
+		return fullOutput.Bytes(), nil
 	}
 
 	var prevBound Bound
 	prevIndex := 0
 	skipping := false // this means we are currently coalescing ranges into skip
 
-	partialOutput := NewStringHexWriter(make([]byte, 0, 100))
+	partialOutput := bytes.NewBuffer(make([]byte, 0, 100))
 	for reader.Len() > 0 {
 		partialOutput.Reset()
 
@@ -123,11 +129,11 @@ func (n *Negentropy) reconcileAux(reader *StringHexReader) (string, error) {
 
 		currBound, err := n.readBound(reader)
 		if err != nil {
-			return "", fmt.Errorf("failed to decode bound: %w", err)
+			return nil, fmt.Errorf("failed to decode bound: %w", err)
 		}
 		modeVal, err := readVarInt(reader)
 		if err != nil {
-			return "", fmt.Errorf("failed to decode mode: %w", err)
+			return nil, fmt.Errorf("failed to decode mode: %w", err)
 		}
 		mode := Mode(modeVal)
 
@@ -139,9 +145,10 @@ func (n *Negentropy) reconcileAux(reader *StringHexReader) (string, error) {
 			skipping = true
 
 		case FingerprintMode:
-			theirFingerprint, err := reader.ReadString(FingerprintSize * 2)
+			theirFingerprint := [FingerprintSize]byte{}
+			_, err := reader.Read(theirFingerprint[:])
 			if err != nil {
-				return "", fmt.Errorf("failed to read fingerprint: %w", err)
+				return nil, fmt.Errorf("failed to read fingerprint: %w", err)
 			}
 			ourFingerprint := n.storage.Fingerprint(lower, upper)
 
@@ -155,15 +162,15 @@ func (n *Negentropy) reconcileAux(reader *StringHexReader) (string, error) {
 		case IdListMode:
 			numIds, err := readVarInt(reader)
 			if err != nil {
-				return "", fmt.Errorf("failed to decode number of ids: %w", err)
+				return nil, fmt.Errorf("failed to decode number of ids: %w", err)
 			}
 
 			// what they have
 			theirItems := make(map[nostr.ID]struct{}, numIds)
 			for i := 0; i < numIds; i++ {
 				var id [32]byte
-				if err := reader.ReadHexBytes(id[:]); err != nil {
+				if _, err := reader.Read(id[:]); err != nil {
-					return "", fmt.Errorf("failed to read id (#%d/%d) in list: %w", i, numIds, err)
+					return nil, fmt.Errorf("failed to read id (#%d/%d) in list: %w", i, numIds, err)
 				} else {
 					theirItems[id] = struct{}{}
 				}
@@ -197,33 +204,32 @@ func (n *Negentropy) reconcileAux(reader *StringHexReader) (string, error) {
 				// server got list of ids, reply with their own ids for the same range
 				finishSkip()
 
-				responseIds := strings.Builder{}
+				responseIds := make([]byte, 0, 32*100)
-				responseIds.Grow(64 * 100)
 				responses := 0
 
 				endBound := currBound
 
 				for index, item := range n.storage.Range(lower, upper) {
-					if n.frameSizeLimit-200 < fullOutput.Len()/2+responseIds.Len()/2 {
+					if n.frameSizeLimit-200 < fullOutput.Len()/2+len(responseIds)/2 {
 						endBound = Bound{item.Timestamp, item.ID[:]}
 						upper = index
 						break
 					}
-					responseIds.WriteString(hex.EncodeToString(item.ID[:]))
+					responseIds = append(responseIds, item.ID[:]...)
 					responses++
 				}
 
 				n.writeBound(partialOutput, endBound)
 				partialOutput.WriteByte(byte(IdListMode))
 				writeVarInt(partialOutput, responses)
-				partialOutput.WriteHex(responseIds.String())
+				partialOutput.Write(responseIds)
 
-				fullOutput.WriteHex(partialOutput.Hex())
+				io.Copy(fullOutput, partialOutput)
 				partialOutput.Reset()
 			}
 
 		default:
-			return "", fmt.Errorf("unexpected mode %d", mode)
+			return nil, fmt.Errorf("unexpected mode %d", mode)
 		}
 
 		if n.frameSizeLimit-200 < fullOutput.Len()/2+partialOutput.Len()/2 {
@@ -231,22 +237,22 @@ func (n *Negentropy) reconcileAux(reader *StringHexReader) (string, error) {
 			remainingFingerprint := n.storage.Fingerprint(upper, n.storage.Size())
 			n.writeBound(fullOutput, InfiniteBound)
 			fullOutput.WriteByte(byte(FingerprintMode))
-			fullOutput.WriteHex(remainingFingerprint)
+			fullOutput.Write(remainingFingerprint[:])
 
 			break // stop processing further
 		} else {
 			// append the constructed output for this iteration
-			fullOutput.WriteHex(partialOutput.Hex())
+			io.Copy(fullOutput, partialOutput)
 		}
 
 		prevIndex = upper
 		prevBound = currBound
 	}
 
-	return fullOutput.Hex(), nil
+	return fullOutput.Bytes(), nil
 }
 
-func (n *Negentropy) SplitRange(lower, upper int, upperBound Bound, output *StringHexWriter) {
+func (n *Negentropy) SplitRange(lower, upper int, upperBound Bound, output *bytes.Buffer) {
 	numElems := upper - lower
 
 	if numElems < buckets*2 {
@@ -256,7 +262,7 @@ func (n *Negentropy) SplitRange(lower, upper int, upperBound Bound, output *Stri
 		writeVarInt(output, numElems)
 
 		for _, item := range n.storage.Range(lower, upper) {
-			output.WriteBytes(item.ID[:])
+			output.Write(item.ID[:])
 		}
 	} else {
 		itemsPerBucket := numElems / buckets
@@ -291,7 +297,7 @@ func (n *Negentropy) SplitRange(lower, upper int, upperBound Bound, output *Stri
 
 			n.writeBound(output, nextBound)
 			output.WriteByte(byte(FingerprintMode))
-			output.WriteHex(ourFingerprint)
+			output.Write(ourFingerprint[:])
 		}
 	}
 }

nip77/negentropy/storage.go

@@ -1,13 +0,0 @@
-package negentropy
-
-import (
-	"iter"
-)
-
-type Storage interface {
-	Size() int
-	Range(begin, end int) iter.Seq2[int, Item]
-	FindLowerBound(begin, end int, value Bound) int
-	GetBound(idx int) Bound
-	Fingerprint(begin, end int) string
-}

nip77/negentropy/storage/accumulator.go

@@ -3,7 +3,6 @@ package storage
 import (
 	"crypto/sha256"
 	"encoding/binary"
-	"encoding/hex"
 
 	"fiatjaf.com/nostr/nip77/negentropy"
 )
@@ -41,9 +40,9 @@ func (acc *Accumulator) AddBytes(other []byte) {
 	}
 }
 
-func (acc *Accumulator) GetFingerprint(n int) string {
+func (acc *Accumulator) GetFingerprint(n int) [negentropy.FingerprintSize]byte {
 	input := acc.Buf[:32]
 	input = append(input, negentropy.EncodeVarInt(n)...)
 	hash := sha256.Sum256(input)
-	return hex.EncodeToString(hash[:negentropy.FingerprintSize])
+	return [negentropy.FingerprintSize]byte(hash[:negentropy.FingerprintSize])
 }

nip77/negentropy/storage/empty/empty.go

@@ -23,6 +23,6 @@ func (Empty) GetBound(idx int) negentropy.Bound {
 	return negentropy.InfiniteBound
 }
 
-func (Empty) Fingerprint(begin, end int) string {
+func (Empty) Fingerprint(begin, end int) [negentropy.FingerprintSize]byte {
 	return acc.GetFingerprint(end - begin)
 }

nip77/negentropy/storage/vector/vector.go

@@ -59,7 +59,7 @@ func (v *Vector) FindLowerBound(begin, end int, bound negentropy.Bound) int {
 	return begin + idx
 }
 
-func (v *Vector) Fingerprint(begin, end int) string {
+func (v *Vector) Fingerprint(begin, end int) [negentropy.FingerprintSize]byte {
 	v.acc.Reset()
 
 	for _, item := range v.Range(begin, end) {

nip77/negentropy/types.go

@@ -2,6 +2,7 @@ package negentropy
 
 import (
 	"fmt"
+	"iter"
 
 	"fiatjaf.com/nostr"
 )
@@ -47,3 +48,11 @@ func (b Bound) String() string {
 	}
 	return fmt.Sprintf("Bound<%d:%x>", b.Timestamp, b.IDPrefix)
 }
+
+type Storage interface {
+	Size() int
+	Range(begin, end int) iter.Seq2[int, Item]
+	FindLowerBound(begin, end int, value Bound) int
+	GetBound(idx int) Bound
+	Fingerprint(begin, end int) [FingerprintSize]byte
+}