negentropy.
- a way to handle custom messages from Relay (NEG-whatever etc) - negentropy implementation (adapted from that other one) - nip77 nostr negentropy extension - QueryEvents method for RelayStore that returns a channel (makes negentropy syncing work more seamlessly)
This commit is contained in:
fiatjaf
126
nip77/negentropy/encoding.go
Normal file
126
nip77/negentropy/encoding.go
Normal file
@@ -0,0 +1,126 @@
|
||||
package negentropy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
|
||||
"github.com/nbd-wtf/go-nostr"
|
||||
)
|
||||
|
||||
func (n *Negentropy) DecodeTimestampIn(reader *bytes.Reader) (nostr.Timestamp, error) {
|
||||
t, err := decodeVarInt(reader)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
timestamp := nostr.Timestamp(t)
|
||||
if timestamp == 0 {
|
||||
timestamp = maxTimestamp
|
||||
} else {
|
||||
timestamp--
|
||||
}
|
||||
|
||||
timestamp += n.lastTimestampIn
|
||||
if timestamp < n.lastTimestampIn { // Check for overflow
|
||||
timestamp = maxTimestamp
|
||||
}
|
||||
n.lastTimestampIn = timestamp
|
||||
return timestamp, nil
|
||||
}
|
||||
|
||||
func (n *Negentropy) DecodeBound(reader *bytes.Reader) (Bound, error) {
|
||||
timestamp, err := n.DecodeTimestampIn(reader)
|
||||
if err != nil {
|
||||
return Bound{}, err
|
||||
}
|
||||
|
||||
length, err := decodeVarInt(reader)
|
||||
if err != nil {
|
||||
return Bound{}, err
|
||||
}
|
||||
|
||||
id := make([]byte, length)
|
||||
if _, err = reader.Read(id); err != nil {
|
||||
return Bound{}, err
|
||||
}
|
||||
|
||||
return Bound{Item{timestamp, hex.EncodeToString(id)}}, nil
|
||||
}
|
||||
|
||||
func (n *Negentropy) encodeTimestampOut(timestamp nostr.Timestamp) []byte {
|
||||
if timestamp == maxTimestamp {
|
||||
n.lastTimestampOut = maxTimestamp
|
||||
return encodeVarInt(0)
|
||||
}
|
||||
temp := timestamp
|
||||
timestamp -= n.lastTimestampOut
|
||||
n.lastTimestampOut = temp
|
||||
return encodeVarInt(int(timestamp + 1))
|
||||
}
|
||||
|
||||
func (n *Negentropy) encodeBound(bound Bound) []byte {
|
||||
var output []byte
|
||||
|
||||
t := n.encodeTimestampOut(bound.Timestamp)
|
||||
idlen := encodeVarInt(len(bound.ID) / 2)
|
||||
output = append(output, t...)
|
||||
output = append(output, idlen...)
|
||||
id, _ := hex.DecodeString(bound.Item.ID)
|
||||
|
||||
output = append(output, id...)
|
||||
return output
|
||||
}
|
||||
|
||||
func getMinimalBound(prev, curr Item) Bound {
|
||||
if curr.Timestamp != prev.Timestamp {
|
||||
return Bound{Item{curr.Timestamp, ""}}
|
||||
}
|
||||
|
||||
sharedPrefixBytes := 0
|
||||
|
||||
for i := 0; i < 32; i++ {
|
||||
if curr.ID[i:i+2] != prev.ID[i:i+2] {
|
||||
break
|
||||
}
|
||||
sharedPrefixBytes++
|
||||
}
|
||||
|
||||
// sharedPrefixBytes + 1 to include the first differing byte, or the entire ID if identical.
|
||||
return Bound{Item{curr.Timestamp, curr.ID[:(sharedPrefixBytes+1)*2]}}
|
||||
}
|
||||
|
||||
func decodeVarInt(reader *bytes.Reader) (int, error) {
|
||||
var res int = 0
|
||||
|
||||
for {
|
||||
b, err := reader.ReadByte()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
res = (res << 7) | (int(b) & 127)
|
||||
if (b & 128) == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func encodeVarInt(n int) []byte {
|
||||
if n == 0 {
|
||||
return []byte{0}
|
||||
}
|
||||
|
||||
var o []byte
|
||||
for n != 0 {
|
||||
o = append([]byte{byte(n & 0x7F)}, o...)
|
||||
n >>= 7
|
||||
}
|
||||
|
||||
for i := 0; i < len(o)-1; i++ {
|
||||
o[i] |= 0x80
|
||||
}
|
||||
|
||||
return o
|
||||
}
|
||||
315
nip77/negentropy/negentropy.go
Normal file
315
nip77/negentropy/negentropy.go
Normal file
@@ -0,0 +1,315 @@
|
||||
package negentropy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"unsafe"
|
||||
|
||||
"github.com/nbd-wtf/go-nostr"
|
||||
)
|
||||
|
||||
const (
|
||||
protocolVersion byte = 0x61 // version 1
|
||||
maxTimestamp = nostr.Timestamp(math.MaxInt64)
|
||||
)
|
||||
|
||||
var infiniteBound = Bound{Item: Item{Timestamp: maxTimestamp}}
|
||||
|
||||
type Negentropy struct {
|
||||
storage Storage
|
||||
sealed bool
|
||||
frameSizeLimit int
|
||||
isInitiator bool
|
||||
lastTimestampIn nostr.Timestamp
|
||||
lastTimestampOut nostr.Timestamp
|
||||
|
||||
Haves chan string
|
||||
HaveNots chan string
|
||||
}
|
||||
|
||||
func NewNegentropy(storage Storage, frameSizeLimit int) *Negentropy {
|
||||
return &Negentropy{
|
||||
storage: storage,
|
||||
frameSizeLimit: frameSizeLimit,
|
||||
}
|
||||
}
|
||||
|
||||
func (n *Negentropy) Insert(evt *nostr.Event) {
|
||||
err := n.storage.Insert(evt.CreatedAt, evt.ID)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (n *Negentropy) seal() {
|
||||
if !n.sealed {
|
||||
n.storage.Seal()
|
||||
}
|
||||
n.sealed = true
|
||||
}
|
||||
|
||||
func (n *Negentropy) Initiate() []byte {
|
||||
n.seal()
|
||||
n.isInitiator = true
|
||||
|
||||
n.Haves = make(chan string, n.storage.Size()/2)
|
||||
n.HaveNots = make(chan string, n.storage.Size()/2)
|
||||
|
||||
output := bytes.NewBuffer(make([]byte, 0, 1+n.storage.Size()*32))
|
||||
output.WriteByte(protocolVersion)
|
||||
n.SplitRange(0, n.storage.Size(), infiniteBound, output)
|
||||
|
||||
return output.Bytes()
|
||||
}
|
||||
|
||||
func (n *Negentropy) Reconcile(msg []byte) (output []byte, err error) {
|
||||
n.seal()
|
||||
reader := bytes.NewReader(msg)
|
||||
|
||||
output, err = n.reconcileAux(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(output) == 1 && n.isInitiator {
|
||||
close(n.Haves)
|
||||
close(n.HaveNots)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func (n *Negentropy) reconcileAux(reader *bytes.Reader) ([]byte, error) {
|
||||
n.lastTimestampIn, n.lastTimestampOut = 0, 0 // reset for each message
|
||||
|
||||
fullOutput := bytes.NewBuffer(make([]byte, 0, 5000))
|
||||
fullOutput.WriteByte(protocolVersion)
|
||||
|
||||
pv, err := reader.ReadByte()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if pv < 0x60 || pv > 0x6f {
|
||||
return nil, fmt.Errorf("invalid protocol version byte")
|
||||
}
|
||||
if pv != protocolVersion {
|
||||
if n.isInitiator {
|
||||
return nil, fmt.Errorf("unsupported negentropy protocol version requested")
|
||||
}
|
||||
return fullOutput.Bytes(), nil
|
||||
}
|
||||
|
||||
var prevBound Bound
|
||||
prevIndex := 0
|
||||
skip := false
|
||||
|
||||
partialOutput := bytes.NewBuffer(make([]byte, 0, 100))
|
||||
for reader.Len() > 0 {
|
||||
partialOutput.Reset()
|
||||
|
||||
doSkip := func() {
|
||||
if skip {
|
||||
skip = false
|
||||
encodedBound := n.encodeBound(prevBound)
|
||||
partialOutput.Write(encodedBound)
|
||||
partialOutput.WriteByte(SkipMode)
|
||||
}
|
||||
}
|
||||
|
||||
currBound, err := n.DecodeBound(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
modeVal, err := decodeVarInt(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mode := Mode(modeVal)
|
||||
|
||||
lower := prevIndex
|
||||
upper := n.storage.FindLowerBound(prevIndex, n.storage.Size(), currBound)
|
||||
|
||||
switch mode {
|
||||
case SkipMode:
|
||||
skip = true
|
||||
|
||||
case FingerprintMode:
|
||||
var theirFingerprint [FingerprintSize]byte
|
||||
_, err := reader.Read(theirFingerprint[:])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ourFingerprint, err := n.storage.Fingerprint(lower, upper)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if theirFingerprint == ourFingerprint {
|
||||
skip = true
|
||||
} else {
|
||||
doSkip()
|
||||
n.SplitRange(lower, upper, currBound, partialOutput)
|
||||
}
|
||||
|
||||
case IdListMode:
|
||||
numIds, err := decodeVarInt(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
theirElems := make(map[string]struct{})
|
||||
var idb [32]byte
|
||||
|
||||
for i := 0; i < numIds; i++ {
|
||||
_, err := reader.Read(idb[:])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
id := hex.EncodeToString(idb[:])
|
||||
theirElems[id] = struct{}{}
|
||||
}
|
||||
|
||||
n.storage.Iterate(lower, upper, func(item Item, _ int) bool {
|
||||
id := item.ID
|
||||
if _, exists := theirElems[id]; !exists {
|
||||
if n.isInitiator {
|
||||
n.Haves <- id
|
||||
}
|
||||
} else {
|
||||
delete(theirElems, id)
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if n.isInitiator {
|
||||
skip = true
|
||||
for id := range theirElems {
|
||||
n.HaveNots <- id
|
||||
}
|
||||
} else {
|
||||
doSkip()
|
||||
|
||||
responseIds := make([]byte, 0, 32*n.storage.Size())
|
||||
endBound := currBound
|
||||
|
||||
n.storage.Iterate(lower, upper, func(item Item, index int) bool {
|
||||
if n.frameSizeLimit-200 < fullOutput.Len()+len(responseIds) {
|
||||
endBound = Bound{item}
|
||||
upper = index
|
||||
return false
|
||||
}
|
||||
|
||||
id, _ := hex.DecodeString(item.ID)
|
||||
responseIds = append(responseIds, id...)
|
||||
return true
|
||||
})
|
||||
|
||||
encodedBound := n.encodeBound(endBound)
|
||||
|
||||
partialOutput.Write(encodedBound)
|
||||
partialOutput.WriteByte(IdListMode)
|
||||
partialOutput.Write(encodeVarInt(len(responseIds) / 32))
|
||||
partialOutput.Write(responseIds)
|
||||
|
||||
partialOutput.WriteTo(fullOutput)
|
||||
partialOutput.Reset()
|
||||
}
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected mode %d", mode)
|
||||
}
|
||||
|
||||
if n.frameSizeLimit-200 < fullOutput.Len()+partialOutput.Len() {
|
||||
// frame size limit exceeded, handle by encoding a boundary and fingerprint for the remaining range
|
||||
remainingFingerprint, err := n.storage.Fingerprint(upper, n.storage.Size())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
fullOutput.Write(n.encodeBound(infiniteBound))
|
||||
fullOutput.WriteByte(FingerprintMode)
|
||||
fullOutput.Write(remainingFingerprint[:])
|
||||
|
||||
break // stop processing further
|
||||
} else {
|
||||
// append the constructed output for this iteration
|
||||
partialOutput.WriteTo(fullOutput)
|
||||
}
|
||||
|
||||
prevIndex = upper
|
||||
prevBound = currBound
|
||||
}
|
||||
|
||||
return fullOutput.Bytes(), nil
|
||||
}
|
||||
|
||||
func (n *Negentropy) SplitRange(lower, upper int, upperBound Bound, output *bytes.Buffer) {
|
||||
numElems := upper - lower
|
||||
const buckets = 16
|
||||
|
||||
if numElems < buckets*2 {
|
||||
// we just send the full ids here
|
||||
boundEncoded := n.encodeBound(upperBound)
|
||||
output.Write(boundEncoded)
|
||||
output.WriteByte(IdListMode)
|
||||
output.Write(encodeVarInt(numElems))
|
||||
|
||||
n.storage.Iterate(lower, upper, func(item Item, _ int) bool {
|
||||
id, _ := hex.DecodeString(item.ID)
|
||||
output.Write(id)
|
||||
return true
|
||||
})
|
||||
} else {
|
||||
itemsPerBucket := numElems / buckets
|
||||
bucketsWithExtra := numElems % buckets
|
||||
curr := lower
|
||||
|
||||
for i := 0; i < buckets; i++ {
|
||||
bucketSize := itemsPerBucket
|
||||
if i < bucketsWithExtra {
|
||||
bucketSize++
|
||||
}
|
||||
ourFingerprint, err := n.storage.Fingerprint(curr, curr+bucketSize)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
panic(err)
|
||||
}
|
||||
|
||||
curr += bucketSize
|
||||
|
||||
var nextBound Bound
|
||||
if curr == upper {
|
||||
nextBound = upperBound
|
||||
} else {
|
||||
var prevItem, currItem Item
|
||||
|
||||
n.storage.Iterate(curr-1, curr+1, func(item Item, index int) bool {
|
||||
if index == curr-1 {
|
||||
prevItem = item
|
||||
} else {
|
||||
currItem = item
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
minBound := getMinimalBound(prevItem, currItem)
|
||||
nextBound = minBound
|
||||
}
|
||||
|
||||
boundEncoded := n.encodeBound(nextBound)
|
||||
output.Write(boundEncoded)
|
||||
output.WriteByte(FingerprintMode)
|
||||
output.Write(ourFingerprint[:])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (n *Negentropy) Name() string {
|
||||
p := unsafe.Pointer(n)
|
||||
return fmt.Sprintf("%d", uintptr(p)&127)
|
||||
}
|
||||
111
nip77/negentropy/types.go
Normal file
111
nip77/negentropy/types.go
Normal file
@@ -0,0 +1,111 @@
|
||||
package negentropy
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/nbd-wtf/go-nostr"
|
||||
)
|
||||
|
||||
const FingerprintSize = 16
|
||||
|
||||
type Mode int
|
||||
|
||||
const (
|
||||
SkipMode = 0
|
||||
FingerprintMode = 1
|
||||
IdListMode = 2
|
||||
)
|
||||
|
||||
type Storage interface {
|
||||
Insert(nostr.Timestamp, string) error
|
||||
Seal()
|
||||
Size() int
|
||||
Iterate(begin, end int, cb func(item Item, i int) bool) error
|
||||
FindLowerBound(begin, end int, value Bound) int
|
||||
GetBound(idx int) Bound
|
||||
Fingerprint(begin, end int) ([FingerprintSize]byte, error)
|
||||
}
|
||||
|
||||
type Item struct {
|
||||
Timestamp nostr.Timestamp
|
||||
ID string
|
||||
}
|
||||
|
||||
func itemCompare(a, b Item) int {
|
||||
if a.Timestamp != b.Timestamp {
|
||||
return int(a.Timestamp - b.Timestamp)
|
||||
}
|
||||
return strings.Compare(a.ID, b.ID)
|
||||
}
|
||||
|
||||
func (i Item) String() string { return fmt.Sprintf("Item<%d:%s>", i.Timestamp, i.ID) }
|
||||
|
||||
type Bound struct{ Item }
|
||||
|
||||
func (b Bound) String() string {
|
||||
if b.Timestamp == infiniteBound.Timestamp {
|
||||
return "Bound<infinite>"
|
||||
}
|
||||
return fmt.Sprintf("Bound<%d:%s>", b.Timestamp, b.ID)
|
||||
}
|
||||
|
||||
type Accumulator struct {
|
||||
Buf []byte
|
||||
}
|
||||
|
||||
func (acc *Accumulator) SetToZero() {
|
||||
acc.Buf = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
|
||||
}
|
||||
|
||||
func (acc *Accumulator) Add(id string) {
|
||||
b, _ := hex.DecodeString(id)
|
||||
acc.AddBytes(b)
|
||||
}
|
||||
|
||||
func (acc *Accumulator) AddAccumulator(other Accumulator) {
|
||||
acc.AddBytes(other.Buf)
|
||||
}
|
||||
|
||||
func (acc *Accumulator) AddBytes(other []byte) {
|
||||
var currCarry, nextCarry uint32
|
||||
|
||||
if len(acc.Buf) < 32 {
|
||||
newBuf := make([]byte, 32)
|
||||
copy(newBuf, acc.Buf)
|
||||
acc.Buf = newBuf
|
||||
}
|
||||
|
||||
for i := 0; i < 8; i++ {
|
||||
offset := i * 4
|
||||
orig := binary.LittleEndian.Uint32(acc.Buf[offset:])
|
||||
otherV := binary.LittleEndian.Uint32(other[offset:])
|
||||
|
||||
next := orig + currCarry + otherV
|
||||
if next < orig || next < otherV {
|
||||
nextCarry = 1
|
||||
}
|
||||
|
||||
binary.LittleEndian.PutUint32(acc.Buf[offset:], next&0xFFFFFFFF)
|
||||
currCarry = nextCarry
|
||||
nextCarry = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (acc *Accumulator) SV() []byte {
|
||||
return acc.Buf[:]
|
||||
}
|
||||
|
||||
func (acc *Accumulator) GetFingerprint(n int) [FingerprintSize]byte {
|
||||
input := acc.SV()
|
||||
input = append(input, encodeVarInt(n)...)
|
||||
|
||||
hash := sha256.Sum256(input)
|
||||
|
||||
var fingerprint [FingerprintSize]byte
|
||||
copy(fingerprint[:], hash[:FingerprintSize])
|
||||
return fingerprint
|
||||
}
|
||||
74
nip77/negentropy/vector.go
Normal file
74
nip77/negentropy/vector.go
Normal file
@@ -0,0 +1,74 @@
|
||||
package negentropy
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
"github.com/nbd-wtf/go-nostr"
|
||||
)
|
||||
|
||||
type Vector struct {
|
||||
items []Item
|
||||
sealed bool
|
||||
}
|
||||
|
||||
func NewVector() *Vector {
|
||||
return &Vector{
|
||||
items: make([]Item, 0, 30),
|
||||
}
|
||||
}
|
||||
|
||||
func (v *Vector) Insert(createdAt nostr.Timestamp, id string) error {
|
||||
if len(id)/2 != 32 {
|
||||
return fmt.Errorf("bad id size for added item: expected %d, got %d", 32, len(id)/2)
|
||||
}
|
||||
|
||||
item := Item{createdAt, id}
|
||||
v.items = append(v.items, item)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *Vector) Size() int { return len(v.items) }
|
||||
|
||||
func (v *Vector) Seal() {
|
||||
if v.sealed {
|
||||
panic("trying to seal an already sealed vector")
|
||||
}
|
||||
v.sealed = true
|
||||
slices.SortFunc(v.items, itemCompare)
|
||||
}
|
||||
|
||||
func (v *Vector) GetBound(idx int) Bound {
|
||||
if idx < len(v.items) {
|
||||
return Bound{v.items[idx]}
|
||||
}
|
||||
return infiniteBound
|
||||
}
|
||||
|
||||
func (v *Vector) Iterate(begin, end int, cb func(Item, int) bool) error {
|
||||
for i := begin; i < end; i++ {
|
||||
if !cb(v.items[i], i) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *Vector) FindLowerBound(begin, end int, bound Bound) int {
|
||||
idx, _ := slices.BinarySearchFunc(v.items[begin:end], bound.Item, itemCompare)
|
||||
return begin + idx
|
||||
}
|
||||
|
||||
func (v *Vector) Fingerprint(begin, end int) ([FingerprintSize]byte, error) {
|
||||
var out Accumulator
|
||||
out.SetToZero()
|
||||
|
||||
if err := v.Iterate(begin, end, func(item Item, _ int) bool {
|
||||
out.Add(item.ID)
|
||||
return true
|
||||
}); err != nil {
|
||||
return [FingerprintSize]byte{}, err
|
||||
}
|
||||
|
||||
return out.GetFingerprint(end - begin), nil
|
||||
}
|
||||
180
nip77/negentropy/whatever_test.go
Normal file
180
nip77/negentropy/whatever_test.go
Normal file
@@ -0,0 +1,180 @@
|
||||
package negentropy
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/nbd-wtf/go-nostr"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSuperSmall(t *testing.T) {
|
||||
runTestWith(t,
|
||||
4,
|
||||
[][]int{{0, 3}}, [][]int{{2, 4}},
|
||||
[][]int{{3, 4}}, [][]int{{0, 2}},
|
||||
)
|
||||
}
|
||||
|
||||
func TestNoNeedToSync(t *testing.T) {
|
||||
runTestWith(t,
|
||||
50,
|
||||
[][]int{{0, 50}}, [][]int{{0, 50}},
|
||||
[][]int{}, [][]int{},
|
||||
)
|
||||
}
|
||||
|
||||
func TestSmallNumbers(t *testing.T) {
|
||||
runTestWith(t,
|
||||
20,
|
||||
[][]int{{2, 15}}, [][]int{{0, 7}, {10, 20}},
|
||||
[][]int{{0, 2}, {15, 20}}, [][]int{{7, 10}},
|
||||
)
|
||||
}
|
||||
|
||||
func TestBigNumbers(t *testing.T) {
|
||||
runTestWith(t,
|
||||
200,
|
||||
[][]int{{20, 150}}, [][]int{{0, 70}, {100, 200}},
|
||||
[][]int{{0, 20}, {150, 200}}, [][]int{{70, 100}},
|
||||
)
|
||||
}
|
||||
|
||||
func TestMuchBiggerNumbersAndConfusion(t *testing.T) {
|
||||
runTestWith(t,
|
||||
20000,
|
||||
[][]int{{20, 150}, {1700, 3400}, {7000, 8100}, {13800, 13816}, {13817, 14950}, {19800, 20000}}, // n1
|
||||
[][]int{{0, 2000}, {3000, 3600}, {10000, 12200}, {13799, 13801}, {14800, 19900}}, // n2
|
||||
[][]int{{0, 20}, {150, 1700}, {3400, 3600}, {10000, 12200}, {13799, 13800}, {14950, 19800}}, // n1 need
|
||||
[][]int{{2000, 3000}, {7000, 8100}, {13801, 13816}, {13817, 14800}, {19900, 20000}}, // n1 have
|
||||
)
|
||||
}
|
||||
|
||||
func runTestWith(t *testing.T,
|
||||
totalEvents int,
|
||||
n1Ranges [][]int, n2Ranges [][]int,
|
||||
expectedN1NeedRanges [][]int, expectedN1HaveRanges [][]int,
|
||||
) {
|
||||
var err error
|
||||
var q []byte
|
||||
var n1 *Negentropy
|
||||
var n2 *Negentropy
|
||||
|
||||
events := make([]*nostr.Event, totalEvents)
|
||||
for i := range events {
|
||||
evt := nostr.Event{}
|
||||
evt.Content = fmt.Sprintf("event %d", i)
|
||||
evt.Kind = 1
|
||||
evt.CreatedAt = nostr.Timestamp(i)
|
||||
evt.ID = fmt.Sprintf("%064d", i)
|
||||
events[i] = &evt
|
||||
}
|
||||
|
||||
{
|
||||
n1 = NewNegentropy(NewVector(), 1<<16)
|
||||
for _, r := range n1Ranges {
|
||||
for i := r[0]; i < r[1]; i++ {
|
||||
n1.Insert(events[i])
|
||||
}
|
||||
}
|
||||
|
||||
q = n1.Initiate()
|
||||
}
|
||||
|
||||
{
|
||||
n2 = NewNegentropy(NewVector(), 1<<16)
|
||||
for _, r := range n2Ranges {
|
||||
for i := r[0]; i < r[1]; i++ {
|
||||
n2.Insert(events[i])
|
||||
}
|
||||
}
|
||||
|
||||
q, err = n2.Reconcile(q)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
invert := map[*Negentropy]*Negentropy{
|
||||
n1: n2,
|
||||
n2: n1,
|
||||
}
|
||||
i := 1
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(3)
|
||||
|
||||
go func() {
|
||||
wg.Done()
|
||||
for n := n1; q != nil; n = invert[n] {
|
||||
i++
|
||||
|
||||
q, err = n.Reconcile(q)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
return
|
||||
}
|
||||
|
||||
if q == nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
expectedHave := make([]string, 0, 100)
|
||||
for _, r := range expectedN1HaveRanges {
|
||||
for i := r[0]; i < r[1]; i++ {
|
||||
expectedHave = append(expectedHave, events[i].ID)
|
||||
}
|
||||
}
|
||||
haves := make([]string, 0, 100)
|
||||
for item := range n1.Haves {
|
||||
if slices.Contains(haves, item) {
|
||||
continue
|
||||
}
|
||||
haves = append(haves, item)
|
||||
}
|
||||
require.ElementsMatch(t, expectedHave, haves, "wrong have")
|
||||
}()
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
expectedNeed := make([]string, 0, 100)
|
||||
for _, r := range expectedN1NeedRanges {
|
||||
for i := r[0]; i < r[1]; i++ {
|
||||
expectedNeed = append(expectedNeed, events[i].ID)
|
||||
}
|
||||
}
|
||||
havenots := make([]string, 0, 100)
|
||||
for item := range n1.HaveNots {
|
||||
if slices.Contains(havenots, item) {
|
||||
continue
|
||||
}
|
||||
havenots = append(havenots, item)
|
||||
}
|
||||
require.ElementsMatch(t, expectedNeed, havenots, "wrong need")
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func hexedBytes(o []byte) string {
|
||||
s := strings.Builder{}
|
||||
s.Grow(2 + 1 + len(o)*5)
|
||||
s.WriteString("[ ")
|
||||
for _, b := range o {
|
||||
x := hex.EncodeToString([]byte{b})
|
||||
s.WriteString("0x")
|
||||
s.WriteString(x)
|
||||
s.WriteString(" ")
|
||||
}
|
||||
s.WriteString("]")
|
||||
return s.String()
|
||||
}
|
||||
Reference in New Issue
Block a user