Coracle/nostrlib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

eventstore: fuzz testing found us a bug!

Browse Source
This commit is contained in:
fiatjaf
2025-08-05 22:56:55 -03:00
parent 47ca205e9e
commit 030cad168d
13 changed files with 1203 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
eventstore/boltdb/helpers.go
View File

@@ -113,14 +113,12 @@ type iterators []*iterator
// i.e. [1, 700, 25, 312, 44, 28] with k=3 becomes something like [700, 312, 44, 1, 25, 28] // i.e. [1, 700, 25, 312, 44, 28] with k=3 becomes something like [700, 312, 44, 1, 25, 28]
// in this case it's hardcoded to use the 'last' field of the iterator // in this case it's hardcoded to use the 'last' field of the iterator
// copied from https://github.com/chrislee87/go-quickselect // copied from https://github.com/chrislee87/go-quickselect
// this is modified to also return the highest 'last' (because it's not guaranteed it will be the first item) func (its iterators) quickselect(k int) {
func (its iterators) quickselect(k int) uint32 {
if len(its) == 0 || k >= len(its) { if len(its) == 0 || k >= len(its) {
return 0 return
} }
left, right := 0, len(its)-1 left, right := 0, len(its)-1
for { for {
// insertion sort for small ranges // insertion sort for small ranges
if right-left <= 20 { if right-left <= 20 {
@@ -129,7 +127,7 @@ func (its iterators) quickselect(k int) uint32 {
its[j], its[j-1] = its[j-1], its[j] its[j], its[j-1] = its[j-1], its[j]
} }
} }
return its[0].last return
} }
// median-of-three to choose pivot // median-of-three to choose pivot
@@ -165,14 +163,7 @@ func (its iterators) quickselect(k int) uint32 {
pivotIndex = rr pivotIndex = rr
if k == pivotIndex { if k == pivotIndex {
// now that stuff is selected we get the highest "last" return
highest := its[0].last
for i := 1; i < k; i++ {
if its[i].last > highest {
highest = its[i].last
}
}
return highest
} }
if k < pivotIndex { if k < pivotIndex {
@@ -183,6 +174,17 @@ func (its iterators) quickselect(k int) uint32 {
} }
} }
// return the highest 'last' value among the first k items in its
func (its iterators) threshold(k int) uint32 {
highest := its[0].last
for i := 1; i < k; i++ {
if its[i].last > highest {
highest = its[i].last
}
}
return highest
}
type key struct { type key struct {
bucket []byte bucket []byte
fullkey []byte fullkey []byte

4
eventstore/boltdb/query.go
View File

@@ -110,7 +110,9 @@ func (b *BoltBackend) query(txn *bbolt.Tx, filter nostr.Filter, limit int, yield
// after pulling from all iterators once we now find out what iterators are // after pulling from all iterators once we now find out what iterators are
// the ones we should keep pulling from next (i.e. which one's last emitted timestamp is the highest) // the ones we should keep pulling from next (i.e. which one's last emitted timestamp is the highest)
threshold := iterators.quickselect(min(numberOfIteratorsToPullOnEachRound, len(iterators))) k := min(numberOfIteratorsToPullOnEachRound, len(iterators))
iterators.quickselect(k)
threshold := iterators.threshold(k)
// so we can emit all the events higher than the threshold // so we can emit all the events higher than the threshold
for i := range iterators { for i := range iterators {

8
eventstore/boltdb/testdata/fuzz/FuzzQuery/bca5f7631b59619d vendored Normal file
View File

@@ -0,0 +1,8 @@
go test fuzz v1
uint(200)
uint(15)
uint(11)
uint(49)
uint(2)
uint(91)
uint(1)

67
eventstore/boltdb/utils_test.go
View File

@@ -1,67 +0,0 @@
package boltdb
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestQuickselect(t *testing.T) {
{
its := iterators{
{last: 781},
{last: 900},
{last: 1},
{last: 81},
{last: 325},
{last: 781},
{last: 562},
{last: 81},
{last: 444},
}
its.quickselect(3)
require.ElementsMatch(t,
[]uint32{its[0].last, its[1].last, its[2].last},
[]uint32{900, 781, 781},
)
}
{
its := iterators{
{last: 781},
{last: 781},
{last: 900},
{last: 1},
{last: 87},
{last: 315},
{last: 789},
{last: 500},
{last: 812},
{last: 306},
{last: 612},
{last: 444},
{last: 59},
{last: 441},
{last: 901},
{last: 901},
{last: 2},
{last: 81},
{last: 325},
{last: 781},
{last: 562},
{last: 81},
{last: 326},
{last: 662},
{last: 444},
{last: 81},
{last: 444},
}
its.quickselect(6)
require.ElementsMatch(t,
[]uint32{its[0].last, its[1].last, its[2].last, its[3].last, its[4].last, its[5].last},
[]uint32{901, 900, 901, 781, 812, 789},
)
}
}

9
eventstore/internal/helpers.go
View File

@@ -83,9 +83,12 @@ func BatchSizePerNumberOfQueries(totalFilterLimit int, numberOfQueries int) int
return totalFilterLimit return totalFilterLimit
} }
return int( return max(
math.Ceil( 4,
math.Pow(float64(totalFilterLimit), 0.80) / math.Pow(float64(numberOfQueries), 0.71), int(
math.Ceil(
math.Pow(float64(totalFilterLimit), 0.80)/math.Pow(float64(numberOfQueries), 0.71),
),
), ),
) )
} }

28
eventstore/lmdb/helpers.go
View File

@@ -98,14 +98,12 @@ type iterators []*iterator
// i.e. [1, 700, 25, 312, 44, 28] with k=3 becomes something like [700, 312, 44, 1, 25, 28] // i.e. [1, 700, 25, 312, 44, 28] with k=3 becomes something like [700, 312, 44, 1, 25, 28]
// in this case it's hardcoded to use the 'last' field of the iterator // in this case it's hardcoded to use the 'last' field of the iterator
// copied from https://github.com/chrislee87/go-quickselect // copied from https://github.com/chrislee87/go-quickselect
// this is modified to also return the highest 'last' (because it's not guaranteed it will be the first item) func (its iterators) quickselect(k int) {
func (its iterators) quickselect(k int) uint32 {
if len(its) == 0 || k >= len(its) { if len(its) == 0 || k >= len(its) {
return 0 return
} }
left, right := 0, len(its)-1 left, right := 0, len(its)-1
for { for {
// insertion sort for small ranges // insertion sort for small ranges
if right-left <= 20 { if right-left <= 20 {
@@ -114,7 +112,7 @@ func (its iterators) quickselect(k int) uint32 {
its[j], its[j-1] = its[j-1], its[j] its[j], its[j-1] = its[j-1], its[j]
} }
} }
return its[0].last return
} }
// median-of-three to choose pivot // median-of-three to choose pivot
@@ -150,14 +148,7 @@ func (its iterators) quickselect(k int) uint32 {
pivotIndex = rr pivotIndex = rr
if k == pivotIndex { if k == pivotIndex {
// now that stuff is selected we get the highest "last" return
highest := its[0].last
for i := 1; i < k; i++ {
if its[i].last > highest {
highest = its[i].last
}
}
return highest
} }
if k < pivotIndex { if k < pivotIndex {
@@ -168,6 +159,17 @@ func (its iterators) quickselect(k int) uint32 {
} }
} }
// return the highest 'last' value among the first k items in its
func (its iterators) threshold(k int) uint32 {
highest := its[0].last
for i := 1; i < k; i++ {
if its[i].last > highest {
highest = its[i].last
}
}
return highest
}
type key struct { type key struct {
dbi lmdb.DBI dbi lmdb.DBI
key []byte key []byte

4
eventstore/lmdb/query.go
View File

@@ -114,7 +114,9 @@ func (b *LMDBBackend) query(txn *lmdb.Txn, filter nostr.Filter, limit int, yield
// after pulling from all iterators once we now find out what iterators are // after pulling from all iterators once we now find out what iterators are
// the ones we should keep pulling from next (i.e. which one's last emitted timestamp is the highest) // the ones we should keep pulling from next (i.e. which one's last emitted timestamp is the highest)
threshold := iterators.quickselect(min(numberOfIteratorsToPullOnEachRound, len(iterators))) k := min(numberOfIteratorsToPullOnEachRound, len(iterators))
iterators.quickselect(k)
threshold := iterators.threshold(k)
// so we can emit all the events higher than the threshold // so we can emit all the events higher than the threshold
for i := range iterators { for i := range iterators {

8
eventstore/lmdb/testdata/fuzz/FuzzQuery/bca5f7631b59619d vendored Normal file
View File

@@ -0,0 +1,8 @@
go test fuzz v1
uint(200)
uint(15)
uint(11)
uint(49)
uint(2)
uint(91)
uint(1)

1126
eventstore/lmdb/utils_test.go
View File

File diff suppressed because it is too large Load Diff

28
eventstore/mmm/helpers.go
View File

@@ -96,14 +96,12 @@ type iterators []*iterator
// i.e. [1, 700, 25, 312, 44, 28] with k=3 becomes something like [700, 312, 44, 1, 25, 28] // i.e. [1, 700, 25, 312, 44, 28] with k=3 becomes something like [700, 312, 44, 1, 25, 28]
// in this case it's hardcoded to use the 'last' field of the iterator // in this case it's hardcoded to use the 'last' field of the iterator
// copied from https://github.com/chrislee87/go-quickselect // copied from https://github.com/chrislee87/go-quickselect
// this is modified to also return the highest 'last' (because it's not guaranteed it will be the first item) func (its iterators) quickselect(k int) {
func (its iterators) quickselect(k int) uint32 {
if len(its) == 0 || k >= len(its) { if len(its) == 0 || k >= len(its) {
return 0 return
} }
left, right := 0, len(its)-1 left, right := 0, len(its)-1
for { for {
// insertion sort for small ranges // insertion sort for small ranges
if right-left <= 20 { if right-left <= 20 {
@@ -112,7 +110,7 @@ func (its iterators) quickselect(k int) uint32 {
its[j], its[j-1] = its[j-1], its[j] its[j], its[j-1] = its[j-1], its[j]
} }
} }
return its[0].last return
} }
// median-of-three to choose pivot // median-of-three to choose pivot
@@ -148,14 +146,7 @@ func (its iterators) quickselect(k int) uint32 {
pivotIndex = rr pivotIndex = rr
if k == pivotIndex { if k == pivotIndex {
// now that stuff is selected we get the highest "last" return
highest := its[0].last
for i := 1; i < k; i++ {
if its[i].last > highest {
highest = its[i].last
}
}
return highest
} }
if k < pivotIndex { if k < pivotIndex {
@@ -166,6 +157,17 @@ func (its iterators) quickselect(k int) uint32 {
} }
} }
// return the highest 'last' value among the first k items in its
func (its iterators) threshold(k int) uint32 {
highest := its[0].last
for i := 1; i < k; i++ {
if its[i].last > highest {
highest = its[i].last
}
}
return highest
}
type key struct { type key struct {
dbi lmdb.DBI dbi lmdb.DBI
key []byte key []byte

4
eventstore/mmm/query.go
View File

@@ -139,7 +139,9 @@ func (il *IndexingLayer) query(txn *lmdb.Txn, filter nostr.Filter, limit int, yi
// after pulling from all iterators once we now find out what iterators are // after pulling from all iterators once we now find out what iterators are
// the ones we should keep pulling from next (i.e. which one's last emitted timestamp is the highest) // the ones we should keep pulling from next (i.e. which one's last emitted timestamp is the highest)
threshold := iterators.quickselect(min(numberOfIteratorsToPullOnEachRound, len(iterators))) k := min(numberOfIteratorsToPullOnEachRound, len(iterators))
iterators.quickselect(k)
threshold := iterators.threshold(k)
// so we can emit all the events higher than the threshold // so we can emit all the events higher than the threshold
for i := range iterators { for i := range iterators {

3
go.mod
View File

@@ -3,7 +3,6 @@ module fiatjaf.com/nostr
go 1.24.1 go 1.24.1
require ( require (
fiatjaf.com/lib v0.2.0
github.com/FastFilter/xorfilter v0.2.1 github.com/FastFilter/xorfilter v0.2.1
github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3
github.com/PowerDNS/lmdb-go v1.9.3 github.com/PowerDNS/lmdb-go v1.9.3
@@ -33,6 +32,7 @@ require (
github.com/tyler-smith/go-bip39 v1.1.0 github.com/tyler-smith/go-bip39 v1.1.0
github.com/urfave/cli/v3 v3.0.0-beta1 github.com/urfave/cli/v3 v3.0.0-beta1
github.com/valyala/fasthttp v1.59.0 github.com/valyala/fasthttp v1.59.0
go.etcd.io/bbolt v1.4.2
golang.org/x/crypto v0.36.0 golang.org/x/crypto v0.36.0
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 golang.org/x/exp v0.0.0-20250305212735-054e65f0b394
golang.org/x/net v0.37.0 golang.org/x/net v0.37.0
@@ -86,7 +86,6 @@ require (
github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/x448/float16 v0.8.4 // indirect github.com/x448/float16 v0.8.4 // indirect
go.etcd.io/bbolt v1.4.2 // indirect
go.opencensus.io v0.24.0 // indirect go.opencensus.io v0.24.0 // indirect
golang.org/x/sys v0.34.0 // indirect golang.org/x/sys v0.34.0 // indirect
google.golang.org/protobuf v1.36.2 // indirect google.golang.org/protobuf v1.36.2 // indirect

4
go.sum
View File

@@ -1,6 +1,4 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
fiatjaf.com/lib v0.2.0 h1:TgIJESbbND6GjOgGHxF5jsO6EMjuAxIzZHPo5DXYexs=
fiatjaf.com/lib v0.2.0/go.mod h1:Ycqq3+mJ9jAWu7XjbQI1cVr+OFgnHn79dQR5oTII47g=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e h1:ahyvB3q25YnZWly5Gq1ekg6jcmWaGj/vG/MhF4aisoc= github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e h1:ahyvB3q25YnZWly5Gq1ekg6jcmWaGj/vG/MhF4aisoc=
github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e/go.mod h1:kGUqhHd//musdITWjFvNTHn90WG9bMLBEPQZ17Cmlpw= github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e/go.mod h1:kGUqhHd//musdITWjFvNTHn90WG9bMLBEPQZ17Cmlpw=
@@ -355,8 +353,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=