diff --git a/eventstore/boltdb/helpers.go b/eventstore/boltdb/helpers.go index b2cc9dc..b5baaf7 100644 --- a/eventstore/boltdb/helpers.go +++ b/eventstore/boltdb/helpers.go @@ -113,14 +113,12 @@ type iterators []*iterator // i.e. [1, 700, 25, 312, 44, 28] with k=3 becomes something like [700, 312, 44, 1, 25, 28] // in this case it's hardcoded to use the 'last' field of the iterator // copied from https://github.com/chrislee87/go-quickselect -// this is modified to also return the highest 'last' (because it's not guaranteed it will be the first item) -func (its iterators) quickselect(k int) uint32 { +func (its iterators) quickselect(k int) { if len(its) == 0 || k >= len(its) { - return 0 + return } left, right := 0, len(its)-1 - for { // insertion sort for small ranges if right-left <= 20 { @@ -129,7 +127,7 @@ func (its iterators) quickselect(k int) uint32 { its[j], its[j-1] = its[j-1], its[j] } } - return its[0].last + return } // median-of-three to choose pivot @@ -165,14 +163,7 @@ func (its iterators) quickselect(k int) uint32 { pivotIndex = rr if k == pivotIndex { - // now that stuff is selected we get the highest "last" - highest := its[0].last - for i := 1; i < k; i++ { - if its[i].last > highest { - highest = its[i].last - } - } - return highest + return } if k < pivotIndex { @@ -183,6 +174,17 @@ func (its iterators) quickselect(k int) uint32 { } } +// return the highest 'last' value among the first k items in its +func (its iterators) threshold(k int) uint32 { + highest := its[0].last + for i := 1; i < k; i++ { + if its[i].last > highest { + highest = its[i].last + } + } + return highest +} + type key struct { bucket []byte fullkey []byte diff --git a/eventstore/boltdb/query.go b/eventstore/boltdb/query.go index 8508f70..962d224 100644 --- a/eventstore/boltdb/query.go +++ b/eventstore/boltdb/query.go @@ -110,7 +110,9 @@ func (b *BoltBackend) query(txn *bbolt.Tx, filter nostr.Filter, limit int, yield // after pulling from all iterators once we now find out what iterators are // the ones we should keep pulling from next (i.e. which one's last emitted timestamp is the highest) - threshold := iterators.quickselect(min(numberOfIteratorsToPullOnEachRound, len(iterators))) + k := min(numberOfIteratorsToPullOnEachRound, len(iterators)) + iterators.quickselect(k) + threshold := iterators.threshold(k) // so we can emit all the events higher than the threshold for i := range iterators { diff --git a/eventstore/boltdb/testdata/fuzz/FuzzQuery/bca5f7631b59619d b/eventstore/boltdb/testdata/fuzz/FuzzQuery/bca5f7631b59619d new file mode 100644 index 0000000..6c68277 --- /dev/null +++ b/eventstore/boltdb/testdata/fuzz/FuzzQuery/bca5f7631b59619d @@ -0,0 +1,8 @@ +go test fuzz v1 +uint(200) +uint(15) +uint(11) +uint(49) +uint(2) +uint(91) +uint(1) diff --git a/eventstore/boltdb/utils_test.go b/eventstore/boltdb/utils_test.go deleted file mode 100644 index 8037adc..0000000 --- a/eventstore/boltdb/utils_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package boltdb - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestQuickselect(t *testing.T) { - { - its := iterators{ - {last: 781}, - {last: 900}, - {last: 1}, - {last: 81}, - {last: 325}, - {last: 781}, - {last: 562}, - {last: 81}, - {last: 444}, - } - - its.quickselect(3) - require.ElementsMatch(t, - []uint32{its[0].last, its[1].last, its[2].last}, - []uint32{900, 781, 781}, - ) - } - - { - its := iterators{ - {last: 781}, - {last: 781}, - {last: 900}, - {last: 1}, - {last: 87}, - {last: 315}, - {last: 789}, - {last: 500}, - {last: 812}, - {last: 306}, - {last: 612}, - {last: 444}, - {last: 59}, - {last: 441}, - {last: 901}, - {last: 901}, - {last: 2}, - {last: 81}, - {last: 325}, - {last: 781}, - {last: 562}, - {last: 81}, - {last: 326}, - {last: 662}, - {last: 444}, - {last: 81}, - {last: 444}, - } - - its.quickselect(6) - require.ElementsMatch(t, - []uint32{its[0].last, its[1].last, its[2].last, its[3].last, its[4].last, its[5].last}, - []uint32{901, 900, 901, 781, 812, 789}, - ) - } -} diff --git a/eventstore/internal/helpers.go b/eventstore/internal/helpers.go index 8e362f2..6e5b2aa 100644 --- a/eventstore/internal/helpers.go +++ b/eventstore/internal/helpers.go @@ -83,9 +83,12 @@ func BatchSizePerNumberOfQueries(totalFilterLimit int, numberOfQueries int) int return totalFilterLimit } - return int( - math.Ceil( - math.Pow(float64(totalFilterLimit), 0.80) / math.Pow(float64(numberOfQueries), 0.71), + return max( + 4, + int( + math.Ceil( + math.Pow(float64(totalFilterLimit), 0.80)/math.Pow(float64(numberOfQueries), 0.71), + ), ), ) } diff --git a/eventstore/lmdb/helpers.go b/eventstore/lmdb/helpers.go index d84b93c..9e51c69 100644 --- a/eventstore/lmdb/helpers.go +++ b/eventstore/lmdb/helpers.go @@ -98,14 +98,12 @@ type iterators []*iterator // i.e. [1, 700, 25, 312, 44, 28] with k=3 becomes something like [700, 312, 44, 1, 25, 28] // in this case it's hardcoded to use the 'last' field of the iterator // copied from https://github.com/chrislee87/go-quickselect -// this is modified to also return the highest 'last' (because it's not guaranteed it will be the first item) -func (its iterators) quickselect(k int) uint32 { +func (its iterators) quickselect(k int) { if len(its) == 0 || k >= len(its) { - return 0 + return } left, right := 0, len(its)-1 - for { // insertion sort for small ranges if right-left <= 20 { @@ -114,7 +112,7 @@ func (its iterators) quickselect(k int) uint32 { its[j], its[j-1] = its[j-1], its[j] } } - return its[0].last + return } // median-of-three to choose pivot @@ -150,14 +148,7 @@ func (its iterators) quickselect(k int) uint32 { pivotIndex = rr if k == pivotIndex { - // now that stuff is selected we get the highest "last" - highest := its[0].last - for i := 1; i < k; i++ { - if its[i].last > highest { - highest = its[i].last - } - } - return highest + return } if k < pivotIndex { @@ -168,6 +159,17 @@ func (its iterators) quickselect(k int) uint32 { } } +// return the highest 'last' value among the first k items in its +func (its iterators) threshold(k int) uint32 { + highest := its[0].last + for i := 1; i < k; i++ { + if its[i].last > highest { + highest = its[i].last + } + } + return highest +} + type key struct { dbi lmdb.DBI key []byte diff --git a/eventstore/lmdb/query.go b/eventstore/lmdb/query.go index 80894b4..dfed295 100644 --- a/eventstore/lmdb/query.go +++ b/eventstore/lmdb/query.go @@ -114,7 +114,9 @@ func (b *LMDBBackend) query(txn *lmdb.Txn, filter nostr.Filter, limit int, yield // after pulling from all iterators once we now find out what iterators are // the ones we should keep pulling from next (i.e. which one's last emitted timestamp is the highest) - threshold := iterators.quickselect(min(numberOfIteratorsToPullOnEachRound, len(iterators))) + k := min(numberOfIteratorsToPullOnEachRound, len(iterators)) + iterators.quickselect(k) + threshold := iterators.threshold(k) // so we can emit all the events higher than the threshold for i := range iterators { diff --git a/eventstore/lmdb/testdata/fuzz/FuzzQuery/bca5f7631b59619d b/eventstore/lmdb/testdata/fuzz/FuzzQuery/bca5f7631b59619d new file mode 100644 index 0000000..6c68277 --- /dev/null +++ b/eventstore/lmdb/testdata/fuzz/FuzzQuery/bca5f7631b59619d @@ -0,0 +1,8 @@ +go test fuzz v1 +uint(200) +uint(15) +uint(11) +uint(49) +uint(2) +uint(91) +uint(1) diff --git a/eventstore/lmdb/utils_test.go b/eventstore/lmdb/utils_test.go index d4ba105..13ca5b6 100644 --- a/eventstore/lmdb/utils_test.go +++ b/eventstore/lmdb/utils_test.go @@ -64,4 +64,1130 @@ func TestQuickselect(t *testing.T) { []uint32{901, 900, 901, 781, 812, 789}, ) } + + { + its := iterators{ + {last: 229}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 85}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 121}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 157}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 193}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 329}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 185}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 221}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 257}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 293}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 429}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 285}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 321}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 357}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 393}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 529}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 385}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 421}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 457}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 493}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 629}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 485}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 521}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 557}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 593}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 729}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 585}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 621}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 657}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 693}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + {last: 0}, + } + + its.quickselect(18) + require.ElementsMatch(t, + []uint32{ + its[0].last, its[1].last, its[2].last, its[3].last, its[4].last, its[5].last, + its[6].last, its[7].last, its[8].last, its[9].last, its[10].last, its[11].last, + its[12].last, its[13].last, its[14].last, its[15].last, its[16].last, its[17].last, + }, + []uint32{ + 729, + 693, + 657, + 629, + 621, + 593, + 585, + 557, + 529, + 521, + 493, + 485, + 457, + 429, + 421, + 393, + 385, + 357, + }, + ) + } } diff --git a/eventstore/mmm/helpers.go b/eventstore/mmm/helpers.go index 30fecd3..d57fa8b 100644 --- a/eventstore/mmm/helpers.go +++ b/eventstore/mmm/helpers.go @@ -96,14 +96,12 @@ type iterators []*iterator // i.e. [1, 700, 25, 312, 44, 28] with k=3 becomes something like [700, 312, 44, 1, 25, 28] // in this case it's hardcoded to use the 'last' field of the iterator // copied from https://github.com/chrislee87/go-quickselect -// this is modified to also return the highest 'last' (because it's not guaranteed it will be the first item) -func (its iterators) quickselect(k int) uint32 { +func (its iterators) quickselect(k int) { if len(its) == 0 || k >= len(its) { - return 0 + return } left, right := 0, len(its)-1 - for { // insertion sort for small ranges if right-left <= 20 { @@ -112,7 +110,7 @@ func (its iterators) quickselect(k int) uint32 { its[j], its[j-1] = its[j-1], its[j] } } - return its[0].last + return } // median-of-three to choose pivot @@ -148,14 +146,7 @@ func (its iterators) quickselect(k int) uint32 { pivotIndex = rr if k == pivotIndex { - // now that stuff is selected we get the highest "last" - highest := its[0].last - for i := 1; i < k; i++ { - if its[i].last > highest { - highest = its[i].last - } - } - return highest + return } if k < pivotIndex { @@ -166,6 +157,17 @@ func (its iterators) quickselect(k int) uint32 { } } +// return the highest 'last' value among the first k items in its +func (its iterators) threshold(k int) uint32 { + highest := its[0].last + for i := 1; i < k; i++ { + if its[i].last > highest { + highest = its[i].last + } + } + return highest +} + type key struct { dbi lmdb.DBI key []byte diff --git a/eventstore/mmm/query.go b/eventstore/mmm/query.go index 292eaa9..a2b0379 100644 --- a/eventstore/mmm/query.go +++ b/eventstore/mmm/query.go @@ -139,7 +139,9 @@ func (il *IndexingLayer) query(txn *lmdb.Txn, filter nostr.Filter, limit int, yi // after pulling from all iterators once we now find out what iterators are // the ones we should keep pulling from next (i.e. which one's last emitted timestamp is the highest) - threshold := iterators.quickselect(min(numberOfIteratorsToPullOnEachRound, len(iterators))) + k := min(numberOfIteratorsToPullOnEachRound, len(iterators)) + iterators.quickselect(k) + threshold := iterators.threshold(k) // so we can emit all the events higher than the threshold for i := range iterators { diff --git a/go.mod b/go.mod index be23983..0cc9c7f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module fiatjaf.com/nostr go 1.24.1 require ( - fiatjaf.com/lib v0.2.0 github.com/FastFilter/xorfilter v0.2.1 github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 github.com/PowerDNS/lmdb-go v1.9.3 @@ -33,6 +32,7 @@ require ( github.com/tyler-smith/go-bip39 v1.1.0 github.com/urfave/cli/v3 v3.0.0-beta1 github.com/valyala/fasthttp v1.59.0 + go.etcd.io/bbolt v1.4.2 golang.org/x/crypto v0.36.0 golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 golang.org/x/net v0.37.0 @@ -86,7 +86,6 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/x448/float16 v0.8.4 // indirect - go.etcd.io/bbolt v1.4.2 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/sys v0.34.0 // indirect google.golang.org/protobuf v1.36.2 // indirect diff --git a/go.sum b/go.sum index cfb0223..ee7fccf 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,4 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -fiatjaf.com/lib v0.2.0 h1:TgIJESbbND6GjOgGHxF5jsO6EMjuAxIzZHPo5DXYexs= -fiatjaf.com/lib v0.2.0/go.mod h1:Ycqq3+mJ9jAWu7XjbQI1cVr+OFgnHn79dQR5oTII47g= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e h1:ahyvB3q25YnZWly5Gq1ekg6jcmWaGj/vG/MhF4aisoc= github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e/go.mod h1:kGUqhHd//musdITWjFvNTHn90WG9bMLBEPQZ17Cmlpw= @@ -355,8 +353,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=