mmm: run it on the eventstore/test suite and fix id query logic.
This commit is contained in:
@@ -29,18 +29,18 @@ func FuzzTest(f *testing.F) {
|
|||||||
rnd := rand.New(rand.NewPCG(uint64(seed), 0))
|
rnd := rand.New(rand.NewPCG(uint64(seed), 0))
|
||||||
|
|
||||||
// initialize MMM
|
// initialize MMM
|
||||||
mmm := &MultiMmapManager{
|
mmmm := &MultiMmapManager{
|
||||||
Dir: tmpDir,
|
Dir: tmpDir,
|
||||||
Logger: &logger,
|
Logger: &logger,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = mmm.Init()
|
err = mmmm.Init()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer mmm.Close()
|
defer mmmm.Close()
|
||||||
|
|
||||||
for i := range nlayers {
|
for i := range nlayers {
|
||||||
name := string([]byte{97 + byte(i)})
|
name := string([]byte{97 + byte(i)})
|
||||||
err = mmm.EnsureLayer(name, &IndexingLayer{})
|
err = mmmm.EnsureLayer(name, &IndexingLayer{})
|
||||||
require.NoError(t, err, "layer %s/%d", name, i)
|
require.NoError(t, err, "layer %s/%d", name, i)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,7 +73,7 @@ func FuzzTest(f *testing.F) {
|
|||||||
}
|
}
|
||||||
evt.Sign(sk)
|
evt.Sign(sk)
|
||||||
|
|
||||||
for _, layer := range mmm.layers {
|
for _, layer := range mmmm.layers {
|
||||||
if evt.Tags.FindWithValue("t", layer.name) != nil {
|
if evt.Tags.FindWithValue("t", layer.name) != nil {
|
||||||
err := layer.SaveEvent(evt)
|
err := layer.SaveEvent(evt)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -86,7 +86,7 @@ func FuzzTest(f *testing.F) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// verify each layer has the correct events
|
// verify each layer has the correct events
|
||||||
for _, layer := range mmm.layers {
|
for _, layer := range mmmm.layers {
|
||||||
count := 0
|
count := 0
|
||||||
for evt := range layer.QueryEvents(nostr.Filter{}, 500) {
|
for evt := range layer.QueryEvents(nostr.Filter{}, 500) {
|
||||||
require.True(t, evt.Tags.ContainsAny("t", []string{layer.name}))
|
require.True(t, evt.Tags.ContainsAny("t", []string{layer.name}))
|
||||||
@@ -100,9 +100,9 @@ func FuzzTest(f *testing.F) {
|
|||||||
|
|
||||||
for range ndeletes {
|
for range ndeletes {
|
||||||
id := storedIds[rnd.Int()%len(storedIds)]
|
id := storedIds[rnd.Int()%len(storedIds)]
|
||||||
layer := mmm.layers[rnd.Int()%len(mmm.layers)]
|
layer := mmmm.layers[rnd.Int()%len(mmmm.layers)]
|
||||||
|
|
||||||
evt, layers := mmm.GetByID(id)
|
evt, layers := mmmm.GetByID(id)
|
||||||
if slices.Contains(deleted[id], layer) {
|
if slices.Contains(deleted[id], layer) {
|
||||||
// already deleted from this layer
|
// already deleted from this layer
|
||||||
require.NotContains(t, layers, layer)
|
require.NotContains(t, layers, layer)
|
||||||
@@ -119,7 +119,7 @@ func FuzzTest(f *testing.F) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for id, deletedlayers := range deleted {
|
for id, deletedlayers := range deleted {
|
||||||
evt, foundlayers := mmm.GetByID(id)
|
evt, foundlayers := mmmm.GetByID(id)
|
||||||
|
|
||||||
for _, layer := range deletedlayers {
|
for _, layer := range deletedlayers {
|
||||||
require.NotContains(t, foundlayers, layer)
|
require.NotContains(t, foundlayers, layer)
|
||||||
@@ -142,7 +142,7 @@ func FuzzTest(f *testing.F) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if evt != nil {
|
if evt != nil {
|
||||||
for _, layer := range mmm.layers {
|
for _, layer := range mmmm.layers {
|
||||||
// verify event still accessible from other layers
|
// verify event still accessible from other layers
|
||||||
if slices.Contains(foundlayers, layer) {
|
if slices.Contains(foundlayers, layer) {
|
||||||
next, stop := iter.Pull(layer.QueryEvents(nostr.Filter{Kinds: []nostr.Kind{evt.Kind}}, 500)) // hack
|
next, stop := iter.Pull(layer.QueryEvents(nostr.Filter{Kinds: []nostr.Kind{evt.Kind}}, 500)) // hack
|
||||||
@@ -161,7 +161,7 @@ func FuzzTest(f *testing.F) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// now delete a layer and events that only exist in that layer should vanish
|
// now delete a layer and events that only exist in that layer should vanish
|
||||||
layer := mmm.layers[rnd.Int()%len(mmm.layers)]
|
layer := mmmm.layers[rnd.Int()%len(mmmm.layers)]
|
||||||
eventsThatShouldVanish := make([]nostr.ID, 0, nevents/2)
|
eventsThatShouldVanish := make([]nostr.ID, 0, nevents/2)
|
||||||
for evt := range layer.QueryEvents(nostr.Filter{}, 500) {
|
for evt := range layer.QueryEvents(nostr.Filter{}, 500) {
|
||||||
if len(evt.Tags) == 1+len(deleted[evt.ID]) {
|
if len(evt.Tags) == 1+len(deleted[evt.ID]) {
|
||||||
@@ -169,11 +169,11 @@ func FuzzTest(f *testing.F) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err = mmm.DropLayer(layer.name)
|
err = mmmm.DropLayer(layer.name)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
for _, id := range eventsThatShouldVanish {
|
for _, id := range eventsThatShouldVanish {
|
||||||
v, ils := mmm.GetByID(id)
|
v, ils := mmmm.GetByID(id)
|
||||||
require.Nil(t, v)
|
require.Nil(t, v)
|
||||||
require.Empty(t, ils)
|
require.Empty(t, ils)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -34,7 +34,11 @@ func (b *MultiMmapManager) GetByID(id nostr.ID) (*nostr.Event, IndexingLayers) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// queryByIDs emits the events of the given id to the given channel if they exist anywhere in this mmm.
|
// queryByIDs emits the events of the given id to the given channel if they exist anywhere in this mmm.
|
||||||
func (b *MultiMmapManager) queryByIDs(ids []nostr.ID, yield func(nostr.Event) bool, withLayers bool) (layers []uint16) {
|
func (b *MultiMmapManager) queryByIDs(
|
||||||
|
ids []nostr.ID,
|
||||||
|
yield func(nostr.Event) bool,
|
||||||
|
withLayers bool,
|
||||||
|
) (layers []uint16) {
|
||||||
b.lmdbEnv.View(func(txn *lmdb.Txn) error {
|
b.lmdbEnv.View(func(txn *lmdb.Txn) error {
|
||||||
txn.RawRead = true
|
txn.RawRead = true
|
||||||
|
|
||||||
@@ -47,7 +51,7 @@ func (b *MultiMmapManager) queryByIDs(ids []nostr.ID, yield func(nostr.Event) bo
|
|||||||
panic(fmt.Errorf("failed to decode event from %v: %w", pos, err))
|
panic(fmt.Errorf("failed to decode event from %v: %w", pos, err))
|
||||||
}
|
}
|
||||||
|
|
||||||
stop := yield(evt)
|
keepGoing := yield(evt)
|
||||||
|
|
||||||
if withLayers {
|
if withLayers {
|
||||||
layers = make([]uint16, 0, (len(val)-12)/2)
|
layers = make([]uint16, 0, (len(val)-12)/2)
|
||||||
@@ -56,7 +60,7 @@ func (b *MultiMmapManager) queryByIDs(ids []nostr.ID, yield func(nostr.Event) bo
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if stop {
|
if !keepGoing {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -70,7 +74,7 @@ func (b *MultiMmapManager) queryByIDs(ids []nostr.ID, yield func(nostr.Event) bo
|
|||||||
|
|
||||||
func (il *IndexingLayer) QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] {
|
func (il *IndexingLayer) QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] {
|
||||||
return func(yield func(nostr.Event) bool) {
|
return func(yield func(nostr.Event) bool) {
|
||||||
if len(filter.IDs) > 0 {
|
if filter.IDs != nil {
|
||||||
il.mmmm.queryByIDs(filter.IDs, yield, false)
|
il.mmmm.queryByIDs(filter.IDs, yield, false)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,10 @@ import (
|
|||||||
"fiatjaf.com/nostr"
|
"fiatjaf.com/nostr"
|
||||||
"fiatjaf.com/nostr/eventstore"
|
"fiatjaf.com/nostr/eventstore"
|
||||||
"fiatjaf.com/nostr/eventstore/lmdb"
|
"fiatjaf.com/nostr/eventstore/lmdb"
|
||||||
|
"fiatjaf.com/nostr/eventstore/mmm"
|
||||||
"fiatjaf.com/nostr/eventstore/slicestore"
|
"fiatjaf.com/nostr/eventstore/slicestore"
|
||||||
|
"github.com/rs/zerolog"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -42,3 +45,26 @@ func TestLMDB(t *testing.T) {
|
|||||||
t.Run(test.name, func(t *testing.T) { test.run(t, &lmdb.LMDBBackend{Path: dbpath + "lmdb"}) })
|
t.Run(test.name, func(t *testing.T) { test.run(t, &lmdb.LMDBBackend{Path: dbpath + "lmdb"}) })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMMM(t *testing.T) {
|
||||||
|
for _, test := range tests {
|
||||||
|
os.RemoveAll(dbpath + "mmm")
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
logger := zerolog.Nop()
|
||||||
|
|
||||||
|
mmmm := &mmm.MultiMmapManager{
|
||||||
|
Dir: dbpath + "mmm",
|
||||||
|
Logger: &logger,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := mmmm.Init()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
il := mmm.IndexingLayer{}
|
||||||
|
err = mmmm.EnsureLayer("test", &il)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
test.run(t, &il)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user