mmm: run it on the eventstore/test suite and fix id query logic.

This commit is contained in:
fiatjaf
2025-08-05 07:36:20 -03:00
parent 17c3d19a98
commit 34225e5dfe
3 changed files with 47 additions and 17 deletions

View File

@@ -29,18 +29,18 @@ func FuzzTest(f *testing.F) {
rnd := rand.New(rand.NewPCG(uint64(seed), 0)) rnd := rand.New(rand.NewPCG(uint64(seed), 0))
// initialize MMM // initialize MMM
mmm := &MultiMmapManager{ mmmm := &MultiMmapManager{
Dir: tmpDir, Dir: tmpDir,
Logger: &logger, Logger: &logger,
} }
err = mmm.Init() err = mmmm.Init()
require.NoError(t, err) require.NoError(t, err)
defer mmm.Close() defer mmmm.Close()
for i := range nlayers { for i := range nlayers {
name := string([]byte{97 + byte(i)}) name := string([]byte{97 + byte(i)})
err = mmm.EnsureLayer(name, &IndexingLayer{}) err = mmmm.EnsureLayer(name, &IndexingLayer{})
require.NoError(t, err, "layer %s/%d", name, i) require.NoError(t, err, "layer %s/%d", name, i)
} }
@@ -73,7 +73,7 @@ func FuzzTest(f *testing.F) {
} }
evt.Sign(sk) evt.Sign(sk)
for _, layer := range mmm.layers { for _, layer := range mmmm.layers {
if evt.Tags.FindWithValue("t", layer.name) != nil { if evt.Tags.FindWithValue("t", layer.name) != nil {
err := layer.SaveEvent(evt) err := layer.SaveEvent(evt)
require.NoError(t, err) require.NoError(t, err)
@@ -86,7 +86,7 @@ func FuzzTest(f *testing.F) {
} }
// verify each layer has the correct events // verify each layer has the correct events
for _, layer := range mmm.layers { for _, layer := range mmmm.layers {
count := 0 count := 0
for evt := range layer.QueryEvents(nostr.Filter{}, 500) { for evt := range layer.QueryEvents(nostr.Filter{}, 500) {
require.True(t, evt.Tags.ContainsAny("t", []string{layer.name})) require.True(t, evt.Tags.ContainsAny("t", []string{layer.name}))
@@ -100,9 +100,9 @@ func FuzzTest(f *testing.F) {
for range ndeletes { for range ndeletes {
id := storedIds[rnd.Int()%len(storedIds)] id := storedIds[rnd.Int()%len(storedIds)]
layer := mmm.layers[rnd.Int()%len(mmm.layers)] layer := mmmm.layers[rnd.Int()%len(mmmm.layers)]
evt, layers := mmm.GetByID(id) evt, layers := mmmm.GetByID(id)
if slices.Contains(deleted[id], layer) { if slices.Contains(deleted[id], layer) {
// already deleted from this layer // already deleted from this layer
require.NotContains(t, layers, layer) require.NotContains(t, layers, layer)
@@ -119,7 +119,7 @@ func FuzzTest(f *testing.F) {
} }
for id, deletedlayers := range deleted { for id, deletedlayers := range deleted {
evt, foundlayers := mmm.GetByID(id) evt, foundlayers := mmmm.GetByID(id)
for _, layer := range deletedlayers { for _, layer := range deletedlayers {
require.NotContains(t, foundlayers, layer) require.NotContains(t, foundlayers, layer)
@@ -142,7 +142,7 @@ func FuzzTest(f *testing.F) {
} }
if evt != nil { if evt != nil {
for _, layer := range mmm.layers { for _, layer := range mmmm.layers {
// verify event still accessible from other layers // verify event still accessible from other layers
if slices.Contains(foundlayers, layer) { if slices.Contains(foundlayers, layer) {
next, stop := iter.Pull(layer.QueryEvents(nostr.Filter{Kinds: []nostr.Kind{evt.Kind}}, 500)) // hack next, stop := iter.Pull(layer.QueryEvents(nostr.Filter{Kinds: []nostr.Kind{evt.Kind}}, 500)) // hack
@@ -161,7 +161,7 @@ func FuzzTest(f *testing.F) {
} }
// now delete a layer and events that only exist in that layer should vanish // now delete a layer and events that only exist in that layer should vanish
layer := mmm.layers[rnd.Int()%len(mmm.layers)] layer := mmmm.layers[rnd.Int()%len(mmmm.layers)]
eventsThatShouldVanish := make([]nostr.ID, 0, nevents/2) eventsThatShouldVanish := make([]nostr.ID, 0, nevents/2)
for evt := range layer.QueryEvents(nostr.Filter{}, 500) { for evt := range layer.QueryEvents(nostr.Filter{}, 500) {
if len(evt.Tags) == 1+len(deleted[evt.ID]) { if len(evt.Tags) == 1+len(deleted[evt.ID]) {
@@ -169,11 +169,11 @@ func FuzzTest(f *testing.F) {
} }
} }
err = mmm.DropLayer(layer.name) err = mmmm.DropLayer(layer.name)
require.NoError(t, err) require.NoError(t, err)
for _, id := range eventsThatShouldVanish { for _, id := range eventsThatShouldVanish {
v, ils := mmm.GetByID(id) v, ils := mmmm.GetByID(id)
require.Nil(t, v) require.Nil(t, v)
require.Empty(t, ils) require.Empty(t, ils)
} }

View File

@@ -34,7 +34,11 @@ func (b *MultiMmapManager) GetByID(id nostr.ID) (*nostr.Event, IndexingLayers) {
} }
// queryByIDs emits the events of the given id to the given channel if they exist anywhere in this mmm. // queryByIDs emits the events of the given id to the given channel if they exist anywhere in this mmm.
func (b *MultiMmapManager) queryByIDs(ids []nostr.ID, yield func(nostr.Event) bool, withLayers bool) (layers []uint16) { func (b *MultiMmapManager) queryByIDs(
ids []nostr.ID,
yield func(nostr.Event) bool,
withLayers bool,
) (layers []uint16) {
b.lmdbEnv.View(func(txn *lmdb.Txn) error { b.lmdbEnv.View(func(txn *lmdb.Txn) error {
txn.RawRead = true txn.RawRead = true
@@ -47,7 +51,7 @@ func (b *MultiMmapManager) queryByIDs(ids []nostr.ID, yield func(nostr.Event) bo
panic(fmt.Errorf("failed to decode event from %v: %w", pos, err)) panic(fmt.Errorf("failed to decode event from %v: %w", pos, err))
} }
stop := yield(evt) keepGoing := yield(evt)
if withLayers { if withLayers {
layers = make([]uint16, 0, (len(val)-12)/2) layers = make([]uint16, 0, (len(val)-12)/2)
@@ -56,7 +60,7 @@ func (b *MultiMmapManager) queryByIDs(ids []nostr.ID, yield func(nostr.Event) bo
} }
} }
if stop { if !keepGoing {
return nil return nil
} }
} }
@@ -70,7 +74,7 @@ func (b *MultiMmapManager) queryByIDs(ids []nostr.ID, yield func(nostr.Event) bo
func (il *IndexingLayer) QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] { func (il *IndexingLayer) QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] {
return func(yield func(nostr.Event) bool) { return func(yield func(nostr.Event) bool) {
if len(filter.IDs) > 0 { if filter.IDs != nil {
il.mmmm.queryByIDs(filter.IDs, yield, false) il.mmmm.queryByIDs(filter.IDs, yield, false)
return return
} }

View File

@@ -8,7 +8,10 @@ import (
"fiatjaf.com/nostr" "fiatjaf.com/nostr"
"fiatjaf.com/nostr/eventstore" "fiatjaf.com/nostr/eventstore"
"fiatjaf.com/nostr/eventstore/lmdb" "fiatjaf.com/nostr/eventstore/lmdb"
"fiatjaf.com/nostr/eventstore/mmm"
"fiatjaf.com/nostr/eventstore/slicestore" "fiatjaf.com/nostr/eventstore/slicestore"
"github.com/rs/zerolog"
"github.com/stretchr/testify/require"
) )
var ( var (
@@ -42,3 +45,26 @@ func TestLMDB(t *testing.T) {
t.Run(test.name, func(t *testing.T) { test.run(t, &lmdb.LMDBBackend{Path: dbpath + "lmdb"}) }) t.Run(test.name, func(t *testing.T) { test.run(t, &lmdb.LMDBBackend{Path: dbpath + "lmdb"}) })
} }
} }
func TestMMM(t *testing.T) {
for _, test := range tests {
os.RemoveAll(dbpath + "mmm")
t.Run(test.name, func(t *testing.T) {
logger := zerolog.Nop()
mmmm := &mmm.MultiMmapManager{
Dir: dbpath + "mmm",
Logger: &logger,
}
err := mmmm.Init()
require.NoError(t, err)
il := mmm.IndexingLayer{}
err = mmmm.EnsureLayer("test", &il)
require.NoError(t, err)
test.run(t, &il)
})
}
}