eventstore: a COUNT test and fix many bugs.

eventstore/boltdb/count.go

@@ -33,7 +33,7 @@ func (b *BoltBackend) CountEvents(filter nostr.Filter) (uint32, error) {
 
 			for {
 				// we already have a k and a v and an err from the cursor setup, so check and use these
-				if !bytes.HasPrefix(it.key, q.prefix) {
+				if !bytes.HasPrefix(it.key, q.prefix) || it.exhausted {
 					// either iteration has errored or we reached the end of this prefix
 					break // stop this cursor and move to the next one
 				}
@@ -54,31 +54,35 @@ func (b *BoltBackend) CountEvents(filter nostr.Filter) (uint32, error) {
 					}
 
 					// check it against pubkeys without decoding the entire thing
-					if !slices.Contains(extraAuthors, betterbinary.GetPubKey(bin)) {
+					if extraAuthors != nil && !slices.Contains(extraAuthors, betterbinary.GetPubKey(bin)) {
 						it.next()
 						continue
 					}
 
 					// check it against kinds without decoding the entire thing
-					if !slices.Contains(extraKinds, betterbinary.GetKind(bin)) {
+					if extraKinds != nil && !slices.Contains(extraKinds, betterbinary.GetKind(bin)) {
 						it.next()
 						continue
 					}
 
-					evt := &nostr.Event{}
+					if extraTagKey != "" {
-					if err := betterbinary.Unmarshal(bin, evt); err != nil {
+						evt := &nostr.Event{}
-						it.next()
+						if err := betterbinary.Unmarshal(bin, evt); err != nil {
-						continue
+							it.next()
-					}
+							continue
+						}
 
-					// if there is still a tag to be checked, do it now
+						// if there is still a tag to be checked, do it now
-					if !evt.Tags.ContainsAny(extraTagKey, extraTagValues) {
+						if !evt.Tags.ContainsAny(extraTagKey, extraTagValues) {
-						it.next()
+							it.next()
-						continue
+							continue
+						}
 					}
 
 					count++
 				}
+
+				it.next()
 			}
 		}
 

eventstore/lmdb/count.go

@@ -34,7 +34,8 @@ func (b *LMDBBackend) CountEvents(filter nostr.Filter) (uint32, error) {
 
 			for {
 				// we already have a k and a v and an err from the cursor setup, so check and use these
-				if it.err != nil ||
+				if it.exhausted ||
+					it.err != nil ||
 					len(it.key) != q.keySize ||
 					!bytes.HasPrefix(it.key, q.prefix) {
 					// either iteration has errored or we reached the end of this prefix
@@ -59,31 +60,35 @@ func (b *LMDBBackend) CountEvents(filter nostr.Filter) (uint32, error) {
 					}
 
 					// check it against pubkeys without decoding the entire thing
-					if !slices.Contains(extraAuthors, betterbinary.GetPubKey(bin)) {
+					if extraAuthors != nil && !slices.Contains(extraAuthors, betterbinary.GetPubKey(bin)) {
 						it.next()
 						continue
 					}
 
 					// check it against kinds without decoding the entire thing
-					if !slices.Contains(extraKinds, betterbinary.GetKind(bin)) {
+					if extraKinds != nil && !slices.Contains(extraKinds, betterbinary.GetKind(bin)) {
 						it.next()
 						continue
 					}
 
-					evt := &nostr.Event{}
+					if extraTagKey != "" {
-					if err := betterbinary.Unmarshal(bin, evt); err != nil {
+						evt := &nostr.Event{}
-						it.next()
+						if err := betterbinary.Unmarshal(bin, evt); err != nil {
-						continue
+							it.next()
-					}
+							continue
+						}
 
-					// if there is still a tag to be checked, do it now
+						// if there is still a tag to be checked, do it now
-					if !evt.Tags.ContainsAny(extraTagKey, extraTagValues) {
+						if !evt.Tags.ContainsAny(extraTagKey, extraTagValues) {
-						it.next()
+							it.next()
-						continue
+							continue
+						}
 					}
 
 					count++
 				}
+
+				it.next()
 			}
 		}
 

eventstore/mmm/count.go

@@ -31,7 +31,8 @@ func (il *IndexingLayer) CountEvents(filter nostr.Filter) (uint32, error) {
 
 			for {
 				// we already have a k and a v and an err from the cursor setup, so check and use these
-				if it.err != nil ||
+				if it.exhausted ||
+					it.err != nil ||
 					len(it.key) != q.keySize ||
 					!bytes.HasPrefix(it.key, q.prefix) {
 					// either iteration has errored or we reached the end of this prefix
@@ -66,16 +67,18 @@ func (il *IndexingLayer) CountEvents(filter nostr.Filter) (uint32, error) {
 					}
 
 					// decode the entire thing (TODO: do a conditional decode while also checking the extra tag)
-					event := &nostr.Event{}
+					if extraTagKey != "" {
-					if err := betterbinary.Unmarshal(bin, event); err != nil {
+						event := &nostr.Event{}
-						it.next()
+						if err := betterbinary.Unmarshal(bin, event); err != nil {
-						continue
+							it.next()
-					}
+							continue
+						}
 
-					// if there is still a tag to be checked, do it now
+						// if there is still a tag to be checked, do it now
-					if !event.Tags.ContainsAny(extraTagKey, extraTagValues) {
+						if !event.Tags.ContainsAny(extraTagKey, extraTagValues) {
-						it.next()
+							it.next()
-						continue
+							continue
+						}
 					}
 
 					count++

eventstore/mmm/mmmm.go

@@ -175,6 +175,21 @@ func (b *MultiMmapManager) EnsureLayer(name string) (*IndexingLayer, error) {
 		err = b.lmdbEnv.View(func(txn *lmdb.Txn) error {
 			txn.RawRead = true
 
+			nameb := []byte(name)
+			if idv, err := txn.Get(b.knownLayers, nameb); err == nil {
+				if err := il.Init(); err != nil {
+					return fmt.Errorf("failed to init read-only layer %s: %w", name, err)
+				}
+				il.id = binary.BigEndian.Uint16(idv)
+				return nil
+			} else {
+				return err
+			}
+		})
+	} else {
+		err = b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
+			txn.RawRead = true
+
 			nameb := []byte(name)
 			if idv, err := txn.Get(b.knownLayers, nameb); lmdb.IsNotFound(err) {
 				if id, err := b.getNextAvailableLayerId(txn); err != nil {
@@ -195,21 +210,6 @@ func (b *MultiMmapManager) EnsureLayer(name string) (*IndexingLayer, error) {
 					return fmt.Errorf("failed to init old layer %s: %w", name, err)
 				}
 
-				return nil
-			} else {
-				return err
-			}
-		})
-	} else {
-		err = b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
-			txn.RawRead = true
-
-			nameb := []byte(name)
-			if idv, err := txn.Get(b.knownLayers, nameb); err == nil {
-				if err := il.Init(); err != nil {
-					return fmt.Errorf("failed to init read-only layer %s: %w", name, err)
-				}
-				il.id = binary.BigEndian.Uint16(idv)
 				return nil
 			} else {
 				return err

eventstore/test/count_test.go

@@ -0,0 +1,151 @@
+package test
+
+import (
+	"testing"
+
+	"fiatjaf.com/nostr"
+	"fiatjaf.com/nostr/eventstore"
+	"github.com/stretchr/testify/require"
+)
+
+func countTest(t *testing.T, db eventstore.Store) {
+	err := db.Init()
+	require.NoError(t, err)
+
+	pk3 := nostr.GetPublicKey(sk3)
+	pk4 := nostr.GetPublicKey(sk4)
+
+	// create test events for counting
+	events := []nostr.Event{
+		{
+			CreatedAt: 700,
+			Content:   "count test event 1",
+			Kind:      1,
+			PubKey:    pk3,
+			Tags: nostr.Tags{
+				{"t", "plec"},
+				{"g", "bbbbbb"},
+				{"r", "https://z.com"},
+			},
+		},
+		{
+			CreatedAt: 701,
+			Content:   "count test event 2",
+			Kind:      1,
+			PubKey:    pk3,
+			Tags: nostr.Tags{
+				{"t", "plec"},
+				{"g", "aaaaaa"},
+				{"r", "https://z.com"},
+			},
+		},
+		{
+			CreatedAt: 702,
+			Content:   "count test event 3",
+			Kind:      2,
+			PubKey:    pk4,
+			Tags: nostr.Tags{
+				{"t", "test"},
+			},
+		},
+	}
+
+	// sign and save events
+	for _, evt := range events {
+		if evt.PubKey == pk3 {
+			evt.Sign(sk3)
+		} else {
+			evt.Sign(sk4)
+		}
+		err = db.SaveEvent(evt)
+		require.NoError(t, err)
+	}
+
+	// test count all events
+	count, err := db.CountEvents(nostr.Filter{})
+	require.NoError(t, err)
+	require.Equal(t, uint32(3), count, "should count the 3 new events")
+
+	// test count by kind 1
+	count, err = db.CountEvents(nostr.Filter{
+		Kinds: []nostr.Kind{1},
+	})
+	require.NoError(t, err)
+	require.Equal(t, uint32(2), count, "should count 2 events of kind 1")
+
+	// test count by author
+	count, err = db.CountEvents(nostr.Filter{
+		Authors: []nostr.PubKey{pk3},
+	})
+	require.NoError(t, err)
+	require.Equal(t, uint32(2), count, "should count 2 events from sk3")
+
+	// test count by author and tag
+	count, err = db.CountEvents(nostr.Filter{
+		Authors: []nostr.PubKey{pk4},
+		Tags:    nostr.TagMap{"t": []string{"test"}},
+	})
+	require.NoError(t, err)
+	require.Equal(t, uint32(1), count, "should count 1 event")
+
+	count, err = db.CountEvents(nostr.Filter{
+		Authors: []nostr.PubKey{pk4},
+		Tags:    nostr.TagMap{"t": []string{"somethingelse"}},
+	})
+	require.NoError(t, err)
+	require.Equal(t, uint32(0), count, "should count 0 events")
+
+	count, err = db.CountEvents(nostr.Filter{
+		Authors: []nostr.PubKey{pk3},
+		Tags:    nostr.TagMap{"t": []string{"test"}},
+	})
+	require.NoError(t, err)
+	require.Equal(t, uint32(0), count, "should count 0 events")
+
+	// test double tag
+	count, err = db.CountEvents(nostr.Filter{
+		Kinds: []nostr.Kind{1, 2},
+		Tags:  nostr.TagMap{"t": []string{"plec"}, "g": []string{"aaaaaa"}},
+	})
+	require.NoError(t, err)
+	require.Equal(t, uint32(1), count, "should count 1 event")
+
+	count, err = db.CountEvents(nostr.Filter{
+		Tags: nostr.TagMap{"t": []string{"plec"}, "g": []string{"aaaaaa"}},
+	})
+	require.NoError(t, err)
+	require.Equal(t, uint32(1), count, "should count 1 event")
+
+	count, err = db.CountEvents(nostr.Filter{
+		Kinds: []nostr.Kind{1},
+		Tags:  nostr.TagMap{"t": []string{"plec"}, "g": []string{"aaaaaa"}},
+	})
+	require.NoError(t, err)
+	require.Equal(t, uint32(1), count, "should count 1 event")
+
+	count, err = db.CountEvents(nostr.Filter{
+		Kinds: []nostr.Kind{1},
+		Tags:  nostr.TagMap{"t": []string{"plec"}, "r": []string{"https://z.com"}},
+	})
+	require.NoError(t, err)
+	require.Equal(t, uint32(2), count, "should count 2 events")
+
+	count, err = db.CountEvents(nostr.Filter{
+		Tags: nostr.TagMap{"t": []string{"plec"}, "r": []string{"https://z.com"}},
+	})
+	require.NoError(t, err)
+	require.Equal(t, uint32(2), count, "should count 2 events")
+
+	count, err = db.CountEvents(nostr.Filter{
+		Kinds: []nostr.Kind{1, 2},
+		Tags:  nostr.TagMap{"t": []string{"plec"}, "r": []string{"https://z.com"}},
+	})
+	require.NoError(t, err)
+	require.Equal(t, uint32(2), count, "should count 2 events")
+
+	count, err = db.CountEvents(nostr.Filter{
+		Tags: nostr.TagMap{"t": []string{"banana"}, "r": []string{"https://z.com"}},
+	})
+	require.NoError(t, err)
+	require.Equal(t, uint32(0), count, "should count 0 events")
+}

eventstore/test/db_test.go

@@ -32,6 +32,7 @@ var tests = []struct {
 	{"second", runSecondTestOn},
 	{"manyauthors", manyAuthorsTest},
 	{"unbalanced", unbalancedTest},
+	{"count", countTest},
 }
 
 func TestSliceStore(t *testing.T) {