eventstore: index tag letter together with the value.

eventstore/badger/helpers.go

@@ -11,30 +11,35 @@ import (
 	"fiatjaf.com/nostr"
 )
 
-func getTagIndexPrefix(tagValue string) ([]byte, int) {
+func getTagIndexPrefix(tagName string, tagValue string) ([]byte, int) {
 	var k []byte   // the key with full length for created_at and idx at the end, but not filled with these
 	var offset int // the offset -- i.e. where the prefix ends and the created_at and idx would start
 
+	letterPrefix := byte(int(tagName[0]) % 256)
+
 	if kind, pkb, d := getAddrTagElements(tagValue); len(pkb) == 32 {
-		// store value in the new special "a" tag index
-		k = make([]byte, 1+2+8+len(d)+4+4)
+		// store value in the new special "a"-style tag index
+		k = make([]byte, 1+1+2+8+len(d)+4+4)
 		k[0] = indexTagAddrPrefix
-		binary.BigEndian.PutUint16(k[1:], uint16(kind))
-		copy(k[1+2:], pkb[0:8])
-		copy(k[1+2+8:], d)
-		offset = 1 + 2 + 8 + len(d)
+		k[1] = letterPrefix
+		binary.BigEndian.PutUint16(k[1+1:], uint16(kind))
+		copy(k[1+1+2:], pkb[0:8])
+		copy(k[1+1+2+8:], d)
+		offset = 1 + 1 + 2 + 8 + len(d)
 	} else if vb, _ := hex.DecodeString(tagValue); len(vb) == 32 {
-		// store value as bytes
-		k = make([]byte, 1+8+4+4)
+		// store value as bytes with tag name prefix
+		k = make([]byte, 1+1+8+4+4)
 		k[0] = indexTag32Prefix
-		copy(k[1:], vb[0:8])
-		offset = 1 + 8
+		k[1] = letterPrefix
+		copy(k[2:], vb[0:8])
+		offset = 1 + 1 + 8
 	} else {
-		// store whatever as utf-8
-		k = make([]byte, 1+len(tagValue)+4+4)
+		// store whatever as utf-8 with tag name prefix
+		k = make([]byte, 1+1+len(tagValue)+4+4)
 		k[0] = indexTagPrefix
-		copy(k[1:], tagValue)
-		offset = 1 + len(tagValue)
+		k[1] = letterPrefix
+		copy(k[2:], tagValue)
+		offset = 1 + 1 + len(tagValue)
 	}
 
 	return k, offset
@@ -102,7 +107,9 @@ func (b *BadgerBackend) getIndexKeysForEvent(evt nostr.Event, idx []byte) iter.S
 				}
 			}
 
-			firstIndex := slices.IndexFunc(evt.Tags, func(t nostr.Tag) bool { return len(t) >= 2 && t[1] == tag[1] })
+			firstIndex := slices.IndexFunc(evt.Tags, func(t nostr.Tag) bool {
+				return len(t) >= 2 && t[0] == tag[0] && t[1] == tag[1]
+			})
 			if firstIndex != i {
 				// duplicate
 				continue
@@ -114,7 +121,7 @@ func (b *BadgerBackend) getIndexKeysForEvent(evt nostr.Event, idx []byte) iter.S
 			}
 
 			// get key prefix (with full length) and offset where to write the last parts
-			k, offset := getTagIndexPrefix(tag[1])
+			k, offset := getTagIndexPrefix(tag[0], tag[1])
 
 			// write the last parts (created_at and idx)
 			binary.BigEndian.PutUint32(k[offset:], uint32(evt.CreatedAt))

eventstore/badger/migrations.go

@@ -2,41 +2,115 @@ package badger
 
 import (
 	"encoding/binary"
+	"fmt"
+	"log"
 
+	"fiatjaf.com/nostr"
+	"fiatjaf.com/nostr/eventstore/codec/betterbinary"
 	"github.com/dgraph-io/badger/v4"
 )
 
 func (b *BadgerBackend) runMigrations() error {
 	return b.Update(func(txn *badger.Txn) error {
-		var version uint16
+		item, err := txn.Get([]byte("version"))
+		if err != nil && err != badger.ErrKeyNotFound {
+			return fmt.Errorf("failed to get db version: %w", err)
+		}
 
-		item, err := txn.Get([]byte{dbVersionKey})
-		if err == badger.ErrKeyNotFound {
-			version = 0
-		} else if err != nil {
-			return err
-		} else {
-			item.Value(func(val []byte) error {
+		var version uint16 = 0
+		if err == nil {
+			err = item.Value(func(val []byte) error {
 				version = binary.BigEndian.Uint16(val)
 				return nil
 			})
+			if err != nil {
+				return fmt.Errorf("failed to read db version: %w", err)
+			}
 		}
 
 		// do the migrations in increasing steps (there is no rollback)
 		//
 
-		if version < 1 {
-			// ...
-		}
+		// the 5 first migrations go to trash because on version 5 we need to export and import all the data anyway
+		if version < 5 {
+			log.Println("[badger] migration 5: delete all indexes and recreate them")
 
-		// b.bumpVersion(txn, 1)
+			// delete all index entries
+			prefixes := []byte{
+				indexIdPrefix,
+				indexCreatedAtPrefix,
+				indexKindPrefix,
+				indexPubkeyPrefix,
+				indexPubkeyKindPrefix,
+				indexTagPrefix,
+				indexTag32Prefix,
+				indexTagAddrPrefix,
+			}
+
+			for _, prefix := range prefixes {
+				it := txn.NewIterator(badger.IteratorOptions{
+					PrefetchValues: false,
+					Prefix:         []byte{prefix},
+				})
+				defer it.Close()
+
+				var keysToDelete [][]byte
+				for it.Seek([]byte{prefix}); it.ValidForPrefix([]byte{prefix}); it.Next() {
+					key := it.Item().Key()
+					keyCopy := make([]byte, len(key))
+					copy(keyCopy, key)
+					keysToDelete = append(keysToDelete, keyCopy)
+				}
+
+				for _, key := range keysToDelete {
+					if err := txn.Delete(key); err != nil {
+						return fmt.Errorf("failed to delete index key %x: %w", key, err)
+					}
+				}
+			}
+
+			// iterate through all events and recreate indexes
+			it := txn.NewIterator(badger.IteratorOptions{
+				PrefetchValues: true,
+				Prefix:         []byte{rawEventStorePrefix},
+			})
+			defer it.Close()
+
+			for it.Seek([]byte{rawEventStorePrefix}); it.ValidForPrefix([]byte{rawEventStorePrefix}); it.Next() {
+				item := it.Item()
+				idx := item.Key()
+
+				err := item.Value(func(val []byte) error {
+					evt := nostr.Event{}
+					if err := betterbinary.Unmarshal(val, &evt); err != nil {
+						return fmt.Errorf("error decoding event %x on migration 5: %w", idx, err)
+					}
+
+					for key := range b.getIndexKeysForEvent(evt, idx[1:]) {
+						if err := txn.Set(key, nil); err != nil {
+							return fmt.Errorf("failed to save index for event %s on migration 5: %w", evt.ID, err)
+						}
+					}
+
+					return nil
+				})
+				if err != nil {
+					return err
+				}
+			}
+
+			// bump version
+			if err := b.bumpVersion(txn, 5); err != nil {
+				return err
+			}
+		}
 
 		return nil
 	})
 }
 
-func (b *BadgerBackend) bumpVersion(txn *badger.Txn, version uint16) error {
-	buf := make([]byte, 2)
-	binary.BigEndian.PutUint16(buf, version)
-	return txn.Set([]byte{dbVersionKey}, buf)
+func (b *BadgerBackend) bumpVersion(txn *badger.Txn, v uint16) error {
+	var newVersion [2]byte
+	binary.BigEndian.PutUint16(newVersion[:], v)
+	return txn.Set([]byte("version"), newVersion[:])
 }

eventstore/badger/query_planner.go

@@ -71,11 +71,10 @@ func prepareQueries(filter nostr.Filter) (
 		queries = make([]query, len(tagValues))
 		for i, value := range tagValues {
 			// get key prefix (with full length) and offset where to write the created_at
-			k, offset := getTagIndexPrefix(value)
+			k, offset := getTagIndexPrefix(tagKey, value)
 			// remove the last parts part to get just the prefix we want here
 			prefix := k[0:offset]
 			queries[i] = query{i: i, prefix: prefix}
-			i++
 		}
 
 		extraFilter = &nostr.Filter{