Coracle/nostrlib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

define a nostr.Kind type for event kinds, make adjustments everywhere and fix some horrible bugs with mmm, lmdb and badger querying and deleting.

Browse Source
This commit is contained in:
fiatjaf
2025-04-20 11:14:39 -03:00
parent 27f40c2cf2
commit 15c6093c9b
74 changed files with 689 additions and 404 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
eventstore/lmdb/count.go
View File

@@ -53,25 +53,25 @@ func (b *LMDBBackend) CountEvents(filter nostr.Filter) (uint32, error) {
count++
} else {
// fetch actual event
val, err := txn.Get(b.rawEventStore, it.valIdx)
bin, err := txn.Get(b.rawEventStore, it.valIdx)
if err != nil {
panic(err)
}
// check it against pubkeys without decoding the entire thing
if !slices.Contains(extraAuthors, [32]byte(val[32:64])) {
if !slices.Contains(extraAuthors, betterbinary.GetPubKey(bin)) {
it.next()
continue
}
// check it against kinds without decoding the entire thing
if !slices.Contains(extraKinds, [2]byte(val[132:134])) {
if !slices.Contains(extraKinds, betterbinary.GetKind(bin)) {
it.next()
continue
}
evt := &nostr.Event{}
if err := betterbinary.Unmarshal(val, evt); err != nil {
if err := betterbinary.Unmarshal(bin, evt); err != nil {
it.next()
continue
}
@@ -139,7 +139,7 @@ func (b *LMDBBackend) CountEventsHLL(filter nostr.Filter, offset int) (uint32, *
}
// fetch actual event (we need it regardless because we need the pubkey for the hll)
val, err := txn.Get(b.rawEventStore, it.valIdx)
bin, err := txn.Get(b.rawEventStore, it.valIdx)
if err != nil {
panic(err)
}
@@ -147,16 +147,16 @@ func (b *LMDBBackend) CountEventsHLL(filter nostr.Filter, offset int) (uint32, *
if extraKinds == nil && extraTagValues == nil {
// nothing extra to check
count++
hll.AddBytes(nostr.PubKey(val[32:64]))
hll.AddBytes(betterbinary.GetPubKey(bin))
} else {
// check it against kinds without decoding the entire thing
if !slices.Contains(extraKinds, [2]byte(val[132:134])) {
if !slices.Contains(extraKinds, betterbinary.GetKind(bin)) {
it.next()
continue
}
evt := &nostr.Event{}
if err := betterbinary.Unmarshal(val, evt); err != nil {
if err := betterbinary.Unmarshal(bin, evt); err != nil {
it.next()
continue
}

4
eventstore/lmdb/delete.go
View File

@@ -26,13 +26,13 @@ func (b *LMDBBackend) delete(txn *lmdb.Txn, id nostr.ID) error {
}
// if we do, get it so we can compute the indexes
buf, err := txn.Get(b.rawEventStore, idx)
bin, err := txn.Get(b.rawEventStore, idx)
if err != nil {
return fmt.Errorf("failed to get raw event %x to delete: %w", id, err)
}
var evt nostr.Event
if err := betterbinary.Unmarshal(buf, &evt); err != nil {
if err := betterbinary.Unmarshal(bin, &evt); err != nil {
return fmt.Errorf("failed to unmarshal raw event %x to delete: %w", id, err)
}

8
eventstore/lmdb/fuzz_test.go
View File

@@ -49,11 +49,11 @@ func FuzzQuery(f *testing.F) {
Authors: make([]nostr.PubKey, authors),
Limit: int(limit),
}
var maxKind uint16 = 1
var maxKind nostr.Kind = 1
if kinds > 0 {
filter.Kinds = make([]uint16, kinds)
filter.Kinds = make([]nostr.Kind, kinds)
for i := range filter.Kinds {
filter.Kinds[i] = uint16(int(kindFactor) * i)
filter.Kinds[i] = nostr.Kind(int(kindFactor) * i)
}
maxKind = filter.Kinds[len(filter.Kinds)-1]
}
@@ -75,7 +75,7 @@ func FuzzQuery(f *testing.F) {
CreatedAt: nostr.Timestamp(skseed)*nostr.Timestamp(timestampAuthorFactor) + nostr.Timestamp(i),
Content: fmt.Sprintf("unbalanced %d", i),
Tags: nostr.Tags{},
Kind: uint16(i) % maxKind,
Kind: nostr.Kind(i) % maxKind,
}
err := evt.Sign(sk)
require.NoError(t, err)

3
eventstore/lmdb/lib.go
View File

@@ -6,6 +6,7 @@ import (
"os"
"sync/atomic"
"fiatjaf.com/nostr"
"fiatjaf.com/nostr/eventstore"
"github.com/PowerDNS/lmdb-go/lmdb"
)
@@ -34,7 +35,7 @@ type LMDBBackend struct {
indexPTagKind lmdb.DBI
hllCache lmdb.DBI
EnableHLLCacheFor func(kind uint16) (useCache bool, skipSavingActualEvent bool)
EnableHLLCacheFor func(kind nostr.Kind) (useCache bool, skipSavingActualEvent bool)
lastId atomic.Uint32
}

17
eventstore/lmdb/query.go
View File

@@ -113,9 +113,10 @@ func (b *LMDBBackend) query(txn *lmdb.Txn, filter nostr.Filter, limit int) ([]in
if oldest.Q == q && remainingUnexhausted > 1 {
continue
}
// fmt.Println(" query", q, unsafe.Pointer(&results[q]), b.dbiName(query.dbi), hex.EncodeToString(query.prefix), len(results[q]))
// fmt.Println(" query", q, unsafe.Pointer(&results[q]), b.dbiName(query.dbi), hex.EncodeToString(query.prefix), hex.EncodeToString(query.startingPoint), len(results[q]))
it := iterators[q]
// fmt.Println(" ", q, unsafe.Pointer(iterators[q]), it.err)
pulledThisIteration := 0
for {
@@ -124,7 +125,7 @@ func (b *LMDBBackend) query(txn *lmdb.Txn, filter nostr.Filter, limit int) ([]in
len(it.key) != query.keySize ||
!bytes.HasPrefix(it.key, query.prefix) {
// either iteration has errored or we reached the end of this prefix
// fmt.Println(" reached end", it.key, query.keySize, query.prefix)
// fmt.Println(" reached end", hex.EncodeToString(it.key), query.keySize, hex.EncodeToString(query.prefix), it.err)
exhaust(q)
break
}
@@ -140,7 +141,7 @@ func (b *LMDBBackend) query(txn *lmdb.Txn, filter nostr.Filter, limit int) ([]in
}
// fetch actual event
val, err := txn.Get(b.rawEventStore, it.valIdx)
bin, err := txn.Get(b.rawEventStore, it.valIdx)
if err != nil {
log.Printf(
"lmdb: failed to get %x based on prefix %x, index key %x from raw event store: %s\n",
@@ -149,26 +150,26 @@ func (b *LMDBBackend) query(txn *lmdb.Txn, filter nostr.Filter, limit int) ([]in
}
// check it against pubkeys without decoding the entire thing
if extraAuthors != nil && !slices.Contains(extraAuthors, [32]byte(val[32:64])) {
if extraAuthors != nil && !slices.Contains(extraAuthors, betterbinary.GetPubKey(bin)) {
it.next()
continue
}
// check it against kinds without decoding the entire thing
if extraKinds != nil && !slices.Contains(extraKinds, [2]byte(val[132:134])) {
if extraKinds != nil && !slices.Contains(extraKinds, betterbinary.GetKind(bin)) {
it.next()
continue
}
// decode the entire thing
event := nostr.Event{}
if err := betterbinary.Unmarshal(val, &event); err != nil {
log.Printf("lmdb: value read error (id %x) on query prefix %x sp %x dbi %d: %s\n", val[0:32],
if err := betterbinary.Unmarshal(bin, &event); err != nil {
log.Printf("lmdb: value read error (id %x) on query prefix %x sp %x dbi %d: %s\n", betterbinary.GetID(bin),
query.prefix, query.startingPoint, query.dbi, err)
return nil, fmt.Errorf("event read error: %w", err)
}
// fmt.Println(" event", hex.EncodeToString(val[0:4]), "kind", binary.BigEndian.Uint16(val[132:134]), "author", hex.EncodeToString(val[32:36]), "ts", nostr.Timestamp(binary.BigEndian.Uint32(val[128:132])), hex.EncodeToString(it.key), it.valIdx)
// fmt.Println(" event", betterbinary.GetID(bin), "kind", betterbinary.GetKind(bin).Num(), "author", betterbinary.GetPubKey(bin), "ts", betterbinary.GetCreatedAt(bin), hex.EncodeToString(it.key), it.valIdx)
// if there is still a tag to be checked, do it now
if extraTagValues != nil && !event.Tags.ContainsAny(extraTagKey, extraTagValues) {

10
eventstore/lmdb/query_planner.go
View File

@@ -22,8 +22,8 @@ type query struct {
func (b *LMDBBackend) prepareQueries(filter nostr.Filter) (
queries []query,
extraAuthors [][32]byte,
extraKinds [][2]byte,
extraAuthors []nostr.PubKey,
extraKinds []nostr.Kind,
extraTagKey string,
extraTagValues []string,
since uint32,
@@ -127,16 +127,16 @@ func (b *LMDBBackend) prepareQueries(filter nostr.Filter) (
// add an extra kind filter if available (only do this on plain tag index, not on ptag-kind index)
if filter.Kinds != nil {
extraKinds = make([][2]byte, len(filter.Kinds))
extraKinds = make([]nostr.Kind, len(filter.Kinds))
for i, kind := range filter.Kinds {
binary.BigEndian.PutUint16(extraKinds[i][0:2], uint16(kind))
extraKinds[i] = kind
}
}
}
// add an extra author search if possible
if filter.Authors != nil {
extraAuthors = make([][32]byte, len(filter.Authors))
extraAuthors = make([]nostr.PubKey, len(filter.Authors))
for i, pk := range filter.Authors {
extraAuthors[i] = pk
}

4
eventstore/lmdb/replace.go
View File

@@ -16,8 +16,8 @@ func (b *LMDBBackend) ReplaceEvent(evt nostr.Event) error {
}
return b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
filter := nostr.Filter{Limit: 1, Kinds: []uint16{evt.Kind}, Authors: []nostr.PubKey{evt.PubKey}}
if nostr.IsAddressableKind(evt.Kind) {
filter := nostr.Filter{Limit: 1, Kinds: []nostr.Kind{evt.Kind}, Authors: []nostr.PubKey{evt.PubKey}}
if evt.Kind.IsAddressable() {
// when addressable, add the "d" tag to the filter
filter.Tags = nostr.TagMap{"d": []string{evt.Tags.GetD()}}
}