From f60fc08f8d1949f9603306c27041a2643374e269 Mon Sep 17 00:00:00 2001 From: fiatjaf Date: Sun, 11 May 2025 09:36:59 -0300 Subject: [PATCH] eventstore: QueryEvents() to take a maxLimit param now so everything is clearer. --- eventstore/badger/lib.go | 11 ----------- eventstore/badger/query.go | 17 ++++++++--------- eventstore/bluge/query.go | 19 +++++++++---------- eventstore/bluge/replace.go | 6 +++--- eventstore/cmd/eventstore/main.go | 4 ++-- eventstore/cmd/eventstore/neg.go | 2 +- eventstore/cmd/eventstore/query-or-save.go | 2 +- eventstore/cmd/eventstore/query.go | 2 +- eventstore/lmdb/lib.go | 15 ++------------- eventstore/lmdb/query.go | 18 ++++++++---------- eventstore/mmm/indexinglayer.go | 7 +------ eventstore/mmm/query.go | 18 ++++++++---------- eventstore/nullstore/lib.go | 2 +- eventstore/slicestore/lib.go | 13 ++++--------- eventstore/store.go | 2 +- eventstore/wrappers/publisher.go | 6 ++++++ filter.go | 15 ++++++++------- khatru/README.md | 6 +++--- khatru/blossom/eventstorewrapper.go | 9 ++++----- khatru/docs/cookbook/search.md | 15 --------------- khatru/docs/core/eventstore.md | 6 +----- khatru/docs/core/routing.md | 2 +- khatru/docs/getting-started/index.md | 6 +++--- khatru/docs/index.md | 4 ++-- khatru/examples/basic-badger/main.go | 2 +- khatru/examples/basic-lmdb/main.go | 2 +- khatru/examples/blossom/main.go | 2 +- khatru/examples/exclusive/main.go | 2 +- khatru/examples/routing/main.go | 6 +++--- khatru/relay.go | 9 +++++++-- khatru/relay_fuzz_test.go | 2 +- khatru/relay_test.go | 4 ++-- nip77/example/example.go | 3 ++- sdk/feeds.go | 5 ++--- sdk/feeds_test.go | 2 +- sdk/list.go | 5 ++++- sdk/metadata.go | 2 +- sdk/set.go | 2 +- sdk/specific_event.go | 2 +- sdk/system.go | 2 +- 40 files changed, 108 insertions(+), 151 deletions(-) diff --git a/eventstore/badger/lib.go b/eventstore/badger/lib.go index fefb01d..f3b20b5 100644 --- a/eventstore/badger/lib.go +++ b/eventstore/badger/lib.go @@ -27,8 +27,6 @@ var _ eventstore.Store = (*BadgerBackend)(nil) type BadgerBackend struct { Path string - MaxLimit int - MaxLimitNegentropy int BadgerOptionsModifier func(badger.Options) badger.Options // Experimental @@ -57,15 +55,6 @@ func (b *BadgerBackend) Init() error { return fmt.Errorf("error running migrations: %w", err) } - if b.MaxLimit != 0 { - b.MaxLimitNegentropy = b.MaxLimit - } else { - b.MaxLimit = 1000 - if b.MaxLimitNegentropy == 0 { - b.MaxLimitNegentropy = 16777216 - } - } - if err := b.DB.View(func(txn *badger.Txn) error { it := txn.NewIterator(badger.IteratorOptions{ Prefix: []byte{0}, diff --git a/eventstore/badger/query.go b/eventstore/badger/query.go index bfe561c..f654375 100644 --- a/eventstore/badger/query.go +++ b/eventstore/badger/query.go @@ -16,26 +16,25 @@ import ( var batchFilled = errors.New("batch-filled") -func (b *BadgerBackend) QueryEvents(filter nostr.Filter) iter.Seq[nostr.Event] { +func (b *BadgerBackend) QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] { return func(yield func(nostr.Event) bool) { if filter.Search != "" { return } // max number of events we'll return - limit := b.MaxLimit / 4 - if filter.Limit > 0 && filter.Limit <= b.MaxLimit { - limit = filter.Limit - } - if tlimit := nostr.GetTheoreticalLimit(filter); tlimit == 0 { + if tlimit := filter.GetTheoreticalLimit(); tlimit == 0 || filter.LimitZero { return - } else if tlimit > 0 { - limit = tlimit + } else if tlimit < maxLimit { + maxLimit = tlimit + } + if filter.Limit < maxLimit { + maxLimit = filter.Limit } // fmt.Println("limit", limit) b.View(func(txn *badger.Txn) error { - results, err := b.query(txn, filter, limit) + results, err := b.query(txn, filter, maxLimit) if err != nil { return err } diff --git a/eventstore/bluge/query.go b/eventstore/bluge/query.go index c1e561f..c5e9265 100644 --- a/eventstore/bluge/query.go +++ b/eventstore/bluge/query.go @@ -10,8 +10,15 @@ import ( "github.com/blugelabs/bluge/search" ) -func (b *BlugeBackend) QueryEvents(filter nostr.Filter) iter.Seq[nostr.Event] { +func (b *BlugeBackend) QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] { return func(yield func(nostr.Event) bool) { + limit := maxLimit + if filter.LimitZero { + return + } else if filter.Limit < limit { + limit = filter.Limit + } + if len(filter.Search) < 2 { return } @@ -69,14 +76,6 @@ func (b *BlugeBackend) QueryEvents(filter nostr.Filter) iter.Seq[nostr.Event] { q = complicatedQuery } - limit := 40 - if filter.Limit != 0 { - limit = filter.Limit - if filter.Limit > 150 { - limit = 150 - } - } - req := bluge.NewTopNSearch(limit, q) dmi, err := reader.Search(context.Background(), req) @@ -92,7 +91,7 @@ func (b *BlugeBackend) QueryEvents(filter nostr.Filter) iter.Seq[nostr.Event] { next.VisitStoredFields(func(field string, value []byte) bool { id, err := nostr.IDFromHex(string(value)) if err == nil { - for evt := range b.RawEventStore.QueryEvents(nostr.Filter{IDs: []nostr.ID{id}}) { + for evt := range b.RawEventStore.QueryEvents(nostr.Filter{IDs: []nostr.ID{id}}, 1) { yield(evt) } } diff --git a/eventstore/bluge/replace.go b/eventstore/bluge/replace.go index 6f3368f..0ee30fb 100644 --- a/eventstore/bluge/replace.go +++ b/eventstore/bluge/replace.go @@ -12,13 +12,13 @@ func (b *BlugeBackend) ReplaceEvent(evt nostr.Event) error { b.Lock() defer b.Unlock() - filter := nostr.Filter{Limit: 1, Kinds: []nostr.Kind{evt.Kind}, Authors: []nostr.PubKey{evt.PubKey}} - if evt.Kind.IsReplaceable() { + filter := nostr.Filter{Kinds: []nostr.Kind{evt.Kind}, Authors: []nostr.PubKey{evt.PubKey}} + if evt.Kind.IsAddressable() { filter.Tags = nostr.TagMap{"d": []string{evt.Tags.GetD()}} } shouldStore := true - for previous := range b.QueryEvents(filter) { + for previous := range b.QueryEvents(filter, 1) { if internal.IsOlder(previous, evt) { if err := b.DeleteEvent(previous.ID); err != nil { return fmt.Errorf("failed to delete event for replacing: %w", err) diff --git a/eventstore/cmd/eventstore/main.go b/eventstore/cmd/eventstore/main.go index d2108f4..3323093 100644 --- a/eventstore/cmd/eventstore/main.go +++ b/eventstore/cmd/eventstore/main.go @@ -70,9 +70,9 @@ var app = &cli.Command{ switch typ { case "lmdb": - db = &lmdb.LMDBBackend{Path: path, MaxLimit: 1_000_000} + db = &lmdb.LMDBBackend{Path: path} case "badger": - db = &badger.BadgerBackend{Path: path, MaxLimit: 1_000_000} + db = &badger.BadgerBackend{Path: path} case "mmm": var err error if db, err = doMmmInit(path); err != nil { diff --git a/eventstore/cmd/eventstore/neg.go b/eventstore/cmd/eventstore/neg.go index b90cc71..6ec5f85 100644 --- a/eventstore/cmd/eventstore/neg.go +++ b/eventstore/cmd/eventstore/neg.go @@ -44,7 +44,7 @@ var neg = &cli.Command{ // create negentropy object and initialize it with events vec := vector.New() neg := negentropy.New(vec, frameSizeLimit) - for evt := range db.QueryEvents(filter) { + for evt := range db.QueryEvents(filter, math.MaxInt) { vec.Insert(evt.CreatedAt, evt.ID) } diff --git a/eventstore/cmd/eventstore/query-or-save.go b/eventstore/cmd/eventstore/query-or-save.go index e8ce403..920b6e7 100644 --- a/eventstore/cmd/eventstore/query-or-save.go +++ b/eventstore/cmd/eventstore/query-or-save.go @@ -47,7 +47,7 @@ func doSave(ctx context.Context, line string, evt nostr.Event) error { } func doQuery(ctx context.Context, f *nostr.Filter) error { - for evt := range db.QueryEvents(*f) { + for evt := range db.QueryEvents(*f, 1_000_000) { fmt.Println(evt) } return nil diff --git a/eventstore/cmd/eventstore/query.go b/eventstore/cmd/eventstore/query.go index a9a6017..7e88a64 100644 --- a/eventstore/cmd/eventstore/query.go +++ b/eventstore/cmd/eventstore/query.go @@ -25,7 +25,7 @@ var query = &cli.Command{ continue } - for evt := range db.QueryEvents(filter) { + for evt := range db.QueryEvents(filter, 1_000_000) { fmt.Println(evt) } } diff --git a/eventstore/lmdb/lib.go b/eventstore/lmdb/lib.go index 17bf4f3..8f0ee27 100644 --- a/eventstore/lmdb/lib.go +++ b/eventstore/lmdb/lib.go @@ -14,10 +14,8 @@ import ( var _ eventstore.Store = (*LMDBBackend)(nil) type LMDBBackend struct { - Path string - MaxLimit int - MaxLimitNegentropy int - MapSize int64 + Path string + MapSize int64 lmdbEnv *lmdb.Env extraFlags uint // (for debugging and testing) @@ -41,15 +39,6 @@ type LMDBBackend struct { } func (b *LMDBBackend) Init() error { - if b.MaxLimit != 0 { - b.MaxLimitNegentropy = b.MaxLimit - } else { - b.MaxLimit = 1500 - if b.MaxLimitNegentropy == 0 { - b.MaxLimitNegentropy = 16777216 - } - } - // create directory if it doesn't exist and open it if err := os.MkdirAll(b.Path, 0755); err != nil { return err diff --git a/eventstore/lmdb/query.go b/eventstore/lmdb/query.go index 4935b55..7d95779 100644 --- a/eventstore/lmdb/query.go +++ b/eventstore/lmdb/query.go @@ -14,27 +14,25 @@ import ( "github.com/PowerDNS/lmdb-go/lmdb" ) -func (b *LMDBBackend) QueryEvents(filter nostr.Filter) iter.Seq[nostr.Event] { +func (b *LMDBBackend) QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] { return func(yield func(nostr.Event) bool) { if filter.Search != "" { return } // max number of events we'll return - var limit int - limit = b.MaxLimit / 4 - if filter.Limit > 0 && filter.Limit <= b.MaxLimit { - limit = filter.Limit - } - if tlimit := nostr.GetTheoreticalLimit(filter); tlimit == 0 { + if tlimit := filter.GetTheoreticalLimit(); tlimit == 0 || filter.LimitZero { return - } else if tlimit > 0 { - limit = tlimit + } else if tlimit < maxLimit { + maxLimit = tlimit + } + if filter.Limit < maxLimit { + maxLimit = filter.Limit } b.lmdbEnv.View(func(txn *lmdb.Txn) error { txn.RawRead = true - results, err := b.query(txn, filter, limit) + results, err := b.query(txn, filter, maxLimit) for _, ie := range results { if !yield(ie.Event) { diff --git a/eventstore/mmm/indexinglayer.go b/eventstore/mmm/indexinglayer.go index 112dfd7..7dcfdcc 100644 --- a/eventstore/mmm/indexinglayer.go +++ b/eventstore/mmm/indexinglayer.go @@ -14,8 +14,7 @@ type IndexingLayer struct { isInitialized bool name string - MaxLimit int - mmmm *MultiMmapManager + mmmm *MultiMmapManager // this is stored in the knownLayers db as a value, and used to keep track of which layer owns each event id uint16 @@ -53,10 +52,6 @@ func (il *IndexingLayer) Init() error { path := filepath.Join(il.mmmm.Dir, il.name) - if il.MaxLimit == 0 { - il.MaxLimit = 500 - } - // open lmdb env, err := lmdb.NewEnv() if err != nil { diff --git a/eventstore/mmm/query.go b/eventstore/mmm/query.go index a0a15cc..f16b4b9 100644 --- a/eventstore/mmm/query.go +++ b/eventstore/mmm/query.go @@ -73,32 +73,30 @@ func (b *MultiMmapManager) queryByIDs(yield func(nostr.Event) bool, ids []nostr. }) } -func (il *IndexingLayer) QueryEvents(filter nostr.Filter) iter.Seq[nostr.Event] { +func (il *IndexingLayer) QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] { return func(yield func(nostr.Event) bool) { if len(filter.IDs) > 0 { il.mmmm.queryByIDs(yield, filter.IDs, nil) return } - if filter.Search != "" { return } // max number of events we'll return - limit := il.MaxLimit / 4 - if filter.Limit > 0 && filter.Limit < il.MaxLimit { - limit = filter.Limit - } - if tlimit := nostr.GetTheoreticalLimit(filter); tlimit == 0 { + if tlimit := filter.GetTheoreticalLimit(); tlimit == 0 || filter.LimitZero { return - } else if tlimit > 0 { - limit = tlimit + } else if tlimit < maxLimit { + maxLimit = tlimit + } + if filter.Limit < maxLimit { + maxLimit = filter.Limit } il.lmdbEnv.View(func(txn *lmdb.Txn) error { txn.RawRead = true - results, err := il.query(txn, filter, limit) + results, err := il.query(txn, filter, filter.Limit) for _, ie := range results { if !yield(ie.Event) { diff --git a/eventstore/nullstore/lib.go b/eventstore/nullstore/lib.go index e3275ba..68e660d 100644 --- a/eventstore/nullstore/lib.go +++ b/eventstore/nullstore/lib.go @@ -21,7 +21,7 @@ func (b NullStore) DeleteEvent(id nostr.ID) error { return nil } -func (b NullStore) QueryEvents(filter nostr.Filter) iter.Seq[nostr.Event] { +func (b NullStore) QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] { return func(yield func(nostr.Event) bool) {} } diff --git a/eventstore/slicestore/lib.go b/eventstore/slicestore/lib.go index 1cc2a49..f7ee806 100644 --- a/eventstore/slicestore/lib.go +++ b/eventstore/slicestore/lib.go @@ -18,24 +18,19 @@ var _ eventstore.Store = (*SliceStore)(nil) type SliceStore struct { sync.Mutex internal []nostr.Event - - MaxLimit int } func (b *SliceStore) Init() error { b.internal = make([]nostr.Event, 0, 5000) - if b.MaxLimit == 0 { - b.MaxLimit = 500 - } return nil } func (b *SliceStore) Close() {} -func (b *SliceStore) QueryEvents(filter nostr.Filter) iter.Seq[nostr.Event] { +func (b *SliceStore) QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] { return func(yield func(nostr.Event) bool) { - if filter.Limit > b.MaxLimit || (filter.Limit == 0 && !filter.LimitZero) { - filter.Limit = b.MaxLimit + if filter.Limit > maxLimit || (filter.Limit == 0 && !filter.LimitZero) { + filter.Limit = maxLimit } // efficiently determine where to start and end @@ -136,7 +131,7 @@ func (b *SliceStore) ReplaceEvent(evt nostr.Event) error { } shouldStore := true - for previous := range b.QueryEvents(filter) { + for previous := range b.QueryEvents(filter, 1) { if internal.IsOlder(previous, evt) { if err := b.delete(previous.ID); err != nil { return fmt.Errorf("failed to delete event for replacing: %w", err) diff --git a/eventstore/store.go b/eventstore/store.go index b3b86da..ef8eadc 100644 --- a/eventstore/store.go +++ b/eventstore/store.go @@ -16,7 +16,7 @@ type Store interface { Close() // QueryEvents returns events that match the filter - QueryEvents(nostr.Filter) iter.Seq[nostr.Event] + QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] // DeleteEvent deletes an event atomically by ID DeleteEvent(nostr.ID) error diff --git a/eventstore/wrappers/publisher.go b/eventstore/wrappers/publisher.go index 632a7e5..3c1c1b1 100644 --- a/eventstore/wrappers/publisher.go +++ b/eventstore/wrappers/publisher.go @@ -3,6 +3,7 @@ package wrappers import ( "context" "fmt" + "iter" "fiatjaf.com/nostr" "fiatjaf.com/nostr/eventstore" @@ -12,6 +13,11 @@ var _ nostr.Publisher = StorePublisher{} type StorePublisher struct { eventstore.Store + MaxLimit int +} + +func (w StorePublisher) QueryEvents(filter nostr.Filter) iter.Seq[nostr.Event] { + return w.Store.QueryEvents(filter, w.MaxLimit) } func (w StorePublisher) Publish(ctx context.Context, evt nostr.Event) error { diff --git a/filter.go b/filter.go index 2a80b48..c716047 100644 --- a/filter.go +++ b/filter.go @@ -1,6 +1,7 @@ package nostr import ( + "math" "slices" "github.com/mailru/easyjson" @@ -148,19 +149,19 @@ func (ef Filter) Clone() Filter { // GetTheoreticalLimit gets the maximum number of events that a normal filter would ever return, for example, if // there is a number of "ids" in the filter, the theoretical limit will be that number of ids. // -// It returns -1 if there are no theoretical limits. +// It returns math.MaxInt if there are no theoretical limits. // // The given .Limit present in the filter is ignored. -func GetTheoreticalLimit(filter Filter) int { - if len(filter.IDs) > 0 { +func (filter Filter) GetTheoreticalLimit() int { + if filter.IDs != nil { return len(filter.IDs) } - if len(filter.Kinds) == 0 { - return -1 + if filter.Kinds != nil { + return math.MaxInt } - if len(filter.Authors) > 0 { + if filter.Authors != nil { allAreReplaceable := true for _, kind := range filter.Kinds { if !kind.IsReplaceable() { @@ -186,5 +187,5 @@ func GetTheoreticalLimit(filter Filter) int { } } - return -1 + return math.MaxInt } diff --git a/khatru/README.md b/khatru/README.md index 1ae8221..a559ff1 100644 --- a/khatru/README.md +++ b/khatru/README.md @@ -108,15 +108,15 @@ func main() { ### But I don't want to write my own database! -Fear no more. Using the https://fiatjaf.com/nostr/eventstore module you get a bunch of compatible databases out of the box and you can just plug them into your relay. For example, [sqlite](https://pkg.go.dev/fiatjaf.com/nostr/eventstore/sqlite3): +Fear no more. Using the https://fiatjaf.com/nostr/eventstore module you get a bunch of compatible databases out of the box and you can just plug them into your relay. For example, [sqlite](https://pkg.go.dev/fiatjaf.com/nostr/eventstore/lmdb): ```go - db := sqlite3.SQLite3Backend{DatabaseURL: "/tmp/khatru-sqlite-tmp"} + db := lmdb.LMDBackend{Path: "/tmp/khatru-lmdb-tmp"} if err := db.Init(); err != nil { panic(err) } - relay.UseEventstore(db) + relay.UseEventstore(db, 500) ``` ### But I don't want to write a bunch of custom policies! diff --git a/khatru/blossom/eventstorewrapper.go b/khatru/blossom/eventstorewrapper.go index c5f52a4..5cfde0b 100644 --- a/khatru/blossom/eventstorewrapper.go +++ b/khatru/blossom/eventstorewrapper.go @@ -18,7 +18,7 @@ type EventStoreBlobIndexWrapper struct { func (es EventStoreBlobIndexWrapper) Keep(ctx context.Context, blob BlobDescriptor, pubkey nostr.PubKey) error { next, stop := iter.Pull( - es.Store.QueryEvents(nostr.Filter{Authors: []nostr.PubKey{pubkey}, Kinds: []nostr.Kind{24242}, Tags: nostr.TagMap{"x": []string{blob.SHA256}}}), + es.Store.QueryEvents(nostr.Filter{Authors: []nostr.PubKey{pubkey}, Kinds: []nostr.Kind{24242}, Tags: nostr.TagMap{"x": []string{blob.SHA256}}}, 1), ) defer stop() @@ -46,7 +46,7 @@ func (es EventStoreBlobIndexWrapper) List(ctx context.Context, pubkey nostr.PubK for evt := range es.Store.QueryEvents(nostr.Filter{ Authors: []nostr.PubKey{pubkey}, Kinds: []nostr.Kind{24242}, - }) { + }, 1000) { yield(es.parseEvent(evt)) } } @@ -54,7 +54,7 @@ func (es EventStoreBlobIndexWrapper) List(ctx context.Context, pubkey nostr.PubK func (es EventStoreBlobIndexWrapper) Get(ctx context.Context, sha256 string) (*BlobDescriptor, error) { next, stop := iter.Pull( - es.Store.QueryEvents(nostr.Filter{Tags: nostr.TagMap{"x": []string{sha256}}, Kinds: []nostr.Kind{24242}, Limit: 1}), + es.Store.QueryEvents(nostr.Filter{Tags: nostr.TagMap{"x": []string{sha256}}, Kinds: []nostr.Kind{24242}, Limit: 1}, 1), ) defer stop() @@ -74,8 +74,7 @@ func (es EventStoreBlobIndexWrapper) Delete(ctx context.Context, sha256 string, Tags: nostr.TagMap{"x": []string{sha256}}, Kinds: []nostr.Kind{24242}, Limit: 1, - }, - ), + }, 1), ) defer stop() diff --git a/khatru/docs/cookbook/search.md b/khatru/docs/cookbook/search.md index 0b17f38..b819283 100644 --- a/khatru/docs/cookbook/search.md +++ b/khatru/docs/cookbook/search.md @@ -34,18 +34,3 @@ func main () { ``` Note that in this case we're using the [LMDB](https://pkg.go.dev/fiatjaf.com/nostr/eventstore/lmdb) adapter for normal queries and it explicitly rejects any filter that contains a `Search` field, while [Bluge](https://pkg.go.dev/fiatjaf.com/nostr/eventstore/bluge) rejects any filter _without_ a `Search` value, which make them pair well together. - -Other adapters, like [SQLite](https://pkg.go.dev/fiatjaf.com/nostr/eventstore/sqlite3), implement search functionality on their own, so if you don't want to use that you would have to have a middleware between, like: - -```go - relay.StoreEvent = policies.SeqStore(db.SaveEvent, search.SaveEvent) - relay.QueryStored = func (ctx context.Context, filter nostr.Filter) iter.Seq[nostr.Event] { - if len(filter.Search) > 0 { - return search.QueryEvents(ctx, filter) - } else { - filterNoSearch := filter - filterNoSearch.Search = "" - return normal.QueryEvents(ctx, filterNoSearch) - } - }) -``` diff --git a/khatru/docs/core/eventstore.md b/khatru/docs/core/eventstore.md index 7cdbd58..dd329ea 100644 --- a/khatru/docs/core/eventstore.md +++ b/khatru/docs/core/eventstore.md @@ -35,7 +35,7 @@ func main() { panic(err) } - relay.UseEventstore(db) + relay.UseEventstore(db, 500) fmt.Println("running on :3334") http.ListenAndServe(":3334", relay) @@ -44,10 +44,6 @@ func main() { [LMDB](https://pkg.go.dev/fiatjaf.com/nostr/eventstore/lmdb) works the same way. -[SQLite](https://pkg.go.dev/fiatjaf.com/nostr/eventstore/sqlite3) also stores things locally so it only needs a `Path`. - -[PostgreSQL](https://pkg.go.dev/fiatjaf.com/nostr/eventstore/postgresql) and [MySQL](https://pkg.go.dev/fiatjaf.com/nostr/eventstore/mysql) use remote connections to database servers, so they take a `DatabaseURL` parameter, but after that it's the same. - ## Using two at a time If you want to use two different adapters at the same time that's easy. Just use the `policies.Seq*` functions: diff --git a/khatru/docs/core/routing.md b/khatru/docs/core/routing.md index 57d6b3d..9c6de54 100644 --- a/khatru/docs/core/routing.md +++ b/khatru/docs/core/routing.md @@ -21,7 +21,7 @@ groupsRelay, _ := khatru29.Init(relay29.Options{Domain: "example.com", DB: group publicStore := slicestore.SliceStore{} publicStore.Init() publicRelay := khatru.NewRelay() -publicRelay.UseEventStore(publicStore) +publicRelay.UseEventStore(publicStore, 1000) // ... // a higher-level relay that just routes between the two above diff --git a/khatru/docs/getting-started/index.md b/khatru/docs/getting-started/index.md index 748a0a4..d042a9c 100644 --- a/khatru/docs/getting-started/index.md +++ b/khatru/docs/getting-started/index.md @@ -31,15 +31,15 @@ relay.Info.Description = "this is my custom relay" relay.Info.Icon = "https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2Fliquipedia.net%2Fcommons%2Fimages%2F3%2F35%2FSCProbe.jpg&f=1&nofb=1&ipt=0cbbfef25bce41da63d910e86c3c343e6c3b9d63194ca9755351bb7c2efa3359&ipo=images" ``` -Now we must set up the basic functions for accepting events and answering queries. We could make our own querying engine from scratch, but we can also use [eventstore](https://fiatjaf.com/nostr/eventstore). In this example we'll use the SQLite adapter: +Now we must set up the basic functions for accepting events and answering queries. We could make our own querying engine from scratch, but we can also use [eventstore](https://pkg.go.dev/fiatjaf.com/nostr/eventstore). In this example we'll use the Badger adapter: ```go -db := sqlite3.SQLite3Backend{DatabaseURL: "/tmp/khatru-sqlite-tmp"} +db := badger.BadgerBackend{Path: "/tmp/khatru-badger-tmp"} if err := db.Init(); err != nil { panic(err) } -relay.UseEventstore(db) +relay.UseEventstore(db, 500) ``` These are lists of functions that will be called in order every time an `EVENT` is received, or a `REQ` query is received. You can add more than one handler there, you can have a function that reads from some other server, but just in some cases, you can do anything. diff --git a/khatru/docs/index.md b/khatru/docs/index.md index 16eba43..18af66f 100644 --- a/khatru/docs/index.md +++ b/khatru/docs/index.md @@ -24,7 +24,7 @@ features: - title: It plugs into event stores easily icon: 📦 link: /core/eventstore - details: khatru's companion, the `eventstore` library, provides all methods for storing and querying events efficiently from SQLite, LMDB, Postgres, Badger and others. + details: khatru's companion, the `eventstore` library, provides all methods for storing and querying events efficiently from LMDB, Badger and others. - title: It supports NIP-42 AUTH icon: 🪪 link: /core/auth @@ -48,7 +48,7 @@ func main() { relay := khatru.NewRelay() db := badger.BadgerBackend{Path: "/tmp/khatru-badgern-tmp"} db.Init() - relay.UseEventStore(db) + relay.UseEventStore(db, 400) http.ListenAndServe(":3334", relay) } ``` diff --git a/khatru/examples/basic-badger/main.go b/khatru/examples/basic-badger/main.go index 52a353a..724cfcf 100644 --- a/khatru/examples/basic-badger/main.go +++ b/khatru/examples/basic-badger/main.go @@ -16,7 +16,7 @@ func main() { panic(err) } - relay.UseEventstore(db) + relay.UseEventstore(db, 400) relay.Negentropy = true diff --git a/khatru/examples/basic-lmdb/main.go b/khatru/examples/basic-lmdb/main.go index 4be5073..daf7d8c 100644 --- a/khatru/examples/basic-lmdb/main.go +++ b/khatru/examples/basic-lmdb/main.go @@ -18,7 +18,7 @@ func main() { panic(err) } - relay.UseEventstore(db) + relay.UseEventstore(db, 400) fmt.Println("running on :3334") http.ListenAndServe(":3334", relay) diff --git a/khatru/examples/blossom/main.go b/khatru/examples/blossom/main.go index 2eac465..b857a8a 100644 --- a/khatru/examples/blossom/main.go +++ b/khatru/examples/blossom/main.go @@ -20,7 +20,7 @@ func main() { panic(err) } - relay.UseEventstore(db) + relay.UseEventstore(db, 400) bdb := &badger.BadgerBackend{Path: "/tmp/khatru-badger-blossom-tmp"} if err := bdb.Init(); err != nil { diff --git a/khatru/examples/exclusive/main.go b/khatru/examples/exclusive/main.go index 240c1c6..ab73e5a 100644 --- a/khatru/examples/exclusive/main.go +++ b/khatru/examples/exclusive/main.go @@ -21,7 +21,7 @@ func main() { panic(err) } - relay.UseEventstore(db) + relay.UseEventstore(db, 400) relay.OnEvent = policies.PreventTooManyIndexableTags(10, nil, nil) relay.OnRequest = policies.NoComplexFilters diff --git a/khatru/examples/routing/main.go b/khatru/examples/routing/main.go index 827da7b..9f1305c 100644 --- a/khatru/examples/routing/main.go +++ b/khatru/examples/routing/main.go @@ -15,17 +15,17 @@ func main() { db1 := &slicestore.SliceStore{} db1.Init() r1 := khatru.NewRelay() - r1.UseEventstore(db1) + r1.UseEventstore(db1, 400) db2 := &badger.BadgerBackend{Path: "/tmp/t"} db2.Init() r2 := khatru.NewRelay() - r2.UseEventstore(db2) + r2.UseEventstore(db2, 400) db3 := &slicestore.SliceStore{} db3.Init() r3 := khatru.NewRelay() - r3.UseEventstore(db3) + r3.UseEventstore(db3, 400) router := khatru.NewRouter() diff --git a/khatru/relay.go b/khatru/relay.go index 6476269..925361a 100644 --- a/khatru/relay.go +++ b/khatru/relay.go @@ -117,9 +117,14 @@ type Relay struct { expirationManager *expirationManager } -func (rl *Relay) UseEventstore(store eventstore.Store) { +// UseEventstore hooks up an eventstore.Store into the relay in the default way. +// It should be used in 85% of the cases, when you don't want to do any complicated scheme with your event storage. +// +// maxQueryLimit is the default max limit to be enforced when querying events, to prevent users for downloading way +// too much, setting it to something like 500 or 1000 should be ok in most cases. +func (rl *Relay) UseEventstore(store eventstore.Store, maxQueryLimit int) { rl.QueryStored = func(ctx context.Context, filter nostr.Filter) iter.Seq[nostr.Event] { - return store.QueryEvents(filter) + return store.QueryEvents(filter, maxQueryLimit) } rl.Count = func(ctx context.Context, filter nostr.Filter) (uint32, error) { return store.CountEvents(filter) diff --git a/khatru/relay_fuzz_test.go b/khatru/relay_fuzz_test.go index 150e628..4238419 100644 --- a/khatru/relay_fuzz_test.go +++ b/khatru/relay_fuzz_test.go @@ -24,7 +24,7 @@ func FuzzReplaceableEvents(f *testing.F) { relay := NewRelay() store := &lmdb.LMDBBackend{Path: "/tmp/fuzz"} store.Init() - relay.UseEventstore(store) + relay.UseEventstore(store, 4000) defer store.Close() diff --git a/khatru/relay_test.go b/khatru/relay_test.go index 10ffbe5..169e70b 100644 --- a/khatru/relay_test.go +++ b/khatru/relay_test.go @@ -17,7 +17,7 @@ func TestBasicRelayFunctionality(t *testing.T) { store := &slicestore.SliceStore{} store.Init() - relay.UseEventstore(store) + relay.UseEventstore(store, 400) // start test server server := httptest.NewServer(relay) @@ -239,7 +239,7 @@ func TestBasicRelayFunctionality(t *testing.T) { relay.expirationManager.interval = 3 * time.Second // check every 3 seconds store := &slicestore.SliceStore{} store.Init() - relay.UseEventstore(store) + relay.UseEventstore(store, 400) // start test server server := httptest.NewServer(relay) diff --git a/nip77/example/example.go b/nip77/example/example.go index 2ce7fc7..582a568 100644 --- a/nip77/example/example.go +++ b/nip77/example/example.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "math" "slices" "time" @@ -18,7 +19,7 @@ func main() { db.Init() sk := nostr.Generate() - local := wrappers.StorePublisher{Store: db} + local := wrappers.StorePublisher{Store: db, MaxLimit: math.MaxInt} for { for i := 0; i < 20; i++ { diff --git a/sdk/feeds.go b/sdk/feeds.go index 2cec7d3..7c67466 100644 --- a/sdk/feeds.go +++ b/sdk/feeds.go @@ -124,19 +124,18 @@ func (sys *System) FetchFeedPage( } filter := nostr.Filter{Authors: []nostr.PubKey{pubkey}, Kinds: kinds} - if until > oldestTimestamp { // we can use our local database filter.Until = until count := 0 - for evt := range sys.Store.QueryEvents(filter) { + for evt := range sys.Store.QueryEvents(filter, limitPerKey) { events = append(events, evt) count++ if count >= limitPerKey { // we got enough from the local store wg.Done() - continue + break } } } diff --git a/sdk/feeds_test.go b/sdk/feeds_test.go index c45fab6..521f61e 100644 --- a/sdk/feeds_test.go +++ b/sdk/feeds_test.go @@ -22,7 +22,7 @@ func TestStreamLiveFeed(t *testing.T) { for _, r := range []*khatru.Relay{relay1, relay2, relay3} { db := &slicestore.SliceStore{} db.Init() - r.UseEventstore(db) + r.UseEventstore(db, 4000) defer db.Close() } diff --git a/sdk/list.go b/sdk/list.go index bdafbaf..4e34bcc 100644 --- a/sdk/list.go +++ b/sdk/list.go @@ -57,7 +57,10 @@ func fetchGenericList[V comparable, I TagItemWithValue[V]]( v := GenericList[V, I]{PubKey: pubkey} - for evt := range sys.Store.QueryEvents(nostr.Filter{Kinds: []nostr.Kind{actualKind}, Authors: []nostr.PubKey{pubkey}}) { + for evt := range sys.Store.QueryEvents(nostr.Filter{ + Kinds: []nostr.Kind{actualKind}, + Authors: []nostr.PubKey{pubkey}, + }, 1) { // ok, we found something locally items := parseItemsFromEventTags(evt, parseTag) v.Event = &evt diff --git a/sdk/metadata.go b/sdk/metadata.go index 75722ba..957b358 100644 --- a/sdk/metadata.go +++ b/sdk/metadata.go @@ -111,7 +111,7 @@ func (sys *System) FetchProfileMetadata(ctx context.Context, pubkey nostr.PubKey pm.PubKey = pubkey - for evt := range sys.Store.QueryEvents(nostr.Filter{Kinds: []nostr.Kind{0}, Authors: []nostr.PubKey{pubkey}}) { + for evt := range sys.Store.QueryEvents(nostr.Filter{Kinds: []nostr.Kind{0}, Authors: []nostr.PubKey{pubkey}}, 1) { // ok, we found something locally pm, _ = ParseMetadata(evt) pm.PubKey = pubkey diff --git a/sdk/set.go b/sdk/set.go index a517ef6..598d5a4 100644 --- a/sdk/set.go +++ b/sdk/set.go @@ -47,7 +47,7 @@ func fetchGenericSets[V comparable, I TagItemWithValue[V]]( v := GenericSets[V, I]{PubKey: pubkey} events := slices.Collect( - sys.Store.QueryEvents(nostr.Filter{Kinds: []nostr.Kind{actualKind}, Authors: []nostr.PubKey{pubkey}}), + sys.Store.QueryEvents(nostr.Filter{Kinds: []nostr.Kind{actualKind}, Authors: []nostr.PubKey{pubkey}}, 100), ) if len(events) != 0 { // ok, we found something locally diff --git a/sdk/specific_event.go b/sdk/specific_event.go index f66ce44..ecc3194 100644 --- a/sdk/specific_event.go +++ b/sdk/specific_event.go @@ -95,7 +95,7 @@ func (sys *System) FetchSpecificEvent( // try to fetch in our internal eventstore first if !params.SkipLocalStore { - for evt := range sys.Store.QueryEvents(filter) { + for evt := range sys.Store.QueryEvents(filter, 1) { return &evt, nil, nil } } diff --git a/sdk/system.go b/sdk/system.go index c855b7d..4923396 100644 --- a/sdk/system.go +++ b/sdk/system.go @@ -136,7 +136,7 @@ func NewSystem() *System { sys.Store = &nullstore.NullStore{} sys.Store.Init() } - sys.Publisher = wrappers.StorePublisher{Store: sys.Store} + sys.Publisher = wrappers.StorePublisher{Store: sys.Store, MaxLimit: 1000} sys.initializeReplaceableDataloaders() sys.initializeAddressableDataloaders()