bring in khatru and eventstore.

.github/workflows/test.yml

@@ -1,21 +0,0 @@
-name: test every commit
-on:
-  - push
-  - pull_request
-
-jobs:
-  test:
-    # temporary using newest ubuntu instead of ubuntu-latest since
-    #   libsecp256k1-dev does not have secp256k1_schnorrsig_sign32 in jammy
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install libsecp256k1-dev
-        run: sudo apt-get install libsecp256k1-dev
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: ./go.mod
-
-      - run: go test ./... -tags=sonic

LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <https://unlicense.org>

LICENSE.md

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2022 nbd
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

eventstore/.gitignore

@@ -0,0 +1 @@
+knowledge.md

eventstore/README.md

@@ -0,0 +1,31 @@
+# eventstore
+
+A collection of reusable database connectors, wrappers and schemas that store Nostr events and expose a simple Go interface:
+
+```go
+type Store interface {
+	// Init is called at the very beginning by [Server.Start], after [Relay.Init],
+	// allowing a storage to initialize its internal resources.
+	Init() error
+
+	// Close must be called after you're done using the store, to free up resources and so on.
+	Close()
+
+	// QueryEvents is invoked upon a client's REQ as described in NIP-01.
+	// it should return a channel with the events as they're recovered from a database.
+	// the channel should be closed after the events are all delivered.
+	QueryEvents(context.Context, nostr.Filter) (chan *nostr.Event, error)
+
+	// DeleteEvent is used to handle deletion events, as per NIP-09.
+	DeleteEvent(context.Context, *nostr.Event) error
+
+	// SaveEvent is called once Relay.AcceptEvent reports true.
+	SaveEvent(context.Context, *nostr.Event) error
+}
+```
+
+[![Go Reference](https://pkg.go.dev/badge/github.com/fiatjaf/eventstore.svg)](https://pkg.go.dev/github.com/fiatjaf/eventstore) [![Run Tests](https://github.com/fiatjaf/eventstore/actions/workflows/test.yml/badge.svg)](https://github.com/fiatjaf/eventstore/actions/workflows/test.yml)
+
+## command-line tool
+
+There is an [`eventstore` command-line tool](cmd/eventstore) that can be used to query these databases directly.

eventstore/badger/count.go

@@ -0,0 +1,168 @@
+package badger
+
+import (
+	"context"
+	"encoding/binary"
+	"log"
+
+	"github.com/dgraph-io/badger/v4"
+	bin "github.com/fiatjaf/eventstore/internal/binary"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/nbd-wtf/go-nostr/nip45/hyperloglog"
+)
+
+func (b *BadgerBackend) CountEvents(ctx context.Context, filter nostr.Filter) (int64, error) {
+	var count int64 = 0
+
+	queries, extraFilter, since, err := prepareQueries(filter)
+	if err != nil {
+		return 0, err
+	}
+
+	err = b.View(func(txn *badger.Txn) error {
+		// iterate only through keys and in reverse order
+		opts := badger.IteratorOptions{
+			Reverse: true,
+		}
+
+		// actually iterate
+		for _, q := range queries {
+			it := txn.NewIterator(opts)
+			defer it.Close()
+
+			for it.Seek(q.startingPoint); it.ValidForPrefix(q.prefix); it.Next() {
+				item := it.Item()
+				key := item.Key()
+
+				idxOffset := len(key) - 4 // this is where the idx actually starts
+
+				// "id" indexes don't contain a timestamp
+				if !q.skipTimestamp {
+					createdAt := binary.BigEndian.Uint32(key[idxOffset-4 : idxOffset])
+					if createdAt < since {
+						break
+					}
+				}
+
+				idx := make([]byte, 5)
+				idx[0] = rawEventStorePrefix
+				copy(idx[1:], key[idxOffset:])
+
+				if extraFilter == nil {
+					count++
+				} else {
+					// fetch actual event
+					item, err := txn.Get(idx)
+					if err != nil {
+						if err == badger.ErrDiscardedTxn {
+							return err
+						}
+						log.Printf("badger: count (%v) failed to get %d from raw event store: %s\n", q, idx, err)
+						return err
+					}
+
+					err = item.Value(func(val []byte) error {
+						evt := &nostr.Event{}
+						if err := bin.Unmarshal(val, evt); err != nil {
+							return err
+						}
+
+						// check if this matches the other filters that were not part of the index
+						if extraFilter.Matches(evt) {
+							count++
+						}
+
+						return nil
+					})
+					if err != nil {
+						log.Printf("badger: count value read error: %s\n", err)
+					}
+				}
+			}
+		}
+
+		return nil
+	})
+
+	return count, err
+}
+
+func (b *BadgerBackend) CountEventsHLL(ctx context.Context, filter nostr.Filter, offset int) (int64, *hyperloglog.HyperLogLog, error) {
+	var count int64 = 0
+
+	queries, extraFilter, since, err := prepareQueries(filter)
+	if err != nil {
+		return 0, nil, err
+	}
+
+	hll := hyperloglog.New(offset)
+
+	err = b.View(func(txn *badger.Txn) error {
+		// iterate only through keys and in reverse order
+		opts := badger.IteratorOptions{
+			Reverse: true,
+		}
+
+		// actually iterate
+		for _, q := range queries {
+			it := txn.NewIterator(opts)
+			defer it.Close()
+
+			for it.Seek(q.startingPoint); it.ValidForPrefix(q.prefix); it.Next() {
+				item := it.Item()
+				key := item.Key()
+
+				idxOffset := len(key) - 4 // this is where the idx actually starts
+
+				// "id" indexes don't contain a timestamp
+				if !q.skipTimestamp {
+					createdAt := binary.BigEndian.Uint32(key[idxOffset-4 : idxOffset])
+					if createdAt < since {
+						break
+					}
+				}
+
+				idx := make([]byte, 5)
+				idx[0] = rawEventStorePrefix
+				copy(idx[1:], key[idxOffset:])
+
+				// fetch actual event
+				item, err := txn.Get(idx)
+				if err != nil {
+					if err == badger.ErrDiscardedTxn {
+						return err
+					}
+					log.Printf("badger: count (%v) failed to get %d from raw event store: %s\n", q, idx, err)
+					return err
+				}
+
+				err = item.Value(func(val []byte) error {
+					if extraFilter == nil {
+						hll.AddBytes(val[32:64])
+						count++
+						return nil
+					}
+
+					evt := &nostr.Event{}
+					if err := bin.Unmarshal(val, evt); err != nil {
+						return err
+					}
+					if extraFilter.Matches(evt) {
+						hll.Add(evt.PubKey)
+						count++
+						return nil
+					}
+
+					return nil
+				})
+				if err != nil {
+					log.Printf("badger: count value read error: %s\n", err)
+				}
+			}
+		}
+
+		return nil
+	})
+
+	return count, hll, err
+}

eventstore/badger/delete.go

@@ -0,0 +1,72 @@
+package badger
+
+import (
+	"context"
+	"encoding/hex"
+	"log"
+
+	"github.com/dgraph-io/badger/v4"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+var serialDelete uint32 = 0
+
+func (b *BadgerBackend) DeleteEvent(ctx context.Context, evt *nostr.Event) error {
+	deletionHappened := false
+
+	err := b.Update(func(txn *badger.Txn) error {
+		var err error
+		deletionHappened, err = b.delete(txn, evt)
+		return err
+	})
+	if err != nil {
+		return err
+	}
+
+	// after deleting, run garbage collector (sometimes)
+	if deletionHappened {
+		serialDelete = (serialDelete + 1) % 256
+		if serialDelete == 0 {
+			if err := b.RunValueLogGC(0.8); err != nil && err != badger.ErrNoRewrite {
+				log.Println("badger gc errored:" + err.Error())
+			}
+		}
+	}
+
+	return nil
+}
+
+func (b *BadgerBackend) delete(txn *badger.Txn, evt *nostr.Event) (bool, error) {
+	idx := make([]byte, 1, 5)
+	idx[0] = rawEventStorePrefix
+
+	// query event by id to get its idx
+	idPrefix8, _ := hex.DecodeString(evt.ID[0 : 8*2])
+	prefix := make([]byte, 1+8)
+	prefix[0] = indexIdPrefix
+	copy(prefix[1:], idPrefix8)
+	opts := badger.IteratorOptions{
+		PrefetchValues: false,
+	}
+	it := txn.NewIterator(opts)
+	it.Seek(prefix)
+	if it.ValidForPrefix(prefix) {
+		idx = append(idx, it.Item().Key()[1+8:]...)
+	}
+	it.Close()
+
+	// if no idx was found, end here, this event doesn't exist
+	if len(idx) == 1 {
+		return false, nil
+	}
+
+	// calculate all index keys we have for this event and delete them
+	for k := range b.getIndexKeysForEvent(evt, idx[1:]) {
+		if err := txn.Delete(k); err != nil {
+			return false, err
+		}
+	}
+
+	// delete the raw event
+	return true, txn.Delete(idx)
+}

eventstore/badger/fuzz_test.go

@@ -0,0 +1,158 @@
+package badger
+
+import (
+	"cmp"
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/dgraph-io/badger/v4"
+	"github.com/fiatjaf/eventstore"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/slices"
+)
+
+func FuzzQuery(f *testing.F) {
+	ctx := context.Background()
+
+	f.Add(uint(200), uint(50), uint(13), uint(2), uint(2), uint(0), uint(1))
+	f.Fuzz(func(t *testing.T, total, limit, authors, timestampAuthorFactor, seedFactor, kinds, kindFactor uint) {
+		total++
+		authors++
+		seedFactor++
+		kindFactor++
+		if kinds == 1 {
+			kinds++
+		}
+		if limit == 0 {
+			return
+		}
+
+		// ~ setup db
+
+		bdb, err := badger.Open(badger.DefaultOptions("").WithInMemory(true))
+		if err != nil {
+			t.Fatalf("failed to create database: %s", err)
+			return
+		}
+		db := &BadgerBackend{}
+		db.DB = bdb
+
+		if err := db.runMigrations(); err != nil {
+			t.Fatalf("error: %s", err)
+			return
+		}
+
+		if err := db.DB.View(func(txn *badger.Txn) error {
+			it := txn.NewIterator(badger.IteratorOptions{
+				Prefix:  []byte{0},
+				Reverse: true,
+			})
+			it.Seek([]byte{1})
+			if it.Valid() {
+				key := it.Item().Key()
+				idx := key[1:]
+				serial := binary.BigEndian.Uint32(idx)
+				db.serial.Store(serial)
+			}
+			it.Close()
+			return nil
+		}); err != nil {
+			t.Fatalf("failed to initialize serial: %s", err)
+			return
+		}
+
+		db.MaxLimit = 500
+		defer db.Close()
+
+		// ~ start actual test
+
+		filter := nostr.Filter{
+			Authors: make([]string, authors),
+			Limit:   int(limit),
+		}
+		maxKind := 1
+		if kinds > 0 {
+			filter.Kinds = make([]int, kinds)
+			for i := range filter.Kinds {
+				filter.Kinds[i] = int(kindFactor) * i
+			}
+			maxKind = filter.Kinds[len(filter.Kinds)-1]
+		}
+
+		for i := 0; i < int(authors); i++ {
+			sk := make([]byte, 32)
+			binary.BigEndian.PutUint32(sk, uint32(i%int(authors*seedFactor))+1)
+			pk, _ := nostr.GetPublicKey(hex.EncodeToString(sk))
+			filter.Authors[i] = pk
+		}
+
+		expected := make([]*nostr.Event, 0, total)
+		for i := 0; i < int(total); i++ {
+			skseed := uint32(i%int(authors*seedFactor)) + 1
+			sk := make([]byte, 32)
+			binary.BigEndian.PutUint32(sk, skseed)
+
+			evt := &nostr.Event{
+				CreatedAt: nostr.Timestamp(skseed)*nostr.Timestamp(timestampAuthorFactor) + nostr.Timestamp(i),
+				Content:   fmt.Sprintf("unbalanced %d", i),
+				Tags:      nostr.Tags{},
+				Kind:      i % maxKind,
+			}
+			err := evt.Sign(hex.EncodeToString(sk))
+			require.NoError(t, err)
+
+			err = db.SaveEvent(ctx, evt)
+			require.NoError(t, err)
+
+			if filter.Matches(evt) {
+				expected = append(expected, evt)
+			}
+		}
+
+		slices.SortFunc(expected, nostr.CompareEventPtrReverse)
+		if len(expected) > int(limit) {
+			expected = expected[0:limit]
+		}
+
+		w := eventstore.RelayWrapper{Store: db}
+
+		start := time.Now()
+		// fmt.Println(filter)
+		res, err := w.QuerySync(ctx, filter)
+		end := time.Now()
+
+		require.NoError(t, err)
+		require.Equal(t, len(expected), len(res), "number of results is different than expected")
+
+		require.Less(t, end.Sub(start).Milliseconds(), int64(1500), "query took too long")
+		require.True(t, slices.IsSortedFunc(res, func(a, b *nostr.Event) int { return cmp.Compare(b.CreatedAt, a.CreatedAt) }), "results are not sorted")
+
+		nresults := len(expected)
+
+		getTimestamps := func(events []*nostr.Event) []nostr.Timestamp {
+			res := make([]nostr.Timestamp, len(events))
+			for i, evt := range events {
+				res[i] = evt.CreatedAt
+			}
+			return res
+		}
+
+		// fmt.Println(" expected   result")
+		// for i := range expected {
+		// 	fmt.Println(" ", expected[i].CreatedAt, expected[i].ID[0:8], "  ", res[i].CreatedAt, res[i].ID[0:8], "           ", i)
+		// }
+
+		require.Equal(t, expected[0].CreatedAt, res[0].CreatedAt, "first result is wrong")
+		require.Equal(t, expected[nresults-1].CreatedAt, res[nresults-1].CreatedAt, "last result is wrong")
+		require.Equal(t, getTimestamps(expected), getTimestamps(res))
+
+		for _, evt := range res {
+			require.True(t, filter.Matches(evt), "event %s doesn't match filter %s", evt, filter)
+		}
+	})
+}

eventstore/badger/helpers.go

@@ -0,0 +1,162 @@
+package badger
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"iter"
+	"strconv"
+	"strings"
+
+	"github.com/nbd-wtf/go-nostr"
+	"golang.org/x/exp/slices"
+)
+
+func getTagIndexPrefix(tagValue string) ([]byte, int) {
+	var k []byte   // the key with full length for created_at and idx at the end, but not filled with these
+	var offset int // the offset -- i.e. where the prefix ends and the created_at and idx would start
+
+	if kind, pkb, d := getAddrTagElements(tagValue); len(pkb) == 32 {
+		// store value in the new special "a" tag index
+		k = make([]byte, 1+2+8+len(d)+4+4)
+		k[0] = indexTagAddrPrefix
+		binary.BigEndian.PutUint16(k[1:], kind)
+		copy(k[1+2:], pkb[0:8])
+		copy(k[1+2+8:], d)
+		offset = 1 + 2 + 8 + len(d)
+	} else if vb, _ := hex.DecodeString(tagValue); len(vb) == 32 {
+		// store value as bytes
+		k = make([]byte, 1+8+4+4)
+		k[0] = indexTag32Prefix
+		copy(k[1:], vb[0:8])
+		offset = 1 + 8
+	} else {
+		// store whatever as utf-8
+		k = make([]byte, 1+len(tagValue)+4+4)
+		k[0] = indexTagPrefix
+		copy(k[1:], tagValue)
+		offset = 1 + len(tagValue)
+	}
+
+	return k, offset
+}
+
+func (b *BadgerBackend) getIndexKeysForEvent(evt *nostr.Event, idx []byte) iter.Seq[[]byte] {
+	return func(yield func([]byte) bool) {
+		{
+			// ~ by id
+			idPrefix8, _ := hex.DecodeString(evt.ID[0 : 8*2])
+			k := make([]byte, 1+8+4)
+			k[0] = indexIdPrefix
+			copy(k[1:], idPrefix8)
+			copy(k[1+8:], idx)
+			if !yield(k) {
+				return
+			}
+		}
+
+		{
+			// ~ by pubkey+date
+			pubkeyPrefix8, _ := hex.DecodeString(evt.PubKey[0 : 8*2])
+			k := make([]byte, 1+8+4+4)
+			k[0] = indexPubkeyPrefix
+			copy(k[1:], pubkeyPrefix8)
+			binary.BigEndian.PutUint32(k[1+8:], uint32(evt.CreatedAt))
+			copy(k[1+8+4:], idx)
+			if !yield(k) {
+				return
+			}
+		}
+
+		{
+			// ~ by kind+date
+			k := make([]byte, 1+2+4+4)
+			k[0] = indexKindPrefix
+			binary.BigEndian.PutUint16(k[1:], uint16(evt.Kind))
+			binary.BigEndian.PutUint32(k[1+2:], uint32(evt.CreatedAt))
+			copy(k[1+2+4:], idx)
+			if !yield(k) {
+				return
+			}
+		}
+
+		{
+			// ~ by pubkey+kind+date
+			pubkeyPrefix8, _ := hex.DecodeString(evt.PubKey[0 : 8*2])
+			k := make([]byte, 1+8+2+4+4)
+			k[0] = indexPubkeyKindPrefix
+			copy(k[1:], pubkeyPrefix8)
+			binary.BigEndian.PutUint16(k[1+8:], uint16(evt.Kind))
+			binary.BigEndian.PutUint32(k[1+8+2:], uint32(evt.CreatedAt))
+			copy(k[1+8+2+4:], idx)
+			if !yield(k) {
+				return
+			}
+		}
+
+		// ~ by tagvalue+date
+		customIndex := b.IndexLongerTag != nil
+		customSkip := b.SkipIndexingTag != nil
+
+		for i, tag := range evt.Tags {
+			if len(tag) < 2 || len(tag[0]) != 1 || len(tag[1]) == 0 || len(tag[1]) > 100 {
+				if !customIndex || !b.IndexLongerTag(evt, tag[0], tag[1]) {
+					// not indexable
+					continue
+				}
+			}
+
+			firstIndex := slices.IndexFunc(evt.Tags, func(t nostr.Tag) bool { return len(t) >= 2 && t[1] == tag[1] })
+			if firstIndex != i {
+				// duplicate
+				continue
+			}
+
+			if customSkip && b.SkipIndexingTag(evt, tag[0], tag[1]) {
+				// purposefully skipped
+				continue
+			}
+
+			// get key prefix (with full length) and offset where to write the last parts
+			k, offset := getTagIndexPrefix(tag[1])
+
+			// write the last parts (created_at and idx)
+			binary.BigEndian.PutUint32(k[offset:], uint32(evt.CreatedAt))
+			copy(k[offset+4:], idx)
+			if !yield(k) {
+				return
+			}
+		}
+
+		{
+			// ~ by date only
+			k := make([]byte, 1+4+4)
+			k[0] = indexCreatedAtPrefix
+			binary.BigEndian.PutUint32(k[1:], uint32(evt.CreatedAt))
+			copy(k[1+4:], idx)
+			if !yield(k) {
+				return
+			}
+		}
+	}
+}
+
+func getAddrTagElements(tagValue string) (kind uint16, pkb []byte, d string) {
+	spl := strings.Split(tagValue, ":")
+	if len(spl) == 3 {
+		if pkb, _ := hex.DecodeString(spl[1]); len(pkb) == 32 {
+			if kind, err := strconv.ParseUint(spl[0], 10, 16); err == nil {
+				return uint16(kind), pkb, spl[2]
+			}
+		}
+	}
+	return 0, nil, ""
+}
+
+func filterMatchesTags(ef *nostr.Filter, event *nostr.Event) bool {
+	for f, v := range ef.Tags {
+		if v != nil && !event.Tags.ContainsAny(f, v) {
+			return false
+		}
+	}
+	return true
+}

eventstore/badger/lib.go

@@ -0,0 +1,100 @@
+package badger
+
+import (
+	"encoding/binary"
+	"fmt"
+	"sync/atomic"
+
+	"github.com/dgraph-io/badger/v4"
+	"github.com/fiatjaf/eventstore"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+const (
+	dbVersionKey          byte = 255
+	rawEventStorePrefix   byte = 0
+	indexCreatedAtPrefix  byte = 1
+	indexIdPrefix         byte = 2
+	indexKindPrefix       byte = 3
+	indexPubkeyPrefix     byte = 4
+	indexPubkeyKindPrefix byte = 5
+	indexTagPrefix        byte = 6
+	indexTag32Prefix      byte = 7
+	indexTagAddrPrefix    byte = 8
+)
+
+var _ eventstore.Store = (*BadgerBackend)(nil)
+
+type BadgerBackend struct {
+	Path                  string
+	MaxLimit              int
+	MaxLimitNegentropy    int
+	BadgerOptionsModifier func(badger.Options) badger.Options
+
+	// Experimental
+	SkipIndexingTag func(event *nostr.Event, tagName string, tagValue string) bool
+	// Experimental
+	IndexLongerTag func(event *nostr.Event, tagName string, tagValue string) bool
+
+	*badger.DB
+
+	serial atomic.Uint32
+}
+
+func (b *BadgerBackend) Init() error {
+	opts := badger.DefaultOptions(b.Path)
+	if b.BadgerOptionsModifier != nil {
+		opts = b.BadgerOptionsModifier(opts)
+	}
+
+	db, err := badger.Open(opts)
+	if err != nil {
+		return err
+	}
+	b.DB = db
+
+	if err := b.runMigrations(); err != nil {
+		return fmt.Errorf("error running migrations: %w", err)
+	}
+
+	if b.MaxLimit != 0 {
+		b.MaxLimitNegentropy = b.MaxLimit
+	} else {
+		b.MaxLimit = 1000
+		if b.MaxLimitNegentropy == 0 {
+			b.MaxLimitNegentropy = 16777216
+		}
+	}
+
+	if err := b.DB.View(func(txn *badger.Txn) error {
+		it := txn.NewIterator(badger.IteratorOptions{
+			Prefix:  []byte{0},
+			Reverse: true,
+		})
+		it.Seek([]byte{1})
+		if it.Valid() {
+			key := it.Item().Key()
+			idx := key[1:]
+			serial := binary.BigEndian.Uint32(idx)
+			b.serial.Store(serial)
+		}
+		it.Close()
+		return nil
+	}); err != nil {
+		return fmt.Errorf("error initializing serial: %w", err)
+	}
+
+	return nil
+}
+
+func (b *BadgerBackend) Close() {
+	b.DB.Close()
+}
+
+func (b *BadgerBackend) Serial() []byte {
+	next := b.serial.Add(1)
+	vb := make([]byte, 5)
+	vb[0] = rawEventStorePrefix
+	binary.BigEndian.PutUint32(vb[1:], next)
+	return vb
+}

eventstore/badger/migrations.go

@@ -0,0 +1,66 @@
+package badger
+
+import (
+	"encoding/binary"
+	"fmt"
+
+	"github.com/dgraph-io/badger/v4"
+)
+
+func (b *BadgerBackend) runMigrations() error {
+	return b.Update(func(txn *badger.Txn) error {
+		var version uint16
+
+		item, err := txn.Get([]byte{dbVersionKey})
+		if err == badger.ErrKeyNotFound {
+			version = 0
+		} else if err != nil {
+			return err
+		} else {
+			item.Value(func(val []byte) error {
+				version = binary.BigEndian.Uint16(val)
+				return nil
+			})
+		}
+
+		// do the migrations in increasing steps (there is no rollback)
+		//
+
+		// the 3 first migrations go to trash because on version 3 we need to export and import all the data anyway
+		if version < 3 {
+			// if there is any data in the relay we will stop and notify the user,
+			// otherwise we just set version to 3 and proceed
+			prefix := []byte{indexIdPrefix}
+			it := txn.NewIterator(badger.IteratorOptions{
+				PrefetchValues: true,
+				PrefetchSize:   100,
+				Prefix:         prefix,
+			})
+			defer it.Close()
+
+			hasAnyEntries := false
+			for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
+				hasAnyEntries = true
+				break
+			}
+
+			if hasAnyEntries {
+				return fmt.Errorf("your database is at version %d, but in order to migrate up to version 3 you must manually export all the events and then import again: run an old version of this software, export the data, then delete the database files, run the new version, import the data back in.", version)
+			}
+
+			b.bumpVersion(txn, 3)
+		}
+
+		if version < 4 {
+			// ...
+		}
+
+		return nil
+	})
+}
+
+func (b *BadgerBackend) bumpVersion(txn *badger.Txn, version uint16) error {
+	buf := make([]byte, 2)
+	binary.BigEndian.PutUint16(buf, version)
+	return txn.Set([]byte{dbVersionKey}, buf)
+}

eventstore/badger/query.go

@@ -0,0 +1,432 @@
+package badger
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"log"
+
+	"github.com/dgraph-io/badger/v4"
+	"github.com/fiatjaf/eventstore"
+	"github.com/fiatjaf/eventstore/internal"
+	bin "github.com/fiatjaf/eventstore/internal/binary"
+	"github.com/nbd-wtf/go-nostr"
+	"golang.org/x/exp/slices"
+)
+
+var batchFilled = errors.New("batch-filled")
+
+func (b *BadgerBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (chan *nostr.Event, error) {
+	ch := make(chan *nostr.Event)
+
+	if filter.Search != "" {
+		close(ch)
+		return ch, nil
+	}
+
+	// max number of events we'll return
+	maxLimit := b.MaxLimit
+	var limit int
+	if eventstore.IsNegentropySession(ctx) {
+		maxLimit = b.MaxLimitNegentropy
+		limit = maxLimit
+	} else {
+		limit = maxLimit / 4
+	}
+	if filter.Limit > 0 && filter.Limit <= maxLimit {
+		limit = filter.Limit
+	}
+	if tlimit := nostr.GetTheoreticalLimit(filter); tlimit == 0 {
+		close(ch)
+		return ch, nil
+	} else if tlimit > 0 {
+		limit = tlimit
+	}
+
+	// fmt.Println("limit", limit)
+
+	go b.View(func(txn *badger.Txn) error {
+		defer close(ch)
+
+		results, err := b.query(txn, filter, limit)
+		if err != nil {
+			return err
+		}
+
+		for _, evt := range results {
+			ch <- evt.Event
+		}
+
+		return nil
+	})
+
+	return ch, nil
+}
+
+func (b *BadgerBackend) query(txn *badger.Txn, filter nostr.Filter, limit int) ([]internal.IterEvent, error) {
+	queries, extraFilter, since, err := prepareQueries(filter)
+	if err != nil {
+		return nil, err
+	}
+
+	iterators := make([]*badger.Iterator, len(queries))
+	exhausted := make([]bool, len(queries)) // indicates that a query won't be used anymore
+	results := make([][]internal.IterEvent, len(queries))
+	pulledPerQuery := make([]int, len(queries))
+
+	// these are kept updated so we never pull from the iterator that is at further distance
+	// (i.e. the one that has the oldest event among all)
+	// we will continue to pull from it as soon as some other iterator takes the position
+	oldest := internal.IterEvent{Q: -1}
+
+	secondPhase := false // after we have gathered enough events we will change the way we iterate
+	secondBatch := make([][]internal.IterEvent, 0, len(queries)+1)
+	secondPhaseParticipants := make([]int, 0, len(queries)+1)
+
+	// while merging results in the second phase we will alternate between these two lists
+	//   to avoid having to create new lists all the time
+	var secondPhaseResultsA []internal.IterEvent
+	var secondPhaseResultsB []internal.IterEvent
+	var secondPhaseResultsToggle bool // this is just a dummy thing we use to keep track of the alternating
+	var secondPhaseHasResultsPending bool
+
+	remainingUnexhausted := len(queries) // when all queries are exhausted we can finally end this thing
+	batchSizePerQuery := internal.BatchSizePerNumberOfQueries(limit, remainingUnexhausted)
+	firstPhaseTotalPulled := 0
+
+	exhaust := func(q int) {
+		exhausted[q] = true
+		remainingUnexhausted--
+		if q == oldest.Q {
+			oldest = internal.IterEvent{Q: -1}
+		}
+	}
+
+	var firstPhaseResults []internal.IterEvent
+
+	for q := range queries {
+		iterators[q] = txn.NewIterator(badger.IteratorOptions{
+			Reverse:        true,
+			PrefetchValues: false, // we don't even have values, only keys
+			Prefix:         queries[q].prefix,
+		})
+		defer iterators[q].Close()
+		iterators[q].Seek(queries[q].startingPoint)
+		results[q] = make([]internal.IterEvent, 0, batchSizePerQuery*2)
+	}
+
+	// we will reuse this throughout the iteration
+	valIdx := make([]byte, 5)
+
+	// fmt.Println("queries", len(queries))
+
+	for c := 0; ; c++ {
+		batchSizePerQuery = internal.BatchSizePerNumberOfQueries(limit, remainingUnexhausted)
+
+		// fmt.Println("  iteration", c, "remaining", remainingUnexhausted, "batchsize", batchSizePerQuery)
+		// we will go through all the iterators in batches until we have pulled all the required results
+		for q, query := range queries {
+			if exhausted[q] {
+				continue
+			}
+			if oldest.Q == q && remainingUnexhausted > 1 {
+				continue
+			}
+			// fmt.Println("    query", q, unsafe.Pointer(&results[q]), hex.EncodeToString(query.prefix), len(results[q]))
+
+			it := iterators[q]
+			pulledThisIteration := 0
+
+			for {
+				if !it.Valid() {
+					// fmt.Println("      reached end")
+					exhaust(q)
+					break
+				}
+
+				item := it.Item()
+				key := item.Key()
+
+				idxOffset := len(key) - 4 // this is where the idx actually starts
+
+				// "id" indexes don't contain a timestamp
+				if !query.skipTimestamp {
+					createdAt := binary.BigEndian.Uint32(key[idxOffset-4 : idxOffset])
+					if createdAt < since {
+						// fmt.Println("        reached since", createdAt, "<", since)
+						exhaust(q)
+						break
+					}
+				}
+
+				valIdx[0] = rawEventStorePrefix
+				copy(valIdx[1:], key[idxOffset:])
+
+				// fetch actual event
+				item, err := txn.Get(valIdx)
+				if err != nil {
+					if err == badger.ErrDiscardedTxn {
+						return nil, err
+					}
+					log.Printf("badger: failed to get %x based on prefix %x, index key %x from raw event store: %s\n",
+						valIdx, query.prefix, key, err)
+					return nil, err
+				}
+
+				if err := item.Value(func(val []byte) error {
+					// fmt.Println("      event", hex.EncodeToString(val[0:4]), "kind", binary.BigEndian.Uint16(val[132:134]), "author", hex.EncodeToString(val[32:36]), "ts", nostr.Timestamp(binary.BigEndian.Uint32(val[128:132])))
+
+					// check it against pubkeys without decoding the entire thing
+					if extraFilter != nil && extraFilter.Authors != nil &&
+						!slices.Contains(extraFilter.Authors, hex.EncodeToString(val[32:64])) {
+						// fmt.Println("        skipped (authors)")
+						return nil
+					}
+
+					// check it against kinds without decoding the entire thing
+					if extraFilter != nil && extraFilter.Kinds != nil &&
+						!slices.Contains(extraFilter.Kinds, int(binary.BigEndian.Uint16(val[132:134]))) {
+						// fmt.Println("        skipped (kinds)")
+						return nil
+					}
+
+					event := &nostr.Event{}
+					if err := bin.Unmarshal(val, event); err != nil {
+						log.Printf("badger: value read error (id %x): %s\n", val[0:32], err)
+						return err
+					}
+
+					// check if this matches the other filters that were not part of the index
+					if extraFilter != nil && !filterMatchesTags(extraFilter, event) {
+						// fmt.Println("        skipped (filter)", extraFilter, event)
+						return nil
+					}
+
+					// this event is good to be used
+					evt := internal.IterEvent{Event: event, Q: q}
+					//
+					//
+					if secondPhase {
+						// do the process described below at HIWAWVRTP.
+						// if we've reached here this means we've already passed the `since` check.
+						// now we have to eliminate the event currently at the `since` threshold.
+						nextThreshold := firstPhaseResults[len(firstPhaseResults)-2]
+						if oldest.Event == nil {
+							// fmt.Println("          b1")
+							// BRANCH WHEN WE DON'T HAVE THE OLDEST EVENT (BWWDHTOE)
+							// when we don't have the oldest set, we will keep the results
+							//   and not change the cutting point -- it's bad, but hopefully not that bad.
+							results[q] = append(results[q], evt)
+							secondPhaseHasResultsPending = true
+						} else if nextThreshold.CreatedAt > oldest.CreatedAt {
+							// fmt.Println("          b2", nextThreshold.CreatedAt, ">", oldest.CreatedAt)
+							// one of the events we have stored is the actual next threshold
+							// eliminate last, update since with oldest
+							firstPhaseResults = firstPhaseResults[0 : len(firstPhaseResults)-1]
+							since = uint32(oldest.CreatedAt)
+							// fmt.Println("            new since", since)
+							//  we null the oldest Event as we can't rely on it anymore
+							//   (we'll fall under BWWDHTOE above) until we have a new oldest set.
+							oldest = internal.IterEvent{Q: -1}
+							// anything we got that would be above this won't trigger an update to
+							//   the oldest anyway, because it will be discarded as being after the limit.
+							//
+							// finally
+							// add this to the results to be merged later
+							results[q] = append(results[q], evt)
+							secondPhaseHasResultsPending = true
+						} else if nextThreshold.CreatedAt < evt.CreatedAt {
+							// the next last event in the firstPhaseResults is the next threshold
+							// fmt.Println("          b3", nextThreshold.CreatedAt, "<", oldest.CreatedAt)
+							// eliminate last, update since with the antelast
+							firstPhaseResults = firstPhaseResults[0 : len(firstPhaseResults)-1]
+							since = uint32(nextThreshold.CreatedAt)
+							// fmt.Println("            new since", since)
+							// add this to the results to be merged later
+							results[q] = append(results[q], evt)
+							secondPhaseHasResultsPending = true
+							// update the oldest event
+							if evt.CreatedAt < oldest.CreatedAt {
+								oldest = evt
+							}
+						} else {
+							// fmt.Println("          b4")
+							// oops, _we_ are the next `since` threshold
+							firstPhaseResults[len(firstPhaseResults)-1] = evt
+							since = uint32(evt.CreatedAt)
+							// fmt.Println("            new since", since)
+							// do not add us to the results to be merged later
+							//   as we're already inhabiting the firstPhaseResults slice
+						}
+					} else {
+						results[q] = append(results[q], evt)
+						firstPhaseTotalPulled++
+
+						// update the oldest event
+						if oldest.Event == nil || evt.CreatedAt < oldest.CreatedAt {
+							oldest = evt
+						}
+					}
+
+					pulledPerQuery[q]++
+					pulledThisIteration++
+					if pulledThisIteration > batchSizePerQuery {
+						return batchFilled
+					}
+					if pulledPerQuery[q] >= limit {
+						exhaust(q)
+						return batchFilled
+					}
+
+					return nil
+				}); err == batchFilled {
+					// fmt.Println("      #")
+					it.Next()
+					break
+				} else if err != nil {
+					return nil, fmt.Errorf("iteration error: %w", err)
+				}
+
+				it.Next()
+			}
+		}
+
+		// we will do this check if we don't accumulated the requested number of events yet
+		// fmt.Println("oldest", oldest.Event, "from iter", oldest.Q)
+		if secondPhase && secondPhaseHasResultsPending && (oldest.Event == nil || remainingUnexhausted == 0) {
+			// fmt.Println("second phase aggregation!")
+			// when we are in the second phase we will aggressively aggregate results on every iteration
+			//
+			secondBatch = secondBatch[:0]
+			for s := 0; s < len(secondPhaseParticipants); s++ {
+				q := secondPhaseParticipants[s]
+
+				if len(results[q]) > 0 {
+					secondBatch = append(secondBatch, results[q])
+				}
+
+				if exhausted[q] {
+					secondPhaseParticipants = internal.SwapDelete(secondPhaseParticipants, s)
+					s--
+				}
+			}
+
+			// every time we get here we will alternate between these A and B lists
+			//   combining everything we have into a new partial results list.
+			// after we've done that we can again set the oldest.
+			// fmt.Println("  xxx", secondPhaseResultsToggle)
+			if secondPhaseResultsToggle {
+				secondBatch = append(secondBatch, secondPhaseResultsB)
+				secondPhaseResultsA = internal.MergeSortMultiple(secondBatch, limit, secondPhaseResultsA)
+				oldest = secondPhaseResultsA[len(secondPhaseResultsA)-1]
+				// fmt.Println("  new aggregated a", len(secondPhaseResultsB))
+			} else {
+				secondBatch = append(secondBatch, secondPhaseResultsA)
+				secondPhaseResultsB = internal.MergeSortMultiple(secondBatch, limit, secondPhaseResultsB)
+				oldest = secondPhaseResultsB[len(secondPhaseResultsB)-1]
+				// fmt.Println("  new aggregated b", len(secondPhaseResultsB))
+			}
+			secondPhaseResultsToggle = !secondPhaseResultsToggle
+
+			since = uint32(oldest.CreatedAt)
+			// fmt.Println("  new since", since)
+
+			// reset the `results` list so we can keep using it
+			results = results[:len(queries)]
+			for _, q := range secondPhaseParticipants {
+				results[q] = results[q][:0]
+			}
+		} else if !secondPhase && firstPhaseTotalPulled >= limit && remainingUnexhausted > 0 {
+			// fmt.Println("have enough!", firstPhaseTotalPulled, "/", limit, "remaining", remainingUnexhausted)
+
+			// we will exclude this oldest number as it is not relevant anymore
+			// (we now want to keep track only of the oldest among the remaining iterators)
+			oldest = internal.IterEvent{Q: -1}
+
+			// HOW IT WORKS AFTER WE'VE REACHED THIS POINT (HIWAWVRTP)
+			// now we can combine the results we have and check what is our current oldest event.
+			// we also discard anything that is after the current cutting point (`limit`).
+			// so if we have [1,2,3], [10, 15, 20] and [7, 21, 49] but we only want 6 total
+			//   we can just keep [1,2,3,7,10,15] and discard [20, 21, 49],
+			//   and also adjust our `since` parameter to `15`, discarding anything we get after it
+			//   and immediately declaring that iterator exhausted.
+			// also every time we get result that is more recent than this updated `since` we can
+			//   keep it but also discard the previous since, moving the needle one back -- for example,
+			//   if we get an `8` we can keep it and move the `since` parameter to `10`, discarding `15`
+			//   in the process.
+			all := make([][]internal.IterEvent, len(results))
+			copy(all, results) // we have to use this otherwise mergeSortMultiple will scramble our results slice
+			firstPhaseResults = internal.MergeSortMultiple(all, limit, nil)
+			oldest = firstPhaseResults[limit-1]
+			since = uint32(oldest.CreatedAt)
+			// fmt.Println("new since", since)
+
+			for q := range queries {
+				if exhausted[q] {
+					continue
+				}
+
+				// we also automatically exhaust any of the iterators that have already passed the
+				// cutting point (`since`)
+				if results[q][len(results[q])-1].CreatedAt < oldest.CreatedAt {
+					exhausted[q] = true
+					remainingUnexhausted--
+					continue
+				}
+
+				// for all the remaining iterators,
+				// since we have merged all the events in this `firstPhaseResults` slice, we can empty the
+				//   current `results` slices and reuse them.
+				results[q] = results[q][:0]
+
+				// build this index of indexes with everybody who remains
+				secondPhaseParticipants = append(secondPhaseParticipants, q)
+			}
+
+			// we create these two lists and alternate between them so we don't have to create a
+			//   a new one every time
+			secondPhaseResultsA = make([]internal.IterEvent, 0, limit*2)
+			secondPhaseResultsB = make([]internal.IterEvent, 0, limit*2)
+
+			// from now on we won't run this block anymore
+			secondPhase = true
+		}
+
+		// fmt.Println("remaining", remainingUnexhausted)
+		if remainingUnexhausted == 0 {
+			break
+		}
+	}
+
+	// fmt.Println("is secondPhase?", secondPhase)
+
+	var combinedResults []internal.IterEvent
+
+	if secondPhase {
+		// fmt.Println("ending second phase")
+		// when we reach this point either secondPhaseResultsA or secondPhaseResultsB will be full of stuff,
+		//   the other will be empty
+		var secondPhaseResults []internal.IterEvent
+		// fmt.Println("xxx", secondPhaseResultsToggle, len(secondPhaseResultsA), len(secondPhaseResultsB))
+		if secondPhaseResultsToggle {
+			secondPhaseResults = secondPhaseResultsB
+			combinedResults = secondPhaseResultsA[0:limit] // reuse this
+			// fmt.Println("  using b", len(secondPhaseResultsA))
+		} else {
+			secondPhaseResults = secondPhaseResultsA
+			combinedResults = secondPhaseResultsB[0:limit] // reuse this
+			// fmt.Println("  using a", len(secondPhaseResultsA))
+		}
+
+		all := [][]internal.IterEvent{firstPhaseResults, secondPhaseResults}
+		combinedResults = internal.MergeSortMultiple(all, limit, combinedResults)
+		// fmt.Println("final combinedResults", len(combinedResults), cap(combinedResults), limit)
+	} else {
+		combinedResults = make([]internal.IterEvent, limit)
+		combinedResults = internal.MergeSortMultiple(results, limit, combinedResults)
+	}
+
+	return combinedResults, nil
+}

eventstore/badger/query_planner.go

@@ -0,0 +1,147 @@
+package badger
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/fiatjaf/eventstore/internal"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+type query struct {
+	i             int
+	prefix        []byte
+	startingPoint []byte
+	skipTimestamp bool
+}
+
+func prepareQueries(filter nostr.Filter) (
+	queries []query,
+	extraFilter *nostr.Filter,
+	since uint32,
+	err error,
+) {
+	// these things have to run for every result we return
+	defer func() {
+		if queries == nil {
+			return
+		}
+
+		var until uint32 = 4294967295
+		if filter.Until != nil {
+			if fu := uint32(*filter.Until); fu < until {
+				until = fu + 1
+			}
+		}
+
+		for i, q := range queries {
+			queries[i].startingPoint = binary.BigEndian.AppendUint32(q.prefix, uint32(until))
+		}
+
+		// this is where we'll end the iteration
+		if filter.Since != nil {
+			if fs := uint32(*filter.Since); fs > since {
+				since = fs
+			}
+		}
+	}()
+
+	var index byte
+
+	if len(filter.IDs) > 0 {
+		queries = make([]query, len(filter.IDs))
+		for i, idHex := range filter.IDs {
+			prefix := make([]byte, 1+8)
+			prefix[0] = indexIdPrefix
+			if len(idHex) != 64 {
+				return nil, nil, 0, fmt.Errorf("invalid id '%s'", idHex)
+			}
+			hex.Decode(prefix[1:], []byte(idHex[0:8*2]))
+			queries[i] = query{i: i, prefix: prefix, skipTimestamp: true}
+		}
+
+		return queries, extraFilter, since, nil
+	}
+
+	if len(filter.Tags) > 0 {
+		// we will select ONE tag to query with
+		tagKey, tagValues, goodness := internal.ChooseNarrowestTag(filter)
+
+		// we won't use a tag index for this as long as we have something else to match with
+		if goodness < 3 && (len(filter.Authors) > 0 || len(filter.Kinds) > 0) {
+			goto pubkeyMatching
+		}
+
+		queries = make([]query, len(tagValues))
+		for i, value := range tagValues {
+			// get key prefix (with full length) and offset where to write the created_at
+			k, offset := getTagIndexPrefix(value)
+			// remove the last parts part to get just the prefix we want here
+			prefix := k[0:offset]
+			queries[i] = query{i: i, prefix: prefix}
+			i++
+		}
+
+		extraFilter = &nostr.Filter{
+			Kinds:   filter.Kinds,
+			Authors: filter.Authors,
+			Tags:    internal.CopyMapWithoutKey(filter.Tags, tagKey),
+		}
+
+		return queries, extraFilter, since, nil
+	}
+
+pubkeyMatching:
+	if len(filter.Authors) > 0 {
+		if len(filter.Kinds) == 0 {
+			queries = make([]query, len(filter.Authors))
+			for i, pubkeyHex := range filter.Authors {
+				if len(pubkeyHex) != 64 {
+					return nil, nil, 0, fmt.Errorf("invalid pubkey '%s'", pubkeyHex)
+				}
+				prefix := make([]byte, 1+8)
+				prefix[0] = indexPubkeyPrefix
+				hex.Decode(prefix[1:], []byte(pubkeyHex[0:8*2]))
+				queries[i] = query{i: i, prefix: prefix}
+			}
+		} else {
+			queries = make([]query, len(filter.Authors)*len(filter.Kinds))
+			i := 0
+			for _, pubkeyHex := range filter.Authors {
+				for _, kind := range filter.Kinds {
+					if len(pubkeyHex) != 64 {
+						return nil, nil, 0, fmt.Errorf("invalid pubkey '%s'", pubkeyHex)
+					}
+
+					prefix := make([]byte, 1+8+2)
+					prefix[0] = indexPubkeyKindPrefix
+					hex.Decode(prefix[1:], []byte(pubkeyHex[0:8*2]))
+					binary.BigEndian.PutUint16(prefix[1+8:], uint16(kind))
+					queries[i] = query{i: i, prefix: prefix}
+					i++
+				}
+			}
+		}
+		extraFilter = &nostr.Filter{Tags: filter.Tags}
+	} else if len(filter.Kinds) > 0 {
+		index = indexKindPrefix
+		queries = make([]query, len(filter.Kinds))
+		for i, kind := range filter.Kinds {
+			prefix := make([]byte, 1+2)
+			prefix[0] = index
+			binary.BigEndian.PutUint16(prefix[1:], uint16(kind))
+			queries[i] = query{i: i, prefix: prefix}
+		}
+		extraFilter = &nostr.Filter{Tags: filter.Tags}
+	} else {
+		index = indexCreatedAtPrefix
+		queries = make([]query, 1)
+		prefix := make([]byte, 1)
+		prefix[0] = index
+		queries[0] = query{i: 0, prefix: prefix}
+		extraFilter = nil
+	}
+
+	return queries, extraFilter, since, nil
+}

eventstore/badger/replace.go

@@ -0,0 +1,49 @@
+package badger
+
+import (
+	"context"
+	"fmt"
+	"math"
+
+	"github.com/dgraph-io/badger/v4"
+	"github.com/fiatjaf/eventstore/internal"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *BadgerBackend) ReplaceEvent(ctx context.Context, evt *nostr.Event) error {
+	// sanity checking
+	if evt.CreatedAt > math.MaxUint32 || evt.Kind > math.MaxUint16 {
+		return fmt.Errorf("event with values out of expected boundaries")
+	}
+
+	return b.Update(func(txn *badger.Txn) error {
+		filter := nostr.Filter{Limit: 1, Kinds: []int{evt.Kind}, Authors: []string{evt.PubKey}}
+		if nostr.IsAddressableKind(evt.Kind) {
+			// when addressable, add the "d" tag to the filter
+			filter.Tags = nostr.TagMap{"d": []string{evt.Tags.GetD()}}
+		}
+
+		// now we fetch the past events, whatever they are, delete them and then save the new
+		results, err := b.query(txn, filter, 10) // in theory limit could be just 1 and this should work
+		if err != nil {
+			return fmt.Errorf("failed to query past events with %s: %w", filter, err)
+		}
+
+		shouldStore := true
+		for _, previous := range results {
+			if internal.IsOlder(previous.Event, evt) {
+				if _, err := b.delete(txn, previous.Event); err != nil {
+					return fmt.Errorf("failed to delete event %s for replacing: %w", previous.Event.ID, err)
+				}
+			} else {
+				// there is a newer event already stored, so we won't store this
+				shouldStore = false
+			}
+		}
+		if shouldStore {
+			return b.save(txn, evt)
+		}
+
+		return nil
+	})
+}

eventstore/badger/save.go

@@ -0,0 +1,59 @@
+package badger
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"math"
+
+	"github.com/dgraph-io/badger/v4"
+	"github.com/fiatjaf/eventstore"
+	bin "github.com/fiatjaf/eventstore/internal/binary"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *BadgerBackend) SaveEvent(ctx context.Context, evt *nostr.Event) error {
+	// sanity checking
+	if evt.CreatedAt > math.MaxUint32 || evt.Kind > math.MaxUint16 {
+		return fmt.Errorf("event with values out of expected boundaries")
+	}
+
+	return b.Update(func(txn *badger.Txn) error {
+		// query event by id to ensure we don't save duplicates
+		id, _ := hex.DecodeString(evt.ID)
+		prefix := make([]byte, 1+8)
+		prefix[0] = indexIdPrefix
+		copy(prefix[1:], id)
+		it := txn.NewIterator(badger.IteratorOptions{})
+		defer it.Close()
+		it.Seek(prefix)
+		if it.ValidForPrefix(prefix) {
+			// event exists
+			return eventstore.ErrDupEvent
+		}
+
+		return b.save(txn, evt)
+	})
+}
+
+func (b *BadgerBackend) save(txn *badger.Txn, evt *nostr.Event) error {
+	// encode to binary
+	bin, err := bin.Marshal(evt)
+	if err != nil {
+		return err
+	}
+
+	idx := b.Serial()
+	// raw event store
+	if err := txn.Set(idx, bin); err != nil {
+		return err
+	}
+
+	for k := range b.getIndexKeysForEvent(evt, idx[1:]) {
+		if err := txn.Set(k, nil); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

eventstore/badger/testdata/fuzz/FuzzQuery

@@ -0,0 +1 @@
+../../../internal/testdata/fuzz/FuzzQuery

eventstore/bluge/bluge_test.go

@@ -0,0 +1,81 @@
+package bluge
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/fiatjaf/eventstore/badger"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBlugeFlow(t *testing.T) {
+	os.RemoveAll("/tmp/blugetest-badger")
+	os.RemoveAll("/tmp/blugetest-bluge")
+
+	bb := &badger.BadgerBackend{Path: "/tmp/blugetest-badger"}
+	bb.Init()
+	defer bb.Close()
+
+	bl := BlugeBackend{
+		Path:          "/tmp/blugetest-bluge",
+		RawEventStore: bb,
+	}
+	bl.Init()
+	defer bl.Close()
+
+	ctx := context.Background()
+
+	willDelete := make([]*nostr.Event, 0, 3)
+
+	for i, content := range []string{
+		"good morning mr paper maker",
+		"good night",
+		"I'll see you again in the paper house",
+		"tonight we dine in my house",
+		"the paper in this house if very good, mr",
+	} {
+		evt := &nostr.Event{Content: content, Tags: nostr.Tags{}}
+		evt.Sign("0000000000000000000000000000000000000000000000000000000000000001")
+
+		bb.SaveEvent(ctx, evt)
+		bl.SaveEvent(ctx, evt)
+
+		if i%2 == 0 {
+			willDelete = append(willDelete, evt)
+		}
+	}
+
+	{
+		ch, err := bl.QueryEvents(ctx, nostr.Filter{Search: "good"})
+		if err != nil {
+			t.Fatalf("QueryEvents error: %s", err)
+			return
+		}
+		n := 0
+		for range ch {
+			n++
+		}
+		assert.Equal(t, 3, n)
+	}
+
+	for _, evt := range willDelete {
+		bl.DeleteEvent(ctx, evt)
+	}
+
+	{
+		ch, err := bl.QueryEvents(ctx, nostr.Filter{Search: "good"})
+		if err != nil {
+			t.Fatalf("QueryEvents error: %s", err)
+			return
+		}
+		n := 0
+		for res := range ch {
+			n++
+			assert.Equal(t, res.Content, "good night")
+			assert.Equal(t, res.PubKey, "79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798")
+		}
+		assert.Equal(t, 1, n)
+	}
+}

eventstore/bluge/delete.go

@@ -0,0 +1,11 @@
+package bluge
+
+import (
+	"context"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *BlugeBackend) DeleteEvent(ctx context.Context, evt *nostr.Event) error {
+	return b.writer.Delete(eventIdentifier(evt.ID))
+}

eventstore/bluge/helpers.go

@@ -0,0 +1,23 @@
+package bluge
+
+import "encoding/hex"
+
+const (
+	contentField   = "c"
+	kindField      = "k"
+	createdAtField = "a"
+	pubkeyField    = "p"
+)
+
+type eventIdentifier string
+
+const idField = "i"
+
+func (id eventIdentifier) Field() string {
+	return idField
+}
+
+func (id eventIdentifier) Term() []byte {
+	v, _ := hex.DecodeString(string(id))
+	return v
+}

eventstore/bluge/lib.go

@@ -0,0 +1,52 @@
+package bluge
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/blugelabs/bluge"
+	"github.com/blugelabs/bluge/analysis/token"
+	"github.com/fiatjaf/eventstore"
+	"golang.org/x/text/unicode/norm"
+)
+
+var _ eventstore.Store = (*BlugeBackend)(nil)
+
+type BlugeBackend struct {
+	sync.Mutex
+	// Path is where the index will be saved
+	Path string
+
+	// RawEventStore is where we'll fetch the raw events from
+	// bluge will only store ids, so the actual events must be somewhere else
+	RawEventStore eventstore.Store
+
+	searchConfig bluge.Config
+	writer       *bluge.Writer
+}
+
+func (b *BlugeBackend) Close() {
+	defer b.writer.Close()
+}
+
+func (b *BlugeBackend) Init() error {
+	if b.Path == "" {
+		return fmt.Errorf("missing Path")
+	}
+	if b.RawEventStore == nil {
+		return fmt.Errorf("missing RawEventStore")
+	}
+
+	b.searchConfig = bluge.DefaultConfig(b.Path)
+	b.searchConfig.DefaultSearchAnalyzer.TokenFilters = append(b.searchConfig.DefaultSearchAnalyzer.TokenFilters,
+		token.NewUnicodeNormalizeFilter(norm.NFKC),
+	)
+
+	var err error
+	b.writer, err = bluge.OpenWriter(b.searchConfig)
+	if err != nil {
+		return fmt.Errorf("error opening writer: %w", err)
+	}
+
+	return nil
+}

eventstore/bluge/query.go

@@ -0,0 +1,117 @@
+package bluge
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"strconv"
+
+	"github.com/blugelabs/bluge"
+	"github.com/blugelabs/bluge/search"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *BlugeBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (chan *nostr.Event, error) {
+	ch := make(chan *nostr.Event)
+
+	if len(filter.Search) < 2 {
+		close(ch)
+		return ch, nil
+	}
+
+	reader, err := b.writer.Reader()
+	if err != nil {
+		close(ch)
+		return nil, fmt.Errorf("unable to open reader: %w", err)
+	}
+
+	searchQ := bluge.NewMatchQuery(filter.Search)
+	searchQ.SetField(contentField)
+	var q bluge.Query = searchQ
+
+	complicatedQuery := bluge.NewBooleanQuery().AddMust(searchQ)
+
+	if len(filter.Kinds) > 0 {
+		eitherKind := bluge.NewBooleanQuery()
+		eitherKind.SetMinShould(1)
+		for _, kind := range filter.Kinds {
+			kindQ := bluge.NewTermQuery(strconv.Itoa(kind))
+			kindQ.SetField(kindField)
+			eitherKind.AddShould(kindQ)
+		}
+		complicatedQuery.AddMust(eitherKind)
+		q = complicatedQuery
+	}
+
+	if len(filter.Authors) > 0 {
+		eitherPubkey := bluge.NewBooleanQuery()
+		eitherPubkey.SetMinShould(1)
+		for _, pubkey := range filter.Authors {
+			if len(pubkey) != 64 {
+				continue
+			}
+			pubkeyQ := bluge.NewTermQuery(pubkey[56:])
+			pubkeyQ.SetField(pubkeyField)
+			eitherPubkey.AddShould(pubkeyQ)
+		}
+		complicatedQuery.AddMust(eitherPubkey)
+		q = complicatedQuery
+	}
+
+	if filter.Since != nil || filter.Until != nil {
+		min := 0.0
+		if filter.Since != nil {
+			min = float64(*filter.Since)
+		}
+		max := float64(nostr.Now())
+		if filter.Until != nil {
+			max = float64(*filter.Until)
+		}
+		dateRangeQ := bluge.NewNumericRangeInclusiveQuery(min, max, true, true)
+		dateRangeQ.SetField(createdAtField)
+		complicatedQuery.AddMust(dateRangeQ)
+		q = complicatedQuery
+	}
+
+	limit := 40
+	if filter.Limit != 0 {
+		limit = filter.Limit
+		if filter.Limit > 150 {
+			limit = 150
+		}
+	}
+
+	req := bluge.NewTopNSearch(limit, q)
+
+	dmi, err := reader.Search(context.Background(), req)
+	if err != nil {
+		close(ch)
+		reader.Close()
+		return ch, fmt.Errorf("error executing search: %w", err)
+	}
+
+	go func() {
+		defer reader.Close()
+		defer close(ch)
+
+		var next *search.DocumentMatch
+		for next, err = dmi.Next(); next != nil; next, err = dmi.Next() {
+			next.VisitStoredFields(func(field string, value []byte) bool {
+				id := hex.EncodeToString(value)
+				rawch, err := b.RawEventStore.QueryEvents(ctx, nostr.Filter{IDs: []string{id}})
+				if err != nil {
+					return false
+				}
+				for evt := range rawch {
+					ch <- evt
+				}
+				return false
+			})
+		}
+		if err != nil {
+			return
+		}
+	}()
+
+	return ch, nil
+}

eventstore/bluge/replace.go

@@ -0,0 +1,44 @@
+package bluge
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/fiatjaf/eventstore/internal"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *BlugeBackend) ReplaceEvent(ctx context.Context, evt *nostr.Event) error {
+	b.Lock()
+	defer b.Unlock()
+
+	filter := nostr.Filter{Limit: 1, Kinds: []int{evt.Kind}, Authors: []string{evt.PubKey}}
+	if nostr.IsAddressableKind(evt.Kind) {
+		filter.Tags = nostr.TagMap{"d": []string{evt.Tags.GetD()}}
+	}
+
+	ch, err := b.QueryEvents(ctx, filter)
+	if err != nil {
+		return fmt.Errorf("failed to query before replacing: %w", err)
+	}
+
+	shouldStore := true
+	for previous := range ch {
+		if internal.IsOlder(previous, evt) {
+			if err := b.DeleteEvent(ctx, previous); err != nil {
+				return fmt.Errorf("failed to delete event for replacing: %w", err)
+			}
+		} else {
+			shouldStore = false
+		}
+	}
+
+	if shouldStore {
+		if err := b.SaveEvent(ctx, evt); err != nil && err != eventstore.ErrDupEvent {
+			return fmt.Errorf("failed to save: %w", err)
+		}
+	}
+
+	return nil
+}

eventstore/bluge/save.go

@@ -0,0 +1,28 @@
+package bluge
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"github.com/blugelabs/bluge"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *BlugeBackend) SaveEvent(ctx context.Context, evt *nostr.Event) error {
+	id := eventIdentifier(evt.ID)
+	doc := &bluge.Document{
+		bluge.NewKeywordFieldBytes(id.Field(), id.Term()).Sortable().StoreValue(),
+	}
+
+	doc.AddField(bluge.NewTextField(contentField, evt.Content))
+	doc.AddField(bluge.NewTextField(kindField, strconv.Itoa(evt.Kind)))
+	doc.AddField(bluge.NewTextField(pubkeyField, evt.PubKey[56:]))
+	doc.AddField(bluge.NewNumericField(createdAtField, float64(evt.CreatedAt)))
+
+	if err := b.writer.Update(doc.ID(), doc); err != nil {
+		return fmt.Errorf("failed to write '%s' document: %w", evt.ID, err)
+	}
+
+	return nil
+}

eventstore/cmd/eventstore/.gitignore

@@ -0,0 +1 @@
+eventstore

eventstore/cmd/eventstore/README.md

@@ -0,0 +1,39 @@
+# eventstore command-line tool
+
+```
+go install github.com/fiatjaf/eventstore/cmd/eventstore@latest
+```
+
+## Usage
+
+This should be pretty straightforward. You pipe events or filters, as JSON, to the `eventstore` command, and they yield something. You can use [nak](https://github.com/fiatjaf/nak) to generate these events or filters easily.
+
+### Querying the last 100 events of kind 1
+
+```fish
+~> nak req -k 1 -l 100 --bare | eventstore -d /path/to/store query
+~> # or
+~> echo '{"kinds":[1],"limit":100}' | eventstore -d /path/to/store query
+```
+
+This will automatically determine the storage type being used at `/path/to/store`, but you can also specify it manually using the `-t` option (`-t lmdb`, `-t sqlite` etc).
+
+### Saving an event to the store
+
+```fish
+~> nak event -k 1 -c hello | eventstore -d /path/to/store save
+~> # or
+~> echo '{"id":"35369e6bae5f77c4e1745c2eb5db84c4493e87f6e449aee62a261bbc1fea2788","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","created_at":1701193836,"kind":1,"tags":[],"content":"hello","sig":"ef08d559e042d9af4cdc3328a064f737603d86ec4f929f193d5a3ce9ea22a3fb8afc1923ee3c3742fd01856065352c5632e91f633528c80e9c5711fa1266824c"}' | eventstore -d /path/to/store save
+```
+
+You can also create a database from scratch if it's a disk database, but then you have to specify `-t` to `sqlite`, `badger` or `lmdb`.
+
+### Connecting to Postgres, MySQL and other remote databases
+
+You should be able to connect by just passing the database connection URI to `-d`:
+
+```bash
+~> eventstore -d 'postgres://myrelay:38yg4o83yf48a3s7g@localhost:5432/myrelay?sslmode=disable' <query|save|delete>
+```
+
+That should be prefixed with `postgres://` for Postgres, `mysql://` for MySQL and `https://` for ElasticSearch.

eventstore/cmd/eventstore/delete.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/urfave/cli/v3"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+var delete_ = &cli.Command{
+	Name:        "delete",
+	ArgsUsage:   "[<id>]",
+	Usage:       "deletes an event by id and all its associated index entries",
+	Description: "takes an id either as an argument or reads a stream of ids from stdin and deletes them from the currently open eventstore.",
+	Action: func(ctx context.Context, c *cli.Command) error {
+		hasError := false
+		for line := range getStdinLinesOrFirstArgument(c) {
+			f := nostr.Filter{IDs: []string{line}}
+			ch, err := db.QueryEvents(ctx, f)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "error querying for %s: %s\n", f, err)
+				hasError = true
+			}
+			for evt := range ch {
+				if err := db.DeleteEvent(ctx, evt); err != nil {
+					fmt.Fprintf(os.Stderr, "error deleting %s: %s\n", evt, err)
+					hasError = true
+				}
+			}
+		}
+
+		if hasError {
+			os.Exit(123)
+		}
+		return nil
+	},
+}

eventstore/cmd/eventstore/helpers.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/urfave/cli/v3"
+)
+
+const (
+	LINE_PROCESSING_ERROR = iota
+)
+
+func detect(dir string) (string, error) {
+	mayBeMMM := false
+	if n := strings.Index(dir, "/"); n > 0 {
+		mayBeMMM = true
+		dir = filepath.Dir(dir)
+	}
+
+	f, err := os.Stat(dir)
+	if err != nil {
+		return "", err
+	}
+	if !f.IsDir() {
+		f, err := os.Open(dir)
+		if err != nil {
+			return "", err
+		}
+		buf := make([]byte, 15)
+		f.Read(buf)
+		if string(buf) == "SQLite format 3" {
+			return "sqlite", nil
+		}
+
+		return "", fmt.Errorf("unknown db format")
+	}
+
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return "", err
+	}
+
+	if mayBeMMM {
+		for _, entry := range entries {
+			if entry.Name() == "mmmm" {
+				if entries, err := os.ReadDir(filepath.Join(dir, "mmmm")); err == nil {
+					for _, e := range entries {
+						if strings.HasSuffix(e.Name(), ".mdb") {
+							return "mmm", nil
+						}
+					}
+				}
+			}
+		}
+	}
+	for _, entry := range entries {
+		if strings.HasSuffix(entry.Name(), ".mdb") {
+			return "lmdb", nil
+		}
+		if strings.HasSuffix(entry.Name(), ".vlog") {
+			return "badger", nil
+		}
+	}
+
+	return "", fmt.Errorf("undetected")
+}
+
+func getStdin() string {
+	stat, _ := os.Stdin.Stat()
+	if (stat.Mode() & os.ModeCharDevice) == 0 {
+		read := bytes.NewBuffer(make([]byte, 0, 1000))
+		_, err := io.Copy(read, os.Stdin)
+		if err == nil {
+			return read.String()
+		}
+	}
+	return ""
+}
+
+func isPiped() bool {
+	stat, _ := os.Stdin.Stat()
+	return stat.Mode()&os.ModeCharDevice == 0
+}
+
+func getStdinLinesOrFirstArgument(c *cli.Command) chan string {
+	// try the first argument
+	target := c.Args().First()
+	if target != "" {
+		single := make(chan string, 1)
+		single <- target
+		close(single)
+		return single
+	}
+
+	// try the stdin
+	multi := make(chan string)
+	writeStdinLinesOrNothing(multi)
+	return multi
+}
+
+func getStdinLinesOrBlank() chan string {
+	multi := make(chan string)
+	if hasStdinLines := writeStdinLinesOrNothing(multi); !hasStdinLines {
+		single := make(chan string, 1)
+		single <- ""
+		close(single)
+		return single
+	} else {
+		return multi
+	}
+}
+
+func writeStdinLinesOrNothing(ch chan string) (hasStdinLines bool) {
+	if isPiped() {
+		// piped
+		go func() {
+			scanner := bufio.NewScanner(os.Stdin)
+			for scanner.Scan() {
+				ch <- strings.TrimSpace(scanner.Text())
+			}
+			close(ch)
+		}()
+		return true
+	} else {
+		// not piped
+		return false
+	}
+}

eventstore/cmd/eventstore/main.go

@@ -0,0 +1,168 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/fiatjaf/eventstore/badger"
+	"github.com/fiatjaf/eventstore/elasticsearch"
+	"github.com/fiatjaf/eventstore/lmdb"
+	"github.com/fiatjaf/eventstore/mysql"
+	"github.com/fiatjaf/eventstore/postgresql"
+	"github.com/fiatjaf/eventstore/slicestore"
+	"github.com/fiatjaf/eventstore/sqlite3"
+	"github.com/fiatjaf/eventstore/strfry"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/urfave/cli/v3"
+)
+
+var db eventstore.Store
+
+var app = &cli.Command{
+	Name:      "eventstore",
+	Usage:     "a CLI for all the eventstore backends",
+	UsageText: "eventstore -d ./data/sqlite <query|save|delete> ...",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:     "store",
+			Aliases:  []string{"d"},
+			Usage:    "path to the database file or directory or database connection uri",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:    "type",
+			Aliases: []string{"t"},
+			Usage:   "store type ('sqlite', 'lmdb', 'badger', 'postgres', 'mysql', 'elasticsearch', 'mmm')",
+		},
+	},
+	Before: func(ctx context.Context, c *cli.Command) (context.Context, error) {
+		path := strings.Trim(c.String("store"), "/")
+		typ := c.String("type")
+		if typ != "" {
+			// bypass automatic detection
+			// this also works for creating disk databases from scratch
+		} else {
+			// try to detect based on url scheme
+			switch {
+			case strings.HasPrefix(path, "postgres://"), strings.HasPrefix(path, "postgresql://"):
+				typ = "postgres"
+			case strings.HasPrefix(path, "mysql://"):
+				typ = "mysql"
+			case strings.HasPrefix(path, "https://"):
+				// if we ever add something else that uses URLs we'll have to modify this
+				typ = "elasticsearch"
+			case strings.HasSuffix(path, ".conf"):
+				typ = "strfry"
+			case strings.HasSuffix(path, ".jsonl"):
+				typ = "file"
+			default:
+				// try to detect based on the form and names of disk files
+				dbname, err := detect(path)
+				if err != nil {
+					if os.IsNotExist(err) {
+						return ctx, fmt.Errorf(
+							"'%s' does not exist, to create a store there specify the --type argument", path)
+					}
+					return ctx, fmt.Errorf("failed to detect store type: %w", err)
+				}
+				typ = dbname
+			}
+		}
+
+		switch typ {
+		case "sqlite":
+			db = &sqlite3.SQLite3Backend{
+				DatabaseURL:       path,
+				QueryLimit:        1_000_000,
+				QueryAuthorsLimit: 1_000_000,
+				QueryKindsLimit:   1_000_000,
+				QueryIDsLimit:     1_000_000,
+				QueryTagsLimit:    1_000_000,
+			}
+		case "lmdb":
+			db = &lmdb.LMDBBackend{Path: path, MaxLimit: 1_000_000}
+		case "badger":
+			db = &badger.BadgerBackend{Path: path, MaxLimit: 1_000_000}
+		case "mmm":
+			var err error
+			if db, err = doMmmInit(path); err != nil {
+				return ctx, err
+			}
+		case "postgres", "postgresql":
+			db = &postgresql.PostgresBackend{
+				DatabaseURL:       path,
+				QueryLimit:        1_000_000,
+				QueryAuthorsLimit: 1_000_000,
+				QueryKindsLimit:   1_000_000,
+				QueryIDsLimit:     1_000_000,
+				QueryTagsLimit:    1_000_000,
+			}
+		case "mysql":
+			db = &mysql.MySQLBackend{
+				DatabaseURL:       path,
+				QueryLimit:        1_000_000,
+				QueryAuthorsLimit: 1_000_000,
+				QueryKindsLimit:   1_000_000,
+				QueryIDsLimit:     1_000_000,
+				QueryTagsLimit:    1_000_000,
+			}
+		case "elasticsearch":
+			db = &elasticsearch.ElasticsearchStorage{URL: path}
+		case "strfry":
+			db = &strfry.StrfryBackend{ConfigPath: path}
+		case "file":
+			db = &slicestore.SliceStore{}
+
+			// run this after we've called db.Init()
+			defer func() {
+				f, err := os.Open(path)
+				if err != nil {
+					log.Printf("failed to file at '%s': %s\n", path, err)
+					os.Exit(3)
+				}
+				scanner := bufio.NewScanner(f)
+				scanner.Buffer(make([]byte, 16*1024*1024), 256*1024*1024)
+				i := 0
+				for scanner.Scan() {
+					var evt nostr.Event
+					if err := json.Unmarshal(scanner.Bytes(), &evt); err != nil {
+						log.Printf("invalid event read at line %d: %s (`%s`)\n", i, err, scanner.Text())
+					}
+					db.SaveEvent(ctx, &evt)
+					i++
+				}
+			}()
+		case "":
+			return ctx, fmt.Errorf("couldn't determine store type, you can use --type to specify it manually")
+		default:
+			return ctx, fmt.Errorf("'%s' store type is not supported by this CLI", typ)
+		}
+
+		if err := db.Init(); err != nil {
+			return ctx, err
+		}
+
+		return ctx, nil
+	},
+	Commands: []*cli.Command{
+		queryOrSave,
+		query,
+		save,
+		delete_,
+		neg,
+	},
+	DefaultCommand: "query-or-save",
+}
+
+func main() {
+	if err := app.Run(context.Background(), os.Args); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}

eventstore/cmd/eventstore/main_mmm.go

@@ -0,0 +1,34 @@
+//go:build !windows
+
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/fiatjaf/eventstore/mmm"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/rs/zerolog"
+)
+
+func doMmmInit(path string) (eventstore.Store, error) {
+	logger := zerolog.New(zerolog.NewConsoleWriter(func(w *zerolog.ConsoleWriter) {
+		w.Out = os.Stderr
+	}))
+	mmmm := mmm.MultiMmapManager{
+		Dir:    filepath.Dir(path),
+		Logger: &logger,
+	}
+	if err := mmmm.Init(); err != nil {
+		return nil, err
+	}
+	il := &mmm.IndexingLayer{
+		ShouldIndex: func(ctx context.Context, e *nostr.Event) bool { return false },
+	}
+	if err := mmmm.EnsureLayer(filepath.Base(path), il); err != nil {
+		return nil, err
+	}
+	return il, nil
+}

eventstore/cmd/eventstore/main_other.go

@@ -0,0 +1,14 @@
+//go:build windows
+
+package main
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/fiatjaf/eventstore"
+)
+
+func doMmmInit(path string) (eventstore.Store, error) {
+	return nil, fmt.Errorf("unsupported OSs (%v)", runtime.GOOS)
+}

eventstore/cmd/eventstore/neg.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"os"
+	"sync"
+
+	"github.com/urfave/cli/v3"
+	"github.com/mailru/easyjson"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/nbd-wtf/go-nostr/nip77/negentropy"
+	"github.com/nbd-wtf/go-nostr/nip77/negentropy/storage/vector"
+)
+
+var neg = &cli.Command{
+	Name:        "neg",
+	ArgsUsage:   "<filter-json> [<negentropy-message-hex>]",
+	Usage:       "initiates a negentropy session with a filter or reconciles a received negentropy message",
+	Description: "applies the filter to the currently open eventstore. if no negentropy message was given it will initiate the process and emit one, if one was given either as an argument or via stdin, it will be reconciled against the current eventstore.\nthe next reconciliation message will be emitted on stdout.\na stream of need/have ids (or nothing) will be emitted to stderr.",
+	Flags: []cli.Flag{
+		&cli.UintFlag{
+			Name: "frame-size-limit",
+		},
+	},
+	Action: func(ctx context.Context, c *cli.Command) error {
+		jfilter := c.Args().First()
+		if jfilter == "" {
+			return fmt.Errorf("missing filter argument")
+		}
+
+		filter := nostr.Filter{}
+		if err := easyjson.Unmarshal([]byte(jfilter), &filter); err != nil {
+			return fmt.Errorf("invalid filter %s: %s\n", jfilter, err)
+		}
+
+		frameSizeLimit := int(c.Uint("frame-size-limit"))
+		if frameSizeLimit == 0 {
+			frameSizeLimit = math.MaxInt
+		}
+
+		// create negentropy object and initialize it with events
+		vec := vector.New()
+		neg := negentropy.New(vec, frameSizeLimit)
+		ch, err := db.QueryEvents(ctx, filter)
+		if err != nil {
+			return fmt.Errorf("error querying: %s\n", err)
+		}
+		for evt := range ch {
+			vec.Insert(evt.CreatedAt, evt.ID)
+		}
+
+		wg := sync.WaitGroup{}
+		go func() {
+			defer wg.Done()
+			for item := range neg.Haves {
+				fmt.Fprintf(os.Stderr, "have %s", item)
+			}
+		}()
+		go func() {
+			defer wg.Done()
+			for item := range neg.HaveNots {
+				fmt.Fprintf(os.Stderr, "need %s", item)
+			}
+		}()
+
+		// get negentropy message from argument or stdin pipe
+		var msg string
+		if isPiped() {
+			data, err := io.ReadAll(os.Stdin)
+			if err != nil {
+				return fmt.Errorf("failed to read from stdin: %w", err)
+			}
+			msg = string(data)
+		} else {
+			msg = c.Args().Get(1)
+		}
+
+		if msg == "" {
+			// initiate the process
+			out := neg.Start()
+			fmt.Println(out)
+		} else {
+			// process the message
+			out, err := neg.Reconcile(msg)
+			if err != nil {
+				return fmt.Errorf("negentropy failed: %s", err)
+			}
+			fmt.Println(out)
+		}
+
+		wg.Wait()
+		return nil
+	},
+}

eventstore/cmd/eventstore/query-or-save.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/urfave/cli/v3"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+// this is the default command when no subcommands are given, we will just try everything
+var queryOrSave = &cli.Command{
+	Hidden: true,
+	Name:   "query-or-save",
+	Action: func(ctx context.Context, c *cli.Command) error {
+		line := getStdin()
+
+		ee := &nostr.EventEnvelope{}
+		re := &nostr.ReqEnvelope{}
+		e := &nostr.Event{}
+		f := &nostr.Filter{}
+		if json.Unmarshal([]byte(line), ee) == nil && ee.Event.ID != "" {
+			e = &ee.Event
+			return doSave(ctx, line, e)
+		}
+		if json.Unmarshal([]byte(line), e) == nil && e.ID != "" {
+			return doSave(ctx, line, e)
+		}
+		if json.Unmarshal([]byte(line), re) == nil && len(re.Filters) > 0 {
+			f = &re.Filters[0]
+			return doQuery(ctx, f)
+		}
+		if json.Unmarshal([]byte(line), f) == nil && len(f.String()) > 2 {
+			return doQuery(ctx, f)
+		}
+
+		return fmt.Errorf("couldn't parse input '%s'", line)
+	},
+}
+
+func doSave(ctx context.Context, line string, e *nostr.Event) error {
+	if err := db.SaveEvent(ctx, e); err != nil {
+		return fmt.Errorf("failed to save event '%s': %s", line, err)
+	}
+	fmt.Fprintf(os.Stderr, "saved %s", e.ID)
+	return nil
+}
+
+func doQuery(ctx context.Context, f *nostr.Filter) error {
+	ch, err := db.QueryEvents(ctx, *f)
+	if err != nil {
+		return fmt.Errorf("error querying: %w", err)
+	}
+
+	for evt := range ch {
+		fmt.Println(evt)
+	}
+	return nil
+}

eventstore/cmd/eventstore/query.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/mailru/easyjson"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/urfave/cli/v3"
+)
+
+var query = &cli.Command{
+	Name:        "query",
+	ArgsUsage:   "[<filter-json>]",
+	Usage:       "queries an eventstore for events, takes a filter as argument",
+	Description: "applies the filter to the currently open eventstore, returning up to a million events.\n takes either a filter as an argument or reads a stream of filters from stdin.",
+	Action: func(ctx context.Context, c *cli.Command) error {
+		hasError := false
+		for line := range getStdinLinesOrFirstArgument(c) {
+			filter := nostr.Filter{}
+			if err := easyjson.Unmarshal([]byte(line), &filter); err != nil {
+				fmt.Fprintf(os.Stderr, "invalid filter '%s': %s\n", line, err)
+				hasError = true
+				continue
+			}
+
+			ch, err := db.QueryEvents(ctx, filter)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "error querying: %s\n", err)
+				hasError = true
+				continue
+			}
+
+			for evt := range ch {
+				fmt.Println(evt)
+			}
+		}
+
+		if hasError {
+			os.Exit(123)
+		}
+		return nil
+	},
+}

eventstore/cmd/eventstore/save.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/urfave/cli/v3"
+	"github.com/mailru/easyjson"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+var save = &cli.Command{
+	Name:        "save",
+	ArgsUsage:   "[<event-json>]",
+	Usage:       "stores an event",
+	Description: "takes either an event as an argument or reads a stream of events from stdin and inserts those in the currently opened eventstore.\ndoesn't perform any kind of signature checking or replacement.",
+	Action: func(ctx context.Context, c *cli.Command) error {
+		hasError := false
+		for line := range getStdinLinesOrFirstArgument(c) {
+			var event nostr.Event
+			if err := easyjson.Unmarshal([]byte(line), &event); err != nil {
+				fmt.Fprintf(os.Stderr, "invalid event '%s': %s\n", line, err)
+				hasError = true
+				continue
+			}
+
+			if err := db.SaveEvent(ctx, &event); err != nil {
+				fmt.Fprintf(os.Stderr, "failed to save event '%s': %s\n", line, err)
+				hasError = true
+				continue
+			}
+
+			fmt.Fprintf(os.Stderr, "saved %s\n", event.ID)
+		}
+
+		if hasError {
+			os.Exit(123)
+		}
+		return nil
+	},
+}

eventstore/errors.go

@@ -0,0 +1,5 @@
+package eventstore
+
+import "errors"
+
+var ErrDupEvent = errors.New("duplicate: event already exists")

eventstore/internal/binary/cmd/decode-binary/.gitignore

@@ -0,0 +1 @@
+decode-binary

eventstore/internal/binary/cmd/decode-binary/main.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/fiatjaf/eventstore/internal/binary"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func main() {
+	b, err := io.ReadAll(os.Stdin)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to read from stdin: %s\n", err)
+		os.Exit(1)
+		return
+	}
+	b = bytes.TrimSpace(b)
+
+	if bytes.HasPrefix(b, []byte("0x")) {
+		fromHex := make([]byte, (len(b)-2)/2)
+		_, err := hex.Decode(fromHex, b[2:])
+		if err == nil {
+			b = fromHex
+		}
+	}
+
+	var evt nostr.Event
+	err = binary.Unmarshal(b, &evt)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to decode: %s\n", err)
+		os.Exit(1)
+		return
+	}
+	fmt.Println(evt.String())
+}

eventstore/internal/binary/hybrid.go

@@ -0,0 +1,103 @@
+package binary
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+// Deprecated -- the encoding used here is not very elegant, we'll have a better binary format later.
+func Unmarshal(data []byte, evt *nostr.Event) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("failed to decode binary for event %s from %s at %d: %v", evt.ID, evt.PubKey, evt.CreatedAt, r)
+		}
+	}()
+
+	evt.ID = hex.EncodeToString(data[0:32])
+	evt.PubKey = hex.EncodeToString(data[32:64])
+	evt.Sig = hex.EncodeToString(data[64:128])
+	evt.CreatedAt = nostr.Timestamp(binary.BigEndian.Uint32(data[128:132]))
+	evt.Kind = int(binary.BigEndian.Uint16(data[132:134]))
+	contentLength := int(binary.BigEndian.Uint16(data[134:136]))
+	evt.Content = string(data[136 : 136+contentLength])
+
+	curr := 136 + contentLength
+
+	nTags := binary.BigEndian.Uint16(data[curr : curr+2])
+	curr++
+	evt.Tags = make(nostr.Tags, nTags)
+
+	for t := range evt.Tags {
+		curr++
+		nItems := int(data[curr])
+		tag := make(nostr.Tag, nItems)
+		for i := range tag {
+			curr = curr + 1
+			itemSize := int(binary.BigEndian.Uint16(data[curr : curr+2]))
+			itemStart := curr + 2
+			item := string(data[itemStart : itemStart+itemSize])
+			tag[i] = item
+			curr = itemStart + itemSize
+		}
+		evt.Tags[t] = tag
+	}
+
+	return err
+}
+
+// Deprecated -- the encoding used here is not very elegant, we'll have a better binary format later.
+func Marshal(evt *nostr.Event) ([]byte, error) {
+	content := []byte(evt.Content)
+	buf := make([]byte, 32+32+64+4+2+2+len(content)+65536+len(evt.Tags)*40 /* blergh */)
+
+	hex.Decode(buf[0:32], []byte(evt.ID))
+	hex.Decode(buf[32:64], []byte(evt.PubKey))
+	hex.Decode(buf[64:128], []byte(evt.Sig))
+
+	if evt.CreatedAt > MaxCreatedAt {
+		return nil, fmt.Errorf("created_at is too big: %d", evt.CreatedAt)
+	}
+	binary.BigEndian.PutUint32(buf[128:132], uint32(evt.CreatedAt))
+
+	if evt.Kind > MaxKind {
+		return nil, fmt.Errorf("kind is too big: %d, max is %d", evt.Kind, MaxKind)
+	}
+	binary.BigEndian.PutUint16(buf[132:134], uint16(evt.Kind))
+
+	if contentLength := len(content); contentLength > MaxContentSize {
+		return nil, fmt.Errorf("content is too large: %d, max is %d", contentLength, MaxContentSize)
+	} else {
+		binary.BigEndian.PutUint16(buf[134:136], uint16(contentLength))
+	}
+	copy(buf[136:], content)
+
+	if tagCount := len(evt.Tags); tagCount > MaxTagCount {
+		return nil, fmt.Errorf("can't encode too many tags: %d, max is %d", tagCount, MaxTagCount)
+	} else {
+		binary.BigEndian.PutUint16(buf[136+len(content):136+len(content)+2], uint16(tagCount))
+	}
+
+	buf = buf[0 : 136+len(content)+2]
+
+	for _, tag := range evt.Tags {
+		if itemCount := len(tag); itemCount > MaxTagItemCount {
+			return nil, fmt.Errorf("can't encode a tag with so many items: %d, max is %d", itemCount, MaxTagItemCount)
+		} else {
+			buf = append(buf, uint8(itemCount))
+		}
+		for _, item := range tag {
+			itemb := []byte(item)
+			itemSize := len(itemb)
+			if itemSize > MaxTagItemSize {
+				return nil, fmt.Errorf("tag item is too large: %d, max is %d", itemSize, MaxTagItemSize)
+			}
+			buf = binary.BigEndian.AppendUint16(buf, uint16(itemSize))
+			buf = append(buf, itemb...)
+			buf = append(buf, 0)
+		}
+	}
+	return buf, nil
+}

eventstore/internal/binary/limits.go

@@ -0,0 +1,35 @@
+package binary
+
+import (
+	"math"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+const (
+	MaxKind         = math.MaxUint16
+	MaxCreatedAt    = math.MaxUint32
+	MaxContentSize  = math.MaxUint16
+	MaxTagCount     = math.MaxUint16
+	MaxTagItemCount = math.MaxUint8
+	MaxTagItemSize  = math.MaxUint16
+)
+
+func EventEligibleForBinaryEncoding(event *nostr.Event) bool {
+	if len(event.Content) > MaxContentSize || event.Kind > MaxKind || event.CreatedAt > MaxCreatedAt || len(event.Tags) > MaxTagCount {
+		return false
+	}
+
+	for _, tag := range event.Tags {
+		if len(tag) > MaxTagItemCount {
+			return false
+		}
+		for _, item := range tag {
+			if len(item) > MaxTagItemSize {
+				return false
+			}
+		}
+	}
+
+	return true
+}

eventstore/internal/checks/interface.go

@@ -0,0 +1,27 @@
+package checks
+
+import (
+	"github.com/fiatjaf/eventstore"
+	"github.com/fiatjaf/eventstore/badger"
+	"github.com/fiatjaf/eventstore/bluge"
+	"github.com/fiatjaf/eventstore/edgedb"
+	"github.com/fiatjaf/eventstore/lmdb"
+	"github.com/fiatjaf/eventstore/mongo"
+	"github.com/fiatjaf/eventstore/mysql"
+	"github.com/fiatjaf/eventstore/postgresql"
+	"github.com/fiatjaf/eventstore/sqlite3"
+	"github.com/fiatjaf/eventstore/strfry"
+)
+
+// compile-time checks to ensure all backends implement Store
+var (
+	_ eventstore.Store = (*badger.BadgerBackend)(nil)
+	_ eventstore.Store = (*lmdb.LMDBBackend)(nil)
+	_ eventstore.Store = (*edgedb.EdgeDBBackend)(nil)
+	_ eventstore.Store = (*postgresql.PostgresBackend)(nil)
+	_ eventstore.Store = (*mongo.MongoDBBackend)(nil)
+	_ eventstore.Store = (*sqlite3.SQLite3Backend)(nil)
+	_ eventstore.Store = (*strfry.StrfryBackend)(nil)
+	_ eventstore.Store = (*bluge.BlugeBackend)(nil)
+	_ eventstore.Store = (*mysql.MySQLBackend)(nil)
+)

eventstore/internal/helpers.go

@@ -0,0 +1,183 @@
+package internal
+
+import (
+	"cmp"
+	"math"
+	"slices"
+	"strings"
+
+	mergesortedslices "fiatjaf.com/lib/merge-sorted-slices"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func IsOlder(previous, next *nostr.Event) bool {
+	return previous.CreatedAt < next.CreatedAt ||
+		(previous.CreatedAt == next.CreatedAt && previous.ID > next.ID)
+}
+
+func ChooseNarrowestTag(filter nostr.Filter) (key string, values []string, goodness int) {
+	var tagKey string
+	var tagValues []string
+	for key, values := range filter.Tags {
+		switch key {
+		case "e", "E", "q":
+			// 'e' and 'q' are the narrowest possible, so if we have that we will use it and that's it
+			tagKey = key
+			tagValues = values
+			goodness = 9
+			break
+		case "a", "A", "i", "I", "g", "r":
+			// these are second-best as they refer to relatively static things
+			goodness = 8
+			tagKey = key
+			tagValues = values
+		case "d":
+			// this is as good as long as we have an "authors"
+			if len(filter.Authors) != 0 && goodness < 7 {
+				goodness = 7
+				tagKey = key
+				tagValues = values
+			} else if goodness < 4 {
+				goodness = 4
+				tagKey = key
+				tagValues = values
+			}
+		case "h", "t", "l", "k", "K":
+			// these things denote "categories", so they are a little more broad
+			if goodness < 6 {
+				goodness = 6
+				tagKey = key
+				tagValues = values
+			}
+		case "p":
+			// this is broad and useless for a pure tag search, but we will still prefer it over others
+			// for secondary filtering
+			if goodness < 2 {
+				goodness = 2
+				tagKey = key
+				tagValues = values
+			}
+		default:
+			// all the other tags are probably too broad and useless
+			if goodness == 0 {
+				tagKey = key
+				tagValues = values
+			}
+		}
+	}
+
+	return tagKey, tagValues, goodness
+}
+
+func CopyMapWithoutKey[K comparable, V any](originalMap map[K]V, key K) map[K]V {
+	newMap := make(map[K]V, len(originalMap)-1)
+	for k, v := range originalMap {
+		if k != key {
+			newMap[k] = v
+		}
+	}
+	return newMap
+}
+
+type IterEvent struct {
+	*nostr.Event
+	Q int
+}
+
+// MergeSortMultipleBatches takes the results of multiple iterators, which are already sorted,
+// and merges them into a single big sorted slice
+func MergeSortMultiple(batches [][]IterEvent, limit int, dst []IterEvent) []IterEvent {
+	// clear up empty lists here while simultaneously computing the total count.
+	// this helps because if there are a bunch of empty lists then this pre-clean
+	//   step will get us in the faster 'merge' branch otherwise we would go to the other.
+	// we would have to do the cleaning anyway inside it.
+	// and even if we still go on the other we save one iteration by already computing the
+	//   total count.
+	total := 0
+	for i := len(batches) - 1; i >= 0; i-- {
+		if len(batches[i]) == 0 {
+			batches = SwapDelete(batches, i)
+		} else {
+			total += len(batches[i])
+		}
+	}
+
+	if limit == -1 {
+		limit = total
+	}
+
+	// this amazing equation will ensure that if one of the two sides goes very small (like 1 or 2)
+	//   the other can go very high (like 500) and we're still in the 'merge' branch.
+	// if values go somewhere in the middle then they may match the 'merge' branch (batches=20,limit=70)
+	//   or not (batches=25, limit=60)
+	if math.Log(float64(len(batches)*2))+math.Log(float64(limit)) < 8 {
+		if dst == nil {
+			dst = make([]IterEvent, limit)
+		} else if cap(dst) < limit {
+			dst = slices.Grow(dst, limit-len(dst))
+		}
+		dst = dst[0:limit]
+		return mergesortedslices.MergeFuncNoEmptyListsIntoSlice(dst, batches, compareIterEvent)
+	} else {
+		if dst == nil {
+			dst = make([]IterEvent, total)
+		} else if cap(dst) < total {
+			dst = slices.Grow(dst, total-len(dst))
+		}
+		dst = dst[0:total]
+
+		// use quicksort in a dumb way that will still be fast because it's cheated
+		lastIndex := 0
+		for _, batch := range batches {
+			copy(dst[lastIndex:], batch)
+			lastIndex += len(batch)
+		}
+
+		slices.SortFunc(dst, compareIterEvent)
+
+		for i, j := 0, total-1; i < j; i, j = i+1, j-1 {
+			dst[i], dst[j] = dst[j], dst[i]
+		}
+
+		if limit < len(dst) {
+			return dst[0:limit]
+		}
+		return dst
+	}
+}
+
+// BatchSizePerNumberOfQueries tries to make an educated guess for the batch size given the total filter limit and
+// the number of abstract queries we'll be conducting at the same time
+func BatchSizePerNumberOfQueries(totalFilterLimit int, numberOfQueries int) int {
+	if numberOfQueries == 1 || totalFilterLimit*numberOfQueries < 50 {
+		return totalFilterLimit
+	}
+
+	return int(
+		math.Ceil(
+			math.Pow(float64(totalFilterLimit), 0.80) / math.Pow(float64(numberOfQueries), 0.71),
+		),
+	)
+}
+
+func SwapDelete[A any](arr []A, i int) []A {
+	arr[i] = arr[len(arr)-1]
+	return arr[:len(arr)-1]
+}
+
+func compareIterEvent(a, b IterEvent) int {
+	if a.Event == nil {
+		if b.Event == nil {
+			return 0
+		} else {
+			return -1
+		}
+	} else if b.Event == nil {
+		return 1
+	}
+
+	if a.CreatedAt == b.CreatedAt {
+		return strings.Compare(a.ID, b.ID)
+	}
+	return cmp.Compare(a.CreatedAt, b.CreatedAt)
+}

eventstore/internal/testdata/fuzz/FuzzQuery/2387982a59ec5d22

@@ -0,0 +1,8 @@
+go test fuzz v1
+uint(256)
+uint(31)
+uint(260)
+uint(2)
+uint(69)
+uint(385)
+uint(1)

eventstore/internal/testdata/fuzz/FuzzQuery/25234b78dd36a5fd

@@ -0,0 +1,8 @@
+go test fuzz v1
+uint(267)
+uint(50)
+uint(355)
+uint(2)
+uint(69)
+uint(213)
+uint(1)

eventstore/internal/testdata/fuzz/FuzzQuery/35a474e7be3cdc57

@@ -0,0 +1,8 @@
+go test fuzz v1
+uint(280)
+uint(0)
+uint(13)
+uint(2)
+uint(2)
+uint(0)
+uint(0)

eventstore/internal/testdata/fuzz/FuzzQuery/6e88633b00eff43d

@@ -0,0 +1,8 @@
+go test fuzz v1
+uint(259)
+uint(126)
+uint(5)
+uint(23)
+uint(0)
+uint(0)
+uint(92)

eventstore/internal/testdata/fuzz/FuzzQuery/70a3844d6c7ec116

@@ -0,0 +1,8 @@
+go test fuzz v1
+uint(201)
+uint(50)
+uint(13)
+uint(97)
+uint(0)
+uint(0)
+uint(77)

eventstore/internal/testdata/fuzz/FuzzQuery/98cca88a26b20e30

@@ -0,0 +1,8 @@
+go test fuzz v1
+uint(164)
+uint(50)
+uint(13)
+uint(1)
+uint(2)
+uint(13)
+uint(0)

eventstore/internal/testdata/fuzz/FuzzQuery/dabb8bfe01b215a2

@@ -0,0 +1,8 @@
+go test fuzz v1
+uint(200)
+uint(50)
+uint(13)
+uint(8)
+uint(2)
+uint(0)
+uint(1)

eventstore/internal/testdata/fuzz/FuzzQuery/debae0ec843d23ec

@@ -0,0 +1,8 @@
+go test fuzz v1
+uint(200)
+uint(117)
+uint(13)
+uint(2)
+uint(2)
+uint(0)
+uint(1)

eventstore/internal/testdata/fuzz/FuzzQuery/f6d74a34318165c2

@@ -0,0 +1,8 @@
+go test fuzz v1
+uint(200)
+uint(50)
+uint(13)
+uint(2)
+uint(2)
+uint(0)
+uint(0)

eventstore/lmdb/count.go

@@ -0,0 +1,241 @@
+package lmdb
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	bin "github.com/fiatjaf/eventstore/internal/binary"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/nbd-wtf/go-nostr/nip45"
+	"github.com/nbd-wtf/go-nostr/nip45/hyperloglog"
+	"golang.org/x/exp/slices"
+)
+
+func (b *LMDBBackend) CountEvents(ctx context.Context, filter nostr.Filter) (int64, error) {
+	var count int64 = 0
+
+	queries, extraAuthors, extraKinds, extraTagKey, extraTagValues, since, err := b.prepareQueries(filter)
+	if err != nil {
+		return 0, err
+	}
+
+	err = b.lmdbEnv.View(func(txn *lmdb.Txn) error {
+		// actually iterate
+		for _, q := range queries {
+			cursor, err := txn.OpenCursor(q.dbi)
+			if err != nil {
+				continue
+			}
+
+			it := &iterator{cursor: cursor}
+			it.seek(q.startingPoint)
+
+			for {
+				// we already have a k and a v and an err from the cursor setup, so check and use these
+				if it.err != nil ||
+					len(it.key) != q.keySize ||
+					!bytes.HasPrefix(it.key, q.prefix) {
+					// either iteration has errored or we reached the end of this prefix
+					break // stop this cursor and move to the next one
+				}
+
+				// "id" indexes don't contain a timestamp
+				if q.timestampSize == 4 {
+					createdAt := binary.BigEndian.Uint32(it.key[len(it.key)-4:])
+					if createdAt < since {
+						break
+					}
+				}
+
+				if extraAuthors == nil && extraKinds == nil && extraTagValues == nil {
+					count++
+				} else {
+					// fetch actual event
+					val, err := txn.Get(b.rawEventStore, it.valIdx)
+					if err != nil {
+						panic(err)
+					}
+
+					// check it against pubkeys without decoding the entire thing
+					if !slices.Contains(extraAuthors, [32]byte(val[32:64])) {
+						it.next()
+						continue
+					}
+
+					// check it against kinds without decoding the entire thing
+					if !slices.Contains(extraKinds, [2]byte(val[132:134])) {
+						it.next()
+						continue
+					}
+
+					evt := &nostr.Event{}
+					if err := bin.Unmarshal(val, evt); err != nil {
+						it.next()
+						continue
+					}
+
+					// if there is still a tag to be checked, do it now
+					if !evt.Tags.ContainsAny(extraTagKey, extraTagValues) {
+						it.next()
+						continue
+					}
+
+					count++
+				}
+			}
+		}
+
+		return nil
+	})
+
+	return count, err
+}
+
+// CountEventsHLL is like CountEvents, but it will build a hyperloglog value while iterating through results, following NIP-45
+func (b *LMDBBackend) CountEventsHLL(ctx context.Context, filter nostr.Filter, offset int) (int64, *hyperloglog.HyperLogLog, error) {
+	if useCache, _ := b.EnableHLLCacheFor(filter.Kinds[0]); useCache {
+		return b.countEventsHLLCached(filter)
+	}
+
+	var count int64 = 0
+
+	// this is different than CountEvents because some of these extra checks are not applicable in HLL-valid filters
+	queries, _, extraKinds, extraTagKey, extraTagValues, since, err := b.prepareQueries(filter)
+	if err != nil {
+		return 0, nil, err
+	}
+
+	hll := hyperloglog.New(offset)
+
+	err = b.lmdbEnv.View(func(txn *lmdb.Txn) error {
+		// actually iterate
+		for _, q := range queries {
+			cursor, err := txn.OpenCursor(q.dbi)
+			if err != nil {
+				continue
+			}
+
+			it := &iterator{cursor: cursor}
+			it.seek(q.startingPoint)
+
+			for {
+				// we already have a k and a v and an err from the cursor setup, so check and use these
+				if it.err != nil ||
+					len(it.key) != q.keySize ||
+					!bytes.HasPrefix(it.key, q.prefix) {
+					// either iteration has errored or we reached the end of this prefix
+					break // stop this cursor and move to the next one
+				}
+
+				// "id" indexes don't contain a timestamp
+				if q.timestampSize == 4 {
+					createdAt := binary.BigEndian.Uint32(it.key[len(it.key)-4:])
+					if createdAt < since {
+						break
+					}
+				}
+
+				// fetch actual event (we need it regardless because we need the pubkey for the hll)
+				val, err := txn.Get(b.rawEventStore, it.valIdx)
+				if err != nil {
+					panic(err)
+				}
+
+				if extraKinds == nil && extraTagValues == nil {
+					// nothing extra to check
+					count++
+					hll.AddBytes(val[32:64])
+				} else {
+					// check it against kinds without decoding the entire thing
+					if !slices.Contains(extraKinds, [2]byte(val[132:134])) {
+						it.next()
+						continue
+					}
+
+					evt := &nostr.Event{}
+					if err := bin.Unmarshal(val, evt); err != nil {
+						it.next()
+						continue
+					}
+
+					// if there is still a tag to be checked, do it now
+					if !evt.Tags.ContainsAny(extraTagKey, extraTagValues) {
+						it.next()
+						continue
+					}
+
+					count++
+					hll.Add(evt.PubKey)
+				}
+			}
+		}
+
+		return nil
+	})
+
+	return count, hll, err
+}
+
+// countEventsHLLCached will just return a cached value from disk (and presumably we don't even have the events required to compute this anymore).
+func (b *LMDBBackend) countEventsHLLCached(filter nostr.Filter) (int64, *hyperloglog.HyperLogLog, error) {
+	cacheKey := make([]byte, 2+8)
+	binary.BigEndian.PutUint16(cacheKey[0:2], uint16(filter.Kinds[0]))
+	switch filter.Kinds[0] {
+	case 3:
+		hex.Decode(cacheKey[2:2+8], []byte(filter.Tags["p"][0][0:8*2]))
+	case 7:
+		hex.Decode(cacheKey[2:2+8], []byte(filter.Tags["e"][0][0:8*2]))
+	case 1111:
+		hex.Decode(cacheKey[2:2+8], []byte(filter.Tags["E"][0][0:8*2]))
+	}
+
+	var count int64
+	var hll *hyperloglog.HyperLogLog
+
+	err := b.lmdbEnv.View(func(txn *lmdb.Txn) error {
+		val, err := txn.Get(b.hllCache, cacheKey)
+		if err != nil {
+			if lmdb.IsNotFound(err) {
+				return nil
+			}
+			return err
+		}
+		hll = hyperloglog.NewWithRegisters(val, 0) // offset doesn't matter here
+		count = int64(hll.Count())
+		return nil
+	})
+
+	return count, hll, err
+}
+
+func (b *LMDBBackend) updateHyperLogLogCachedValues(txn *lmdb.Txn, evt *nostr.Event) error {
+	cacheKey := make([]byte, 2+8)
+	binary.BigEndian.PutUint16(cacheKey[0:2], uint16(evt.Kind))
+
+	for ref, offset := range nip45.HyperLogLogEventPubkeyOffsetsAndReferencesForEvent(evt) {
+		// setup cache key (reusing buffer)
+		hex.Decode(cacheKey[2:2+8], []byte(ref[0:8*2]))
+
+		// fetch hll value from cache db
+		hll := hyperloglog.New(offset)
+		val, err := txn.Get(b.hllCache, cacheKey)
+		if err == nil {
+			hll.SetRegisters(val)
+		} else if !lmdb.IsNotFound(err) {
+			return err
+		}
+
+		// add this event
+		hll.Add(evt.PubKey)
+
+		// save values back again
+		if err := txn.Put(b.hllCache, cacheKey, hll.GetRegisters(), 0); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

eventstore/lmdb/delete.go

@@ -0,0 +1,43 @@
+package lmdb
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *LMDBBackend) DeleteEvent(ctx context.Context, evt *nostr.Event) error {
+	return b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
+		return b.delete(txn, evt)
+	})
+}
+
+func (b *LMDBBackend) delete(txn *lmdb.Txn, evt *nostr.Event) error {
+	idPrefix8, _ := hex.DecodeString(evt.ID[0 : 8*2])
+	idx, err := txn.Get(b.indexId, idPrefix8)
+	if lmdb.IsNotFound(err) {
+		// we already do not have this
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("failed to get current idx for deleting %x: %w", evt.ID[0:8*2], err)
+	}
+
+	// calculate all index keys we have for this event and delete them
+	for k := range b.getIndexKeysForEvent(evt) {
+		err := txn.Del(k.dbi, k.key, idx)
+		if err != nil {
+			return fmt.Errorf("failed to delete index entry %s for %x: %w", b.keyName(k), evt.ID[0:8*2], err)
+		}
+	}
+
+	// delete the raw event
+	if err := txn.Del(b.rawEventStore, idx, nil); err != nil {
+		return fmt.Errorf("failed to delete raw event %x (idx %x): %w", evt.ID[0:8*2], idx, err)
+	}
+
+	return nil
+}

eventstore/lmdb/fuzz_test.go

@@ -0,0 +1,137 @@
+package lmdb
+
+import (
+	"cmp"
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/fiatjaf/eventstore"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/slices"
+)
+
+func FuzzQuery(f *testing.F) {
+	ctx := context.Background()
+
+	f.Add(uint(200), uint(50), uint(13), uint(2), uint(2), uint(0), uint(1))
+	f.Fuzz(func(t *testing.T, total, limit, authors, timestampAuthorFactor, seedFactor, kinds, kindFactor uint) {
+		total++
+		authors++
+		seedFactor++
+		kindFactor++
+		if kinds == 1 {
+			kinds++
+		}
+		if limit == 0 {
+			return
+		}
+
+		// ~ setup db
+		if err := os.RemoveAll("/tmp/lmdbtest"); err != nil {
+			t.Fatal(err)
+			return
+		}
+		db := &LMDBBackend{}
+		db.Path = "/tmp/lmdbtest"
+		db.extraFlags = lmdb.NoSync
+		db.MaxLimit = 500
+		if err := db.Init(); err != nil {
+			t.Fatal(err)
+			return
+		}
+		defer db.Close()
+
+		// ~ start actual test
+
+		filter := nostr.Filter{
+			Authors: make([]string, authors),
+			Limit:   int(limit),
+		}
+		maxKind := 1
+		if kinds > 0 {
+			filter.Kinds = make([]int, kinds)
+			for i := range filter.Kinds {
+				filter.Kinds[i] = int(kindFactor) * i
+			}
+			maxKind = filter.Kinds[len(filter.Kinds)-1]
+		}
+
+		for i := 0; i < int(authors); i++ {
+			sk := make([]byte, 32)
+			binary.BigEndian.PutUint32(sk, uint32(i%int(authors*seedFactor))+1)
+			pk, _ := nostr.GetPublicKey(hex.EncodeToString(sk))
+			filter.Authors[i] = pk
+		}
+
+		expected := make([]*nostr.Event, 0, total)
+		for i := 0; i < int(total); i++ {
+			skseed := uint32(i%int(authors*seedFactor)) + 1
+			sk := make([]byte, 32)
+			binary.BigEndian.PutUint32(sk, skseed)
+
+			evt := &nostr.Event{
+				CreatedAt: nostr.Timestamp(skseed)*nostr.Timestamp(timestampAuthorFactor) + nostr.Timestamp(i),
+				Content:   fmt.Sprintf("unbalanced %d", i),
+				Tags:      nostr.Tags{},
+				Kind:      i % maxKind,
+			}
+			err := evt.Sign(hex.EncodeToString(sk))
+			require.NoError(t, err)
+
+			err = db.SaveEvent(ctx, evt)
+			require.NoError(t, err)
+
+			if filter.Matches(evt) {
+				expected = append(expected, evt)
+			}
+		}
+
+		slices.SortFunc(expected, nostr.CompareEventPtrReverse)
+		if len(expected) > int(limit) {
+			expected = expected[0:limit]
+		}
+
+		w := eventstore.RelayWrapper{Store: db}
+
+		start := time.Now()
+
+		res, err := w.QuerySync(ctx, filter)
+		end := time.Now()
+
+		require.NoError(t, err)
+		require.Equal(t, len(expected), len(res), "number of results is different than expected")
+
+		require.Less(t, end.Sub(start).Milliseconds(), int64(1500), "query took too long")
+		nresults := len(expected)
+
+		getTimestamps := func(events []*nostr.Event) []nostr.Timestamp {
+			res := make([]nostr.Timestamp, len(events))
+			for i, evt := range events {
+				res[i] = evt.CreatedAt
+			}
+			return res
+		}
+
+		fmt.Println(" expected   result")
+		for i := range expected {
+			fmt.Println(" ", expected[i].CreatedAt, expected[i].ID[0:8], "  ", res[i].CreatedAt, res[i].ID[0:8], "           ", i)
+		}
+
+		require.Equal(t, expected[0].CreatedAt, res[0].CreatedAt, "first result is wrong")
+		require.Equal(t, expected[nresults-1].CreatedAt, res[nresults-1].CreatedAt, "last result (%d) is wrong", nresults-1)
+		require.Equal(t, getTimestamps(expected), getTimestamps(res))
+
+		for _, evt := range res {
+			require.True(t, filter.Matches(evt), "event %s doesn't match filter %s", evt, filter)
+		}
+
+		require.True(t, slices.IsSortedFunc(res, func(a, b *nostr.Event) int { return cmp.Compare(b.CreatedAt, a.CreatedAt) }), "results are not sorted")
+	})
+}

eventstore/lmdb/helpers.go

@@ -0,0 +1,213 @@
+package lmdb
+
+import (
+	"crypto/md5"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"iter"
+	"strconv"
+	"strings"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/nbd-wtf/go-nostr"
+	"golang.org/x/exp/slices"
+)
+
+// this iterator always goes backwards
+type iterator struct {
+	cursor *lmdb.Cursor
+	key    []byte
+	valIdx []byte
+	err    error
+}
+
+func (it *iterator) seek(key []byte) {
+	if _, _, errsr := it.cursor.Get(key, nil, lmdb.SetRange); errsr != nil {
+		if operr, ok := errsr.(*lmdb.OpError); !ok || operr.Errno != lmdb.NotFound {
+			// in this case it's really an error
+			panic(operr)
+		} else {
+			// we're at the end and we just want notes before this,
+			// so we just need to set the cursor the last key, this is not a real error
+			it.key, it.valIdx, it.err = it.cursor.Get(nil, nil, lmdb.Last)
+		}
+	} else {
+		// move one back as the first step
+		it.key, it.valIdx, it.err = it.cursor.Get(nil, nil, lmdb.Prev)
+	}
+}
+
+func (it *iterator) next() {
+	// move one back (we'll look into k and v and err in the next iteration)
+	it.key, it.valIdx, it.err = it.cursor.Get(nil, nil, lmdb.Prev)
+}
+
+type key struct {
+	dbi lmdb.DBI
+	key []byte
+}
+
+func (b *LMDBBackend) keyName(key key) string {
+	return fmt.Sprintf("<dbi=%s key=%x>", b.dbiName(key.dbi), key.key)
+}
+
+func (b *LMDBBackend) getIndexKeysForEvent(evt *nostr.Event) iter.Seq[key] {
+	return func(yield func(key) bool) {
+		{
+			// ~ by id
+			k := make([]byte, 8)
+			hex.Decode(k[0:8], []byte(evt.ID[0:8*2]))
+			if !yield(key{dbi: b.indexId, key: k[0:8]}) {
+				return
+			}
+		}
+
+		{
+			// ~ by pubkey+date
+			k := make([]byte, 8+4)
+			hex.Decode(k[0:8], []byte(evt.PubKey[0:8*2]))
+			binary.BigEndian.PutUint32(k[8:8+4], uint32(evt.CreatedAt))
+			if !yield(key{dbi: b.indexPubkey, key: k[0 : 8+4]}) {
+				return
+			}
+		}
+
+		{
+			// ~ by kind+date
+			k := make([]byte, 2+4)
+			binary.BigEndian.PutUint16(k[0:2], uint16(evt.Kind))
+			binary.BigEndian.PutUint32(k[2:2+4], uint32(evt.CreatedAt))
+			if !yield(key{dbi: b.indexKind, key: k[0 : 2+4]}) {
+				return
+			}
+		}
+
+		{
+			// ~ by pubkey+kind+date
+			k := make([]byte, 8+2+4)
+			hex.Decode(k[0:8], []byte(evt.PubKey[0:8*2]))
+			binary.BigEndian.PutUint16(k[8:8+2], uint16(evt.Kind))
+			binary.BigEndian.PutUint32(k[8+2:8+2+4], uint32(evt.CreatedAt))
+			if !yield(key{dbi: b.indexPubkeyKind, key: k[0 : 8+2+4]}) {
+				return
+			}
+		}
+
+		// ~ by tagvalue+date
+		// ~ by p-tag+kind+date
+		for i, tag := range evt.Tags {
+			if len(tag) < 2 || len(tag[0]) != 1 || len(tag[1]) == 0 || len(tag[1]) > 100 {
+				// not indexable
+				continue
+			}
+			firstIndex := slices.IndexFunc(evt.Tags, func(t nostr.Tag) bool { return len(t) >= 2 && t[1] == tag[1] })
+			if firstIndex != i {
+				// duplicate
+				continue
+			}
+
+			// get key prefix (with full length) and offset where to write the created_at
+			dbi, k, offset := b.getTagIndexPrefix(tag[1])
+			binary.BigEndian.PutUint32(k[offset:], uint32(evt.CreatedAt))
+			if !yield(key{dbi: dbi, key: k}) {
+				return
+			}
+
+			// now the p-tag+kind+date
+			if dbi == b.indexTag32 && tag[0] == "p" {
+				k := make([]byte, 8+2+4)
+				hex.Decode(k[0:8], []byte(tag[1][0:8*2]))
+				binary.BigEndian.PutUint16(k[8:8+2], uint16(evt.Kind))
+				binary.BigEndian.PutUint32(k[8+2:8+2+4], uint32(evt.CreatedAt))
+				dbi := b.indexPTagKind
+				if !yield(key{dbi: dbi, key: k[0 : 8+2+4]}) {
+					return
+				}
+			}
+		}
+
+		{
+			// ~ by date only
+			k := make([]byte, 4)
+			binary.BigEndian.PutUint32(k[0:4], uint32(evt.CreatedAt))
+			if !yield(key{dbi: b.indexCreatedAt, key: k[0:4]}) {
+				return
+			}
+		}
+	}
+}
+
+func (b *LMDBBackend) getTagIndexPrefix(tagValue string) (lmdb.DBI, []byte, int) {
+	var k []byte   // the key with full length for created_at and idx at the end, but not filled with these
+	var offset int // the offset -- i.e. where the prefix ends and the created_at and idx would start
+	var dbi lmdb.DBI
+
+	// if it's 32 bytes as hex, save it as bytes
+	if len(tagValue) == 64 {
+		// but we actually only use the first 8 bytes
+		k = make([]byte, 8+4)
+		if _, err := hex.Decode(k[0:8], []byte(tagValue[0:8*2])); err == nil {
+			offset = 8
+			dbi = b.indexTag32
+			return dbi, k[0 : 8+4], offset
+		}
+	}
+
+	// if it looks like an "a" tag, index it in this special format
+	spl := strings.Split(tagValue, ":")
+	if len(spl) == 3 && len(spl[1]) == 64 {
+		k = make([]byte, 2+8+30)
+		if _, err := hex.Decode(k[2:2+8], []byte(tagValue[0:8*2])); err == nil {
+			if kind, err := strconv.ParseUint(spl[0], 10, 16); err == nil {
+				k[0] = byte(kind >> 8)
+				k[1] = byte(kind)
+				// limit "d" identifier to 30 bytes (so we don't have to grow our byte slice)
+				n := copy(k[2+8:2+8+30], spl[2])
+				offset = 2 + 8 + n
+				return dbi, k[0 : offset+4], offset
+			}
+		}
+	}
+
+	// index whatever else as a md5 hash of the contents
+	h := md5.New()
+	h.Write([]byte(tagValue))
+	k = make([]byte, 0, 16+4)
+	k = h.Sum(k)
+	offset = 16
+	dbi = b.indexTag
+
+	return dbi, k[0 : 16+4], offset
+}
+
+func (b *LMDBBackend) dbiName(dbi lmdb.DBI) string {
+	switch dbi {
+	case b.hllCache:
+		return "hllCache"
+	case b.settingsStore:
+		return "settingsStore"
+	case b.rawEventStore:
+		return "rawEventStore"
+	case b.indexCreatedAt:
+		return "indexCreatedAt"
+	case b.indexId:
+		return "indexId"
+	case b.indexKind:
+		return "indexKind"
+	case b.indexPubkey:
+		return "indexPubkey"
+	case b.indexPubkeyKind:
+		return "indexPubkeyKind"
+	case b.indexTag:
+		return "indexTag"
+	case b.indexTag32:
+		return "indexTag32"
+	case b.indexTagAddr:
+		return "indexTagAddr"
+	case b.indexPTagKind:
+		return "indexPTagKind"
+	default:
+		return "<unexpected>"
+	}
+}

eventstore/lmdb/lib.go

@@ -0,0 +1,208 @@
+package lmdb
+
+import (
+	"encoding/binary"
+	"fmt"
+	"os"
+	"sync/atomic"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/fiatjaf/eventstore"
+)
+
+var _ eventstore.Store = (*LMDBBackend)(nil)
+
+type LMDBBackend struct {
+	Path               string
+	MaxLimit           int
+	MaxLimitNegentropy int
+	MapSize            int64
+
+	lmdbEnv    *lmdb.Env
+	extraFlags uint // (for debugging and testing)
+
+	settingsStore   lmdb.DBI
+	rawEventStore   lmdb.DBI
+	indexCreatedAt  lmdb.DBI
+	indexId         lmdb.DBI
+	indexKind       lmdb.DBI
+	indexPubkey     lmdb.DBI
+	indexPubkeyKind lmdb.DBI
+	indexTag        lmdb.DBI
+	indexTag32      lmdb.DBI
+	indexTagAddr    lmdb.DBI
+	indexPTagKind   lmdb.DBI
+
+	hllCache          lmdb.DBI
+	EnableHLLCacheFor func(kind int) (useCache bool, skipSavingActualEvent bool)
+
+	lastId atomic.Uint32
+}
+
+func (b *LMDBBackend) Init() error {
+	if b.MaxLimit != 0 {
+		b.MaxLimitNegentropy = b.MaxLimit
+	} else {
+		b.MaxLimit = 1500
+		if b.MaxLimitNegentropy == 0 {
+			b.MaxLimitNegentropy = 16777216
+		}
+	}
+
+	// create directory if it doesn't exist and open it
+	if err := os.MkdirAll(b.Path, 0755); err != nil {
+		return err
+	}
+
+	return b.initialize()
+}
+
+func (b *LMDBBackend) Close() {
+	b.lmdbEnv.Close()
+}
+
+func (b *LMDBBackend) Serial() []byte {
+	v := b.lastId.Add(1)
+	vb := make([]byte, 4)
+	binary.BigEndian.PutUint32(vb[:], uint32(v))
+	return vb
+}
+
+// Compact can only be called when the database is not being used because it will overwrite everything.
+// It will temporarily move the database to a new location, then move it back.
+// If something goes wrong crash the process and look for the copy of the data on tmppath.
+func (b *LMDBBackend) Compact(tmppath string) error {
+	if err := os.MkdirAll(tmppath, 0755); err != nil {
+		return err
+	}
+
+	if err := b.lmdbEnv.Copy(tmppath); err != nil {
+		return fmt.Errorf("failed to copy: %w", err)
+	}
+
+	if err := b.lmdbEnv.Close(); err != nil {
+		return err
+	}
+	if err := os.RemoveAll(b.Path); err != nil {
+		return err
+	}
+	if err := os.Rename(tmppath, b.Path); err != nil {
+		return err
+	}
+
+	return b.initialize()
+}
+
+func (b *LMDBBackend) initialize() error {
+	env, err := lmdb.NewEnv()
+	if err != nil {
+		return err
+	}
+
+	env.SetMaxDBs(12)
+	env.SetMaxReaders(1000)
+	if b.MapSize == 0 {
+		env.SetMapSize(1 << 38) // ~273GB
+	} else {
+		env.SetMapSize(b.MapSize)
+	}
+
+	if err := env.Open(b.Path, lmdb.NoTLS|lmdb.WriteMap|b.extraFlags, 0644); err != nil {
+		return err
+	}
+	b.lmdbEnv = env
+
+	var multiIndexCreationFlags uint = lmdb.Create | lmdb.DupSort | lmdb.DupFixed
+
+	// open each db
+	if err := b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
+		if dbi, err := txn.OpenDBI("settings", lmdb.Create); err != nil {
+			return err
+		} else {
+			b.settingsStore = dbi
+		}
+		if dbi, err := txn.OpenDBI("raw", lmdb.Create); err != nil {
+			return err
+		} else {
+			b.rawEventStore = dbi
+		}
+		if dbi, err := txn.OpenDBI("created_at", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			b.indexCreatedAt = dbi
+		}
+		if dbi, err := txn.OpenDBI("id", lmdb.Create); err != nil {
+			return err
+		} else {
+			b.indexId = dbi
+		}
+		if dbi, err := txn.OpenDBI("kind", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			b.indexKind = dbi
+		}
+		if dbi, err := txn.OpenDBI("pubkey", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			b.indexPubkey = dbi
+		}
+		if dbi, err := txn.OpenDBI("pubkeyKind", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			b.indexPubkeyKind = dbi
+		}
+		if dbi, err := txn.OpenDBI("tag", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			b.indexTag = dbi
+		}
+		if dbi, err := txn.OpenDBI("tag32", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			b.indexTag32 = dbi
+		}
+		if dbi, err := txn.OpenDBI("tagaddr", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			b.indexTagAddr = dbi
+		}
+		if dbi, err := txn.OpenDBI("ptagKind", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			b.indexPTagKind = dbi
+		}
+		if dbi, err := txn.OpenDBI("hllCache", lmdb.Create); err != nil {
+			return err
+		} else {
+			b.hllCache = dbi
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	// get lastId
+	if err := b.lmdbEnv.View(func(txn *lmdb.Txn) error {
+		txn.RawRead = true
+		cursor, err := txn.OpenCursor(b.rawEventStore)
+		if err != nil {
+			return err
+		}
+		defer cursor.Close()
+		k, _, err := cursor.Get(nil, nil, lmdb.Last)
+		if lmdb.IsNotFound(err) {
+			// nothing found, so we're at zero
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		b.lastId.Store(binary.BigEndian.Uint32(k))
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return b.runMigrations()
+}

eventstore/lmdb/migration.go

@@ -0,0 +1,147 @@
+package lmdb
+
+import (
+	"encoding/binary"
+	"fmt"
+	"log"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	bin "github.com/fiatjaf/eventstore/internal/binary"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+const (
+	DB_VERSION byte = 'v'
+)
+
+func (b *LMDBBackend) runMigrations() error {
+	return b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
+		var version uint16
+		v, err := txn.Get(b.settingsStore, []byte{DB_VERSION})
+		if err != nil {
+			if lmdb.IsNotFound(err) {
+				version = 0
+			} else if v == nil {
+				return fmt.Errorf("failed to read database version: %w", err)
+			}
+		} else {
+			version = binary.BigEndian.Uint16(v)
+		}
+
+		// all previous migrations are useless because we will just reindex everything
+		if version == 0 {
+			// if there is any data in the relay we will just set the version to the max without saying anything
+			cursor, err := txn.OpenCursor(b.rawEventStore)
+			if err != nil {
+				return fmt.Errorf("failed to open cursor in migration: %w", err)
+			}
+			defer cursor.Close()
+
+			hasAnyEntries := false
+			_, _, err = cursor.Get(nil, nil, lmdb.First)
+			for err == nil {
+				hasAnyEntries = true
+				break
+			}
+
+			if !hasAnyEntries {
+				b.setVersion(txn, 8)
+				version = 8
+				return nil
+			}
+		}
+
+		// do the migrations in increasing steps (there is no rollback)
+		//
+
+		// this is when we reindex everything
+		if version < 8 {
+			log.Println("[lmdb] migration 8: reindex everything")
+
+			if err := txn.Drop(b.indexId, false); err != nil {
+				return err
+			}
+			if err := txn.Drop(b.indexCreatedAt, false); err != nil {
+				return err
+			}
+			if err := txn.Drop(b.indexKind, false); err != nil {
+				return err
+			}
+			if err := txn.Drop(b.indexPTagKind, false); err != nil {
+				return err
+			}
+			if err := txn.Drop(b.indexPubkey, false); err != nil {
+				return err
+			}
+			if err := txn.Drop(b.indexPubkeyKind, false); err != nil {
+				return err
+			}
+			if err := txn.Drop(b.indexTag, false); err != nil {
+				return err
+			}
+			if err := txn.Drop(b.indexTag32, false); err != nil {
+				return err
+			}
+			if err := txn.Drop(b.indexTagAddr, false); err != nil {
+				return err
+			}
+
+			cursor, err := txn.OpenCursor(b.rawEventStore)
+			if err != nil {
+				return fmt.Errorf("failed to open cursor in migration 8: %w", err)
+			}
+			defer cursor.Close()
+
+			seen := make(map[[32]byte]struct{})
+
+			idx, val, err := cursor.Get(nil, nil, lmdb.First)
+			for err == nil {
+				idp := *(*[32]byte)(val[0:32])
+				if _, isDup := seen[idp]; isDup {
+					// do not index, but delete this entry
+					if err := txn.Del(b.rawEventStore, idx, nil); err != nil {
+						return err
+					}
+
+					// next
+					idx, val, err = cursor.Get(nil, nil, lmdb.Next)
+					continue
+				}
+
+				seen[idp] = struct{}{}
+
+				evt := &nostr.Event{}
+				if err := bin.Unmarshal(val, evt); err != nil {
+					return fmt.Errorf("error decoding event %x on migration 5: %w", idx, err)
+				}
+
+				for key := range b.getIndexKeysForEvent(evt) {
+					if err := txn.Put(key.dbi, key.key, idx, 0); err != nil {
+						return fmt.Errorf("failed to save index %s for event %s (%v) on migration 8: %w",
+							b.keyName(key), evt.ID, idx, err)
+					}
+				}
+
+				// next
+				idx, val, err = cursor.Get(nil, nil, lmdb.Next)
+			}
+			if lmdbErr, ok := err.(*lmdb.OpError); ok && lmdbErr.Errno != lmdb.NotFound {
+				// exited the loop with an error different from NOTFOUND
+				return err
+			}
+
+			// bump version
+			if err := b.setVersion(txn, 8); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+func (b *LMDBBackend) setVersion(txn *lmdb.Txn, version uint16) error {
+	buf, err := txn.PutReserve(b.settingsStore, []byte{DB_VERSION}, 4, 0)
+	binary.BigEndian.PutUint16(buf, version)
+	return err
+}

eventstore/lmdb/query.go

@@ -0,0 +1,410 @@
+package lmdb
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"fmt"
+	"log"
+	"slices"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/fiatjaf/eventstore"
+	"github.com/fiatjaf/eventstore/internal"
+	bin "github.com/fiatjaf/eventstore/internal/binary"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *LMDBBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (chan *nostr.Event, error) {
+	ch := make(chan *nostr.Event)
+
+	if filter.Search != "" {
+		close(ch)
+		return ch, nil
+	}
+
+	// max number of events we'll return
+	maxLimit := b.MaxLimit
+	var limit int
+	if eventstore.IsNegentropySession(ctx) {
+		maxLimit = b.MaxLimitNegentropy
+		limit = maxLimit
+	} else {
+		limit = maxLimit / 4
+	}
+	if filter.Limit > 0 && filter.Limit <= maxLimit {
+		limit = filter.Limit
+	}
+	if tlimit := nostr.GetTheoreticalLimit(filter); tlimit == 0 {
+		close(ch)
+		return ch, nil
+	} else if tlimit > 0 {
+		limit = tlimit
+	}
+
+	go b.lmdbEnv.View(func(txn *lmdb.Txn) error {
+		txn.RawRead = true
+		defer close(ch)
+		results, err := b.query(txn, filter, limit)
+
+		for _, ie := range results {
+			ch <- ie.Event
+		}
+
+		return err
+	})
+
+	return ch, nil
+}
+
+func (b *LMDBBackend) query(txn *lmdb.Txn, filter nostr.Filter, limit int) ([]internal.IterEvent, error) {
+	queries, extraAuthors, extraKinds, extraTagKey, extraTagValues, since, err := b.prepareQueries(filter)
+	if err != nil {
+		return nil, err
+	}
+
+	iterators := make([]*iterator, len(queries))
+	exhausted := make([]bool, len(queries)) // indicates that a query won't be used anymore
+	results := make([][]internal.IterEvent, len(queries))
+	pulledPerQuery := make([]int, len(queries))
+
+	// these are kept updated so we never pull from the iterator that is at further distance
+	// (i.e. the one that has the oldest event among all)
+	// we will continue to pull from it as soon as some other iterator takes the position
+	oldest := internal.IterEvent{Q: -1}
+
+	secondPhase := false // after we have gathered enough events we will change the way we iterate
+	secondBatch := make([][]internal.IterEvent, 0, len(queries)+1)
+	secondPhaseParticipants := make([]int, 0, len(queries)+1)
+
+	// while merging results in the second phase we will alternate between these two lists
+	//   to avoid having to create new lists all the time
+	var secondPhaseResultsA []internal.IterEvent
+	var secondPhaseResultsB []internal.IterEvent
+	var secondPhaseResultsToggle bool // this is just a dummy thing we use to keep track of the alternating
+	var secondPhaseHasResultsPending bool
+
+	remainingUnexhausted := len(queries) // when all queries are exhausted we can finally end this thing
+	batchSizePerQuery := internal.BatchSizePerNumberOfQueries(limit, remainingUnexhausted)
+	firstPhaseTotalPulled := 0
+
+	exhaust := func(q int) {
+		exhausted[q] = true
+		remainingUnexhausted--
+		if q == oldest.Q {
+			oldest = internal.IterEvent{Q: -1}
+		}
+	}
+
+	var firstPhaseResults []internal.IterEvent
+
+	for q := range queries {
+		cursor, err := txn.OpenCursor(queries[q].dbi)
+		if err != nil {
+			return nil, err
+		}
+		iterators[q] = &iterator{cursor: cursor}
+		defer cursor.Close()
+		iterators[q].seek(queries[q].startingPoint)
+		results[q] = make([]internal.IterEvent, 0, batchSizePerQuery*2)
+	}
+
+	// fmt.Println("queries", len(queries))
+
+	for c := 0; ; c++ {
+		batchSizePerQuery = internal.BatchSizePerNumberOfQueries(limit, remainingUnexhausted)
+
+		// fmt.Println("  iteration", c, "remaining", remainingUnexhausted, "batchsize", batchSizePerQuery)
+		// we will go through all the iterators in batches until we have pulled all the required results
+		for q, query := range queries {
+			if exhausted[q] {
+				continue
+			}
+			if oldest.Q == q && remainingUnexhausted > 1 {
+				continue
+			}
+			// fmt.Println("    query", q, unsafe.Pointer(&results[q]), hex.EncodeToString(query.prefix), len(results[q]))
+
+			it := iterators[q]
+			pulledThisIteration := 0
+
+			for {
+				// we already have a k and a v and an err from the cursor setup, so check and use these
+				if it.err != nil ||
+					len(it.key) != query.keySize ||
+					!bytes.HasPrefix(it.key, query.prefix) {
+					// either iteration has errored or we reached the end of this prefix
+					// fmt.Println("      reached end", it.key, query.keySize, query.prefix)
+					exhaust(q)
+					break
+				}
+
+				// "id" indexes don't contain a timestamp
+				if query.timestampSize == 4 {
+					createdAt := binary.BigEndian.Uint32(it.key[len(it.key)-4:])
+					if createdAt < since {
+						// fmt.Println("        reached since", createdAt, "<", since)
+						exhaust(q)
+						break
+					}
+				}
+
+				// fetch actual event
+				val, err := txn.Get(b.rawEventStore, it.valIdx)
+				if err != nil {
+					log.Printf(
+						"lmdb: failed to get %x based on prefix %x, index key %x from raw event store: %s\n",
+						it.valIdx, query.prefix, it.key, err)
+					return nil, fmt.Errorf("iteration error: %w", err)
+				}
+
+				// check it against pubkeys without decoding the entire thing
+				if extraAuthors != nil && !slices.Contains(extraAuthors, [32]byte(val[32:64])) {
+					it.next()
+					continue
+				}
+
+				// check it against kinds without decoding the entire thing
+				if extraKinds != nil && !slices.Contains(extraKinds, [2]byte(val[132:134])) {
+					it.next()
+					continue
+				}
+
+				// decode the entire thing
+				event := &nostr.Event{}
+				if err := bin.Unmarshal(val, event); err != nil {
+					log.Printf("lmdb: value read error (id %x) on query prefix %x sp %x dbi %d: %s\n", val[0:32],
+						query.prefix, query.startingPoint, query.dbi, err)
+					return nil, fmt.Errorf("event read error: %w", err)
+				}
+
+				// fmt.Println("      event", hex.EncodeToString(val[0:4]), "kind", binary.BigEndian.Uint16(val[132:134]), "author", hex.EncodeToString(val[32:36]), "ts", nostr.Timestamp(binary.BigEndian.Uint32(val[128:132])), hex.EncodeToString(it.key), it.valIdx)
+
+				// if there is still a tag to be checked, do it now
+				if extraTagValues != nil && !event.Tags.ContainsAny(extraTagKey, extraTagValues) {
+					it.next()
+					continue
+				}
+
+				// this event is good to be used
+				evt := internal.IterEvent{Event: event, Q: q}
+				//
+				//
+				if secondPhase {
+					// do the process described below at HIWAWVRTP.
+					// if we've reached here this means we've already passed the `since` check.
+					// now we have to eliminate the event currently at the `since` threshold.
+					nextThreshold := firstPhaseResults[len(firstPhaseResults)-2]
+					if oldest.Event == nil {
+						// fmt.Println("          b1", evt.ID[0:8])
+						// BRANCH WHEN WE DON'T HAVE THE OLDEST EVENT (BWWDHTOE)
+						// when we don't have the oldest set, we will keep the results
+						//   and not change the cutting point -- it's bad, but hopefully not that bad.
+						results[q] = append(results[q], evt)
+						secondPhaseHasResultsPending = true
+					} else if nextThreshold.CreatedAt > oldest.CreatedAt {
+						// fmt.Println("          b2", nextThreshold.CreatedAt, ">", oldest.CreatedAt, evt.ID[0:8])
+						// one of the events we have stored is the actual next threshold
+						// eliminate last, update since with oldest
+						firstPhaseResults = firstPhaseResults[0 : len(firstPhaseResults)-1]
+						since = uint32(oldest.CreatedAt)
+						// fmt.Println("            new since", since, evt.ID[0:8])
+						//  we null the oldest Event as we can't rely on it anymore
+						//   (we'll fall under BWWDHTOE above) until we have a new oldest set.
+						oldest = internal.IterEvent{Q: -1}
+						// anything we got that would be above this won't trigger an update to
+						//   the oldest anyway, because it will be discarded as being after the limit.
+						//
+						// finally
+						// add this to the results to be merged later
+						results[q] = append(results[q], evt)
+						secondPhaseHasResultsPending = true
+					} else if nextThreshold.CreatedAt < evt.CreatedAt {
+						// the next last event in the firstPhaseResults is the next threshold
+						// fmt.Println("          b3", nextThreshold.CreatedAt, "<", oldest.CreatedAt, evt.ID[0:8])
+						// eliminate last, update since with the antelast
+						firstPhaseResults = firstPhaseResults[0 : len(firstPhaseResults)-1]
+						since = uint32(nextThreshold.CreatedAt)
+						// fmt.Println("            new since", since)
+						// add this to the results to be merged later
+						results[q] = append(results[q], evt)
+						secondPhaseHasResultsPending = true
+						// update the oldest event
+						if evt.CreatedAt < oldest.CreatedAt {
+							oldest = evt
+						}
+					} else {
+						// fmt.Println("          b4", evt.ID[0:8])
+						// oops, _we_ are the next `since` threshold
+						firstPhaseResults[len(firstPhaseResults)-1] = evt
+						since = uint32(evt.CreatedAt)
+						// fmt.Println("            new since", since)
+						// do not add us to the results to be merged later
+						//   as we're already inhabiting the firstPhaseResults slice
+					}
+				} else {
+					results[q] = append(results[q], evt)
+					firstPhaseTotalPulled++
+
+					// update the oldest event
+					if oldest.Event == nil || evt.CreatedAt < oldest.CreatedAt {
+						oldest = evt
+					}
+				}
+
+				pulledPerQuery[q]++
+				pulledThisIteration++
+				if pulledThisIteration > batchSizePerQuery {
+					// batch filled
+					it.next()
+					// fmt.Println("        filled", hex.EncodeToString(it.key), it.valIdx)
+					break
+				}
+				if pulledPerQuery[q] >= limit {
+					// batch filled + reached limit for this query (which is the global limit)
+					exhaust(q)
+					it.next()
+					break
+				}
+
+				it.next()
+			}
+		}
+
+		// we will do this check if we don't accumulated the requested number of events yet
+		// fmt.Println("oldest", oldest.Event, "from iter", oldest.Q)
+		if secondPhase && secondPhaseHasResultsPending && (oldest.Event == nil || remainingUnexhausted == 0) {
+			// fmt.Println("second phase aggregation!")
+			// when we are in the second phase we will aggressively aggregate results on every iteration
+			//
+			secondBatch = secondBatch[:0]
+			for s := 0; s < len(secondPhaseParticipants); s++ {
+				q := secondPhaseParticipants[s]
+
+				if len(results[q]) > 0 {
+					secondBatch = append(secondBatch, results[q])
+				}
+
+				if exhausted[q] {
+					secondPhaseParticipants = internal.SwapDelete(secondPhaseParticipants, s)
+					s--
+				}
+			}
+
+			// every time we get here we will alternate between these A and B lists
+			//   combining everything we have into a new partial results list.
+			// after we've done that we can again set the oldest.
+			// fmt.Println("  xxx", secondPhaseResultsToggle)
+			if secondPhaseResultsToggle {
+				secondBatch = append(secondBatch, secondPhaseResultsB)
+				secondPhaseResultsA = internal.MergeSortMultiple(secondBatch, limit, secondPhaseResultsA)
+				oldest = secondPhaseResultsA[len(secondPhaseResultsA)-1]
+				// fmt.Println("  new aggregated a", len(secondPhaseResultsB))
+			} else {
+				secondBatch = append(secondBatch, secondPhaseResultsA)
+				secondPhaseResultsB = internal.MergeSortMultiple(secondBatch, limit, secondPhaseResultsB)
+				oldest = secondPhaseResultsB[len(secondPhaseResultsB)-1]
+				// fmt.Println("  new aggregated b", len(secondPhaseResultsB))
+			}
+			secondPhaseResultsToggle = !secondPhaseResultsToggle
+
+			since = uint32(oldest.CreatedAt)
+			// fmt.Println("  new since", since)
+
+			// reset the `results` list so we can keep using it
+			results = results[:len(queries)]
+			for _, q := range secondPhaseParticipants {
+				results[q] = results[q][:0]
+			}
+		} else if !secondPhase && firstPhaseTotalPulled >= limit && remainingUnexhausted > 0 {
+			// fmt.Println("have enough!", firstPhaseTotalPulled, "/", limit, "remaining", remainingUnexhausted)
+
+			// we will exclude this oldest number as it is not relevant anymore
+			// (we now want to keep track only of the oldest among the remaining iterators)
+			oldest = internal.IterEvent{Q: -1}
+
+			// HOW IT WORKS AFTER WE'VE REACHED THIS POINT (HIWAWVRTP)
+			// now we can combine the results we have and check what is our current oldest event.
+			// we also discard anything that is after the current cutting point (`limit`).
+			// so if we have [1,2,3], [10, 15, 20] and [7, 21, 49] but we only want 6 total
+			//   we can just keep [1,2,3,7,10,15] and discard [20, 21, 49],
+			//   and also adjust our `since` parameter to `15`, discarding anything we get after it
+			//   and immediately declaring that iterator exhausted.
+			// also every time we get result that is more recent than this updated `since` we can
+			//   keep it but also discard the previous since, moving the needle one back -- for example,
+			//   if we get an `8` we can keep it and move the `since` parameter to `10`, discarding `15`
+			//   in the process.
+			all := make([][]internal.IterEvent, len(results))
+			copy(all, results) // we have to use this otherwise internal.MergeSortMultiple will scramble our results slice
+			firstPhaseResults = internal.MergeSortMultiple(all, limit, nil)
+			oldest = firstPhaseResults[limit-1]
+			since = uint32(oldest.CreatedAt)
+			// fmt.Println("new since", since)
+
+			for q := range queries {
+				if exhausted[q] {
+					continue
+				}
+
+				// we also automatically exhaust any of the iterators that have already passed the
+				// cutting point (`since`)
+				if results[q][len(results[q])-1].CreatedAt < oldest.CreatedAt {
+					exhausted[q] = true
+					remainingUnexhausted--
+					continue
+				}
+
+				// for all the remaining iterators,
+				// since we have merged all the events in this `firstPhaseResults` slice, we can empty the
+				//   current `results` slices and reuse them.
+				results[q] = results[q][:0]
+
+				// build this index of indexes with everybody who remains
+				secondPhaseParticipants = append(secondPhaseParticipants, q)
+			}
+
+			// we create these two lists and alternate between them so we don't have to create a
+			//   a new one every time
+			secondPhaseResultsA = make([]internal.IterEvent, 0, limit*2)
+			secondPhaseResultsB = make([]internal.IterEvent, 0, limit*2)
+
+			// from now on we won't run this block anymore
+			secondPhase = true
+		}
+
+		// fmt.Println("remaining", remainingUnexhausted)
+		if remainingUnexhausted == 0 {
+			break
+		}
+	}
+
+	// fmt.Println("is secondPhase?", secondPhase)
+
+	var combinedResults []internal.IterEvent
+
+	if secondPhase {
+		// fmt.Println("ending second phase")
+		// when we reach this point either secondPhaseResultsA or secondPhaseResultsB will be full of stuff,
+		//   the other will be empty
+		var secondPhaseResults []internal.IterEvent
+		// fmt.Println("xxx", secondPhaseResultsToggle, len(secondPhaseResultsA), len(secondPhaseResultsB))
+		if secondPhaseResultsToggle {
+			secondPhaseResults = secondPhaseResultsB
+			combinedResults = secondPhaseResultsA[0:limit] // reuse this
+			// fmt.Println("  using b", len(secondPhaseResultsA))
+		} else {
+			secondPhaseResults = secondPhaseResultsA
+			combinedResults = secondPhaseResultsB[0:limit] // reuse this
+			// fmt.Println("  using a", len(secondPhaseResultsA))
+		}
+
+		all := [][]internal.IterEvent{firstPhaseResults, secondPhaseResults}
+		combinedResults = internal.MergeSortMultiple(all, limit, combinedResults)
+		// fmt.Println("final combinedResults", len(combinedResults), cap(combinedResults), limit)
+	} else {
+		combinedResults = make([]internal.IterEvent, limit)
+		combinedResults = internal.MergeSortMultiple(results, limit, combinedResults)
+	}
+
+	return combinedResults, nil
+}

eventstore/lmdb/query_planner.go

@@ -0,0 +1,218 @@
+package lmdb
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/fiatjaf/eventstore/internal"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+type query struct {
+	i             int
+	dbi           lmdb.DBI
+	prefix        []byte
+	results       chan *nostr.Event
+	keySize       int
+	timestampSize int
+	startingPoint []byte
+}
+
+func (b *LMDBBackend) prepareQueries(filter nostr.Filter) (
+	queries []query,
+	extraAuthors [][32]byte,
+	extraKinds [][2]byte,
+	extraTagKey string,
+	extraTagValues []string,
+	since uint32,
+	err error,
+) {
+	// we will apply this to every query we return
+	defer func() {
+		if queries == nil {
+			return
+		}
+
+		var until uint32 = 4294967295
+		if filter.Until != nil {
+			if fu := uint32(*filter.Until); fu < until {
+				until = fu + 1
+			}
+		}
+		for i, q := range queries {
+			sp := make([]byte, len(q.prefix))
+			sp = sp[0:len(q.prefix)]
+			copy(sp, q.prefix)
+			queries[i].startingPoint = binary.BigEndian.AppendUint32(sp, uint32(until))
+			queries[i].results = make(chan *nostr.Event, 12)
+		}
+	}()
+
+	if filter.IDs != nil {
+		// when there are ids we ignore everything else
+		queries = make([]query, len(filter.IDs))
+		for i, idHex := range filter.IDs {
+			if len(idHex) != 64 {
+				return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid id '%s'", idHex)
+			}
+			prefix := make([]byte, 8)
+			if _, err := hex.Decode(prefix[0:8], []byte(idHex[0:8*2])); err != nil {
+				return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid id '%s'", idHex)
+			}
+			queries[i] = query{i: i, dbi: b.indexId, prefix: prefix[0:8], keySize: 8, timestampSize: 0}
+		}
+		return queries, nil, nil, "", nil, 0, nil
+	}
+
+	// this is where we'll end the iteration
+	if filter.Since != nil {
+		if fs := uint32(*filter.Since); fs > since {
+			since = fs
+		}
+	}
+
+	if len(filter.Tags) > 0 {
+		// we will select ONE tag to query for and ONE extra tag to do further narrowing, if available
+		tagKey, tagValues, goodness := internal.ChooseNarrowestTag(filter)
+
+		// we won't use a tag index for this as long as we have something else to match with
+		if goodness < 2 && (len(filter.Authors) > 0 || len(filter.Kinds) > 0) {
+			goto pubkeyMatching
+		}
+
+		// only "p" tag has a goodness of 2, so
+		if goodness == 2 {
+			// this means we got a "p" tag, so we will use the ptag-kind index
+			i := 0
+			if filter.Kinds != nil {
+				queries = make([]query, len(tagValues)*len(filter.Kinds))
+				for _, value := range tagValues {
+					if len(value) != 64 {
+						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
+					}
+
+					for _, kind := range filter.Kinds {
+						k := make([]byte, 8+2)
+						if _, err := hex.Decode(k[0:8], []byte(value[0:8*2])); err != nil {
+							return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
+						}
+						binary.BigEndian.PutUint16(k[8:8+2], uint16(kind))
+						queries[i] = query{i: i, dbi: b.indexPTagKind, prefix: k[0 : 8+2], keySize: 8 + 2 + 4, timestampSize: 4}
+						i++
+					}
+				}
+			} else {
+				// even if there are no kinds, in that case we will just return any kind and not care
+				queries = make([]query, len(tagValues))
+				for i, value := range tagValues {
+					if len(value) != 64 {
+						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
+					}
+
+					k := make([]byte, 8)
+					if _, err := hex.Decode(k[0:8], []byte(value[0:8*2])); err != nil {
+						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
+					}
+					queries[i] = query{i: i, dbi: b.indexPTagKind, prefix: k[0:8], keySize: 8 + 2 + 4, timestampSize: 4}
+				}
+			}
+		} else {
+			// otherwise we will use a plain tag index
+			queries = make([]query, len(tagValues))
+			for i, value := range tagValues {
+				// get key prefix (with full length) and offset where to write the created_at
+				dbi, k, offset := b.getTagIndexPrefix(value)
+				// remove the last parts part to get just the prefix we want here
+				prefix := k[0:offset]
+				queries[i] = query{i: i, dbi: dbi, prefix: prefix, keySize: len(prefix) + 4, timestampSize: 4}
+				i++
+			}
+
+			// add an extra kind filter if available (only do this on plain tag index, not on ptag-kind index)
+			if filter.Kinds != nil {
+				extraKinds = make([][2]byte, len(filter.Kinds))
+				for i, kind := range filter.Kinds {
+					binary.BigEndian.PutUint16(extraKinds[i][0:2], uint16(kind))
+				}
+			}
+		}
+
+		// add an extra author search if possible
+		if filter.Authors != nil {
+			extraAuthors = make([][32]byte, len(filter.Authors))
+			for i, pk := range filter.Authors {
+				hex.Decode(extraAuthors[i][:], []byte(pk))
+			}
+		}
+
+		// add an extra useless tag if available
+		filter.Tags = internal.CopyMapWithoutKey(filter.Tags, tagKey)
+		if len(filter.Tags) > 0 {
+			extraTagKey, extraTagValues, _ = internal.ChooseNarrowestTag(filter)
+		}
+
+		return queries, extraAuthors, extraKinds, extraTagKey, extraTagValues, since, nil
+	}
+
+pubkeyMatching:
+	if len(filter.Authors) > 0 {
+		if len(filter.Kinds) == 0 {
+			// will use pubkey index
+			queries = make([]query, len(filter.Authors))
+			for i, pubkeyHex := range filter.Authors {
+				if len(pubkeyHex) != 64 {
+					return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
+				}
+				prefix := make([]byte, 8)
+				if _, err := hex.Decode(prefix[0:8], []byte(pubkeyHex[0:8*2])); err != nil {
+					return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
+				}
+				queries[i] = query{i: i, dbi: b.indexPubkey, prefix: prefix[0:8], keySize: 8 + 4, timestampSize: 4}
+			}
+		} else {
+			// will use pubkeyKind index
+			queries = make([]query, len(filter.Authors)*len(filter.Kinds))
+			i := 0
+			for _, pubkeyHex := range filter.Authors {
+				for _, kind := range filter.Kinds {
+					if len(pubkeyHex) != 64 {
+						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
+					}
+					prefix := make([]byte, 8+2)
+					if _, err := hex.Decode(prefix[0:8], []byte(pubkeyHex[0:8*2])); err != nil {
+						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
+					}
+					binary.BigEndian.PutUint16(prefix[8:8+2], uint16(kind))
+					queries[i] = query{i: i, dbi: b.indexPubkeyKind, prefix: prefix[0 : 8+2], keySize: 10 + 4, timestampSize: 4}
+					i++
+				}
+			}
+		}
+
+		// potentially with an extra useless tag filtering
+		extraTagKey, extraTagValues, _ = internal.ChooseNarrowestTag(filter)
+		return queries, nil, nil, extraTagKey, extraTagValues, since, nil
+	}
+
+	if len(filter.Kinds) > 0 {
+		// will use a kind index
+		queries = make([]query, len(filter.Kinds))
+		for i, kind := range filter.Kinds {
+			prefix := make([]byte, 2)
+			binary.BigEndian.PutUint16(prefix[0:2], uint16(kind))
+			queries[i] = query{i: i, dbi: b.indexKind, prefix: prefix[0:2], keySize: 2 + 4, timestampSize: 4}
+		}
+
+		// potentially with an extra useless tag filtering
+		tagKey, tagValues, _ := internal.ChooseNarrowestTag(filter)
+		return queries, nil, nil, tagKey, tagValues, since, nil
+	}
+
+	// if we got here our query will have nothing to filter with
+	queries = make([]query, 1)
+	prefix := make([]byte, 0)
+	queries[0] = query{i: 0, dbi: b.indexCreatedAt, prefix: prefix, keySize: 0 + 4, timestampSize: 4}
+	return queries, nil, nil, "", nil, since, nil
+}

eventstore/lmdb/replace.go

@@ -0,0 +1,49 @@
+package lmdb
+
+import (
+	"context"
+	"fmt"
+	"math"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/fiatjaf/eventstore/internal"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *LMDBBackend) ReplaceEvent(ctx context.Context, evt *nostr.Event) error {
+	// sanity checking
+	if evt.CreatedAt > math.MaxUint32 || evt.Kind > math.MaxUint16 {
+		return fmt.Errorf("event with values out of expected boundaries")
+	}
+
+	return b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
+		filter := nostr.Filter{Limit: 1, Kinds: []int{evt.Kind}, Authors: []string{evt.PubKey}}
+		if nostr.IsAddressableKind(evt.Kind) {
+			// when addressable, add the "d" tag to the filter
+			filter.Tags = nostr.TagMap{"d": []string{evt.Tags.GetD()}}
+		}
+
+		// now we fetch the past events, whatever they are, delete them and then save the new
+		results, err := b.query(txn, filter, 10) // in theory limit could be just 1 and this should work
+		if err != nil {
+			return fmt.Errorf("failed to query past events with %s: %w", filter, err)
+		}
+
+		shouldStore := true
+		for _, previous := range results {
+			if internal.IsOlder(previous.Event, evt) {
+				if err := b.delete(txn, previous.Event); err != nil {
+					return fmt.Errorf("failed to delete event %s for replacing: %w", previous.Event.ID, err)
+				}
+			} else {
+				// there is a newer event already stored, so we won't store this
+				shouldStore = false
+			}
+		}
+		if shouldStore {
+			return b.save(txn, evt)
+		}
+
+		return nil
+	})
+}

eventstore/lmdb/save.go

@@ -0,0 +1,71 @@
+package lmdb
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"math"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/fiatjaf/eventstore"
+	bin "github.com/fiatjaf/eventstore/internal/binary"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *LMDBBackend) SaveEvent(ctx context.Context, evt *nostr.Event) error {
+	// sanity checking
+	if evt.CreatedAt > math.MaxUint32 || evt.Kind > math.MaxUint16 {
+		return fmt.Errorf("event with values out of expected boundaries")
+	}
+
+	return b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
+		if b.EnableHLLCacheFor != nil {
+			// modify hyperloglog caches relative to this
+			useCache, skipSaving := b.EnableHLLCacheFor(evt.Kind)
+
+			if useCache {
+				err := b.updateHyperLogLogCachedValues(txn, evt)
+				if err != nil {
+					return fmt.Errorf("failed to update hll cache: %w", err)
+				}
+				if skipSaving {
+					return nil
+				}
+			}
+		}
+
+		// check if we already have this id
+		id, _ := hex.DecodeString(evt.ID)
+		_, err := txn.Get(b.indexId, id)
+		if operr, ok := err.(*lmdb.OpError); ok && operr.Errno != lmdb.NotFound {
+			// we will only proceed if we get a NotFound
+			return eventstore.ErrDupEvent
+		}
+
+		return b.save(txn, evt)
+	})
+}
+
+func (b *LMDBBackend) save(txn *lmdb.Txn, evt *nostr.Event) error {
+	// encode to binary form so we'll save it
+	bin, err := bin.Marshal(evt)
+	if err != nil {
+		return err
+	}
+
+	idx := b.Serial()
+	// raw event store
+	if err := txn.Put(b.rawEventStore, idx, bin, 0); err != nil {
+		return err
+	}
+
+	// put indexes
+	for k := range b.getIndexKeysForEvent(evt) {
+		err := txn.Put(k.dbi, k.key, idx, 0)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

eventstore/lmdb/testdata/fuzz/FuzzQuery

@@ -0,0 +1 @@
+../../../internal/testdata/fuzz/FuzzQuery

eventstore/mmm/betterbinary/codec.go

@@ -0,0 +1,139 @@
+package betterbinary
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"math"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+const (
+	MaxKind         = math.MaxUint16
+	MaxCreatedAt    = math.MaxUint32
+	MaxContentSize  = math.MaxUint16
+	MaxTagCount     = math.MaxUint16
+	MaxTagItemCount = math.MaxUint8
+	MaxTagItemSize  = math.MaxUint16
+)
+
+func Measure(evt nostr.Event) int {
+	n := 135 // static base
+
+	n += 2 + // tag section length
+		2 + // number of tags
+		len(evt.Tags)*3 // each tag offset + each tag item count
+	for _, tag := range evt.Tags {
+		n += len(tag) * 2 // item length for each item in this tag
+		for _, item := range tag {
+			n += len(item) // actual tag item
+		}
+	}
+
+	// content length and actual content
+	n += 2 + len(evt.Content)
+
+	return n
+}
+
+func Marshal(evt nostr.Event, buf []byte) error {
+	buf[0] = 0
+
+	if evt.Kind > MaxKind {
+		return fmt.Errorf("kind is too big: %d, max is %d", evt.Kind, MaxKind)
+	}
+	binary.LittleEndian.PutUint16(buf[1:3], uint16(evt.Kind))
+
+	if evt.CreatedAt > MaxCreatedAt {
+		return fmt.Errorf("created_at is too big: %d, max is %d", evt.CreatedAt, MaxCreatedAt)
+	}
+	binary.LittleEndian.PutUint32(buf[3:7], uint32(evt.CreatedAt))
+
+	hex.Decode(buf[7:39], []byte(evt.ID))
+	hex.Decode(buf[39:71], []byte(evt.PubKey))
+	hex.Decode(buf[71:135], []byte(evt.Sig))
+
+	tagBase := 135
+	// buf[135:137] (tagsSectionLength) will be set later when we know the absolute size of the tags section
+
+	ntags := len(evt.Tags)
+	if ntags > MaxTagCount {
+		return fmt.Errorf("can't encode too many tags: %d, max is %d", ntags, MaxTagCount)
+	}
+	binary.LittleEndian.PutUint16(buf[137:139], uint16(ntags))
+
+	tagOffset := 2 + 2 + ntags*2
+	for t, tag := range evt.Tags {
+		binary.LittleEndian.PutUint16(buf[tagBase+2+2+t*2:], uint16(tagOffset))
+
+		itemCount := len(tag)
+		if itemCount > MaxTagItemCount {
+			return fmt.Errorf("can't encode a tag with so many items: %d, max is %d", itemCount, MaxTagItemCount)
+		}
+		buf[tagBase+tagOffset] = uint8(itemCount)
+
+		itemOffset := 1
+		for _, item := range tag {
+			itemSize := len(item)
+			if itemSize > MaxTagItemSize {
+				return fmt.Errorf("tag item is too large: %d, max is %d", itemSize, MaxTagItemSize)
+			}
+
+			binary.LittleEndian.PutUint16(buf[tagBase+tagOffset+itemOffset:], uint16(itemSize))
+			copy(buf[tagBase+tagOffset+itemOffset+2:], []byte(item))
+			itemOffset += 2 + len(item)
+		}
+		tagOffset += itemOffset
+	}
+
+	tagsSectionLength := tagOffset
+	binary.LittleEndian.PutUint16(buf[tagBase:], uint16(tagsSectionLength))
+
+	// content
+	if contentLength := len(evt.Content); contentLength > MaxContentSize {
+		return fmt.Errorf("content is too large: %d, max is %d", contentLength, MaxContentSize)
+	} else {
+		binary.LittleEndian.PutUint16(buf[tagBase+tagsSectionLength:], uint16(contentLength))
+	}
+	copy(buf[tagBase+tagsSectionLength+2:], []byte(evt.Content))
+
+	return nil
+}
+
+func Unmarshal(data []byte, evt *nostr.Event) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("failed to decode binary: %v", r)
+		}
+	}()
+
+	evt.Kind = int(binary.LittleEndian.Uint16(data[1:3]))
+	evt.CreatedAt = nostr.Timestamp(binary.LittleEndian.Uint32(data[3:7]))
+	evt.ID = hex.EncodeToString(data[7:39])
+	evt.PubKey = hex.EncodeToString(data[39:71])
+	evt.Sig = hex.EncodeToString(data[71:135])
+
+	const tagbase = 135
+	tagsSectionLength := binary.LittleEndian.Uint16(data[tagbase:])
+	ntags := binary.LittleEndian.Uint16(data[tagbase+2:])
+	evt.Tags = make(nostr.Tags, ntags)
+	for t := range evt.Tags {
+		offset := binary.LittleEndian.Uint16(data[tagbase+4+t*2:])
+		nitems := int(data[tagbase+offset])
+		tag := make(nostr.Tag, nitems)
+
+		curr := tagbase + offset + 1
+		for i := range tag {
+			length := binary.LittleEndian.Uint16(data[curr:])
+			tag[i] = string(data[curr+2 : curr+2+length])
+			curr += 2 + length
+		}
+		evt.Tags[t] = tag
+	}
+
+	contentLength := binary.LittleEndian.Uint16(data[tagbase+tagsSectionLength:])
+	evt.Content = string(data[tagbase+tagsSectionLength+2 : tagbase+tagsSectionLength+2+contentLength])
+
+	return err
+}

eventstore/mmm/betterbinary/filtering.go

@@ -0,0 +1,33 @@
+package betterbinary
+
+import (
+	"encoding/binary"
+	"slices"
+)
+
+func TagMatches(evtb []byte, key string, vals []string) bool {
+	matches := make([][]byte, 0, len(vals))
+	for _, val := range vals {
+		match := append([]byte{1, 0, key[0], uint8(len(val)), 0}, val...)
+		matches = append(matches, match)
+	}
+
+	ntags := binary.LittleEndian.Uint16(evtb[137:])
+	var t uint16
+	for t = 0; t < ntags; t++ {
+		offset := int(binary.LittleEndian.Uint16(evtb[139+t*2:]))
+		nitems := evtb[135+offset]
+		if nitems >= 2 {
+			for _, match := range matches {
+				if slices.Equal(evtb[135+offset+1:135+offset+1+len(match)], match) {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+func KindMatches(evtb []byte, kind uint16) bool {
+	return binary.LittleEndian.Uint16(evtb[1:3]) == kind
+}

eventstore/mmm/betterbinary/filtering_test.go

@@ -0,0 +1,51 @@
+package betterbinary
+
+import (
+	"testing"
+
+	"github.com/mailru/easyjson"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func TestTagFiltering(t *testing.T) {
+	for _, tc := range []struct {
+		json      string
+		tagKey    string
+		tagValues []string
+		matches   bool
+	}{
+		{
+			`{"id":"a9663055164ab8b30d9524656370c4bf93393bb051b7edf4556f40c5298dc0c7","pubkey":"ee11a5dff40c19a555f41fe42b48f00e618c91225622ae37b6c2bb67b76c4e49","created_at":1681778790,"kind":1,"sig":"4dfea1a6f73141d5691e43afc3234dbe73016db0fb207cf247e0127cc2591ee6b4be5b462272030a9bde75882aae810f359682b1b6ce6cbb97201141c576db42","content":"He got snowed in"}`,
+			"x",
+			[]string{"sadjqw", ""},
+			false,
+		},
+		{
+			`{"id":"a9663055164ab8b30d9524656370c4bf93393bb051b7edf4556f40c5298dc0c7","pubkey":"ee11a5dff40c19a555f41fe42b48f00e618c91225622ae37b6c2bb67b76c4e49","created_at":1681778790,"kind":1,"sig":"4dfea1a6f73141d5691e43afc3234dbe73016db0fb207cf247e0127cc2591ee6b4be5b462272030a9bde75882aae810f359682b1b6ce6cbb97201141c576db42","content":"He got snowed in","tags":[["client","gossip"],["p","e2ccf7cf20403f3f2a4a55b328f0de3be38558a7d5f33632fdaaefc726c1c8eb"],["e","2c86abcc98f7fd8a6750aab8df6c1863903f107206cc2d72e8afeb6c38357aed","wss://nostr-pub.wellorder.net/","root"]]}`,
+			"e",
+			[]string{"2c86abcc98f7fd8a6750aab8df6c1863903f107206cc2d72e8afeb6c38357aed"},
+			true,
+		},
+		{
+			`{"id":"3f551da67788c7aae15360d025595dc2d391f10bb7e759ee5d9b2ad7d64392e4","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","created_at":1712715433,"kind":1,"tags":[["-"],["askdasds"],["t","spam"],["t","nada"]],"content":"ggsgsgsg","sig":"43431f4cf8bd015305c2d484841e5730d261beeb375a86c57a61df3d26e820ce8d6712d2a3c89e3f2298597f14abf58079954e9e658ba59bfc2d7ce6384f25c7"}`,
+			"t",
+			[]string{"nothing", "nada"},
+			true,
+		},
+		{
+			`{"id":"3f551da67788c7aae15360d025595dc2d391f10bb7e759ee5d9b2ad7d64392e4","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","created_at":1712715433,"kind":1,"tags":[["-"],["askdasds"],["t","spam"],["t","nada"]],"content":"ggsgsgsg","sig":"43431f4cf8bd015305c2d484841e5730d261beeb375a86c57a61df3d26e820ce8d6712d2a3c89e3f2298597f14abf58079954e9e658ba59bfc2d7ce6384f25c7"}`,
+			"z",
+			[]string{"nothing", "nada"},
+			false,
+		},
+	} {
+		var evt nostr.Event
+		easyjson.Unmarshal([]byte(tc.json), &evt)
+		bin := make([]byte, Measure(evt))
+		Marshal(evt, bin)
+
+		if res := TagMatches(bin, tc.tagKey, tc.tagValues); res != tc.matches {
+			t.Fatalf("matched incorrectly: %v=>%v over %s was %v, expected %v", tc.tagKey, tc.tagValues, tc.json, res, tc.matches)
+		}
+	}
+}

eventstore/mmm/count.go

@@ -0,0 +1,91 @@
+package mmm
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"slices"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/fiatjaf/eventstore/mmm/betterbinary"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (il *IndexingLayer) CountEvents(ctx context.Context, filter nostr.Filter) (int64, error) {
+	var count int64 = 0
+
+	queries, extraAuthors, extraKinds, extraTagKey, extraTagValues, since, err := il.prepareQueries(filter)
+	if err != nil {
+		return 0, err
+	}
+
+	err = il.lmdbEnv.View(func(txn *lmdb.Txn) error {
+		// actually iterate
+		for _, q := range queries {
+			cursor, err := txn.OpenCursor(q.dbi)
+			if err != nil {
+				continue
+			}
+
+			it := &iterator{cursor: cursor}
+			it.seek(q.startingPoint)
+
+			for {
+				// we already have a k and a v and an err from the cursor setup, so check and use these
+				if it.err != nil ||
+					len(it.key) != q.keySize ||
+					!bytes.HasPrefix(it.key, q.prefix) {
+					// either iteration has errored or we reached the end of this prefix
+					break // stop this cursor and move to the next one
+				}
+
+				// "id" indexes don't contain a timestamp
+				if q.timestampSize == 4 {
+					createdAt := binary.BigEndian.Uint32(it.key[len(it.key)-4:])
+					if createdAt < since {
+						break
+					}
+				}
+
+				if extraAuthors == nil && extraKinds == nil && extraTagValues == nil {
+					count++
+				} else {
+					// fetch actual event
+					pos := positionFromBytes(it.posb)
+					bin := il.mmmm.mmapf[pos.start : pos.start+uint64(pos.size)]
+
+					// check it against pubkeys without decoding the entire thing
+					if extraAuthors != nil && !slices.Contains(extraAuthors, [32]byte(bin[39:71])) {
+						it.next()
+						continue
+					}
+
+					// check it against kinds without decoding the entire thing
+					if extraKinds != nil && !slices.Contains(extraKinds, [2]byte(bin[1:3])) {
+						it.next()
+						continue
+					}
+
+					// decode the entire thing (TODO: do a conditional decode while also checking the extra tag)
+					event := &nostr.Event{}
+					if err := betterbinary.Unmarshal(bin, event); err != nil {
+						it.next()
+						continue
+					}
+
+					// if there is still a tag to be checked, do it now
+					if !event.Tags.ContainsAny(extraTagKey, extraTagValues) {
+						it.next()
+						continue
+					}
+
+					count++
+				}
+			}
+		}
+
+		return nil
+	})
+
+	return count, err
+}

eventstore/mmm/delete.go

@@ -0,0 +1,78 @@
+package mmm
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"slices"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (il *IndexingLayer) DeleteEvent(ctx context.Context, evt *nostr.Event) error {
+	return il.mmmm.lmdbEnv.Update(func(mmmtxn *lmdb.Txn) error {
+		return il.lmdbEnv.Update(func(iltxn *lmdb.Txn) error {
+			return il.delete(mmmtxn, iltxn, evt)
+		})
+	})
+}
+
+func (il *IndexingLayer) delete(mmmtxn *lmdb.Txn, iltxn *lmdb.Txn, evt *nostr.Event) error {
+	zeroRefs := false
+	b := il.mmmm
+
+	b.Logger.Debug().Str("layer", il.name).Uint16("il", il.id).Msg("deleting")
+
+	// first in the mmmm txn we check if we have the event still
+	idPrefix8, _ := hex.DecodeString(evt.ID[0 : 8*2])
+	val, err := mmmtxn.Get(b.indexId, idPrefix8)
+	if err != nil {
+		if lmdb.IsNotFound(err) {
+			// we already do not have this anywhere
+			return nil
+		}
+		return fmt.Errorf("failed to check if we have the event %x: %w", idPrefix8, err)
+	}
+
+	// we have this, but do we have it in the current layer?
+	// val is [posb][il_idx][il_idx...]
+	pos := positionFromBytes(val[0:12])
+
+	// check references
+	currentLayer := binary.BigEndian.AppendUint16(nil, il.id)
+	for i := 12; i < len(val); i += 2 {
+		if slices.Equal(val[i:i+2], currentLayer) {
+			// we will remove the current layer if it's found
+			nextval := make([]byte, len(val)-2)
+			copy(nextval, val[0:i])
+			copy(nextval[i:], val[i+2:])
+
+			if err := mmmtxn.Put(b.indexId, idPrefix8, nextval, 0); err != nil {
+				return fmt.Errorf("failed to update references for %x: %w", idPrefix8, err)
+			}
+
+			// if there are no more layers we will delete everything later
+			zeroRefs = len(nextval) == 12
+
+			break
+		}
+	}
+
+	// calculate all index keys we have for this event and delete them
+	for k := range il.getIndexKeysForEvent(evt) {
+		if err := iltxn.Del(k.dbi, k.key, val[0:12]); err != nil && !lmdb.IsNotFound(err) {
+			return fmt.Errorf("index entry %v/%x deletion failed: %w", k.dbi, k.key, err)
+		}
+	}
+
+	// if there are no more refs we delete the event from the id index and mmap
+	if zeroRefs {
+		if err := b.purge(mmmtxn, idPrefix8, pos); err != nil {
+			panic(err)
+		}
+	}
+
+	return nil
+}

eventstore/mmm/freeranges.go

@@ -0,0 +1,68 @@
+package mmm
+
+import (
+	"fmt"
+	"slices"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+)
+
+func (b *MultiMmapManager) mergeNewFreeRange(pos position) (isAtEnd bool) {
+	// before adding check if we can merge this with some other range
+	// (to merge means to delete the previous and add a new one)
+	toDelete := make([]int, 0, 2)
+	for f, fr := range b.freeRanges {
+		if pos.start+uint64(pos.size) == fr.start {
+			// [new_pos_to_be_freed][existing_fr] -> merge!
+			toDelete = append(toDelete, f)
+			pos.size = pos.size + fr.size
+		} else if fr.start+uint64(fr.size) == pos.start {
+			// [existing_fr][new_pos_to_be_freed] -> merge!
+			toDelete = append(toDelete, f)
+			pos.start = fr.start
+			pos.size = fr.size + pos.size
+		}
+	}
+	slices.SortFunc(toDelete, func(a, b int) int { return b - a })
+	for _, idx := range toDelete {
+		b.freeRanges = slices.Delete(b.freeRanges, idx, idx+1)
+	}
+
+	// when we're at the end of a file we just delete everything and don't add new free ranges
+	// the caller will truncate the mmap file and adjust the position accordingly
+	if pos.start+uint64(pos.size) == b.mmapfEnd {
+		return true
+	}
+
+	b.addNewFreeRange(pos)
+	return false
+}
+
+func (b *MultiMmapManager) addNewFreeRange(pos position) {
+	// update freeranges slice in memory
+	idx, _ := slices.BinarySearchFunc(b.freeRanges, pos, func(item, target position) int {
+		if item.size > target.size {
+			return 1
+		} else if target.size > item.size {
+			return -1
+		} else if item.start > target.start {
+			return 1
+		} else {
+			return -1
+		}
+	})
+	b.freeRanges = slices.Insert(b.freeRanges, idx, pos)
+}
+
+func (b *MultiMmapManager) saveFreeRanges(txn *lmdb.Txn) error {
+	// save to database
+	valReserve, err := txn.PutReserve(b.stuff, FREERANGES_KEY, len(b.freeRanges)*12, 0)
+	if err != nil {
+		return fmt.Errorf("on put freeranges: %w", err)
+	}
+	for f, fr := range b.freeRanges {
+		bytesFromPosition(valReserve[f*12:], fr)
+	}
+
+	return nil
+}

eventstore/mmm/fuzz_test.go

@@ -0,0 +1,191 @@
+package mmm
+
+import (
+	"context"
+	"fmt"
+	"math/rand/v2"
+	"os"
+	"slices"
+	"testing"
+
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/require"
+)
+
+func FuzzTest(f *testing.F) {
+	f.Add(0, uint(84), uint(10), uint(5))
+	f.Fuzz(func(t *testing.T, seed int, nlayers, nevents, ndeletes uint) {
+		nlayers = nlayers%23 + 1
+		nevents = nevents%10000 + 1
+		ndeletes = ndeletes % nevents
+
+		// create a temporary directory for the test
+		tmpDir, err := os.MkdirTemp("", "mmm-test-*")
+		require.NoError(t, err)
+		defer os.RemoveAll(tmpDir)
+
+		logger := zerolog.Nop()
+		rnd := rand.New(rand.NewPCG(uint64(seed), 0))
+
+		// initialize MMM
+		mmm := &MultiMmapManager{
+			Dir:    tmpDir,
+			Logger: &logger,
+		}
+
+		err = mmm.Init()
+		require.NoError(t, err)
+		defer mmm.Close()
+
+		for i := range nlayers {
+			name := string([]byte{97 + byte(i)})
+			err = mmm.EnsureLayer(name, &IndexingLayer{
+				MaxLimit: 1000,
+			})
+			require.NoError(t, err, "layer %s/%d", name, i)
+		}
+
+		// create test events
+		ctx := context.Background()
+		sk := "945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb"
+		storedIds := make([]string, nevents)
+		nTags := make(map[string]int)
+		storedByLayer := make(map[string][]string)
+
+		// create n events with random combinations of tags
+		for i := 0; i < int(nevents); i++ {
+			tags := nostr.Tags{}
+			// randomly add 1-nlayers tags
+			numTags := 1 + (i % int(nlayers))
+			usedTags := make(map[string]bool)
+
+			for j := 0; j < numTags; j++ {
+				tag := string([]byte{97 + byte(i%int(nlayers))})
+				if !usedTags[tag] {
+					tags = append(tags, nostr.Tag{"t", tag})
+					usedTags[tag] = true
+				}
+			}
+
+			evt := &nostr.Event{
+				CreatedAt: nostr.Timestamp(i),
+				Kind:      i, // hack to query by serial id
+				Tags:      tags,
+				Content:   fmt.Sprintf("test content %d", i),
+			}
+			evt.Sign(sk)
+
+			for _, layer := range mmm.layers {
+				if evt.Tags.FindWithValue("t", layer.name) != nil {
+					err := layer.SaveEvent(ctx, evt)
+					require.NoError(t, err)
+					storedByLayer[layer.name] = append(storedByLayer[layer.name], evt.ID)
+				}
+			}
+
+			storedIds = append(storedIds, evt.ID)
+			nTags[evt.ID] = len(evt.Tags)
+		}
+
+		// verify each layer has the correct events
+		for _, layer := range mmm.layers {
+			results, err := layer.QueryEvents(ctx, nostr.Filter{})
+			require.NoError(t, err)
+
+			count := 0
+			for evt := range results {
+				require.True(t, evt.Tags.ContainsAny("t", []string{layer.name}))
+				count++
+			}
+			require.Equal(t, count, len(storedByLayer[layer.name]))
+		}
+
+		// randomly select n events to delete from random layers
+		deleted := make(map[string][]*IndexingLayer)
+
+		for range ndeletes {
+			id := storedIds[rnd.Int()%len(storedIds)]
+			layer := mmm.layers[rnd.Int()%len(mmm.layers)]
+
+			evt, layers := mmm.GetByID(id)
+
+			if slices.Contains(deleted[id], layer) {
+				// already deleted from this layer
+				require.NotContains(t, layers, layer)
+			} else if evt != nil && evt.Tags.FindWithValue("t", layer.name) != nil {
+				require.Contains(t, layers, layer)
+
+				// delete now
+				layer.DeleteEvent(ctx, evt)
+				deleted[id] = append(deleted[id], layer)
+			} else {
+				// was never saved to this in the first place
+				require.NotContains(t, layers, layer)
+			}
+		}
+
+		for id, deletedlayers := range deleted {
+			evt, foundlayers := mmm.GetByID(id)
+
+			for _, layer := range deletedlayers {
+				require.NotContains(t, foundlayers, layer)
+			}
+			for _, layer := range foundlayers {
+				require.NotNil(t, evt.Tags.FindWithValue("t", layer.name))
+			}
+
+			if nTags[id] == len(deletedlayers) && evt != nil {
+				deletedlayersnames := make([]string, len(deletedlayers))
+				for i, layer := range deletedlayers {
+					deletedlayersnames[i] = layer.name
+				}
+
+				t.Fatalf("id %s has %d tags %v, should have been deleted from %v, but wasn't: %s",
+					id, nTags[id], evt.Tags, deletedlayersnames, evt)
+			} else if nTags[id] > len(deletedlayers) {
+				t.Fatalf("id %s should still be available as it had %d tags and was only deleted from %v, but isn't",
+					id, nTags[id], deletedlayers)
+			}
+
+			if evt != nil {
+				for _, layer := range mmm.layers {
+					// verify event still accessible from other layers
+					if slices.Contains(foundlayers, layer) {
+						ch, err := layer.QueryEvents(ctx, nostr.Filter{Kinds: []int{evt.Kind}}) // hack
+						require.NoError(t, err)
+						fetched := <-ch
+						require.NotNil(t, fetched)
+					} else {
+						// and not accessible from this layer we just deleted
+						ch, err := layer.QueryEvents(ctx, nostr.Filter{Kinds: []int{evt.Kind}}) // hack
+						require.NoError(t, err)
+						fetched := <-ch
+						require.Nil(t, fetched)
+					}
+				}
+			}
+		}
+
+		// now delete a layer and events that only exist in that layer should vanish
+		layer := mmm.layers[rnd.Int()%len(mmm.layers)]
+		ch, err := layer.QueryEvents(ctx, nostr.Filter{})
+		require.NoError(t, err)
+
+		eventsThatShouldVanish := make([]string, 0, nevents/2)
+		for evt := range ch {
+			if len(evt.Tags) == 1+len(deleted[evt.ID]) {
+				eventsThatShouldVanish = append(eventsThatShouldVanish, evt.ID)
+			}
+		}
+
+		err = mmm.DropLayer(layer.name)
+		require.NoError(t, err)
+
+		for _, id := range eventsThatShouldVanish {
+			v, ils := mmm.GetByID(id)
+			require.Nil(t, v)
+			require.Empty(t, ils)
+		}
+	})
+}

eventstore/mmm/helpers.go

@@ -0,0 +1,165 @@
+package mmm
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"iter"
+	"slices"
+	"strconv"
+	"strings"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+// this iterator always goes backwards
+type iterator struct {
+	cursor *lmdb.Cursor
+	key    []byte
+	posb   []byte
+	err    error
+}
+
+func (it *iterator) seek(key []byte) {
+	if _, _, errsr := it.cursor.Get(key, nil, lmdb.SetRange); errsr != nil {
+		if operr, ok := errsr.(*lmdb.OpError); !ok || operr.Errno != lmdb.NotFound {
+			// in this case it's really an error
+			panic(operr)
+		} else {
+			// we're at the end and we just want notes before this,
+			// so we just need to set the cursor the last key, this is not a real error
+			it.key, it.posb, it.err = it.cursor.Get(nil, nil, lmdb.Last)
+		}
+	} else {
+		// move one back as the first step
+		it.key, it.posb, it.err = it.cursor.Get(nil, nil, lmdb.Prev)
+	}
+}
+
+func (it *iterator) next() {
+	// move one back (we'll look into k and v and err in the next iteration)
+	it.key, it.posb, it.err = it.cursor.Get(nil, nil, lmdb.Prev)
+}
+
+type key struct {
+	dbi lmdb.DBI
+	key []byte
+}
+
+func (il *IndexingLayer) getIndexKeysForEvent(evt *nostr.Event) iter.Seq[key] {
+	return func(yield func(key) bool) {
+		{
+			// ~ by pubkey+date
+			k := make([]byte, 8+4)
+			hex.Decode(k[0:8], []byte(evt.PubKey[0:8*2]))
+			binary.BigEndian.PutUint32(k[8:8+4], uint32(evt.CreatedAt))
+			if !yield(key{dbi: il.indexPubkey, key: k[0 : 8+4]}) {
+				return
+			}
+		}
+
+		{
+			// ~ by kind+date
+			k := make([]byte, 2+4)
+			binary.BigEndian.PutUint16(k[0:2], uint16(evt.Kind))
+			binary.BigEndian.PutUint32(k[2:2+4], uint32(evt.CreatedAt))
+			if !yield(key{dbi: il.indexKind, key: k[0 : 2+4]}) {
+				return
+			}
+		}
+
+		{
+			// ~ by pubkey+kind+date
+			k := make([]byte, 8+2+4)
+			hex.Decode(k[0:8], []byte(evt.PubKey[0:8*2]))
+			binary.BigEndian.PutUint16(k[8:8+2], uint16(evt.Kind))
+			binary.BigEndian.PutUint32(k[8+2:8+2+4], uint32(evt.CreatedAt))
+			if !yield(key{dbi: il.indexPubkeyKind, key: k[0 : 8+2+4]}) {
+				return
+			}
+		}
+
+		// ~ by tagvalue+date
+		// ~ by p-tag+kind+date
+		for i, tag := range evt.Tags {
+			if len(tag) < 2 || len(tag[0]) != 1 || len(tag[1]) == 0 || len(tag[1]) > 100 {
+				// not indexable
+				continue
+			}
+			firstIndex := slices.IndexFunc(evt.Tags, func(t nostr.Tag) bool { return len(t) >= 2 && t[1] == tag[1] })
+			if firstIndex != i {
+				// duplicate
+				continue
+			}
+
+			// get key prefix (with full length) and offset where to write the created_at
+			dbi, k, offset := il.getTagIndexPrefix(tag[1])
+			binary.BigEndian.PutUint32(k[offset:], uint32(evt.CreatedAt))
+			if !yield(key{dbi: dbi, key: k}) {
+				return
+			}
+
+			// now the p-tag+kind+date
+			if dbi == il.indexTag32 && tag[0] == "p" {
+				k := make([]byte, 8+2+4)
+				hex.Decode(k[0:8], []byte(tag[1][0:8*2]))
+				binary.BigEndian.PutUint16(k[8:8+2], uint16(evt.Kind))
+				binary.BigEndian.PutUint32(k[8+2:8+2+4], uint32(evt.CreatedAt))
+				dbi := il.indexPTagKind
+				if !yield(key{dbi: dbi, key: k[0 : 8+2+4]}) {
+					return
+				}
+			}
+		}
+
+		{
+			// ~ by date only
+			k := make([]byte, 4)
+			binary.BigEndian.PutUint32(k[0:4], uint32(evt.CreatedAt))
+			if !yield(key{dbi: il.indexCreatedAt, key: k[0:4]}) {
+				return
+			}
+		}
+	}
+}
+
+func (il *IndexingLayer) getTagIndexPrefix(tagValue string) (lmdb.DBI, []byte, int) {
+	var k []byte   // the key with full length for created_at and idx at the end, but not filled with these
+	var offset int // the offset -- i.e. where the prefix ends and the created_at and idx would start
+	var dbi lmdb.DBI
+
+	// if it's 32 bytes as hex, save it as bytes
+	if len(tagValue) == 64 {
+		// but we actually only use the first 8 bytes
+		k = make([]byte, 8+4)
+		if _, err := hex.Decode(k[0:8], []byte(tagValue[0:8*2])); err == nil {
+			offset = 8
+			dbi = il.indexTag32
+			return dbi, k[0 : 8+4], offset
+		}
+	}
+
+	// if it looks like an "a" tag, index it in this special format
+	spl := strings.Split(tagValue, ":")
+	if len(spl) == 3 && len(spl[1]) == 64 {
+		k = make([]byte, 2+8+30)
+		if _, err := hex.Decode(k[2:2+8], []byte(tagValue[0:8*2])); err == nil {
+			if kind, err := strconv.ParseUint(spl[0], 10, 16); err == nil {
+				k[0] = byte(kind >> 8)
+				k[1] = byte(kind)
+				// limit "d" identifier to 30 bytes (so we don't have to grow our byte slice)
+				n := copy(k[2+8:2+8+30], spl[2])
+				offset = 2 + 8 + n
+				return dbi, k[0 : offset+4], offset
+			}
+		}
+	}
+
+	// index whatever else as utf-8, but limit it to 40 bytes
+	k = make([]byte, 40+4)
+	n := copy(k[0:40], tagValue)
+	offset = n
+	dbi = il.indexTag
+
+	return dbi, k[0 : n+4], offset
+}

eventstore/mmm/indexinglayer.go

@@ -0,0 +1,200 @@
+package mmm
+
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/fiatjaf/eventstore"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+var _ eventstore.Store = (*IndexingLayer)(nil)
+
+type IndexingLayer struct {
+	isInitialized bool
+	name          string
+
+	ShouldIndex func(context.Context, *nostr.Event) bool
+	MaxLimit    int
+
+	mmmm *MultiMmapManager
+
+	// this is stored in the knownLayers db as a value, and used to keep track of which layer owns each event
+	id uint16
+
+	lmdbEnv *lmdb.Env
+
+	indexCreatedAt  lmdb.DBI
+	indexKind       lmdb.DBI
+	indexPubkey     lmdb.DBI
+	indexPubkeyKind lmdb.DBI
+	indexTag        lmdb.DBI
+	indexTag32      lmdb.DBI
+	indexTagAddr    lmdb.DBI
+	indexPTagKind   lmdb.DBI
+}
+
+type IndexingLayers []*IndexingLayer
+
+func (ils IndexingLayers) ByID(ilid uint16) *IndexingLayer {
+	for _, il := range ils {
+		if il.id == ilid {
+			return il
+		}
+	}
+	return nil
+}
+
+const multiIndexCreationFlags uint = lmdb.Create | lmdb.DupSort
+
+func (il *IndexingLayer) Init() error {
+	if il.isInitialized {
+		return nil
+	}
+	il.isInitialized = true
+
+	path := filepath.Join(il.mmmm.Dir, il.name)
+
+	if il.MaxLimit == 0 {
+		il.MaxLimit = 500
+	}
+
+	// open lmdb
+	env, err := lmdb.NewEnv()
+	if err != nil {
+		return err
+	}
+
+	env.SetMaxDBs(8)
+	env.SetMaxReaders(1000)
+	env.SetMapSize(1 << 38) // ~273GB
+
+	// create directory if it doesn't exist and open it
+	if err := os.MkdirAll(path, 0755); err != nil {
+		return err
+	}
+
+	err = env.Open(path, lmdb.NoTLS, 0644)
+	if err != nil {
+		return err
+	}
+	il.lmdbEnv = env
+
+	// open each db
+	if err := il.lmdbEnv.Update(func(txn *lmdb.Txn) error {
+		if dbi, err := txn.OpenDBI("created_at", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			il.indexCreatedAt = dbi
+		}
+		if dbi, err := txn.OpenDBI("kind", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			il.indexKind = dbi
+		}
+		if dbi, err := txn.OpenDBI("pubkey", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			il.indexPubkey = dbi
+		}
+		if dbi, err := txn.OpenDBI("pubkeyKind", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			il.indexPubkeyKind = dbi
+		}
+		if dbi, err := txn.OpenDBI("tag", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			il.indexTag = dbi
+		}
+		if dbi, err := txn.OpenDBI("tag32", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			il.indexTag32 = dbi
+		}
+		if dbi, err := txn.OpenDBI("tagaddr", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			il.indexTagAddr = dbi
+		}
+		if dbi, err := txn.OpenDBI("ptagKind", multiIndexCreationFlags); err != nil {
+			return err
+		} else {
+			il.indexPTagKind = dbi
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (il *IndexingLayer) Name() string { return il.name }
+
+func (il *IndexingLayer) runThroughEvents(txn *lmdb.Txn) error {
+	ctx := context.Background()
+	b := il.mmmm
+
+	// run through all events we have and see if this new index wants them
+	cursor, err := txn.OpenCursor(b.indexId)
+	if err != nil {
+		return fmt.Errorf("when opening cursor on %v: %w", b.indexId, err)
+	}
+	defer cursor.Close()
+
+	for {
+		idPrefix8, val, err := cursor.Get(nil, nil, lmdb.Next)
+		if lmdb.IsNotFound(err) {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("when moving the cursor: %w", err)
+		}
+
+		update := false
+
+		posb := val[0:12]
+		pos := positionFromBytes(posb)
+		evt := &nostr.Event{}
+		if err := b.loadEvent(pos, evt); err != nil {
+			return fmt.Errorf("when loading event from mmap: %w", err)
+		}
+
+		if il.ShouldIndex != nil && il.ShouldIndex(ctx, evt) {
+			// add the current reference
+			val = binary.BigEndian.AppendUint16(val, il.id)
+
+			// if we were already updating to remove the reference
+			// now that we've added the reference back we don't really have to update
+			update = !update
+
+			// actually index
+			if err := il.lmdbEnv.Update(func(iltxn *lmdb.Txn) error {
+				for k := range il.getIndexKeysForEvent(evt) {
+					if err := iltxn.Put(k.dbi, k.key, posb, 0); err != nil {
+						return err
+					}
+				}
+				return nil
+			}); err != nil {
+				return fmt.Errorf("failed to index: %w", err)
+			}
+		}
+
+		if update {
+			if err := txn.Put(b.indexId, idPrefix8, val, 0); err != nil {
+				return fmt.Errorf("failed to put updated index+refs: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+func (il *IndexingLayer) Close() {
+	il.lmdbEnv.Close()
+}

eventstore/mmm/mmmm.go

@@ -0,0 +1,335 @@
+package mmm
+
+import (
+	"encoding/binary"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"sync"
+	"syscall"
+	"unsafe"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/fiatjaf/eventstore/mmm/betterbinary"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/rs/zerolog"
+)
+
+type mmap []byte
+
+func (_ mmap) String() string { return "<memory-mapped file>" }
+
+type MultiMmapManager struct {
+	Dir    string
+	Logger *zerolog.Logger
+
+	layers IndexingLayers
+
+	mmapfPath string
+	mmapf     mmap
+	mmapfEnd  uint64
+
+	lmdbEnv     *lmdb.Env
+	stuff       lmdb.DBI
+	knownLayers lmdb.DBI
+	indexId     lmdb.DBI
+
+	freeRanges []position
+
+	mutex sync.Mutex
+}
+
+func (b *MultiMmapManager) String() string {
+	return fmt.Sprintf("<MultiMmapManager on %s with %d layers @ %v>", b.Dir, len(b.layers), unsafe.Pointer(b))
+}
+
+const (
+	MMAP_INFINITE_SIZE = 1 << 40
+	maxuint16          = 65535
+	maxuint32          = 4294967295
+)
+
+var FREERANGES_KEY = []byte{'F'}
+
+func (b *MultiMmapManager) Init() error {
+	// create directory if it doesn't exist
+	dbpath := filepath.Join(b.Dir, "mmmm")
+	if err := os.MkdirAll(dbpath, 0755); err != nil {
+		return fmt.Errorf("failed to create directory %s: %w", dbpath, err)
+	}
+
+	// open a huge mmapped file
+	b.mmapfPath = filepath.Join(b.Dir, "events")
+	file, err := os.OpenFile(b.mmapfPath, os.O_CREATE|os.O_RDWR, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open events file at %s: %w", b.mmapfPath, err)
+	}
+	mmapf, err := syscall.Mmap(int(file.Fd()), 0, MMAP_INFINITE_SIZE,
+		syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
+	if err != nil {
+		return fmt.Errorf("failed to mmap events file at %s: %w", b.mmapfPath, err)
+	}
+	b.mmapf = mmapf
+
+	if stat, err := os.Stat(b.mmapfPath); err != nil {
+		return err
+	} else {
+		b.mmapfEnd = uint64(stat.Size())
+	}
+
+	// open lmdb
+	env, err := lmdb.NewEnv()
+	if err != nil {
+		return err
+	}
+
+	env.SetMaxDBs(3)
+	env.SetMaxReaders(1000)
+	env.SetMapSize(1 << 38) // ~273GB
+
+	err = env.Open(dbpath, lmdb.NoTLS, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open lmdb at %s: %w", dbpath, err)
+	}
+	b.lmdbEnv = env
+
+	if err := b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
+		if dbi, err := txn.OpenDBI("stuff", lmdb.Create); err != nil {
+			return err
+		} else {
+			b.stuff = dbi
+		}
+
+		// this just keeps track of all the layers we know (just their names)
+		// they will be instantiated by the application after their name is read from the database.
+		// new layers created at runtime will be saved here.
+		if dbi, err := txn.OpenDBI("layers", lmdb.Create); err != nil {
+			return err
+		} else {
+			b.knownLayers = dbi
+		}
+
+		// this is a global index of events by id that also keeps references
+		// to all the layers that may be indexing them -- such that whenever
+		// an event is deleted from all layers it can be deleted from global
+		if dbi, err := txn.OpenDBI("id-references", lmdb.Create); err != nil {
+			return err
+		} else {
+			b.indexId = dbi
+		}
+
+		// load all free ranges into memory
+		{
+			data, err := txn.Get(b.stuff, FREERANGES_KEY)
+			if err != nil && !lmdb.IsNotFound(err) {
+				return fmt.Errorf("on freeranges: %w", err)
+			}
+			b.freeRanges = make([]position, len(data)/12)
+			logOp := b.Logger.Debug()
+			for f := range b.freeRanges {
+				pos := positionFromBytes(data[f*12 : (f+1)*12])
+				b.freeRanges[f] = pos
+				if pos.size > 20 {
+					logOp = logOp.Uint32(fmt.Sprintf("%d", pos.start), pos.size)
+				}
+			}
+			slices.SortFunc(b.freeRanges, func(a, b position) int { return int(a.size - b.size) })
+			logOp.Msg("loaded free ranges")
+		}
+
+		return nil
+	}); err != nil {
+		return fmt.Errorf("failed to open and load db data: %w", err)
+	}
+
+	return nil
+}
+
+func (b *MultiMmapManager) EnsureLayer(name string, il *IndexingLayer) error {
+	b.mutex.Lock()
+	defer b.mutex.Unlock()
+
+	il.mmmm = b
+	il.name = name
+
+	err := b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
+		txn.RawRead = true
+
+		nameb := []byte(name)
+		if idv, err := txn.Get(b.knownLayers, nameb); lmdb.IsNotFound(err) {
+			if id, err := b.getNextAvailableLayerId(txn); err != nil {
+				return fmt.Errorf("failed to reserve a layer id for %s: %w", name, err)
+			} else {
+				il.id = id
+			}
+
+			if err := il.Init(); err != nil {
+				return fmt.Errorf("failed to init new layer %s: %w", name, err)
+			}
+
+			if err := il.runThroughEvents(txn); err != nil {
+				return fmt.Errorf("failed to run %s through events: %w", name, err)
+			}
+			return txn.Put(b.knownLayers, []byte(name), binary.BigEndian.AppendUint16(nil, il.id), 0)
+		} else if err == nil {
+			il.id = binary.BigEndian.Uint16(idv)
+
+			if err := il.Init(); err != nil {
+				return fmt.Errorf("failed to init old layer %s: %w", name, err)
+			}
+
+			return nil
+		} else {
+			return err
+		}
+	})
+	if err != nil {
+		return err
+	}
+
+	b.layers = append(b.layers, il)
+	return nil
+}
+
+func (b *MultiMmapManager) DropLayer(name string) error {
+	b.mutex.Lock()
+	defer b.mutex.Unlock()
+
+	// get layer reference
+	idx := slices.IndexFunc(b.layers, func(il *IndexingLayer) bool { return il.name == name })
+	if idx == -1 {
+		return fmt.Errorf("layer '%s' doesn't exist", name)
+	}
+	il := b.layers[idx]
+
+	// remove layer references
+	err := b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
+		if err := b.removeAllReferencesFromLayer(txn, il.id); err != nil {
+			return err
+		}
+
+		return txn.Del(b.knownLayers, []byte(il.name), nil)
+	})
+	if err != nil {
+		return err
+	}
+
+	// delete everything (the indexes) from this layer db actually
+	err = il.lmdbEnv.Update(func(txn *lmdb.Txn) error {
+		for _, dbi := range []lmdb.DBI{
+			il.indexCreatedAt,
+			il.indexKind,
+			il.indexPubkey,
+			il.indexPubkeyKind,
+			il.indexTag,
+			il.indexTag32,
+			il.indexTagAddr,
+			il.indexPTagKind,
+		} {
+			if err := txn.Drop(dbi, true); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return il.lmdbEnv.Close()
+}
+
+func (b *MultiMmapManager) removeAllReferencesFromLayer(txn *lmdb.Txn, layerId uint16) error {
+	cursor, err := txn.OpenCursor(b.indexId)
+	if err != nil {
+		return fmt.Errorf("when opening cursor on %v: %w", b.indexId, err)
+	}
+	defer cursor.Close()
+
+	for {
+		idPrefix8, val, err := cursor.Get(nil, nil, lmdb.Next)
+		if lmdb.IsNotFound(err) {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("when moving the cursor: %w", err)
+		}
+
+		var zeroRefs bool
+		var update bool
+
+		needle := binary.BigEndian.AppendUint16(nil, layerId)
+		for s := 12; s < len(val); s += 2 {
+			if slices.Equal(val[s:s+2], needle) {
+				// swap delete
+				copy(val[s:s+2], val[len(val)-2:])
+				val = val[0 : len(val)-2]
+
+				update = true
+
+				// we must erase this event if its references reach zero
+				zeroRefs = len(val) == 12
+
+				break
+			}
+		}
+
+		if zeroRefs {
+			posb := val[0:12]
+			pos := positionFromBytes(posb)
+
+			if err := b.purge(txn, idPrefix8, pos); err != nil {
+				return fmt.Errorf("failed to purge unreferenced event %x: %w", idPrefix8, err)
+			}
+		} else if update {
+			if err := txn.Put(b.indexId, idPrefix8, val, 0); err != nil {
+				return fmt.Errorf("failed to put updated index+refs: %w", err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (b *MultiMmapManager) loadEvent(pos position, eventReceiver *nostr.Event) error {
+	return betterbinary.Unmarshal(b.mmapf[pos.start:pos.start+uint64(pos.size)], eventReceiver)
+}
+
+// getNextAvailableLayerId iterates through all existing layers to find a vacant id
+func (b *MultiMmapManager) getNextAvailableLayerId(txn *lmdb.Txn) (uint16, error) {
+	cursor, err := txn.OpenCursor(b.knownLayers)
+	if err != nil {
+		return 0, fmt.Errorf("failed to open cursor: %w", err)
+	}
+
+	used := [1 << 16]bool{}
+	_, val, err := cursor.Get(nil, nil, lmdb.First)
+	for err == nil {
+		// something was found
+		used[binary.BigEndian.Uint16(val)] = true
+		// next
+		_, val, err = cursor.Get(nil, nil, lmdb.Next)
+	}
+	if !lmdb.IsNotFound(err) {
+		// a real error
+		return 0, err
+	}
+
+	// loop exited, get the first available
+	var id uint16
+	for num, isUsed := range used {
+		if !isUsed {
+			id = uint16(num)
+			break
+		}
+	}
+	return id, nil
+}
+
+func (b *MultiMmapManager) Close() {
+	b.lmdbEnv.Close()
+	for _, il := range b.layers {
+		il.Close()
+	}
+}

eventstore/mmm/mmmm_test.go

@@ -0,0 +1,386 @@
+package mmm
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMultiLayerIndexing(t *testing.T) {
+	// Create a temporary directory for the test
+	tmpDir := "/tmp/eventstore-mmm-test"
+	os.RemoveAll(tmpDir)
+
+	logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr})
+
+	// initialize MMM with three layers:
+	// 1. odd timestamps layer
+	// 2. even timestamps layer
+	// 3. all events layer
+	mmm := &MultiMmapManager{
+		Dir:    tmpDir,
+		Logger: &logger,
+	}
+
+	err := mmm.Init()
+	require.NoError(t, err)
+	defer mmm.Close()
+
+	// create layers
+	err = mmm.EnsureLayer("odd", &IndexingLayer{
+		MaxLimit: 100,
+		ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool {
+			return evt.CreatedAt%2 == 1
+		},
+	})
+	require.NoError(t, err)
+	err = mmm.EnsureLayer("even", &IndexingLayer{
+		MaxLimit: 100,
+		ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool {
+			return evt.CreatedAt%2 == 0
+		},
+	})
+	require.NoError(t, err)
+	err = mmm.EnsureLayer("all", &IndexingLayer{
+		MaxLimit: 100,
+		ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool {
+			return true
+		},
+	})
+	require.NoError(t, err)
+
+	// create test events
+	ctx := context.Background()
+	baseTime := nostr.Timestamp(0)
+	sk := "945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb"
+	events := make([]*nostr.Event, 10)
+	for i := 0; i < 10; i++ {
+		evt := &nostr.Event{
+			CreatedAt: baseTime + nostr.Timestamp(i),
+			Kind:      1,
+			Tags:      nostr.Tags{},
+			Content:   "test content",
+		}
+		evt.Sign(sk)
+		events[i] = evt
+		stored, err := mmm.StoreGlobal(ctx, evt)
+		require.NoError(t, err)
+		require.True(t, stored)
+	}
+
+	{
+		// query odd layer
+		oddResults, err := mmm.layers[0].QueryEvents(ctx, nostr.Filter{
+			Kinds: []int{1},
+		})
+		require.NoError(t, err)
+
+		oddCount := 0
+		for evt := range oddResults {
+			require.Equal(t, evt.CreatedAt%2, nostr.Timestamp(1))
+			oddCount++
+		}
+		require.Equal(t, 5, oddCount)
+	}
+
+	{
+		// query even layer
+		evenResults, err := mmm.layers[1].QueryEvents(ctx, nostr.Filter{
+			Kinds: []int{1},
+		})
+		require.NoError(t, err)
+
+		evenCount := 0
+		for evt := range evenResults {
+			require.Equal(t, evt.CreatedAt%2, nostr.Timestamp(0))
+			evenCount++
+		}
+		require.Equal(t, 5, evenCount)
+	}
+
+	{
+		// query all layer
+		allResults, err := mmm.layers[2].QueryEvents(ctx, nostr.Filter{
+			Kinds: []int{1},
+		})
+		require.NoError(t, err)
+
+		allCount := 0
+		for range allResults {
+			allCount++
+		}
+		require.Equal(t, 10, allCount)
+	}
+
+	// delete some events
+	err = mmm.layers[0].DeleteEvent(ctx, events[1]) // odd timestamp
+	require.NoError(t, err)
+	err = mmm.layers[1].DeleteEvent(ctx, events[2]) // even timestamp
+
+	// verify deletions
+	{
+		oddResults, err := mmm.layers[0].QueryEvents(ctx, nostr.Filter{
+			Kinds: []int{1},
+		})
+		require.NoError(t, err)
+		oddCount := 0
+		for range oddResults {
+			oddCount++
+		}
+		require.Equal(t, 4, oddCount)
+	}
+
+	{
+		evenResults, err := mmm.layers[1].QueryEvents(ctx, nostr.Filter{
+			Kinds: []int{1},
+		})
+		require.NoError(t, err)
+		evenCount := 0
+		for range evenResults {
+			evenCount++
+		}
+		require.Equal(t, 4, evenCount)
+	}
+
+	{
+		allResults, err := mmm.layers[2].QueryEvents(ctx, nostr.Filter{
+			Kinds: []int{1},
+		})
+		require.NoError(t, err)
+		allCount := 0
+		for range allResults {
+			allCount++
+		}
+		require.Equal(t, 10, allCount)
+	}
+
+	// save events directly to layers regardless of timestamp
+	{
+		oddEvent := &nostr.Event{
+			CreatedAt: baseTime + 100, // even timestamp
+			Kind:      1,
+			Content:   "forced odd",
+		}
+		oddEvent.Sign(sk)
+		err = mmm.layers[0].SaveEvent(ctx, oddEvent) // save even timestamp to odd layer
+		require.NoError(t, err)
+
+		// it is added to the odd il
+		oddResults, err := mmm.layers[0].QueryEvents(ctx, nostr.Filter{
+			Kinds: []int{1},
+		})
+		require.NoError(t, err)
+		oddCount := 0
+		for range oddResults {
+			oddCount++
+		}
+		require.Equal(t, 5, oddCount)
+
+		// it doesn't affect the event il
+		evenResults, err := mmm.layers[1].QueryEvents(ctx, nostr.Filter{
+			Kinds: []int{1},
+		})
+		require.NoError(t, err)
+		evenCount := 0
+		for range evenResults {
+			evenCount++
+		}
+		require.Equal(t, 4, evenCount)
+	}
+
+	// test replaceable events
+	for _, layer := range mmm.layers {
+		replaceable := &nostr.Event{
+			CreatedAt: baseTime + 0,
+			Kind:      0,
+			Content:   fmt.Sprintf("first"),
+		}
+		replaceable.Sign(sk)
+		err := layer.ReplaceEvent(ctx, replaceable)
+		require.NoError(t, err)
+	}
+
+	// replace events alternating between layers
+	for i := range mmm.layers {
+		content := fmt.Sprintf("last %d", i)
+
+		newEvt := &nostr.Event{
+			CreatedAt: baseTime + 1000,
+			Kind:      0,
+			Content:   content,
+		}
+		newEvt.Sign(sk)
+
+		layer := mmm.layers[i]
+		err = layer.ReplaceEvent(ctx, newEvt)
+		require.NoError(t, err)
+
+		// verify replacement in the layer that did it
+		results, err := layer.QueryEvents(ctx, nostr.Filter{
+			Kinds: []int{0},
+		})
+		require.NoError(t, err)
+
+		count := 0
+		for evt := range results {
+			require.Equal(t, content, evt.Content)
+			count++
+		}
+		require.Equal(t, 1, count)
+
+		// verify other layers still have the old version
+		for j := 0; j < 3; j++ {
+			if mmm.layers[j] == layer {
+				continue
+			}
+			results, err := mmm.layers[j].QueryEvents(ctx, nostr.Filter{
+				Kinds: []int{0},
+			})
+			require.NoError(t, err)
+
+			count := 0
+			for evt := range results {
+				if i < j {
+					require.Equal(t, "first", evt.Content)
+				} else {
+					require.Equal(t, evt.Content, fmt.Sprintf("last %d", j))
+				}
+				count++
+			}
+
+			require.Equal(t, 1, count, "%d/%d", i, j)
+		}
+	}
+}
+
+func TestLayerReferenceTracking(t *testing.T) {
+	// Create a temporary directory for the test
+	tmpDir, err := os.MkdirTemp("", "mmm-test-*")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr})
+
+	// initialize MMM with three layers
+	mmm := &MultiMmapManager{
+		Dir:    tmpDir,
+		Logger: &logger,
+	}
+
+	err = mmm.Init()
+	require.NoError(t, err)
+	defer mmm.Close()
+
+	// create three layers
+	err = mmm.EnsureLayer("layer1", &IndexingLayer{
+		MaxLimit:    100,
+		ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool { return true },
+	})
+	require.NoError(t, err)
+	err = mmm.EnsureLayer("layer2", &IndexingLayer{
+		MaxLimit:    100,
+		ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool { return true },
+	})
+	require.NoError(t, err)
+	err = mmm.EnsureLayer("layer3", &IndexingLayer{
+		MaxLimit:    100,
+		ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool { return true },
+	})
+	require.NoError(t, err)
+	err = mmm.EnsureLayer("layer4", &IndexingLayer{
+		MaxLimit:    100,
+		ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool { return true },
+	})
+	require.NoError(t, err)
+
+	// create test events
+	ctx := context.Background()
+	sk := "945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb"
+	evt1 := &nostr.Event{
+		CreatedAt: 1000,
+		Kind:      1,
+		Tags:      nostr.Tags{},
+		Content:   "event 1",
+	}
+	evt1.Sign(sk)
+
+	evt2 := &nostr.Event{
+		CreatedAt: 2000,
+		Kind:      1,
+		Tags:      nostr.Tags{},
+		Content:   "event 2",
+	}
+	evt2.Sign(sk)
+
+	// save evt1 to layer1
+	err = mmm.layers[0].SaveEvent(ctx, evt1)
+	require.NoError(t, err)
+
+	// save evt1 to layer2
+	err = mmm.layers[1].SaveEvent(ctx, evt1)
+	require.NoError(t, err)
+
+	// save evt1 to layer4
+	err = mmm.layers[0].SaveEvent(ctx, evt1)
+	require.NoError(t, err)
+
+	// delete evt1 from layer1
+	err = mmm.layers[0].DeleteEvent(ctx, evt1)
+	require.NoError(t, err)
+
+	// save evt2 to layer3
+	err = mmm.layers[2].SaveEvent(ctx, evt2)
+	require.NoError(t, err)
+
+	// save evt2 to layer4
+	err = mmm.layers[3].SaveEvent(ctx, evt2)
+	require.NoError(t, err)
+
+	// save evt2 to layer3 again
+	err = mmm.layers[2].SaveEvent(ctx, evt2)
+	require.NoError(t, err)
+
+	// delete evt1 from layer4
+	err = mmm.layers[3].DeleteEvent(ctx, evt1)
+	require.NoError(t, err)
+
+	// verify the state of the indexId database
+	err = mmm.lmdbEnv.View(func(txn *lmdb.Txn) error {
+		cursor, err := txn.OpenCursor(mmm.indexId)
+		if err != nil {
+			return err
+		}
+		defer cursor.Close()
+
+		count := 0
+		for k, v, err := cursor.Get(nil, nil, lmdb.First); err == nil; k, v, err = cursor.Get(nil, nil, lmdb.Next) {
+			count++
+			if hex.EncodeToString(k) == evt1.ID[:16] {
+				// evt1 should only reference layer2
+				require.Equal(t, 14, len(v), "evt1 should have one layer reference")
+				layerRef := binary.BigEndian.Uint16(v[12:14])
+				require.Equal(t, mmm.layers[1].id, layerRef, "evt1 should reference layer2")
+			} else if hex.EncodeToString(k) == evt2.ID[:16] {
+				// evt2 should references to layer3 and layer4
+				require.Equal(t, 16, len(v), "evt2 should have two layer references")
+				layer3Ref := binary.BigEndian.Uint16(v[12:14])
+				require.Equal(t, mmm.layers[2].id, layer3Ref, "evt2 should reference layer3")
+				layer4Ref := binary.BigEndian.Uint16(v[14:16])
+				require.Equal(t, mmm.layers[3].id, layer4Ref, "evt2 should reference layer4")
+			} else {
+				t.Errorf("unexpected event in indexId: %x", k)
+			}
+		}
+		require.Equal(t, 2, count, "should have exactly two events in indexId")
+		return nil
+	})
+	require.NoError(t, err)
+}

eventstore/mmm/position.go

@@ -0,0 +1,27 @@
+package mmm
+
+import (
+	"encoding/binary"
+	"fmt"
+)
+
+type position struct {
+	start uint64
+	size  uint32
+}
+
+func (pos position) String() string {
+	return fmt.Sprintf("<%d|%d|%d>", pos.start, pos.size, pos.start+uint64(pos.size))
+}
+
+func positionFromBytes(posb []byte) position {
+	return position{
+		size:  binary.BigEndian.Uint32(posb[0:4]),
+		start: binary.BigEndian.Uint64(posb[4:12]),
+	}
+}
+
+func bytesFromPosition(out []byte, pos position) {
+	binary.BigEndian.PutUint32(out[0:4], pos.size)
+	binary.BigEndian.PutUint64(out[4:12], pos.start)
+}

eventstore/mmm/purge.go

@@ -0,0 +1,36 @@
+package mmm
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+)
+
+func (b *MultiMmapManager) purge(txn *lmdb.Txn, idPrefix8 []byte, pos position) error {
+	b.Logger.Debug().Hex("event", idPrefix8).Stringer("pos", pos).Msg("purging")
+
+	// delete from index
+	if err := txn.Del(b.indexId, idPrefix8, nil); err != nil {
+		return err
+	}
+
+	// will add the current range to free ranges, which means it is "deleted" (or merge with existing)
+	isAtEnd := b.mergeNewFreeRange(pos)
+
+	if isAtEnd {
+		// when at the end, truncate the mmap
+		// [new_pos_to_be_freed][end_of_file] -> shrink file!
+		pos.size = 0 // so we don't try to add this some lines below
+		if err := os.Truncate(b.mmapfPath, int64(pos.start)); err != nil {
+			panic(fmt.Errorf("error decreasing %s: %w", b.mmapfPath, err))
+		}
+		b.mmapfEnd = pos.start
+	} else {
+		// this is for debugging -------------
+		copy(b.mmapf[pos.start:], bytes.Repeat([]byte{'!'}, int(pos.size)))
+	}
+
+	return b.saveFreeRanges(txn)
+}

eventstore/mmm/query.go

@@ -0,0 +1,460 @@
+package mmm
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"log"
+	"slices"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/fiatjaf/eventstore/internal"
+	"github.com/fiatjaf/eventstore/mmm/betterbinary"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+// GetByID returns the event -- if found in this mmm -- and all the IndexingLayers it belongs to.
+func (b *MultiMmapManager) GetByID(id string) (*nostr.Event, IndexingLayers) {
+	events := make(chan *nostr.Event)
+	presence := make(chan []uint16)
+	b.queryByIDs(events, []string{id}, presence)
+	for evt := range events {
+		p := <-presence
+		present := make([]*IndexingLayer, len(p))
+		for i, id := range p {
+			present[i] = b.layers.ByID(id)
+		}
+		return evt, present
+	}
+	return nil, nil
+}
+
+// queryByIDs emits the events of the given id to the given channel if they exist anywhere in this mmm.
+// if presence is given it will also be used to emit slices of the ids of the IndexingLayers this event is stored in.
+// it closes the channels when it ends.
+func (b *MultiMmapManager) queryByIDs(ch chan *nostr.Event, ids []string, presence chan []uint16) {
+	go b.lmdbEnv.View(func(txn *lmdb.Txn) error {
+		txn.RawRead = true
+		defer close(ch)
+		if presence != nil {
+			defer close(presence)
+		}
+
+		for _, id := range ids {
+			if len(id) != 64 {
+				continue
+			}
+
+			idPrefix8, _ := hex.DecodeString(id[0 : 8*2])
+			val, err := txn.Get(b.indexId, idPrefix8)
+			if err == nil {
+				pos := positionFromBytes(val[0:12])
+				evt := &nostr.Event{}
+				if err := b.loadEvent(pos, evt); err != nil {
+					panic(fmt.Errorf("failed to decode event from %v: %w", pos, err))
+				}
+				ch <- evt
+
+				if presence != nil {
+					layers := make([]uint16, 0, (len(val)-12)/2)
+					for s := 12; s < len(val); s += 2 {
+						layers = append(layers, binary.BigEndian.Uint16(val[s:s+2]))
+					}
+					presence <- layers
+				}
+			}
+		}
+
+		return nil
+	})
+}
+
+func (il *IndexingLayer) QueryEvents(ctx context.Context, filter nostr.Filter) (chan *nostr.Event, error) {
+	ch := make(chan *nostr.Event)
+
+	if len(filter.IDs) > 0 {
+		il.mmmm.queryByIDs(ch, filter.IDs, nil)
+		return ch, nil
+	}
+
+	if filter.Search != "" {
+		close(ch)
+		return ch, nil
+	}
+
+	// max number of events we'll return
+	limit := il.MaxLimit / 4
+	if filter.Limit > 0 && filter.Limit < il.MaxLimit {
+		limit = filter.Limit
+	}
+	if tlimit := nostr.GetTheoreticalLimit(filter); tlimit == 0 {
+		close(ch)
+		return ch, nil
+	} else if tlimit > 0 {
+		limit = tlimit
+	}
+
+	go il.lmdbEnv.View(func(txn *lmdb.Txn) error {
+		txn.RawRead = true
+		defer close(ch)
+
+		results, err := il.query(txn, filter, limit)
+
+		for _, ie := range results {
+			ch <- ie.Event
+		}
+
+		return err
+	})
+
+	return ch, nil
+}
+
+func (il *IndexingLayer) query(txn *lmdb.Txn, filter nostr.Filter, limit int) ([]internal.IterEvent, error) {
+	queries, extraAuthors, extraKinds, extraTagKey, extraTagValues, since, err := il.prepareQueries(filter)
+	if err != nil {
+		return nil, err
+	}
+
+	iterators := make([]*iterator, len(queries))
+	exhausted := make([]bool, len(queries)) // indicates that a query won't be used anymore
+	results := make([][]internal.IterEvent, len(queries))
+	pulledPerQuery := make([]int, len(queries))
+
+	// these are kept updated so we never pull from the iterator that is at further distance
+	// (i.e. the one that has the oldest event among all)
+	// we will continue to pull from it as soon as some other iterator takes the position
+	oldest := internal.IterEvent{Q: -1}
+
+	secondPhase := false // after we have gathered enough events we will change the way we iterate
+	secondBatch := make([][]internal.IterEvent, 0, len(queries)+1)
+	secondPhaseParticipants := make([]int, 0, len(queries)+1)
+
+	// while merging results in the second phase we will alternate between these two lists
+	//   to avoid having to create new lists all the time
+	var secondPhaseResultsA []internal.IterEvent
+	var secondPhaseResultsB []internal.IterEvent
+	var secondPhaseResultsToggle bool // this is just a dummy thing we use to keep track of the alternating
+	var secondPhaseHasResultsPending bool
+
+	remainingUnexhausted := len(queries) // when all queries are exhausted we can finally end this thing
+	batchSizePerQuery := internal.BatchSizePerNumberOfQueries(limit, remainingUnexhausted)
+	firstPhaseTotalPulled := 0
+
+	exhaust := func(q int) {
+		exhausted[q] = true
+		remainingUnexhausted--
+		if q == oldest.Q {
+			oldest = internal.IterEvent{Q: -1}
+		}
+	}
+
+	var firstPhaseResults []internal.IterEvent
+
+	for q := range queries {
+		cursor, err := txn.OpenCursor(queries[q].dbi)
+		if err != nil {
+			return nil, err
+		}
+		iterators[q] = &iterator{cursor: cursor}
+		defer cursor.Close()
+		iterators[q].seek(queries[q].startingPoint)
+		results[q] = make([]internal.IterEvent, 0, batchSizePerQuery*2)
+	}
+
+	// fmt.Println("queries", len(queries))
+
+	for c := 0; ; c++ {
+		batchSizePerQuery = internal.BatchSizePerNumberOfQueries(limit, remainingUnexhausted)
+
+		// fmt.Println("  iteration", c, "remaining", remainingUnexhausted, "batchsize", batchSizePerQuery)
+		// we will go through all the iterators in batches until we have pulled all the required results
+		for q, query := range queries {
+			if exhausted[q] {
+				continue
+			}
+			if oldest.Q == q && remainingUnexhausted > 1 {
+				continue
+			}
+			// fmt.Println("    query", q, unsafe.Pointer(&results[q]), hex.EncodeToString(query.prefix), len(results[q]))
+
+			it := iterators[q]
+			pulledThisIteration := 0
+
+			for {
+				// we already have a k and a v and an err from the cursor setup, so check and use these
+				if it.err != nil ||
+					len(it.key) != query.keySize ||
+					!bytes.HasPrefix(it.key, query.prefix) {
+					// either iteration has errored or we reached the end of this prefix
+					// fmt.Println("      reached end", it.key, query.keySize, query.prefix)
+					exhaust(q)
+					break
+				}
+
+				// "id" indexes don't contain a timestamp
+				if query.timestampSize == 4 {
+					createdAt := binary.BigEndian.Uint32(it.key[len(it.key)-4:])
+					if createdAt < since {
+						// fmt.Println("        reached since", createdAt, "<", since)
+						exhaust(q)
+						break
+					}
+				}
+
+				// fetch actual event
+				pos := positionFromBytes(it.posb)
+				bin := il.mmmm.mmapf[pos.start : pos.start+uint64(pos.size)]
+
+				// check it against pubkeys without decoding the entire thing
+				if extraAuthors != nil && !slices.Contains(extraAuthors, [32]byte(bin[39:71])) {
+					it.next()
+					continue
+				}
+
+				// check it against kinds without decoding the entire thing
+				if extraKinds != nil && !slices.Contains(extraKinds, [2]byte(bin[1:3])) {
+					it.next()
+					continue
+				}
+
+				// decode the entire thing (TODO: do a conditional decode while also checking the extra tag)
+				event := &nostr.Event{}
+				if err := betterbinary.Unmarshal(bin, event); err != nil {
+					log.Printf("mmm: value read error (id %x) on query prefix %x sp %x dbi %d: %s\n",
+						bin[0:32], query.prefix, query.startingPoint, query.dbi, err)
+					return nil, fmt.Errorf("event read error: %w", err)
+				}
+
+				// fmt.Println("      event", hex.EncodeToString(val[0:4]), "kind", binary.BigEndian.Uint16(val[132:134]), "author", hex.EncodeToString(val[32:36]), "ts", nostr.Timestamp(binary.BigEndian.Uint32(val[128:132])), hex.EncodeToString(it.key), it.valIdx)
+
+				// if there is still a tag to be checked, do it now
+				if extraTagValues != nil && !event.Tags.ContainsAny(extraTagKey, extraTagValues) {
+					it.next()
+					continue
+				}
+
+				// this event is good to be used
+				evt := internal.IterEvent{Event: event, Q: q}
+				//
+				//
+				if secondPhase {
+					// do the process described below at HIWAWVRTP.
+					// if we've reached here this means we've already passed the `since` check.
+					// now we have to eliminate the event currently at the `since` threshold.
+					nextThreshold := firstPhaseResults[len(firstPhaseResults)-2]
+					if oldest.Event == nil {
+						// fmt.Println("          b1", evt.ID[0:8])
+						// BRANCH WHEN WE DON'T HAVE THE OLDEST EVENT (BWWDHTOE)
+						// when we don't have the oldest set, we will keep the results
+						//   and not change the cutting point -- it's bad, but hopefully not that bad.
+						results[q] = append(results[q], evt)
+						secondPhaseHasResultsPending = true
+					} else if nextThreshold.CreatedAt > oldest.CreatedAt {
+						// fmt.Println("          b2", nextThreshold.CreatedAt, ">", oldest.CreatedAt, evt.ID[0:8])
+						// one of the events we have stored is the actual next threshold
+						// eliminate last, update since with oldest
+						firstPhaseResults = firstPhaseResults[0 : len(firstPhaseResults)-1]
+						since = uint32(oldest.CreatedAt)
+						// fmt.Println("            new since", since, evt.ID[0:8])
+						//  we null the oldest Event as we can't rely on it anymore
+						//   (we'll fall under BWWDHTOE above) until we have a new oldest set.
+						oldest = internal.IterEvent{Q: -1}
+						// anything we got that would be above this won't trigger an update to
+						//   the oldest anyway, because it will be discarded as being after the limit.
+						//
+						// finally
+						// add this to the results to be merged later
+						results[q] = append(results[q], evt)
+						secondPhaseHasResultsPending = true
+					} else if nextThreshold.CreatedAt < evt.CreatedAt {
+						// the next last event in the firstPhaseResults is the next threshold
+						// fmt.Println("          b3", nextThreshold.CreatedAt, "<", oldest.CreatedAt, evt.ID[0:8])
+						// eliminate last, update since with the antelast
+						firstPhaseResults = firstPhaseResults[0 : len(firstPhaseResults)-1]
+						since = uint32(nextThreshold.CreatedAt)
+						// fmt.Println("            new since", since)
+						// add this to the results to be merged later
+						results[q] = append(results[q], evt)
+						secondPhaseHasResultsPending = true
+						// update the oldest event
+						if evt.CreatedAt < oldest.CreatedAt {
+							oldest = evt
+						}
+					} else {
+						// fmt.Println("          b4", evt.ID[0:8])
+						// oops, _we_ are the next `since` threshold
+						firstPhaseResults[len(firstPhaseResults)-1] = evt
+						since = uint32(evt.CreatedAt)
+						// fmt.Println("            new since", since)
+						// do not add us to the results to be merged later
+						//   as we're already inhabiting the firstPhaseResults slice
+					}
+				} else {
+					results[q] = append(results[q], evt)
+					firstPhaseTotalPulled++
+
+					// update the oldest event
+					if oldest.Event == nil || evt.CreatedAt < oldest.CreatedAt {
+						oldest = evt
+					}
+				}
+
+				pulledPerQuery[q]++
+				pulledThisIteration++
+				if pulledThisIteration > batchSizePerQuery {
+					// batch filled
+					it.next()
+					// fmt.Println("        filled", hex.EncodeToString(it.key), it.valIdx)
+					break
+				}
+				if pulledPerQuery[q] >= limit {
+					// batch filled + reached limit for this query (which is the global limit)
+					exhaust(q)
+					it.next()
+					break
+				}
+
+				it.next()
+			}
+		}
+
+		// we will do this check if we don't accumulated the requested number of events yet
+		// fmt.Println("oldest", oldest.Event, "from iter", oldest.Q)
+		if secondPhase && secondPhaseHasResultsPending && (oldest.Event == nil || remainingUnexhausted == 0) {
+			// fmt.Println("second phase aggregation!")
+			// when we are in the second phase we will aggressively aggregate results on every iteration
+			//
+			secondBatch = secondBatch[:0]
+			for s := 0; s < len(secondPhaseParticipants); s++ {
+				q := secondPhaseParticipants[s]
+
+				if len(results[q]) > 0 {
+					secondBatch = append(secondBatch, results[q])
+				}
+
+				if exhausted[q] {
+					secondPhaseParticipants = internal.SwapDelete(secondPhaseParticipants, s)
+					s--
+				}
+			}
+
+			// every time we get here we will alternate between these A and B lists
+			//   combining everything we have into a new partial results list.
+			// after we've done that we can again set the oldest.
+			// fmt.Println("  xxx", secondPhaseResultsToggle)
+			if secondPhaseResultsToggle {
+				secondBatch = append(secondBatch, secondPhaseResultsB)
+				secondPhaseResultsA = internal.MergeSortMultiple(secondBatch, limit, secondPhaseResultsA)
+				oldest = secondPhaseResultsA[len(secondPhaseResultsA)-1]
+				// fmt.Println("  new aggregated a", len(secondPhaseResultsB))
+			} else {
+				secondBatch = append(secondBatch, secondPhaseResultsA)
+				secondPhaseResultsB = internal.MergeSortMultiple(secondBatch, limit, secondPhaseResultsB)
+				oldest = secondPhaseResultsB[len(secondPhaseResultsB)-1]
+				// fmt.Println("  new aggregated b", len(secondPhaseResultsB))
+			}
+			secondPhaseResultsToggle = !secondPhaseResultsToggle
+
+			since = uint32(oldest.CreatedAt)
+			// fmt.Println("  new since", since)
+
+			// reset the `results` list so we can keep using it
+			results = results[:len(queries)]
+			for _, q := range secondPhaseParticipants {
+				results[q] = results[q][:0]
+			}
+		} else if !secondPhase && firstPhaseTotalPulled >= limit && remainingUnexhausted > 0 {
+			// fmt.Println("have enough!", firstPhaseTotalPulled, "/", limit, "remaining", remainingUnexhausted)
+
+			// we will exclude this oldest number as it is not relevant anymore
+			// (we now want to keep track only of the oldest among the remaining iterators)
+			oldest = internal.IterEvent{Q: -1}
+
+			// HOW IT WORKS AFTER WE'VE REACHED THIS POINT (HIWAWVRTP)
+			// now we can combine the results we have and check what is our current oldest event.
+			// we also discard anything that is after the current cutting point (`limit`).
+			// so if we have [1,2,3], [10, 15, 20] and [7, 21, 49] but we only want 6 total
+			//   we can just keep [1,2,3,7,10,15] and discard [20, 21, 49],
+			//   and also adjust our `since` parameter to `15`, discarding anything we get after it
+			//   and immediately declaring that iterator exhausted.
+			// also every time we get result that is more recent than this updated `since` we can
+			//   keep it but also discard the previous since, moving the needle one back -- for example,
+			//   if we get an `8` we can keep it and move the `since` parameter to `10`, discarding `15`
+			//   in the process.
+			all := make([][]internal.IterEvent, len(results))
+			copy(all, results) // we have to use this otherwise internal.MergeSortMultiple will scramble our results slice
+			firstPhaseResults = internal.MergeSortMultiple(all, limit, nil)
+			oldest = firstPhaseResults[limit-1]
+			since = uint32(oldest.CreatedAt)
+			// fmt.Println("new since", since)
+
+			for q := range queries {
+				if exhausted[q] {
+					continue
+				}
+
+				// we also automatically exhaust any of the iterators that have already passed the
+				// cutting point (`since`)
+				if results[q][len(results[q])-1].CreatedAt < oldest.CreatedAt {
+					exhausted[q] = true
+					remainingUnexhausted--
+					continue
+				}
+
+				// for all the remaining iterators,
+				// since we have merged all the events in this `firstPhaseResults` slice, we can empty the
+				//   current `results` slices and reuse them.
+				results[q] = results[q][:0]
+
+				// build this index of indexes with everybody who remains
+				secondPhaseParticipants = append(secondPhaseParticipants, q)
+			}
+
+			// we create these two lists and alternate between them so we don't have to create a
+			//   a new one every time
+			secondPhaseResultsA = make([]internal.IterEvent, 0, limit*2)
+			secondPhaseResultsB = make([]internal.IterEvent, 0, limit*2)
+
+			// from now on we won't run this block anymore
+			secondPhase = true
+		}
+
+		// fmt.Println("remaining", remainingUnexhausted)
+		if remainingUnexhausted == 0 {
+			break
+		}
+	}
+
+	// fmt.Println("is secondPhase?", secondPhase)
+
+	var combinedResults []internal.IterEvent
+
+	if secondPhase {
+		// fmt.Println("ending second phase")
+		// when we reach this point either secondPhaseResultsA or secondPhaseResultsB will be full of stuff,
+		//   the other will be empty
+		var secondPhaseResults []internal.IterEvent
+		// fmt.Println("xxx", secondPhaseResultsToggle, len(secondPhaseResultsA), len(secondPhaseResultsB))
+		if secondPhaseResultsToggle {
+			secondPhaseResults = secondPhaseResultsB
+			combinedResults = secondPhaseResultsA[0:limit] // reuse this
+			// fmt.Println("  using b", len(secondPhaseResultsA))
+		} else {
+			secondPhaseResults = secondPhaseResultsA
+			combinedResults = secondPhaseResultsB[0:limit] // reuse this
+			// fmt.Println("  using a", len(secondPhaseResultsA))
+		}
+
+		all := [][]internal.IterEvent{firstPhaseResults, secondPhaseResults}
+		combinedResults = internal.MergeSortMultiple(all, limit, combinedResults)
+		// fmt.Println("final combinedResults", len(combinedResults), cap(combinedResults), limit)
+	} else {
+		combinedResults = make([]internal.IterEvent, limit)
+		combinedResults = internal.MergeSortMultiple(results, limit, combinedResults)
+	}
+
+	return combinedResults, nil
+}

eventstore/mmm/query_planner.go

@@ -0,0 +1,202 @@
+package mmm
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/fiatjaf/eventstore/internal"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+type query struct {
+	i             int
+	dbi           lmdb.DBI
+	prefix        []byte
+	results       chan *nostr.Event
+	keySize       int
+	timestampSize int
+	startingPoint []byte
+}
+
+func (il *IndexingLayer) prepareQueries(filter nostr.Filter) (
+	queries []query,
+	extraAuthors [][32]byte,
+	extraKinds [][2]byte,
+	extraTagKey string,
+	extraTagValues []string,
+	since uint32,
+	err error,
+) {
+	// we will apply this to every query we return
+	defer func() {
+		if queries == nil {
+			return
+		}
+
+		var until uint32 = 4294967295
+		if filter.Until != nil {
+			if fu := uint32(*filter.Until); fu < until {
+				until = fu + 1
+			}
+		}
+		for i, q := range queries {
+			sp := make([]byte, len(q.prefix))
+			sp = sp[0:len(q.prefix)]
+			copy(sp, q.prefix)
+			queries[i].startingPoint = binary.BigEndian.AppendUint32(sp, uint32(until))
+			queries[i].results = make(chan *nostr.Event, 12)
+		}
+	}()
+
+	// this is where we'll end the iteration
+	if filter.Since != nil {
+		if fs := uint32(*filter.Since); fs > since {
+			since = fs
+		}
+	}
+
+	if len(filter.Tags) > 0 {
+		// we will select ONE tag to query for and ONE extra tag to do further narrowing, if available
+		tagKey, tagValues, goodness := internal.ChooseNarrowestTag(filter)
+
+		// we won't use a tag index for this as long as we have something else to match with
+		if goodness < 2 && (len(filter.Authors) > 0 || len(filter.Kinds) > 0) {
+			goto pubkeyMatching
+		}
+
+		// only "p" tag has a goodness of 2, so
+		if goodness == 2 {
+			// this means we got a "p" tag, so we will use the ptag-kind index
+			i := 0
+			if filter.Kinds != nil {
+				queries = make([]query, len(tagValues)*len(filter.Kinds))
+				for _, value := range tagValues {
+					if len(value) != 64 {
+						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
+					}
+
+					for _, kind := range filter.Kinds {
+						k := make([]byte, 8+2)
+						if _, err := hex.Decode(k[0:8], []byte(value[0:8*2])); err != nil {
+							return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
+						}
+						binary.BigEndian.PutUint16(k[8:8+2], uint16(kind))
+						queries[i] = query{i: i, dbi: il.indexPTagKind, prefix: k[0 : 8+2], keySize: 8 + 2 + 4, timestampSize: 4}
+						i++
+					}
+				}
+			} else {
+				// even if there are no kinds, in that case we will just return any kind and not care
+				queries = make([]query, len(tagValues))
+				for i, value := range tagValues {
+					if len(value) != 64 {
+						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
+					}
+
+					k := make([]byte, 8)
+					if _, err := hex.Decode(k[0:8], []byte(value[0:8*2])); err != nil {
+						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
+					}
+					queries[i] = query{i: i, dbi: il.indexPTagKind, prefix: k[0:8], keySize: 8 + 2 + 4, timestampSize: 4}
+				}
+			}
+		} else {
+			// otherwise we will use a plain tag index
+			queries = make([]query, len(tagValues))
+			for i, value := range tagValues {
+				// get key prefix (with full length) and offset where to write the created_at
+				dbi, k, offset := il.getTagIndexPrefix(value)
+				// remove the last parts part to get just the prefix we want here
+				prefix := k[0:offset]
+				queries[i] = query{i: i, dbi: dbi, prefix: prefix, keySize: len(prefix) + 4, timestampSize: 4}
+				i++
+			}
+
+			// add an extra kind filter if available (only do this on plain tag index, not on ptag-kind index)
+			if filter.Kinds != nil {
+				extraKinds = make([][2]byte, len(filter.Kinds))
+				for i, kind := range filter.Kinds {
+					binary.BigEndian.PutUint16(extraKinds[i][0:2], uint16(kind))
+				}
+			}
+		}
+
+		// add an extra author search if possible
+		if filter.Authors != nil {
+			extraAuthors = make([][32]byte, len(filter.Authors))
+			for i, pk := range filter.Authors {
+				hex.Decode(extraAuthors[i][:], []byte(pk))
+			}
+		}
+
+		// add an extra useless tag if available
+		filter.Tags = internal.CopyMapWithoutKey(filter.Tags, tagKey)
+		if len(filter.Tags) > 0 {
+			extraTagKey, extraTagValues, _ = internal.ChooseNarrowestTag(filter)
+		}
+
+		return queries, extraAuthors, extraKinds, extraTagKey, extraTagValues, since, nil
+	}
+
+pubkeyMatching:
+	if len(filter.Authors) > 0 {
+		if len(filter.Kinds) == 0 {
+			// will use pubkey index
+			queries = make([]query, len(filter.Authors))
+			for i, pubkeyHex := range filter.Authors {
+				if len(pubkeyHex) != 64 {
+					return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
+				}
+				prefix := make([]byte, 8)
+				if _, err := hex.Decode(prefix[0:8], []byte(pubkeyHex[0:8*2])); err != nil {
+					return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
+				}
+				queries[i] = query{i: i, dbi: il.indexPubkey, prefix: prefix[0:8], keySize: 8 + 4, timestampSize: 4}
+			}
+		} else {
+			// will use pubkeyKind index
+			queries = make([]query, len(filter.Authors)*len(filter.Kinds))
+			i := 0
+			for _, pubkeyHex := range filter.Authors {
+				for _, kind := range filter.Kinds {
+					if len(pubkeyHex) != 64 {
+						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
+					}
+					prefix := make([]byte, 8+2)
+					if _, err := hex.Decode(prefix[0:8], []byte(pubkeyHex[0:8*2])); err != nil {
+						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
+					}
+					binary.BigEndian.PutUint16(prefix[8:8+2], uint16(kind))
+					queries[i] = query{i: i, dbi: il.indexPubkeyKind, prefix: prefix[0 : 8+2], keySize: 10 + 4, timestampSize: 4}
+					i++
+				}
+			}
+		}
+
+		// potentially with an extra useless tag filtering
+		extraTagKey, extraTagValues, _ = internal.ChooseNarrowestTag(filter)
+		return queries, nil, nil, extraTagKey, extraTagValues, since, nil
+	}
+
+	if len(filter.Kinds) > 0 {
+		// will use a kind index
+		queries = make([]query, len(filter.Kinds))
+		for i, kind := range filter.Kinds {
+			prefix := make([]byte, 2)
+			binary.BigEndian.PutUint16(prefix[0:2], uint16(kind))
+			queries[i] = query{i: i, dbi: il.indexKind, prefix: prefix[0:2], keySize: 2 + 4, timestampSize: 4}
+		}
+
+		// potentially with an extra useless tag filtering
+		tagKey, tagValues, _ := internal.ChooseNarrowestTag(filter)
+		return queries, nil, nil, tagKey, tagValues, since, nil
+	}
+
+	// if we got here our query will have nothing to filter with
+	queries = make([]query, 1)
+	prefix := make([]byte, 0)
+	queries[0] = query{i: 0, dbi: il.indexCreatedAt, prefix: prefix, keySize: 0 + 4, timestampSize: 4}
+	return queries, nil, nil, "", nil, since, nil
+}

eventstore/mmm/replace.go

@@ -0,0 +1,54 @@
+package mmm
+
+import (
+	"context"
+	"fmt"
+	"math"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/fiatjaf/eventstore/internal"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (il *IndexingLayer) ReplaceEvent(ctx context.Context, evt *nostr.Event) error {
+	// sanity checking
+	if evt.CreatedAt > math.MaxUint32 || evt.Kind > math.MaxUint16 {
+		return fmt.Errorf("event with values out of expected boundaries")
+	}
+
+	filter := nostr.Filter{Limit: 1, Kinds: []int{evt.Kind}, Authors: []string{evt.PubKey}}
+	if nostr.IsAddressableKind(evt.Kind) {
+		// when addressable, add the "d" tag to the filter
+		filter.Tags = nostr.TagMap{"d": []string{evt.Tags.GetD()}}
+	}
+
+	return il.mmmm.lmdbEnv.Update(func(mmmtxn *lmdb.Txn) error {
+		mmmtxn.RawRead = true
+
+		return il.lmdbEnv.Update(func(iltxn *lmdb.Txn) error {
+			// now we fetch the past events, whatever they are, delete them and then save the new
+			prevResults, err := il.query(iltxn, filter, 10) // in theory limit could be just 1 and this should work
+			if err != nil {
+				return fmt.Errorf("failed to query past events with %s: %w", filter, err)
+			}
+
+			shouldStore := true
+			for _, previous := range prevResults {
+				if internal.IsOlder(previous.Event, evt) {
+					if err := il.delete(mmmtxn, iltxn, previous.Event); err != nil {
+						return fmt.Errorf("failed to delete event %s for replacing: %w", previous.Event.ID, err)
+					}
+				} else {
+					// there is a newer event already stored, so we won't store this
+					shouldStore = false
+				}
+			}
+			if shouldStore {
+				_, err := il.mmmm.storeOn(mmmtxn, []*IndexingLayer{il}, []*lmdb.Txn{iltxn}, evt)
+				return err
+			}
+
+			return nil
+		})
+	})
+}

eventstore/mmm/save.go

@@ -0,0 +1,234 @@
+package mmm
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"runtime"
+	"slices"
+	"syscall"
+	"unsafe"
+
+	"github.com/PowerDNS/lmdb-go/lmdb"
+	"github.com/fiatjaf/eventstore/mmm/betterbinary"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *MultiMmapManager) StoreGlobal(ctx context.Context, evt *nostr.Event) (stored bool, err error) {
+	someoneWantsIt := false
+
+	b.mutex.Lock()
+	defer b.mutex.Unlock()
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	// do this just so it's cleaner, we're already locking the thread and the mutex anyway
+	mmmtxn, err := b.lmdbEnv.BeginTxn(nil, 0)
+	if err != nil {
+		return false, fmt.Errorf("failed to begin global transaction: %w", err)
+	}
+	mmmtxn.RawRead = true
+
+	iltxns := make([]*lmdb.Txn, 0, len(b.layers))
+	ils := make([]*IndexingLayer, 0, len(b.layers))
+
+	// ask if any of the indexing layers want this
+	for _, il := range b.layers {
+		if il.ShouldIndex != nil && il.ShouldIndex(ctx, evt) {
+			someoneWantsIt = true
+
+			iltxn, err := il.lmdbEnv.BeginTxn(nil, 0)
+			if err != nil {
+				mmmtxn.Abort()
+				for _, txn := range iltxns {
+					txn.Abort()
+				}
+				return false, fmt.Errorf("failed to start txn on %s: %w", il.name, err)
+			}
+
+			ils = append(ils, il)
+			iltxns = append(iltxns, iltxn)
+		}
+	}
+
+	if !someoneWantsIt {
+		// no one wants it
+		mmmtxn.Abort()
+		return false, fmt.Errorf("not wanted")
+	}
+
+	stored, err = b.storeOn(mmmtxn, ils, iltxns, evt)
+	if stored {
+		mmmtxn.Commit()
+		for _, txn := range iltxns {
+			txn.Commit()
+		}
+	} else {
+		mmmtxn.Abort()
+		for _, txn := range iltxns {
+			txn.Abort()
+		}
+	}
+
+	return stored, err
+}
+
+func (il *IndexingLayer) SaveEvent(ctx context.Context, evt *nostr.Event) error {
+	il.mmmm.mutex.Lock()
+	defer il.mmmm.mutex.Unlock()
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	// do this just so it's cleaner, we're already locking the thread and the mutex anyway
+	mmmtxn, err := il.mmmm.lmdbEnv.BeginTxn(nil, 0)
+	if err != nil {
+		return fmt.Errorf("failed to begin global transaction: %w", err)
+	}
+	mmmtxn.RawRead = true
+
+	iltxn, err := il.lmdbEnv.BeginTxn(nil, 0)
+	if err != nil {
+		mmmtxn.Abort()
+		return fmt.Errorf("failed to start txn on %s: %w", il.name, err)
+	}
+
+	if _, err := il.mmmm.storeOn(mmmtxn, []*IndexingLayer{il}, []*lmdb.Txn{iltxn}, evt); err != nil {
+		mmmtxn.Abort()
+		if iltxn != nil {
+			iltxn.Abort()
+		}
+		return err
+	}
+
+	mmmtxn.Commit()
+	iltxn.Commit()
+	return nil
+}
+
+func (b *MultiMmapManager) storeOn(
+	mmmtxn *lmdb.Txn,
+	ils []*IndexingLayer,
+	iltxns []*lmdb.Txn,
+	evt *nostr.Event,
+) (stored bool, err error) {
+	// sanity checking
+	if evt.CreatedAt > maxuint32 || evt.Kind > maxuint16 {
+		return false, fmt.Errorf("event with values out of expected boundaries")
+	}
+
+	// check if we already have this id
+	idPrefix8, _ := hex.DecodeString(evt.ID[0 : 8*2])
+	val, err := mmmtxn.Get(b.indexId, idPrefix8)
+	if err == nil {
+		// we found the event, now check if it is already indexed by the layers that want to store it
+		for i := len(ils) - 1; i >= 0; i-- {
+			for s := 12; s < len(val); s += 2 {
+				ilid := binary.BigEndian.Uint16(val[s : s+2])
+				if ils[i].id == ilid {
+					// swap delete this il, but keep the deleted ones at the end
+					// (so the caller can successfully finalize the transactions)
+					ils[i], ils[len(ils)-1] = ils[len(ils)-1], ils[i]
+					ils = ils[0 : len(ils)-1]
+					iltxns[i], iltxns[len(iltxns)-1] = iltxns[len(iltxns)-1], iltxns[i]
+					iltxns = iltxns[0 : len(iltxns)-1]
+					break
+				}
+			}
+		}
+	} else if !lmdb.IsNotFound(err) {
+		// now if we got an error from lmdb we will only proceed if we get a NotFound -- for anything else we will error
+		return false, fmt.Errorf("error checking existence: %w", err)
+	}
+
+	// if all ils already have this event indexed (or no il was given) we can end here
+	if len(ils) == 0 {
+		return false, nil
+	}
+
+	// get event binary size
+	pos := position{
+		size: uint32(betterbinary.Measure(*evt)),
+	}
+	if pos.size >= 1<<16 {
+		return false, fmt.Errorf("event too large to store, max %d, got %d", 1<<16, pos.size)
+	}
+
+	// find a suitable place for this to be stored in
+	appendToMmap := true
+	for f, fr := range b.freeRanges {
+		if fr.size >= pos.size {
+			// found the smallest possible place that can fit this event
+			appendToMmap = false
+			pos.start = fr.start
+
+			// modify the free ranges we're keeping track of
+			// (i.e. delete the current and add a new freerange with the remaining space)
+			b.freeRanges = slices.Delete(b.freeRanges, f, f+1)
+
+			if pos.size != fr.size {
+				b.addNewFreeRange(position{
+					start: fr.start + uint64(pos.size),
+					size:  fr.size - pos.size,
+				})
+			}
+
+			if err := b.saveFreeRanges(mmmtxn); err != nil {
+				return false, fmt.Errorf("failed to save modified free ranges: %w", err)
+			}
+
+			break
+		}
+	}
+
+	if appendToMmap {
+		// no free ranges found, so write to the end of the mmap file
+		pos.start = b.mmapfEnd
+		mmapfNewSize := int64(b.mmapfEnd) + int64(pos.size)
+		if err := os.Truncate(b.mmapfPath, mmapfNewSize); err != nil {
+			return false, fmt.Errorf("error increasing %s: %w", b.mmapfPath, err)
+		}
+		b.mmapfEnd = uint64(mmapfNewSize)
+	}
+
+	// write to the mmap
+	if err := betterbinary.Marshal(*evt, b.mmapf[pos.start:]); err != nil {
+		return false, fmt.Errorf("error marshaling to %d: %w", pos.start, err)
+	}
+
+	// prepare value to be saved in the id index (if we didn't have it already)
+	// val: [posb][layerIdRefs...]
+	if val == nil {
+		val = make([]byte, 12, 12+2*len(b.layers))
+		binary.BigEndian.PutUint32(val[0:4], pos.size)
+		binary.BigEndian.PutUint64(val[4:12], pos.start)
+	}
+
+	// each index that was reserved above for the different layers
+	for i, il := range ils {
+		iltxn := iltxns[i]
+
+		for k := range il.getIndexKeysForEvent(evt) {
+			if err := iltxn.Put(k.dbi, k.key, val[0:12] /* pos */, 0); err != nil {
+				b.Logger.Warn().Str("name", il.name).Msg("failed to index event on layer")
+			}
+		}
+
+		val = binary.BigEndian.AppendUint16(val, il.id)
+	}
+
+	// store the id index with the refcounts
+	if err := mmmtxn.Put(b.indexId, idPrefix8, val, 0); err != nil {
+		panic(fmt.Errorf("failed to store %x by id: %w", idPrefix8, err))
+	}
+
+	// msync
+	_, _, errno := syscall.Syscall(syscall.SYS_MSYNC,
+		uintptr(unsafe.Pointer(&b.mmapf[0])), uintptr(len(b.mmapf)), syscall.MS_SYNC)
+	if errno != 0 {
+		panic(fmt.Errorf("msync failed: %w", syscall.Errno(errno)))
+	}
+
+	return true, nil
+}

eventstore/negentropy.go

@@ -0,0 +1,13 @@
+package eventstore
+
+import "context"
+
+var negentropySessionKey = struct{}{}
+
+func IsNegentropySession(ctx context.Context) bool {
+	return ctx.Value(negentropySessionKey) != nil
+}
+
+func SetNegentropy(ctx context.Context) context.Context {
+	return context.WithValue(ctx, negentropySessionKey, struct{}{})
+}

eventstore/nullstore/README.md

@@ -0,0 +1,2 @@
+`nullstore` is an eventstore that doesn't actually do anything.
+It doesn't store anything, it doesn't return anything.

eventstore/nullstore/lib.go

@@ -0,0 +1,36 @@
+package nullstore
+
+import (
+	"context"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+var _ eventstore.Store = NullStore{}
+
+type NullStore struct{}
+
+func (b NullStore) Init() error {
+	return nil
+}
+
+func (b NullStore) Close() {}
+
+func (b NullStore) DeleteEvent(ctx context.Context, evt *nostr.Event) error {
+	return nil
+}
+
+func (b NullStore) QueryEvents(ctx context.Context, filter nostr.Filter) (chan *nostr.Event, error) {
+	ch := make(chan *nostr.Event)
+	close(ch)
+	return ch, nil
+}
+
+func (b NullStore) SaveEvent(ctx context.Context, evt *nostr.Event) error {
+	return nil
+}
+
+func (b NullStore) ReplaceEvent(ctx context.Context, evt *nostr.Event) error {
+	return nil
+}

eventstore/relay_interface.go

@@ -0,0 +1,56 @@
+package eventstore
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+type RelayWrapper struct {
+	Store
+}
+
+var _ nostr.RelayStore = (*RelayWrapper)(nil)
+
+func (w RelayWrapper) Publish(ctx context.Context, evt nostr.Event) error {
+	if nostr.IsEphemeralKind(evt.Kind) {
+		// do not store ephemeral events
+		return nil
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	if nostr.IsRegularKind(evt.Kind) {
+		// regular events are just saved directly
+		if err := w.SaveEvent(ctx, &evt); err != nil && err != ErrDupEvent {
+			return fmt.Errorf("failed to save: %w", err)
+		}
+		return nil
+	}
+
+	// others are replaced
+	w.Store.ReplaceEvent(ctx, &evt)
+
+	return nil
+}
+
+func (w RelayWrapper) QuerySync(ctx context.Context, filter nostr.Filter) ([]*nostr.Event, error) {
+	ch, err := w.Store.QueryEvents(ctx, filter)
+	if err != nil {
+		return nil, fmt.Errorf("failed to query: %w", err)
+	}
+
+	n := filter.Limit
+	if n == 0 {
+		n = 500
+	}
+
+	results := make([]*nostr.Event, 0, n)
+	for evt := range ch {
+		results = append(results, evt)
+	}
+
+	return results, nil
+}

eventstore/slicestore/lib.go

@@ -0,0 +1,157 @@
+package slicestore
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/fiatjaf/eventstore/internal"
+	"github.com/nbd-wtf/go-nostr"
+	"golang.org/x/exp/slices"
+)
+
+var _ eventstore.Store = (*SliceStore)(nil)
+
+type SliceStore struct {
+	sync.Mutex
+	internal []*nostr.Event
+
+	MaxLimit int
+}
+
+func (b *SliceStore) Init() error {
+	b.internal = make([]*nostr.Event, 0, 5000)
+	if b.MaxLimit == 0 {
+		b.MaxLimit = 500
+	}
+	return nil
+}
+
+func (b *SliceStore) Close() {}
+
+func (b *SliceStore) QueryEvents(ctx context.Context, filter nostr.Filter) (chan *nostr.Event, error) {
+	ch := make(chan *nostr.Event)
+	if filter.Limit > b.MaxLimit || (filter.Limit == 0 && !filter.LimitZero) {
+		filter.Limit = b.MaxLimit
+	}
+
+	// efficiently determine where to start and end
+	start := 0
+	end := len(b.internal)
+	if filter.Until != nil {
+		start, _ = slices.BinarySearchFunc(b.internal, *filter.Until, eventTimestampComparator)
+	}
+	if filter.Since != nil {
+		end, _ = slices.BinarySearchFunc(b.internal, *filter.Since, eventTimestampComparator)
+	}
+
+	// ham
+	if end < start {
+		close(ch)
+		return ch, nil
+	}
+
+	count := 0
+	go func() {
+		for _, event := range b.internal[start:end] {
+			if count == filter.Limit {
+				break
+			}
+
+			if filter.Matches(event) {
+				select {
+				case ch <- event:
+				case <-ctx.Done():
+					return
+				}
+				count++
+			}
+		}
+		close(ch)
+	}()
+	return ch, nil
+}
+
+func (b *SliceStore) CountEvents(ctx context.Context, filter nostr.Filter) (int64, error) {
+	var val int64
+	for _, event := range b.internal {
+		if filter.Matches(event) {
+			val++
+		}
+	}
+	return val, nil
+}
+
+func (b *SliceStore) SaveEvent(ctx context.Context, evt *nostr.Event) error {
+	idx, found := slices.BinarySearchFunc(b.internal, evt, eventComparator)
+	if found {
+		return eventstore.ErrDupEvent
+	}
+	// let's insert at the correct place in the array
+	b.internal = append(b.internal, evt) // bogus
+	copy(b.internal[idx+1:], b.internal[idx:])
+	b.internal[idx] = evt
+
+	return nil
+}
+
+func (b *SliceStore) DeleteEvent(ctx context.Context, evt *nostr.Event) error {
+	idx, found := slices.BinarySearchFunc(b.internal, evt, eventComparator)
+	if !found {
+		// we don't have this event
+		return nil
+	}
+
+	// we have it
+	copy(b.internal[idx:], b.internal[idx+1:])
+	b.internal = b.internal[0 : len(b.internal)-1]
+	return nil
+}
+
+func (b *SliceStore) ReplaceEvent(ctx context.Context, evt *nostr.Event) error {
+	b.Lock()
+	defer b.Unlock()
+
+	filter := nostr.Filter{Limit: 1, Kinds: []int{evt.Kind}, Authors: []string{evt.PubKey}}
+	if nostr.IsAddressableKind(evt.Kind) {
+		filter.Tags = nostr.TagMap{"d": []string{evt.Tags.GetD()}}
+	}
+
+	ch, err := b.QueryEvents(ctx, filter)
+	if err != nil {
+		return fmt.Errorf("failed to query before replacing: %w", err)
+	}
+
+	shouldStore := true
+	for previous := range ch {
+		if internal.IsOlder(previous, evt) {
+			if err := b.DeleteEvent(ctx, previous); err != nil {
+				return fmt.Errorf("failed to delete event for replacing: %w", err)
+			}
+		} else {
+			shouldStore = false
+		}
+	}
+
+	if shouldStore {
+		if err := b.SaveEvent(ctx, evt); err != nil && err != eventstore.ErrDupEvent {
+			return fmt.Errorf("failed to save: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func eventTimestampComparator(e *nostr.Event, t nostr.Timestamp) int {
+	return int(t) - int(e.CreatedAt)
+}
+
+func eventComparator(a *nostr.Event, b *nostr.Event) int {
+	c := int(b.CreatedAt) - int(a.CreatedAt)
+	if c != 0 {
+		return c
+	}
+	return strings.Compare(b.ID, a.ID)
+}

eventstore/slicestore/slicestore_test.go

@@ -0,0 +1,60 @@
+package slicestore
+
+import (
+	"context"
+	"testing"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func TestBasicStuff(t *testing.T) {
+	ctx := context.Background()
+	ss := &SliceStore{}
+	ss.Init()
+	defer ss.Close()
+
+	for i := 0; i < 20; i++ {
+		v := i
+		kind := 11
+		if i%2 == 0 {
+			v = i + 10000
+		}
+		if i%3 == 0 {
+			kind = 12
+		}
+		ss.SaveEvent(ctx, &nostr.Event{CreatedAt: nostr.Timestamp(v), Kind: kind})
+	}
+
+	ch, _ := ss.QueryEvents(ctx, nostr.Filter{})
+	list := make([]*nostr.Event, 0, 20)
+	for event := range ch {
+		list = append(list, event)
+	}
+
+	if len(list) != 20 {
+		t.Fatalf("failed to load 20 events")
+	}
+	if list[0].CreatedAt != 10018 || list[1].CreatedAt != 10016 || list[18].CreatedAt != 3 || list[19].CreatedAt != 1 {
+		t.Fatalf("order is incorrect")
+	}
+
+	until := nostr.Timestamp(9999)
+	ch, _ = ss.QueryEvents(ctx, nostr.Filter{Limit: 15, Until: &until, Kinds: []int{11}})
+	list = make([]*nostr.Event, 0, 7)
+	for event := range ch {
+		list = append(list, event)
+	}
+	if len(list) != 7 {
+		t.Fatalf("should have gotten 7, not %d", len(list))
+	}
+
+	since := nostr.Timestamp(10009)
+	ch, _ = ss.QueryEvents(ctx, nostr.Filter{Since: &since})
+	list = make([]*nostr.Event, 0, 5)
+	for event := range ch {
+		list = append(list, event)
+	}
+	if len(list) != 5 {
+		t.Fatalf("should have gotten 5, not %d", len(list))
+	}
+}

eventstore/store.go

@@ -0,0 +1,32 @@
+package eventstore
+
+import (
+	"context"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+// Store is a persistence layer for nostr events handled by a relay.
+type Store interface {
+	// Init is called at the very beginning by [Server.Start], after [Relay.Init],
+	// allowing a storage to initialize its internal resources.
+	Init() error
+
+	// Close must be called after you're done using the store, to free up resources and so on.
+	Close()
+
+	// QueryEvents should return a channel with the events as they're recovered from a database.
+	//   the channel should be closed after the events are all delivered.
+	QueryEvents(context.Context, nostr.Filter) (chan *nostr.Event, error)
+	// DeleteEvent just deletes an event, no side-effects.
+	DeleteEvent(context.Context, *nostr.Event) error
+	// SaveEvent just saves an event, no side-effects.
+	SaveEvent(context.Context, *nostr.Event) error
+	// ReplaceEvent atomically replaces a replaceable or addressable event.
+	// Conceptually it is like a Query->Delete->Save, but streamlined.
+	ReplaceEvent(context.Context, *nostr.Event) error
+}
+
+type Counter interface {
+	CountEvents(context.Context, nostr.Filter) (int64, error)
+}

eventstore/strfry/lib.go

@@ -0,0 +1,164 @@
+package strfry
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/mailru/easyjson"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+var _ eventstore.Store = (*StrfryBackend)(nil)
+
+type StrfryBackend struct {
+	ConfigPath     string
+	ExecutablePath string
+}
+
+func (s *StrfryBackend) Init() error {
+	if s.ExecutablePath == "" {
+		configPath := filepath.Dir(s.ConfigPath)
+		os.Setenv("PATH", configPath+":"+os.Getenv("PATH"))
+		exe, err := exec.LookPath("strfry")
+		if err != nil {
+			return fmt.Errorf("failed to find strfry executable: %w (better provide it manually)", err)
+		}
+		s.ExecutablePath = exe
+	}
+
+	return nil
+}
+
+func (_ StrfryBackend) Close() {}
+
+func (s StrfryBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (chan *nostr.Event, error) {
+	stdout, err := s.baseStrfryScan(ctx, filter)
+	if err != nil {
+		return nil, err
+	}
+
+	ch := make(chan *nostr.Event)
+	go func() {
+		defer close(ch)
+		for {
+			line, err := stdout.ReadBytes('\n')
+			if err != nil {
+				break
+			}
+
+			evt := &nostr.Event{}
+			easyjson.Unmarshal(line, evt)
+			if evt.ID == "" {
+				continue
+			}
+
+			ch <- evt
+		}
+	}()
+
+	return ch, nil
+}
+
+func (s *StrfryBackend) ReplaceEvent(ctx context.Context, evt *nostr.Event) error {
+	return s.SaveEvent(ctx, evt)
+}
+
+func (s StrfryBackend) SaveEvent(ctx context.Context, evt *nostr.Event) error {
+	args := make([]string, 0, 4)
+	if s.ConfigPath != "" {
+		args = append(args, "--config="+s.ConfigPath)
+	}
+	args = append(args, "import")
+	args = append(args, "--show-rejected")
+	args = append(args, "--no-verify")
+
+	cmd := exec.CommandContext(ctx, s.ExecutablePath, args...)
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+
+	// event is sent on stdin
+	j, _ := easyjson.Marshal(evt)
+	cmd.Stdin = bytes.NewBuffer(j)
+
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf(
+			"%s %s failed: %w, (%s)",
+			s.ExecutablePath, strings.Join(args, " "), err, stderr.String(),
+		)
+	}
+
+	return nil
+}
+
+func (s StrfryBackend) DeleteEvent(ctx context.Context, evt *nostr.Event) error {
+	args := make([]string, 0, 3)
+	if s.ConfigPath != "" {
+		args = append(args, "--config="+s.ConfigPath)
+	}
+	args = append(args, "delete")
+	args = append(args, "--filter={\"ids\":[\""+evt.ID+"\"]}")
+
+	cmd := exec.CommandContext(ctx, s.ExecutablePath, args...)
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf(
+			"%s %s failed: %w, (%s)",
+			s.ExecutablePath, strings.Join(args, " "), err, stderr.String(),
+		)
+	}
+
+	return nil
+}
+
+func (s StrfryBackend) CountEvents(ctx context.Context, filter nostr.Filter) (int64, error) {
+	stdout, err := s.baseStrfryScan(ctx, filter)
+	if err != nil {
+		return 0, err
+	}
+
+	var count int64
+	for {
+		_, err := stdout.ReadBytes('\n')
+		if err != nil {
+			break
+		}
+		count++
+	}
+
+	return count, nil
+}
+
+func (s StrfryBackend) baseStrfryScan(ctx context.Context, filter nostr.Filter) (*bytes.Buffer, error) {
+	args := make([]string, 0, 3)
+	if s.ConfigPath != "" {
+		args = append(args, "--config="+s.ConfigPath)
+	}
+	args = append(args, "scan")
+	args = append(args, filter.String())
+
+	cmd := exec.CommandContext(ctx, s.ExecutablePath, args...)
+	var stdout bytes.Buffer
+	cmd.Stdout = &stdout
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	if err != nil {
+		return nil, fmt.Errorf(
+			"%s %s failed: %w, (%s)",
+			s.ExecutablePath, strings.Join(args, " "), err, stderr.String(),
+		)
+	}
+
+	return &stdout, nil
+}

eventstore/test/benchmark_test.go

@@ -0,0 +1,113 @@
+package test
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/fiatjaf/eventstore/badger"
+	"github.com/fiatjaf/eventstore/lmdb"
+	"github.com/fiatjaf/eventstore/slicestore"
+	"github.com/fiatjaf/eventstore/sqlite3"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func BenchmarkSliceStore(b *testing.B) {
+	s := &slicestore.SliceStore{}
+	s.Init()
+	runBenchmarkOn(b, s)
+}
+
+func BenchmarkLMDB(b *testing.B) {
+	os.RemoveAll(dbpath + "lmdb")
+	l := &lmdb.LMDBBackend{Path: dbpath + "lmdb"}
+	l.Init()
+
+	runBenchmarkOn(b, l)
+}
+
+func BenchmarkBadger(b *testing.B) {
+	d := &badger.BadgerBackend{Path: dbpath + "badger"}
+	d.Init()
+	runBenchmarkOn(b, d)
+}
+
+func BenchmarkSQLite(b *testing.B) {
+	os.RemoveAll(dbpath + "sqlite")
+	q := &sqlite3.SQLite3Backend{DatabaseURL: dbpath + "sqlite", QueryTagsLimit: 50}
+	q.Init()
+
+	runBenchmarkOn(b, q)
+}
+
+func runBenchmarkOn(b *testing.B, db eventstore.Store) {
+	for i := 0; i < 10000; i++ {
+		eTag := make([]byte, 32)
+		binary.BigEndian.PutUint16(eTag, uint16(i))
+
+		ref, _ := nostr.GetPublicKey(sk3)
+		if i%3 == 0 {
+			ref, _ = nostr.GetPublicKey(sk4)
+		}
+
+		evt := &nostr.Event{
+			CreatedAt: nostr.Timestamp(i*10 + 2),
+			Content:   fmt.Sprintf("hello %d", i),
+			Tags: nostr.Tags{
+				{"t", fmt.Sprintf("t%d", i)},
+				{"e", hex.EncodeToString(eTag)},
+				{"p", ref},
+			},
+			Kind: i % 10,
+		}
+		sk := sk3
+		if i%3 == 0 {
+			sk = sk4
+		}
+		evt.Sign(sk)
+		db.SaveEvent(ctx, evt)
+	}
+
+	filters := make([]nostr.Filter, 0, 10)
+	filters = append(filters, nostr.Filter{Kinds: []int{1, 4, 8, 16}})
+	pk3, _ := nostr.GetPublicKey(sk3)
+	filters = append(filters, nostr.Filter{Authors: []string{pk3, nostr.GeneratePrivateKey()}})
+	filters = append(filters, nostr.Filter{Authors: []string{pk3, nostr.GeneratePrivateKey()}, Kinds: []int{3, 4}})
+	filters = append(filters, nostr.Filter{})
+	filters = append(filters, nostr.Filter{Limit: 20})
+	filters = append(filters, nostr.Filter{Kinds: []int{8, 9}, Tags: nostr.TagMap{"p": []string{pk3}}})
+	pk4, _ := nostr.GetPublicKey(sk4)
+	filters = append(filters, nostr.Filter{Kinds: []int{8, 9}, Tags: nostr.TagMap{"p": []string{pk3, pk4}}})
+	filters = append(filters, nostr.Filter{Kinds: []int{8, 9}, Tags: nostr.TagMap{"p": []string{pk3, pk4}}})
+	eTags := make([]string, 20)
+	for i := 0; i < 20; i++ {
+		eTag := make([]byte, 32)
+		binary.BigEndian.PutUint16(eTag, uint16(i))
+		eTags[i] = hex.EncodeToString(eTag)
+	}
+	filters = append(filters, nostr.Filter{Kinds: []int{9}, Tags: nostr.TagMap{"e": eTags}})
+	filters = append(filters, nostr.Filter{Kinds: []int{5}, Tags: nostr.TagMap{"e": eTags, "t": []string{"t5"}}})
+	filters = append(filters, nostr.Filter{Tags: nostr.TagMap{"e": eTags}})
+	filters = append(filters, nostr.Filter{Tags: nostr.TagMap{"e": eTags}, Limit: 50})
+
+	b.Run("filter", func(b *testing.B) {
+		for q, filter := range filters {
+			b.Run(fmt.Sprintf("q-%d", q), func(b *testing.B) {
+				for i := 0; i < b.N; i++ {
+					_, _ = db.QueryEvents(ctx, filter)
+				}
+			})
+		}
+	})
+
+	b.Run("insert", func(b *testing.B) {
+		evt := &nostr.Event{Kind: 788, CreatedAt: nostr.Now(), Content: "blergh", Tags: nostr.Tags{{"t", "spam"}}}
+		evt.Sign(sk4)
+		for i := 0; i < b.N; i++ {
+			db.SaveEvent(ctx, evt)
+		}
+	})
+}

eventstore/test/db_test.go

@@ -0,0 +1,77 @@
+package test
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	embeddedpostgres "github.com/fergusstrange/embedded-postgres"
+	"github.com/fiatjaf/eventstore"
+	"github.com/fiatjaf/eventstore/badger"
+	"github.com/fiatjaf/eventstore/lmdb"
+	"github.com/fiatjaf/eventstore/postgresql"
+	"github.com/fiatjaf/eventstore/slicestore"
+	"github.com/fiatjaf/eventstore/sqlite3"
+)
+
+const (
+	dbpath = "/tmp/eventstore-test"
+	sk3    = "0000000000000000000000000000000000000000000000000000000000000003"
+	sk4    = "0000000000000000000000000000000000000000000000000000000000000004"
+)
+
+var ctx = context.Background()
+
+var tests = []struct {
+	name string
+	run  func(*testing.T, eventstore.Store)
+}{
+	{"first", runFirstTestOn},
+	{"second", runSecondTestOn},
+	{"manyauthors", manyAuthorsTest},
+	{"unbalanced", unbalancedTest},
+}
+
+func TestSliceStore(t *testing.T) {
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) { test.run(t, &slicestore.SliceStore{}) })
+	}
+}
+
+func TestLMDB(t *testing.T) {
+	for _, test := range tests {
+		os.RemoveAll(dbpath + "lmdb")
+		t.Run(test.name, func(t *testing.T) { test.run(t, &lmdb.LMDBBackend{Path: dbpath + "lmdb"}) })
+	}
+}
+
+func TestBadger(t *testing.T) {
+	for _, test := range tests {
+		os.RemoveAll(dbpath + "badger")
+		t.Run(test.name, func(t *testing.T) { test.run(t, &badger.BadgerBackend{Path: dbpath + "badger"}) })
+	}
+}
+
+func TestSQLite(t *testing.T) {
+	for _, test := range tests {
+		os.RemoveAll(dbpath + "sqlite")
+		t.Run(test.name, func(t *testing.T) {
+			test.run(t, &sqlite3.SQLite3Backend{DatabaseURL: dbpath + "sqlite", QueryLimit: 1000, QueryTagsLimit: 50, QueryAuthorsLimit: 2000})
+		})
+	}
+}
+
+func TestPostgres(t *testing.T) {
+	for _, test := range tests {
+		postgres := embeddedpostgres.NewDatabase()
+		err := postgres.Start()
+		if err != nil {
+			t.Fatalf("failed to start embedded postgres: %s", err)
+			return
+		}
+		t.Run(test.name, func(t *testing.T) {
+			test.run(t, &postgresql.PostgresBackend{DatabaseURL: "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable", QueryLimit: 1000, QueryTagsLimit: 50, QueryAuthorsLimit: 2000})
+		})
+		postgres.Stop()
+	}
+}

eventstore/test/first_test.go

@@ -0,0 +1,248 @@
+package test
+
+import (
+	"fmt"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/stretchr/testify/require"
+)
+
+func runFirstTestOn(t *testing.T, db eventstore.Store) {
+	err := db.Init()
+	require.NoError(t, err)
+
+	allEvents := make([]*nostr.Event, 0, 10)
+
+	// insert
+	for i := 0; i < 10; i++ {
+		evt := &nostr.Event{
+			CreatedAt: nostr.Timestamp(i*10 + 2),
+			Content:   fmt.Sprintf("hello %d", i),
+			Tags: nostr.Tags{
+				{"t", fmt.Sprintf("%d", i)},
+				{"e", "0" + strconv.Itoa(i) + strings.Repeat("0", 62)},
+			},
+			Kind: 1,
+		}
+		sk := sk3
+		if i%3 == 0 {
+			sk = sk4
+		}
+		if i%2 == 0 {
+			evt.Kind = 9
+		}
+		evt.Sign(sk)
+		allEvents = append(allEvents, evt)
+		err = db.SaveEvent(ctx, evt)
+		require.NoError(t, err)
+	}
+
+	// query
+	w := eventstore.RelayWrapper{Store: db}
+	{
+		results, err := w.QuerySync(ctx, nostr.Filter{})
+		require.NoError(t, err)
+		require.Len(t, results, len(allEvents))
+		require.ElementsMatch(t,
+			allEvents,
+			results,
+			"open-ended query results error")
+	}
+
+	{
+		for i := 0; i < 10; i++ {
+			since := nostr.Timestamp(i*10 + 1)
+			results, err := w.QuerySync(ctx, nostr.Filter{Since: &since})
+			require.NoError(t, err)
+			require.ElementsMatch(t,
+				allEvents[i:],
+				results,
+				"since query results error %d", i)
+		}
+	}
+
+	{
+		results, err := w.QuerySync(ctx, nostr.Filter{IDs: []string{allEvents[7].ID, allEvents[9].ID}})
+		require.NoError(t, err)
+		require.Len(t, results, 2)
+		require.ElementsMatch(t,
+			[]*nostr.Event{allEvents[7], allEvents[9]},
+			results,
+			"id query error")
+	}
+
+	{
+		results, err := w.QuerySync(ctx, nostr.Filter{Kinds: []int{1}})
+		require.NoError(t, err)
+		require.ElementsMatch(t,
+			[]*nostr.Event{allEvents[1], allEvents[3], allEvents[5], allEvents[7], allEvents[9]},
+			results,
+			"kind query error")
+	}
+
+	{
+		results, err := w.QuerySync(ctx, nostr.Filter{Kinds: []int{9}})
+		require.NoError(t, err)
+		require.ElementsMatch(t,
+			[]*nostr.Event{allEvents[0], allEvents[2], allEvents[4], allEvents[6], allEvents[8]},
+			results,
+			"second kind query error")
+	}
+
+	{
+		pk4, _ := nostr.GetPublicKey(sk4)
+		results, err := w.QuerySync(ctx, nostr.Filter{Authors: []string{pk4}})
+		require.NoError(t, err)
+		require.ElementsMatch(t,
+			[]*nostr.Event{allEvents[0], allEvents[3], allEvents[6], allEvents[9]},
+			results,
+			"pubkey query error")
+	}
+
+	{
+		pk3, _ := nostr.GetPublicKey(sk3)
+		results, err := w.QuerySync(ctx, nostr.Filter{Kinds: []int{9}, Authors: []string{pk3}})
+		require.NoError(t, err)
+		require.ElementsMatch(t,
+			[]*nostr.Event{allEvents[2], allEvents[4], allEvents[8]},
+			results,
+			"pubkey kind query error")
+	}
+
+	{
+		pk3, _ := nostr.GetPublicKey(sk3)
+		pk4, _ := nostr.GetPublicKey(sk4)
+		results, err := w.QuerySync(ctx, nostr.Filter{Kinds: []int{9, 5, 7}, Authors: []string{pk3, pk4, pk4[1:] + "a"}})
+		require.NoError(t, err)
+		require.ElementsMatch(t,
+			[]*nostr.Event{allEvents[0], allEvents[2], allEvents[4], allEvents[6], allEvents[8]},
+			results,
+			"2 pubkeys and kind query error")
+	}
+
+	{
+		results, err := w.QuerySync(ctx, nostr.Filter{Tags: nostr.TagMap{"t": []string{"2", "4", "6"}}})
+		require.NoError(t, err)
+		require.ElementsMatch(t,
+			[]*nostr.Event{allEvents[2], allEvents[4], allEvents[6]},
+			results,
+			"tag query error")
+	}
+
+	// delete
+	require.NoError(t, db.DeleteEvent(ctx, allEvents[4]), "delete 1 error")
+	require.NoError(t, db.DeleteEvent(ctx, allEvents[5]), "delete 2 error")
+
+	// query again
+	{
+		results, err := w.QuerySync(ctx, nostr.Filter{})
+		require.NoError(t, err)
+		require.ElementsMatch(t,
+			slices.Concat(allEvents[0:4], allEvents[6:]),
+			results,
+			"second open-ended query error")
+	}
+
+	{
+		results, err := w.QuerySync(ctx, nostr.Filter{Tags: nostr.TagMap{"t": []string{"2", "6"}}})
+		require.NoError(t, err)
+		require.ElementsMatch(t,
+			[]*nostr.Event{allEvents[2], allEvents[6]},
+			results,
+			"second tag query error")
+	}
+
+	{
+		results, err := w.QuerySync(ctx, nostr.Filter{Tags: nostr.TagMap{"e": []string{allEvents[3].Tags[1][1]}}})
+		require.NoError(t, err)
+		require.ElementsMatch(t,
+			[]*nostr.Event{allEvents[3]},
+			results,
+			"'e' tag query error")
+	}
+
+	{
+		for i := 0; i < 4; i++ {
+			until := nostr.Timestamp(i*10 + 1)
+			results, err := w.QuerySync(ctx, nostr.Filter{Until: &until})
+			require.NoError(t, err)
+
+			require.ElementsMatch(t,
+				allEvents[:i],
+				results,
+				"until query results error %d", i)
+		}
+	}
+
+	// test p-tag querying
+	{
+		p := "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"
+		p2 := "2eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"
+
+		newEvents := []*nostr.Event{
+			{Tags: nostr.Tags{nostr.Tag{"p", p}}, Kind: 1984, CreatedAt: nostr.Timestamp(100), Content: "first"},
+			{Tags: nostr.Tags{nostr.Tag{"p", p}, nostr.Tag{"t", "x"}}, Kind: 1984, CreatedAt: nostr.Timestamp(101), Content: "middle"},
+			{Tags: nostr.Tags{nostr.Tag{"p", p}}, Kind: 1984, CreatedAt: nostr.Timestamp(102), Content: "last"},
+			{Tags: nostr.Tags{nostr.Tag{"p", p}}, Kind: 1111, CreatedAt: nostr.Timestamp(101), Content: "bulufas"},
+			{Tags: nostr.Tags{nostr.Tag{"p", p}}, Kind: 1111, CreatedAt: nostr.Timestamp(102), Content: "safulub"},
+			{Tags: nostr.Tags{nostr.Tag{"p", p}}, Kind: 1, CreatedAt: nostr.Timestamp(103), Content: "bololo"},
+			{Tags: nostr.Tags{nostr.Tag{"p", p2}}, Kind: 1, CreatedAt: nostr.Timestamp(104), Content: "wololo"},
+			{Tags: nostr.Tags{nostr.Tag{"p", p}, nostr.Tag{"p", p2}}, Kind: 1, CreatedAt: nostr.Timestamp(104), Content: "trololo"},
+		}
+
+		sk := nostr.GeneratePrivateKey()
+		for _, newEvent := range newEvents {
+			newEvent.Sign(sk)
+			require.NoError(t, db.SaveEvent(ctx, newEvent))
+		}
+
+		{
+			results, err := w.QuerySync(ctx, nostr.Filter{
+				Tags:  nostr.TagMap{"p": []string{p}},
+				Kinds: []int{1984},
+				Limit: 2,
+			})
+			require.NoError(t, err)
+			require.ElementsMatch(t,
+				[]*nostr.Event{newEvents[2], newEvents[1]},
+				results,
+				"'p' tag 1 query error")
+		}
+
+		{
+			results, err := w.QuerySync(ctx, nostr.Filter{
+				Tags:  nostr.TagMap{"p": []string{p}, "t": []string{"x"}},
+				Limit: 4,
+			})
+			require.NoError(t, err)
+			require.ElementsMatch(t,
+				// the results won't be in canonical time order because this query is too awful, needs a kind
+				[]*nostr.Event{newEvents[1]},
+				results,
+				"'p' tag 2 query error")
+		}
+
+		{
+			results, err := w.QuerySync(ctx, nostr.Filter{
+				Tags:  nostr.TagMap{"p": []string{p, p2}},
+				Kinds: []int{1},
+				Limit: 4,
+			})
+			require.NoError(t, err)
+
+			for _, idx := range []int{5, 6, 7} {
+				require.True(t,
+					slices.ContainsFunc(
+						results,
+						func(evt *nostr.Event) bool { return evt.ID == newEvents[idx].ID },
+					),
+					"'p' tag 3 query error")
+			}
+		}
+	}
+}

eventstore/test/manyauthors_test.go

@@ -0,0 +1,68 @@
+package test
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"slices"
+	"testing"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/stretchr/testify/require"
+)
+
+func manyAuthorsTest(t *testing.T, db eventstore.Store) {
+	db.Init()
+
+	const total = 10000
+	const limit = 500
+	const authors = 1700
+	kinds := []int{6, 7, 8}
+
+	bigfilter := nostr.Filter{
+		Authors: make([]string, authors),
+		Kinds:   kinds,
+		Limit:   limit,
+	}
+	for i := 0; i < authors; i++ {
+		sk := make([]byte, 32)
+		binary.BigEndian.PutUint32(sk, uint32(i%(total/5))+1)
+		pk, _ := nostr.GetPublicKey(hex.EncodeToString(sk))
+		bigfilter.Authors[i] = pk
+	}
+
+	ordered := make([]*nostr.Event, 0, total)
+	for i := 0; i < total; i++ {
+		sk := make([]byte, 32)
+		binary.BigEndian.PutUint32(sk, uint32(i%(total/5))+1)
+
+		evt := &nostr.Event{
+			CreatedAt: nostr.Timestamp(i*i) / 4,
+			Content:   fmt.Sprintf("lots of stuff %d", i),
+			Tags:      nostr.Tags{},
+			Kind:      i % 10,
+		}
+		err := evt.Sign(hex.EncodeToString(sk))
+		require.NoError(t, err)
+
+		err = db.SaveEvent(ctx, evt)
+		require.NoError(t, err)
+
+		if bigfilter.Matches(evt) {
+			ordered = append(ordered, evt)
+		}
+	}
+
+	w := eventstore.RelayWrapper{Store: db}
+
+	res, err := w.QuerySync(ctx, bigfilter)
+
+	require.NoError(t, err)
+	require.Len(t, res, limit)
+	require.True(t, slices.IsSortedFunc(res, nostr.CompareEventPtrReverse))
+	slices.SortFunc(ordered, nostr.CompareEventPtrReverse)
+	require.Equal(t, ordered[0], res[0])
+	require.Equal(t, ordered[limit-1], res[limit-1])
+	require.Equal(t, ordered[0:limit], res)
+}

eventstore/test/relaywrapper_test.go

@@ -0,0 +1,49 @@
+package test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/fiatjaf/eventstore/slicestore"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/stretchr/testify/require"
+)
+
+var sk = "486d5f6d4891f4ce3cd5f4d6b62d184ec8ea10db455830ab7918ca43d4d7ad24"
+
+func TestRelayWrapper(t *testing.T) {
+	ctx := context.Background()
+
+	s := &slicestore.SliceStore{}
+	s.Init()
+	defer s.Close()
+
+	w := eventstore.RelayWrapper{Store: s}
+
+	evt1 := nostr.Event{
+		Kind:      3,
+		CreatedAt: 0,
+		Tags:      nostr.Tags{},
+		Content:   "first",
+	}
+	evt1.Sign(sk)
+
+	evt2 := nostr.Event{
+		Kind:      3,
+		CreatedAt: 1,
+		Tags:      nostr.Tags{},
+		Content:   "second",
+	}
+	evt2.Sign(sk)
+
+	for range 200 {
+		go w.Publish(ctx, evt1)
+		go w.Publish(ctx, evt1)
+	}
+	time.Sleep(time.Millisecond * 200)
+
+	evts, _ := w.QuerySync(ctx, nostr.Filter{Kinds: []int{3}})
+	require.Len(t, evts, 1)
+}

eventstore/test/second_test.go

@@ -0,0 +1,82 @@
+package test
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"testing"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/stretchr/testify/require"
+)
+
+func runSecondTestOn(t *testing.T, db eventstore.Store) {
+	db.Init()
+
+	for i := 0; i < 10000; i++ {
+		eTag := make([]byte, 32)
+		binary.BigEndian.PutUint16(eTag, uint16(i))
+
+		ref, _ := nostr.GetPublicKey(sk3)
+		if i%3 == 0 {
+			ref, _ = nostr.GetPublicKey(sk4)
+		}
+
+		evt := &nostr.Event{
+			CreatedAt: nostr.Timestamp(i*10 + 2),
+			Content:   fmt.Sprintf("hello %d", i),
+			Tags: nostr.Tags{
+				{"t", fmt.Sprintf("t%d", i)},
+				{"e", hex.EncodeToString(eTag)},
+				{"p", ref},
+			},
+			Kind: i % 10,
+		}
+		sk := sk3
+		if i%3 == 0 {
+			sk = sk4
+		}
+		evt.Sign(sk)
+		err := db.SaveEvent(ctx, evt)
+		require.NoError(t, err)
+	}
+
+	w := eventstore.RelayWrapper{Store: db}
+	pk3, _ := nostr.GetPublicKey(sk3)
+	pk4, _ := nostr.GetPublicKey(sk4)
+	eTags := make([]string, 20)
+	for i := 0; i < 20; i++ {
+		eTag := make([]byte, 32)
+		binary.BigEndian.PutUint16(eTag, uint16(i))
+		eTags[i] = hex.EncodeToString(eTag)
+	}
+
+	filters := make([]nostr.Filter, 0, 10)
+	filters = append(filters, nostr.Filter{Kinds: []int{1, 4, 8, 16}})
+	filters = append(filters, nostr.Filter{Authors: []string{pk3, nostr.GeneratePrivateKey()}})
+	filters = append(filters, nostr.Filter{Authors: []string{pk3, nostr.GeneratePrivateKey()}, Kinds: []int{3, 4}})
+	filters = append(filters, nostr.Filter{})
+	filters = append(filters, nostr.Filter{Limit: 20})
+	filters = append(filters, nostr.Filter{Kinds: []int{8, 9}, Tags: nostr.TagMap{"p": []string{pk3}}})
+	filters = append(filters, nostr.Filter{Kinds: []int{8, 9}, Tags: nostr.TagMap{"p": []string{pk3, pk4}}})
+	filters = append(filters, nostr.Filter{Kinds: []int{8, 9}, Tags: nostr.TagMap{"p": []string{pk3, pk4}}})
+	filters = append(filters, nostr.Filter{Kinds: []int{9}, Tags: nostr.TagMap{"e": eTags}})
+	filters = append(filters, nostr.Filter{Kinds: []int{5}, Tags: nostr.TagMap{"e": eTags, "t": []string{"t5"}}})
+	filters = append(filters, nostr.Filter{Tags: nostr.TagMap{"e": eTags}})
+	filters = append(filters, nostr.Filter{Tags: nostr.TagMap{"e": eTags}, Limit: 50})
+
+	t.Run("filter", func(t *testing.T) {
+		for q, filter := range filters {
+			q := q
+			filter := filter
+			label := fmt.Sprintf("filter %d: %s", q, filter)
+
+			t.Run(fmt.Sprintf("q-%d", q), func(t *testing.T) {
+				results, err := w.QuerySync(ctx, filter)
+				require.NoError(t, err, filter)
+				require.NotEmpty(t, results, label)
+			})
+		}
+	})
+}

eventstore/test/test_helpers.go

@@ -0,0 +1,13 @@
+package test
+
+import (
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func getTimestamps(events []*nostr.Event) []nostr.Timestamp {
+	res := make([]nostr.Timestamp, len(events))
+	for i, evt := range events {
+		res[i] = evt.CreatedAt
+	}
+	return res
+}

eventstore/test/unbalanced_test.go

@@ -0,0 +1,82 @@
+package test
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"slices"
+	"testing"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/stretchr/testify/require"
+)
+
+// this is testing what happens when most results come from the same abstract query -- but not all
+func unbalancedTest(t *testing.T, db eventstore.Store) {
+	db.Init()
+
+	const total = 10000
+	const limit = 160
+	const authors = 1400
+
+	bigfilter := nostr.Filter{
+		Authors: make([]string, authors),
+		Limit:   limit,
+	}
+	for i := 0; i < authors; i++ {
+		sk := make([]byte, 32)
+		binary.BigEndian.PutUint32(sk, uint32(i%(authors*2))+1)
+		pk, _ := nostr.GetPublicKey(hex.EncodeToString(sk))
+		bigfilter.Authors[i] = pk
+	}
+	// fmt.Println("filter", bigfilter)
+
+	expected := make([]*nostr.Event, 0, total)
+	for i := 0; i < total; i++ {
+		skseed := uint32(i%(authors*2)) + 1
+		sk := make([]byte, 32)
+		binary.BigEndian.PutUint32(sk, skseed)
+
+		evt := &nostr.Event{
+			CreatedAt: nostr.Timestamp(skseed)*1000 + nostr.Timestamp(i),
+			Content:   fmt.Sprintf("unbalanced %d", i),
+			Tags:      nostr.Tags{},
+			Kind:      1,
+		}
+		err := evt.Sign(hex.EncodeToString(sk))
+		require.NoError(t, err)
+
+		err = db.SaveEvent(ctx, evt)
+		require.NoError(t, err)
+
+		if bigfilter.Matches(evt) {
+			expected = append(expected, evt)
+		}
+	}
+
+	slices.SortFunc(expected, nostr.CompareEventPtrReverse)
+	if len(expected) > limit {
+		expected = expected[0:limit]
+	}
+	require.Len(t, expected, limit)
+
+	w := eventstore.RelayWrapper{Store: db}
+
+	res, err := w.QuerySync(ctx, bigfilter)
+
+	require.NoError(t, err)
+	require.Equal(t, limit, len(res))
+	require.True(t, slices.IsSortedFunc(res, nostr.CompareEventPtrReverse))
+	require.Equal(t, expected[0], res[0])
+
+	// fmt.Println(" expected   result")
+	// ets := getTimestamps(expected)
+	// rts := getTimestamps(res)
+	// for i := range ets {
+	// 	fmt.Println(" ", ets[i], "  ", rts[i], "           ", i)
+	// }
+
+	require.Equal(t, expected[limit-1], res[limit-1])
+	require.Equal(t, expected[0:limit], res)
+}

eventstore/wrappers/count/count.go

@@ -0,0 +1,34 @@
+package count
+
+import (
+	"context"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+type Wrapper struct {
+	eventstore.Store
+}
+
+var _ eventstore.Store = (*Wrapper)(nil)
+
+func (w Wrapper) CountEvents(ctx context.Context, filter nostr.Filter) (int64, error) {
+	if counter, ok := w.Store.(eventstore.Counter); ok {
+		return counter.CountEvents(ctx, filter)
+	}
+
+	ch, err := w.Store.QueryEvents(ctx, filter)
+	if err != nil {
+		return 0, err
+	}
+	if ch == nil {
+		return 0, nil
+	}
+
+	var count int64
+	for range ch {
+		count++
+	}
+	return count, nil
+}

eventstore/wrappers/disablesearch/disablesearch.go

@@ -0,0 +1,21 @@
+package disablesearch
+
+import (
+	"context"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+type Wrapper struct {
+	eventstore.Store
+}
+
+var _ eventstore.Store = (*Wrapper)(nil)
+
+func (w Wrapper) QueryEvents(ctx context.Context, filter nostr.Filter) (chan *nostr.Event, error) {
+	if filter.Search != "" {
+		return nil, nil
+	}
+	return w.Store.QueryEvents(ctx, filter)
+}

eventstore/wrappers/skipevent/skipevent.go

@@ -0,0 +1,24 @@
+package skipevent
+
+import (
+	"context"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+type Wrapper struct {
+	eventstore.Store
+
+	Skip func(ctx context.Context, evt *nostr.Event) bool
+}
+
+var _ eventstore.Store = (*Wrapper)(nil)
+
+func (w Wrapper) SaveEvent(ctx context.Context, evt *nostr.Event) error {
+	if w.Skip(ctx, evt) {
+		return nil
+	}
+
+	return w.Store.SaveEvent(ctx, evt)
+}

khatru/.gitignore

@@ -0,0 +1,3 @@
+*.env
+.idea/
+knowledge.md

khatru/README.md

@@ -0,0 +1,141 @@
+# khatru, a relay framework [![docs badge](https://img.shields.io/badge/docs-reference-blue)](https://pkg.go.dev/github.com/fiatjaf/khatru#Relay)
+
+[![Run Tests](https://github.com/fiatjaf/khatru/actions/workflows/test.yml/badge.svg)](https://github.com/fiatjaf/khatru/actions/workflows/test.yml)
+[![Go Reference](https://pkg.go.dev/badge/github.com/fiatjaf/khatru.svg)](https://pkg.go.dev/github.com/fiatjaf/khatru)
+[![Go Report Card](https://goreportcard.com/badge/github.com/fiatjaf/khatru)](https://goreportcard.com/report/github.com/fiatjaf/khatru)
+
+Khatru makes it easy to write very very custom relays:
+
+  - custom event or filter acceptance policies
+  - custom `AUTH` handlers
+  - custom storage and pluggable databases
+  - custom webpages and other HTTP handlers
+
+Here's a sample:
+
+```go
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net/http"
+
+	"github.com/fiatjaf/khatru"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func main() {
+	// create the relay instance
+	relay := khatru.NewRelay()
+
+	// set up some basic properties (will be returned on the NIP-11 endpoint)
+	relay.Info.Name = "my relay"
+	relay.Info.PubKey = "79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798"
+	relay.Info.Description = "this is my custom relay"
+	relay.Info.Icon = "https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2Fliquipedia.net%2Fcommons%2Fimages%2F3%2F35%2FSCProbe.jpg&f=1&nofb=1&ipt=0cbbfef25bce41da63d910e86c3c343e6c3b9d63194ca9755351bb7c2efa3359&ipo=images"
+
+	// you must bring your own storage scheme -- if you want to have any
+	store := make(map[string]*nostr.Event, 120)
+
+	// set up the basic relay functions
+	relay.StoreEvent = append(relay.StoreEvent,
+		func(ctx context.Context, event *nostr.Event) error {
+			store[event.ID] = event
+			return nil
+		},
+	)
+	relay.QueryEvents = append(relay.QueryEvents,
+		func(ctx context.Context, filter nostr.Filter) (chan *nostr.Event, error) {
+			ch := make(chan *nostr.Event)
+			go func() {
+				for _, evt := range store {
+					if filter.Matches(evt) {
+						ch <- evt
+					}
+				}
+				close(ch)
+			}()
+			return ch, nil
+		},
+	)
+	relay.DeleteEvent = append(relay.DeleteEvent,
+		func(ctx context.Context, event *nostr.Event) error {
+			delete(store, event.ID)
+			return nil
+		},
+	)
+
+	// there are many other configurable things you can set
+	relay.RejectEvent = append(relay.RejectEvent,
+		// built-in policies
+		policies.ValidateKind,
+
+		// define your own policies
+		policies.PreventLargeTags(100),
+		func(ctx context.Context, event *nostr.Event) (reject bool, msg string) {
+			if event.PubKey == "fa984bd7dbb282f07e16e7ae87b26a2a7b9b90b7246a44771f0cf5ae58018f52" {
+				return true, "we don't allow this person to write here"
+			}
+			return false, "" // anyone else can
+		},
+	)
+
+	// you can request auth by rejecting an event or a request with the prefix "auth-required: "
+	relay.RejectFilter = append(relay.RejectFilter,
+		// built-in policies
+		policies.NoComplexFilters,
+
+		// define your own policies
+		func(ctx context.Context, filter nostr.Filter) (reject bool, msg string) {
+			if pubkey := khatru.GetAuthed(ctx); pubkey != "" {
+				log.Printf("request from %s\n", pubkey)
+				return false, ""
+			}
+			return true, "auth-required: only authenticated users can read from this relay"
+			// (this will cause an AUTH message to be sent and then a CLOSED message such that clients can
+			//  authenticate and then request again)
+		},
+	)
+	// check the docs for more goodies!
+
+	mux := relay.Router()
+	// set up other http handlers
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("content-type", "text/html")
+		fmt.Fprintf(w, `<b>welcome</b> to my relay!`)
+	})
+
+	// start the server
+	fmt.Println("running on :3334")
+	http.ListenAndServe(":3334", relay)
+}
+```
+
+### But I don't want to write my own database!
+
+Fear no more. Using the https://github.com/fiatjaf/eventstore module you get a bunch of compatible databases out of the box and you can just plug them into your relay. For example, [sqlite](https://pkg.go.dev/github.com/fiatjaf/eventstore/sqlite3):
+
+```go
+	db := sqlite3.SQLite3Backend{DatabaseURL: "/tmp/khatru-sqlite-tmp"}
+	if err := db.Init(); err != nil {
+		panic(err)
+	}
+
+	relay.StoreEvent = append(relay.StoreEvent, db.SaveEvent)
+	relay.QueryEvents = append(relay.QueryEvents, db.QueryEvents)
+	relay.CountEvents = append(relay.CountEvents, db.CountEvents)
+	relay.DeleteEvent = append(relay.DeleteEvent, db.DeleteEvent)
+	relay.ReplaceEvent = append(relay.ReplaceEvent, db.ReplaceEvent)
+```
+
+### But I don't want to write a bunch of custom policies!
+
+Fear no more. We have a bunch of common policies written in the `github.com/fiatjaf/khatru/policies` package and also a handpicked selection of base sane defaults, which you can apply with:
+
+```go
+	policies.ApplySaneDefaults(relay)
+```
+
+Contributions to this are very much welcomed.