Coracle/nostrlib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bring in khatru and eventstore.

Browse Source
This commit is contained in:
fiatjaf
2025-04-15 08:49:28 -03:00
parent 8466a9757b
commit 76032dc089
170 changed files with 15018 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
eventstore/mmm/betterbinary/codec.go Normal file
View File

@@ -0,0 +1,139 @@
package betterbinary
import (
"encoding/binary"
"encoding/hex"
"fmt"
"math"
"github.com/nbd-wtf/go-nostr"
)
const (
MaxKind = math.MaxUint16
MaxCreatedAt = math.MaxUint32
MaxContentSize = math.MaxUint16
MaxTagCount = math.MaxUint16
MaxTagItemCount = math.MaxUint8
MaxTagItemSize = math.MaxUint16
)
func Measure(evt nostr.Event) int {
n := 135 // static base
n += 2 + // tag section length
2 + // number of tags
len(evt.Tags)*3 // each tag offset + each tag item count
for _, tag := range evt.Tags {
n += len(tag) * 2 // item length for each item in this tag
for _, item := range tag {
n += len(item) // actual tag item
}
}
// content length and actual content
n += 2 + len(evt.Content)
return n
}
func Marshal(evt nostr.Event, buf []byte) error {
buf[0] = 0
if evt.Kind > MaxKind {
return fmt.Errorf("kind is too big: %d, max is %d", evt.Kind, MaxKind)
}
binary.LittleEndian.PutUint16(buf[1:3], uint16(evt.Kind))
if evt.CreatedAt > MaxCreatedAt {
return fmt.Errorf("created_at is too big: %d, max is %d", evt.CreatedAt, MaxCreatedAt)
}
binary.LittleEndian.PutUint32(buf[3:7], uint32(evt.CreatedAt))
hex.Decode(buf[7:39], []byte(evt.ID))
hex.Decode(buf[39:71], []byte(evt.PubKey))
hex.Decode(buf[71:135], []byte(evt.Sig))
tagBase := 135
// buf[135:137] (tagsSectionLength) will be set later when we know the absolute size of the tags section
ntags := len(evt.Tags)
if ntags > MaxTagCount {
return fmt.Errorf("can't encode too many tags: %d, max is %d", ntags, MaxTagCount)
}
binary.LittleEndian.PutUint16(buf[137:139], uint16(ntags))
tagOffset := 2 + 2 + ntags*2
for t, tag := range evt.Tags {
binary.LittleEndian.PutUint16(buf[tagBase+2+2+t*2:], uint16(tagOffset))
itemCount := len(tag)
if itemCount > MaxTagItemCount {
return fmt.Errorf("can't encode a tag with so many items: %d, max is %d", itemCount, MaxTagItemCount)
}
buf[tagBase+tagOffset] = uint8(itemCount)
itemOffset := 1
for _, item := range tag {
itemSize := len(item)
if itemSize > MaxTagItemSize {
return fmt.Errorf("tag item is too large: %d, max is %d", itemSize, MaxTagItemSize)
}
binary.LittleEndian.PutUint16(buf[tagBase+tagOffset+itemOffset:], uint16(itemSize))
copy(buf[tagBase+tagOffset+itemOffset+2:], []byte(item))
itemOffset += 2 + len(item)
}
tagOffset += itemOffset
}
tagsSectionLength := tagOffset
binary.LittleEndian.PutUint16(buf[tagBase:], uint16(tagsSectionLength))
// content
if contentLength := len(evt.Content); contentLength > MaxContentSize {
return fmt.Errorf("content is too large: %d, max is %d", contentLength, MaxContentSize)
} else {
binary.LittleEndian.PutUint16(buf[tagBase+tagsSectionLength:], uint16(contentLength))
}
copy(buf[tagBase+tagsSectionLength+2:], []byte(evt.Content))
return nil
}
func Unmarshal(data []byte, evt *nostr.Event) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("failed to decode binary: %v", r)
}
}()
evt.Kind = int(binary.LittleEndian.Uint16(data[1:3]))
evt.CreatedAt = nostr.Timestamp(binary.LittleEndian.Uint32(data[3:7]))
evt.ID = hex.EncodeToString(data[7:39])
evt.PubKey = hex.EncodeToString(data[39:71])
evt.Sig = hex.EncodeToString(data[71:135])
const tagbase = 135
tagsSectionLength := binary.LittleEndian.Uint16(data[tagbase:])
ntags := binary.LittleEndian.Uint16(data[tagbase+2:])
evt.Tags = make(nostr.Tags, ntags)
for t := range evt.Tags {
offset := binary.LittleEndian.Uint16(data[tagbase+4+t*2:])
nitems := int(data[tagbase+offset])
tag := make(nostr.Tag, nitems)
curr := tagbase + offset + 1
for i := range tag {
length := binary.LittleEndian.Uint16(data[curr:])
tag[i] = string(data[curr+2 : curr+2+length])
curr += 2 + length
}
evt.Tags[t] = tag
}
contentLength := binary.LittleEndian.Uint16(data[tagbase+tagsSectionLength:])
evt.Content = string(data[tagbase+tagsSectionLength+2 : tagbase+tagsSectionLength+2+contentLength])
return err
}

182
eventstore/mmm/betterbinary/codec_test.go Normal file
View File

File diff suppressed because one or more lines are too long

33
eventstore/mmm/betterbinary/filtering.go Normal file
View File

@@ -0,0 +1,33 @@
package betterbinary
import (
"encoding/binary"
"slices"
)
func TagMatches(evtb []byte, key string, vals []string) bool {
matches := make([][]byte, 0, len(vals))
for _, val := range vals {
match := append([]byte{1, 0, key[0], uint8(len(val)), 0}, val...)
matches = append(matches, match)
}
ntags := binary.LittleEndian.Uint16(evtb[137:])
var t uint16
for t = 0; t < ntags; t++ {
offset := int(binary.LittleEndian.Uint16(evtb[139+t*2:]))
nitems := evtb[135+offset]
if nitems >= 2 {
for _, match := range matches {
if slices.Equal(evtb[135+offset+1:135+offset+1+len(match)], match) {
return true
}
}
}
}
return false
}
func KindMatches(evtb []byte, kind uint16) bool {
return binary.LittleEndian.Uint16(evtb[1:3]) == kind
}

51
eventstore/mmm/betterbinary/filtering_test.go Normal file
View File

@@ -0,0 +1,51 @@
package betterbinary
import (
"testing"
"github.com/mailru/easyjson"
"github.com/nbd-wtf/go-nostr"
)
func TestTagFiltering(t *testing.T) {
for _, tc := range []struct {
json string
tagKey string
tagValues []string
matches bool
}{
{
`{"id":"a9663055164ab8b30d9524656370c4bf93393bb051b7edf4556f40c5298dc0c7","pubkey":"ee11a5dff40c19a555f41fe42b48f00e618c91225622ae37b6c2bb67b76c4e49","created_at":1681778790,"kind":1,"sig":"4dfea1a6f73141d5691e43afc3234dbe73016db0fb207cf247e0127cc2591ee6b4be5b462272030a9bde75882aae810f359682b1b6ce6cbb97201141c576db42","content":"He got snowed in"}`,
"x",
[]string{"sadjqw", ""},
false,
},
{
`{"id":"a9663055164ab8b30d9524656370c4bf93393bb051b7edf4556f40c5298dc0c7","pubkey":"ee11a5dff40c19a555f41fe42b48f00e618c91225622ae37b6c2bb67b76c4e49","created_at":1681778790,"kind":1,"sig":"4dfea1a6f73141d5691e43afc3234dbe73016db0fb207cf247e0127cc2591ee6b4be5b462272030a9bde75882aae810f359682b1b6ce6cbb97201141c576db42","content":"He got snowed in","tags":[["client","gossip"],["p","e2ccf7cf20403f3f2a4a55b328f0de3be38558a7d5f33632fdaaefc726c1c8eb"],["e","2c86abcc98f7fd8a6750aab8df6c1863903f107206cc2d72e8afeb6c38357aed","wss://nostr-pub.wellorder.net/","root"]]}`,
"e",
[]string{"2c86abcc98f7fd8a6750aab8df6c1863903f107206cc2d72e8afeb6c38357aed"},
true,
},
{
`{"id":"3f551da67788c7aae15360d025595dc2d391f10bb7e759ee5d9b2ad7d64392e4","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","created_at":1712715433,"kind":1,"tags":[["-"],["askdasds"],["t","spam"],["t","nada"]],"content":"ggsgsgsg","sig":"43431f4cf8bd015305c2d484841e5730d261beeb375a86c57a61df3d26e820ce8d6712d2a3c89e3f2298597f14abf58079954e9e658ba59bfc2d7ce6384f25c7"}`,
"t",
[]string{"nothing", "nada"},
true,
},
{
`{"id":"3f551da67788c7aae15360d025595dc2d391f10bb7e759ee5d9b2ad7d64392e4","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","created_at":1712715433,"kind":1,"tags":[["-"],["askdasds"],["t","spam"],["t","nada"]],"content":"ggsgsgsg","sig":"43431f4cf8bd015305c2d484841e5730d261beeb375a86c57a61df3d26e820ce8d6712d2a3c89e3f2298597f14abf58079954e9e658ba59bfc2d7ce6384f25c7"}`,
"z",
[]string{"nothing", "nada"},
false,
},
} {
var evt nostr.Event
easyjson.Unmarshal([]byte(tc.json), &evt)
bin := make([]byte, Measure(evt))
Marshal(evt, bin)
if res := TagMatches(bin, tc.tagKey, tc.tagValues); res != tc.matches {
t.Fatalf("matched incorrectly: %v=>%v over %s was %v, expected %v", tc.tagKey, tc.tagValues, tc.json, res, tc.matches)
}
}
}

91
eventstore/mmm/count.go Normal file
View File

@@ -0,0 +1,91 @@
package mmm
import (
"bytes"
"context"
"encoding/binary"
"slices"
"github.com/PowerDNS/lmdb-go/lmdb"
"github.com/fiatjaf/eventstore/mmm/betterbinary"
"github.com/nbd-wtf/go-nostr"
)
func (il *IndexingLayer) CountEvents(ctx context.Context, filter nostr.Filter) (int64, error) {
var count int64 = 0
queries, extraAuthors, extraKinds, extraTagKey, extraTagValues, since, err := il.prepareQueries(filter)
if err != nil {
return 0, err
}
err = il.lmdbEnv.View(func(txn *lmdb.Txn) error {
// actually iterate
for _, q := range queries {
cursor, err := txn.OpenCursor(q.dbi)
if err != nil {
continue
}
it := &iterator{cursor: cursor}
it.seek(q.startingPoint)
for {
// we already have a k and a v and an err from the cursor setup, so check and use these
if it.err != nil ||
len(it.key) != q.keySize ||
!bytes.HasPrefix(it.key, q.prefix) {
// either iteration has errored or we reached the end of this prefix
break // stop this cursor and move to the next one
}
// "id" indexes don't contain a timestamp
if q.timestampSize == 4 {
createdAt := binary.BigEndian.Uint32(it.key[len(it.key)-4:])
if createdAt < since {
break
}
}
if extraAuthors == nil && extraKinds == nil && extraTagValues == nil {
count++
} else {
// fetch actual event
pos := positionFromBytes(it.posb)
bin := il.mmmm.mmapf[pos.start : pos.start+uint64(pos.size)]
// check it against pubkeys without decoding the entire thing
if extraAuthors != nil && !slices.Contains(extraAuthors, [32]byte(bin[39:71])) {
it.next()
continue
}
// check it against kinds without decoding the entire thing
if extraKinds != nil && !slices.Contains(extraKinds, [2]byte(bin[1:3])) {
it.next()
continue
}
// decode the entire thing (TODO: do a conditional decode while also checking the extra tag)
event := &nostr.Event{}
if err := betterbinary.Unmarshal(bin, event); err != nil {
it.next()
continue
}
// if there is still a tag to be checked, do it now
if !event.Tags.ContainsAny(extraTagKey, extraTagValues) {
it.next()
continue
}
count++
}
}
}
return nil
})
return count, err
}

78
eventstore/mmm/delete.go Normal file
View File

@@ -0,0 +1,78 @@
package mmm
import (
"context"
"encoding/binary"
"encoding/hex"
"fmt"
"slices"
"github.com/PowerDNS/lmdb-go/lmdb"
"github.com/nbd-wtf/go-nostr"
)
func (il *IndexingLayer) DeleteEvent(ctx context.Context, evt *nostr.Event) error {
return il.mmmm.lmdbEnv.Update(func(mmmtxn *lmdb.Txn) error {
return il.lmdbEnv.Update(func(iltxn *lmdb.Txn) error {
return il.delete(mmmtxn, iltxn, evt)
})
})
}
func (il *IndexingLayer) delete(mmmtxn *lmdb.Txn, iltxn *lmdb.Txn, evt *nostr.Event) error {
zeroRefs := false
b := il.mmmm
b.Logger.Debug().Str("layer", il.name).Uint16("il", il.id).Msg("deleting")
// first in the mmmm txn we check if we have the event still
idPrefix8, _ := hex.DecodeString(evt.ID[0 : 8*2])
val, err := mmmtxn.Get(b.indexId, idPrefix8)
if err != nil {
if lmdb.IsNotFound(err) {
// we already do not have this anywhere
return nil
}
return fmt.Errorf("failed to check if we have the event %x: %w", idPrefix8, err)
}
// we have this, but do we have it in the current layer?
// val is [posb][il_idx][il_idx...]
pos := positionFromBytes(val[0:12])
// check references
currentLayer := binary.BigEndian.AppendUint16(nil, il.id)
for i := 12; i < len(val); i += 2 {
if slices.Equal(val[i:i+2], currentLayer) {
// we will remove the current layer if it's found
nextval := make([]byte, len(val)-2)
copy(nextval, val[0:i])
copy(nextval[i:], val[i+2:])
if err := mmmtxn.Put(b.indexId, idPrefix8, nextval, 0); err != nil {
return fmt.Errorf("failed to update references for %x: %w", idPrefix8, err)
}
// if there are no more layers we will delete everything later
zeroRefs = len(nextval) == 12
break
}
}
// calculate all index keys we have for this event and delete them
for k := range il.getIndexKeysForEvent(evt) {
if err := iltxn.Del(k.dbi, k.key, val[0:12]); err != nil && !lmdb.IsNotFound(err) {
return fmt.Errorf("index entry %v/%x deletion failed: %w", k.dbi, k.key, err)
}
}
// if there are no more refs we delete the event from the id index and mmap
if zeroRefs {
if err := b.purge(mmmtxn, idPrefix8, pos); err != nil {
panic(err)
}
}
return nil
}

68
eventstore/mmm/freeranges.go Normal file
View File

@@ -0,0 +1,68 @@
package mmm
import (
"fmt"
"slices"
"github.com/PowerDNS/lmdb-go/lmdb"
)
func (b *MultiMmapManager) mergeNewFreeRange(pos position) (isAtEnd bool) {
// before adding check if we can merge this with some other range
// (to merge means to delete the previous and add a new one)
toDelete := make([]int, 0, 2)
for f, fr := range b.freeRanges {
if pos.start+uint64(pos.size) == fr.start {
// [new_pos_to_be_freed][existing_fr] -> merge!
toDelete = append(toDelete, f)
pos.size = pos.size + fr.size
} else if fr.start+uint64(fr.size) == pos.start {
// [existing_fr][new_pos_to_be_freed] -> merge!
toDelete = append(toDelete, f)
pos.start = fr.start
pos.size = fr.size + pos.size
}
}
slices.SortFunc(toDelete, func(a, b int) int { return b - a })
for _, idx := range toDelete {
b.freeRanges = slices.Delete(b.freeRanges, idx, idx+1)
}
// when we're at the end of a file we just delete everything and don't add new free ranges
// the caller will truncate the mmap file and adjust the position accordingly
if pos.start+uint64(pos.size) == b.mmapfEnd {
return true
}
b.addNewFreeRange(pos)
return false
}
func (b *MultiMmapManager) addNewFreeRange(pos position) {
// update freeranges slice in memory
idx, _ := slices.BinarySearchFunc(b.freeRanges, pos, func(item, target position) int {
if item.size > target.size {
return 1
} else if target.size > item.size {
return -1
} else if item.start > target.start {
return 1
} else {
return -1
}
})
b.freeRanges = slices.Insert(b.freeRanges, idx, pos)
}
func (b *MultiMmapManager) saveFreeRanges(txn *lmdb.Txn) error {
// save to database
valReserve, err := txn.PutReserve(b.stuff, FREERANGES_KEY, len(b.freeRanges)*12, 0)
if err != nil {
return fmt.Errorf("on put freeranges: %w", err)
}
for f, fr := range b.freeRanges {
bytesFromPosition(valReserve[f*12:], fr)
}
return nil
}

191
eventstore/mmm/fuzz_test.go Normal file
View File

@@ -0,0 +1,191 @@
package mmm
import (
"context"
"fmt"
"math/rand/v2"
"os"
"slices"
"testing"
"github.com/nbd-wtf/go-nostr"
"github.com/rs/zerolog"
"github.com/stretchr/testify/require"
)
func FuzzTest(f *testing.F) {
f.Add(0, uint(84), uint(10), uint(5))
f.Fuzz(func(t *testing.T, seed int, nlayers, nevents, ndeletes uint) {
nlayers = nlayers%23 + 1
nevents = nevents%10000 + 1
ndeletes = ndeletes % nevents
// create a temporary directory for the test
tmpDir, err := os.MkdirTemp("", "mmm-test-*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
logger := zerolog.Nop()
rnd := rand.New(rand.NewPCG(uint64(seed), 0))
// initialize MMM
mmm := &MultiMmapManager{
Dir: tmpDir,
Logger: &logger,
}
err = mmm.Init()
require.NoError(t, err)
defer mmm.Close()
for i := range nlayers {
name := string([]byte{97 + byte(i)})
err = mmm.EnsureLayer(name, &IndexingLayer{
MaxLimit: 1000,
})
require.NoError(t, err, "layer %s/%d", name, i)
}
// create test events
ctx := context.Background()
sk := "945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb"
storedIds := make([]string, nevents)
nTags := make(map[string]int)
storedByLayer := make(map[string][]string)
// create n events with random combinations of tags
for i := 0; i < int(nevents); i++ {
tags := nostr.Tags{}
// randomly add 1-nlayers tags
numTags := 1 + (i % int(nlayers))
usedTags := make(map[string]bool)
for j := 0; j < numTags; j++ {
tag := string([]byte{97 + byte(i%int(nlayers))})
if !usedTags[tag] {
tags = append(tags, nostr.Tag{"t", tag})
usedTags[tag] = true
}
}
evt := &nostr.Event{
CreatedAt: nostr.Timestamp(i),
Kind: i, // hack to query by serial id
Tags: tags,
Content: fmt.Sprintf("test content %d", i),
}
evt.Sign(sk)
for _, layer := range mmm.layers {
if evt.Tags.FindWithValue("t", layer.name) != nil {
err := layer.SaveEvent(ctx, evt)
require.NoError(t, err)
storedByLayer[layer.name] = append(storedByLayer[layer.name], evt.ID)
}
}
storedIds = append(storedIds, evt.ID)
nTags[evt.ID] = len(evt.Tags)
}
// verify each layer has the correct events
for _, layer := range mmm.layers {
results, err := layer.QueryEvents(ctx, nostr.Filter{})
require.NoError(t, err)
count := 0
for evt := range results {
require.True(t, evt.Tags.ContainsAny("t", []string{layer.name}))
count++
}
require.Equal(t, count, len(storedByLayer[layer.name]))
}
// randomly select n events to delete from random layers
deleted := make(map[string][]*IndexingLayer)
for range ndeletes {
id := storedIds[rnd.Int()%len(storedIds)]
layer := mmm.layers[rnd.Int()%len(mmm.layers)]
evt, layers := mmm.GetByID(id)
if slices.Contains(deleted[id], layer) {
// already deleted from this layer
require.NotContains(t, layers, layer)
} else if evt != nil && evt.Tags.FindWithValue("t", layer.name) != nil {
require.Contains(t, layers, layer)
// delete now
layer.DeleteEvent(ctx, evt)
deleted[id] = append(deleted[id], layer)
} else {
// was never saved to this in the first place
require.NotContains(t, layers, layer)
}
}
for id, deletedlayers := range deleted {
evt, foundlayers := mmm.GetByID(id)
for _, layer := range deletedlayers {
require.NotContains(t, foundlayers, layer)
}
for _, layer := range foundlayers {
require.NotNil(t, evt.Tags.FindWithValue("t", layer.name))
}
if nTags[id] == len(deletedlayers) && evt != nil {
deletedlayersnames := make([]string, len(deletedlayers))
for i, layer := range deletedlayers {
deletedlayersnames[i] = layer.name
}
t.Fatalf("id %s has %d tags %v, should have been deleted from %v, but wasn't: %s",
id, nTags[id], evt.Tags, deletedlayersnames, evt)
} else if nTags[id] > len(deletedlayers) {
t.Fatalf("id %s should still be available as it had %d tags and was only deleted from %v, but isn't",
id, nTags[id], deletedlayers)
}
if evt != nil {
for _, layer := range mmm.layers {
// verify event still accessible from other layers
if slices.Contains(foundlayers, layer) {
ch, err := layer.QueryEvents(ctx, nostr.Filter{Kinds: []int{evt.Kind}}) // hack
require.NoError(t, err)
fetched := <-ch
require.NotNil(t, fetched)
} else {
// and not accessible from this layer we just deleted
ch, err := layer.QueryEvents(ctx, nostr.Filter{Kinds: []int{evt.Kind}}) // hack
require.NoError(t, err)
fetched := <-ch
require.Nil(t, fetched)
}
}
}
}
// now delete a layer and events that only exist in that layer should vanish
layer := mmm.layers[rnd.Int()%len(mmm.layers)]
ch, err := layer.QueryEvents(ctx, nostr.Filter{})
require.NoError(t, err)
eventsThatShouldVanish := make([]string, 0, nevents/2)
for evt := range ch {
if len(evt.Tags) == 1+len(deleted[evt.ID]) {
eventsThatShouldVanish = append(eventsThatShouldVanish, evt.ID)
}
}
err = mmm.DropLayer(layer.name)
require.NoError(t, err)
for _, id := range eventsThatShouldVanish {
v, ils := mmm.GetByID(id)
require.Nil(t, v)
require.Empty(t, ils)
}
})
}

165
eventstore/mmm/helpers.go Normal file
View File

@@ -0,0 +1,165 @@
package mmm
import (
"encoding/binary"
"encoding/hex"
"iter"
"slices"
"strconv"
"strings"
"github.com/PowerDNS/lmdb-go/lmdb"
"github.com/nbd-wtf/go-nostr"
)
// this iterator always goes backwards
type iterator struct {
cursor *lmdb.Cursor
key []byte
posb []byte
err error
}
func (it *iterator) seek(key []byte) {
if _, _, errsr := it.cursor.Get(key, nil, lmdb.SetRange); errsr != nil {
if operr, ok := errsr.(*lmdb.OpError); !ok || operr.Errno != lmdb.NotFound {
// in this case it's really an error
panic(operr)
} else {
// we're at the end and we just want notes before this,
// so we just need to set the cursor the last key, this is not a real error
it.key, it.posb, it.err = it.cursor.Get(nil, nil, lmdb.Last)
}
} else {
// move one back as the first step
it.key, it.posb, it.err = it.cursor.Get(nil, nil, lmdb.Prev)
}
}
func (it *iterator) next() {
// move one back (we'll look into k and v and err in the next iteration)
it.key, it.posb, it.err = it.cursor.Get(nil, nil, lmdb.Prev)
}
type key struct {
dbi lmdb.DBI
key []byte
}
func (il *IndexingLayer) getIndexKeysForEvent(evt *nostr.Event) iter.Seq[key] {
return func(yield func(key) bool) {
{
// ~ by pubkey+date
k := make([]byte, 8+4)
hex.Decode(k[0:8], []byte(evt.PubKey[0:8*2]))
binary.BigEndian.PutUint32(k[8:8+4], uint32(evt.CreatedAt))
if !yield(key{dbi: il.indexPubkey, key: k[0 : 8+4]}) {
return
}
}
{
// ~ by kind+date
k := make([]byte, 2+4)
binary.BigEndian.PutUint16(k[0:2], uint16(evt.Kind))
binary.BigEndian.PutUint32(k[2:2+4], uint32(evt.CreatedAt))
if !yield(key{dbi: il.indexKind, key: k[0 : 2+4]}) {
return
}
}
{
// ~ by pubkey+kind+date
k := make([]byte, 8+2+4)
hex.Decode(k[0:8], []byte(evt.PubKey[0:8*2]))
binary.BigEndian.PutUint16(k[8:8+2], uint16(evt.Kind))
binary.BigEndian.PutUint32(k[8+2:8+2+4], uint32(evt.CreatedAt))
if !yield(key{dbi: il.indexPubkeyKind, key: k[0 : 8+2+4]}) {
return
}
}
// ~ by tagvalue+date
// ~ by p-tag+kind+date
for i, tag := range evt.Tags {
if len(tag) < 2 || len(tag[0]) != 1 || len(tag[1]) == 0 || len(tag[1]) > 100 {
// not indexable
continue
}
firstIndex := slices.IndexFunc(evt.Tags, func(t nostr.Tag) bool { return len(t) >= 2 && t[1] == tag[1] })
if firstIndex != i {
// duplicate
continue
}
// get key prefix (with full length) and offset where to write the created_at
dbi, k, offset := il.getTagIndexPrefix(tag[1])
binary.BigEndian.PutUint32(k[offset:], uint32(evt.CreatedAt))
if !yield(key{dbi: dbi, key: k}) {
return
}
// now the p-tag+kind+date
if dbi == il.indexTag32 && tag[0] == "p" {
k := make([]byte, 8+2+4)
hex.Decode(k[0:8], []byte(tag[1][0:8*2]))
binary.BigEndian.PutUint16(k[8:8+2], uint16(evt.Kind))
binary.BigEndian.PutUint32(k[8+2:8+2+4], uint32(evt.CreatedAt))
dbi := il.indexPTagKind
if !yield(key{dbi: dbi, key: k[0 : 8+2+4]}) {
return
}
}
}
{
// ~ by date only
k := make([]byte, 4)
binary.BigEndian.PutUint32(k[0:4], uint32(evt.CreatedAt))
if !yield(key{dbi: il.indexCreatedAt, key: k[0:4]}) {
return
}
}
}
}
func (il *IndexingLayer) getTagIndexPrefix(tagValue string) (lmdb.DBI, []byte, int) {
var k []byte // the key with full length for created_at and idx at the end, but not filled with these
var offset int // the offset -- i.e. where the prefix ends and the created_at and idx would start
var dbi lmdb.DBI
// if it's 32 bytes as hex, save it as bytes
if len(tagValue) == 64 {
// but we actually only use the first 8 bytes
k = make([]byte, 8+4)
if _, err := hex.Decode(k[0:8], []byte(tagValue[0:8*2])); err == nil {
offset = 8
dbi = il.indexTag32
return dbi, k[0 : 8+4], offset
}
}
// if it looks like an "a" tag, index it in this special format
spl := strings.Split(tagValue, ":")
if len(spl) == 3 && len(spl[1]) == 64 {
k = make([]byte, 2+8+30)
if _, err := hex.Decode(k[2:2+8], []byte(tagValue[0:8*2])); err == nil {
if kind, err := strconv.ParseUint(spl[0], 10, 16); err == nil {
k[0] = byte(kind >> 8)
k[1] = byte(kind)
// limit "d" identifier to 30 bytes (so we don't have to grow our byte slice)
n := copy(k[2+8:2+8+30], spl[2])
offset = 2 + 8 + n
return dbi, k[0 : offset+4], offset
}
}
}
// index whatever else as utf-8, but limit it to 40 bytes
k = make([]byte, 40+4)
n := copy(k[0:40], tagValue)
offset = n
dbi = il.indexTag
return dbi, k[0 : n+4], offset
}

200
eventstore/mmm/indexinglayer.go Normal file
View File

@@ -0,0 +1,200 @@
package mmm
import (
"context"
"encoding/binary"
"fmt"
"os"
"path/filepath"
"github.com/PowerDNS/lmdb-go/lmdb"
"github.com/fiatjaf/eventstore"
"github.com/nbd-wtf/go-nostr"
)
var _ eventstore.Store = (*IndexingLayer)(nil)
type IndexingLayer struct {
isInitialized bool
name string
ShouldIndex func(context.Context, *nostr.Event) bool
MaxLimit int
mmmm *MultiMmapManager
// this is stored in the knownLayers db as a value, and used to keep track of which layer owns each event
id uint16
lmdbEnv *lmdb.Env
indexCreatedAt lmdb.DBI
indexKind lmdb.DBI
indexPubkey lmdb.DBI
indexPubkeyKind lmdb.DBI
indexTag lmdb.DBI
indexTag32 lmdb.DBI
indexTagAddr lmdb.DBI
indexPTagKind lmdb.DBI
}
type IndexingLayers []*IndexingLayer
func (ils IndexingLayers) ByID(ilid uint16) *IndexingLayer {
for _, il := range ils {
if il.id == ilid {
return il
}
}
return nil
}
const multiIndexCreationFlags uint = lmdb.Create | lmdb.DupSort
func (il *IndexingLayer) Init() error {
if il.isInitialized {
return nil
}
il.isInitialized = true
path := filepath.Join(il.mmmm.Dir, il.name)
if il.MaxLimit == 0 {
il.MaxLimit = 500
}
// open lmdb
env, err := lmdb.NewEnv()
if err != nil {
return err
}
env.SetMaxDBs(8)
env.SetMaxReaders(1000)
env.SetMapSize(1 << 38) // ~273GB
// create directory if it doesn't exist and open it
if err := os.MkdirAll(path, 0755); err != nil {
return err
}
err = env.Open(path, lmdb.NoTLS, 0644)
if err != nil {
return err
}
il.lmdbEnv = env
// open each db
if err := il.lmdbEnv.Update(func(txn *lmdb.Txn) error {
if dbi, err := txn.OpenDBI("created_at", multiIndexCreationFlags); err != nil {
return err
} else {
il.indexCreatedAt = dbi
}
if dbi, err := txn.OpenDBI("kind", multiIndexCreationFlags); err != nil {
return err
} else {
il.indexKind = dbi
}
if dbi, err := txn.OpenDBI("pubkey", multiIndexCreationFlags); err != nil {
return err
} else {
il.indexPubkey = dbi
}
if dbi, err := txn.OpenDBI("pubkeyKind", multiIndexCreationFlags); err != nil {
return err
} else {
il.indexPubkeyKind = dbi
}
if dbi, err := txn.OpenDBI("tag", multiIndexCreationFlags); err != nil {
return err
} else {
il.indexTag = dbi
}
if dbi, err := txn.OpenDBI("tag32", multiIndexCreationFlags); err != nil {
return err
} else {
il.indexTag32 = dbi
}
if dbi, err := txn.OpenDBI("tagaddr", multiIndexCreationFlags); err != nil {
return err
} else {
il.indexTagAddr = dbi
}
if dbi, err := txn.OpenDBI("ptagKind", multiIndexCreationFlags); err != nil {
return err
} else {
il.indexPTagKind = dbi
}
return nil
}); err != nil {
return err
}
return nil
}
func (il *IndexingLayer) Name() string { return il.name }
func (il *IndexingLayer) runThroughEvents(txn *lmdb.Txn) error {
ctx := context.Background()
b := il.mmmm
// run through all events we have and see if this new index wants them
cursor, err := txn.OpenCursor(b.indexId)
if err != nil {
return fmt.Errorf("when opening cursor on %v: %w", b.indexId, err)
}
defer cursor.Close()
for {
idPrefix8, val, err := cursor.Get(nil, nil, lmdb.Next)
if lmdb.IsNotFound(err) {
break
}
if err != nil {
return fmt.Errorf("when moving the cursor: %w", err)
}
update := false
posb := val[0:12]
pos := positionFromBytes(posb)
evt := &nostr.Event{}
if err := b.loadEvent(pos, evt); err != nil {
return fmt.Errorf("when loading event from mmap: %w", err)
}
if il.ShouldIndex != nil && il.ShouldIndex(ctx, evt) {
// add the current reference
val = binary.BigEndian.AppendUint16(val, il.id)
// if we were already updating to remove the reference
// now that we've added the reference back we don't really have to update
update = !update
// actually index
if err := il.lmdbEnv.Update(func(iltxn *lmdb.Txn) error {
for k := range il.getIndexKeysForEvent(evt) {
if err := iltxn.Put(k.dbi, k.key, posb, 0); err != nil {
return err
}
}
return nil
}); err != nil {
return fmt.Errorf("failed to index: %w", err)
}
}
if update {
if err := txn.Put(b.indexId, idPrefix8, val, 0); err != nil {
return fmt.Errorf("failed to put updated index+refs: %w", err)
}
}
}
return nil
}
func (il *IndexingLayer) Close() {
il.lmdbEnv.Close()
}

335
eventstore/mmm/mmmm.go Normal file
View File

@@ -0,0 +1,335 @@
package mmm
import (
"encoding/binary"
"fmt"
"os"
"path/filepath"
"slices"
"sync"
"syscall"
"unsafe"
"github.com/PowerDNS/lmdb-go/lmdb"
"github.com/fiatjaf/eventstore/mmm/betterbinary"
"github.com/nbd-wtf/go-nostr"
"github.com/rs/zerolog"
)
type mmap []byte
func (_ mmap) String() string { return "<memory-mapped file>" }
type MultiMmapManager struct {
Dir string
Logger *zerolog.Logger
layers IndexingLayers
mmapfPath string
mmapf mmap
mmapfEnd uint64
lmdbEnv *lmdb.Env
stuff lmdb.DBI
knownLayers lmdb.DBI
indexId lmdb.DBI
freeRanges []position
mutex sync.Mutex
}
func (b *MultiMmapManager) String() string {
return fmt.Sprintf("<MultiMmapManager on %s with %d layers @ %v>", b.Dir, len(b.layers), unsafe.Pointer(b))
}
const (
MMAP_INFINITE_SIZE = 1 << 40
maxuint16 = 65535
maxuint32 = 4294967295
)
var FREERANGES_KEY = []byte{'F'}
func (b *MultiMmapManager) Init() error {
// create directory if it doesn't exist
dbpath := filepath.Join(b.Dir, "mmmm")
if err := os.MkdirAll(dbpath, 0755); err != nil {
return fmt.Errorf("failed to create directory %s: %w", dbpath, err)
}
// open a huge mmapped file
b.mmapfPath = filepath.Join(b.Dir, "events")
file, err := os.OpenFile(b.mmapfPath, os.O_CREATE|os.O_RDWR, 0644)
if err != nil {
return fmt.Errorf("failed to open events file at %s: %w", b.mmapfPath, err)
}
mmapf, err := syscall.Mmap(int(file.Fd()), 0, MMAP_INFINITE_SIZE,
syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
if err != nil {
return fmt.Errorf("failed to mmap events file at %s: %w", b.mmapfPath, err)
}
b.mmapf = mmapf
if stat, err := os.Stat(b.mmapfPath); err != nil {
return err
} else {
b.mmapfEnd = uint64(stat.Size())
}
// open lmdb
env, err := lmdb.NewEnv()
if err != nil {
return err
}
env.SetMaxDBs(3)
env.SetMaxReaders(1000)
env.SetMapSize(1 << 38) // ~273GB
err = env.Open(dbpath, lmdb.NoTLS, 0644)
if err != nil {
return fmt.Errorf("failed to open lmdb at %s: %w", dbpath, err)
}
b.lmdbEnv = env
if err := b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
if dbi, err := txn.OpenDBI("stuff", lmdb.Create); err != nil {
return err
} else {
b.stuff = dbi
}
// this just keeps track of all the layers we know (just their names)
// they will be instantiated by the application after their name is read from the database.
// new layers created at runtime will be saved here.
if dbi, err := txn.OpenDBI("layers", lmdb.Create); err != nil {
return err
} else {
b.knownLayers = dbi
}
// this is a global index of events by id that also keeps references
// to all the layers that may be indexing them -- such that whenever
// an event is deleted from all layers it can be deleted from global
if dbi, err := txn.OpenDBI("id-references", lmdb.Create); err != nil {
return err
} else {
b.indexId = dbi
}
// load all free ranges into memory
{
data, err := txn.Get(b.stuff, FREERANGES_KEY)
if err != nil && !lmdb.IsNotFound(err) {
return fmt.Errorf("on freeranges: %w", err)
}
b.freeRanges = make([]position, len(data)/12)
logOp := b.Logger.Debug()
for f := range b.freeRanges {
pos := positionFromBytes(data[f*12 : (f+1)*12])
b.freeRanges[f] = pos
if pos.size > 20 {
logOp = logOp.Uint32(fmt.Sprintf("%d", pos.start), pos.size)
}
}
slices.SortFunc(b.freeRanges, func(a, b position) int { return int(a.size - b.size) })
logOp.Msg("loaded free ranges")
}
return nil
}); err != nil {
return fmt.Errorf("failed to open and load db data: %w", err)
}
return nil
}
func (b *MultiMmapManager) EnsureLayer(name string, il *IndexingLayer) error {
b.mutex.Lock()
defer b.mutex.Unlock()
il.mmmm = b
il.name = name
err := b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
txn.RawRead = true
nameb := []byte(name)
if idv, err := txn.Get(b.knownLayers, nameb); lmdb.IsNotFound(err) {
if id, err := b.getNextAvailableLayerId(txn); err != nil {
return fmt.Errorf("failed to reserve a layer id for %s: %w", name, err)
} else {
il.id = id
}
if err := il.Init(); err != nil {
return fmt.Errorf("failed to init new layer %s: %w", name, err)
}
if err := il.runThroughEvents(txn); err != nil {
return fmt.Errorf("failed to run %s through events: %w", name, err)
}
return txn.Put(b.knownLayers, []byte(name), binary.BigEndian.AppendUint16(nil, il.id), 0)
} else if err == nil {
il.id = binary.BigEndian.Uint16(idv)
if err := il.Init(); err != nil {
return fmt.Errorf("failed to init old layer %s: %w", name, err)
}
return nil
} else {
return err
}
})
if err != nil {
return err
}
b.layers = append(b.layers, il)
return nil
}
func (b *MultiMmapManager) DropLayer(name string) error {
b.mutex.Lock()
defer b.mutex.Unlock()
// get layer reference
idx := slices.IndexFunc(b.layers, func(il *IndexingLayer) bool { return il.name == name })
if idx == -1 {
return fmt.Errorf("layer '%s' doesn't exist", name)
}
il := b.layers[idx]
// remove layer references
err := b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
if err := b.removeAllReferencesFromLayer(txn, il.id); err != nil {
return err
}
return txn.Del(b.knownLayers, []byte(il.name), nil)
})
if err != nil {
return err
}
// delete everything (the indexes) from this layer db actually
err = il.lmdbEnv.Update(func(txn *lmdb.Txn) error {
for _, dbi := range []lmdb.DBI{
il.indexCreatedAt,
il.indexKind,
il.indexPubkey,
il.indexPubkeyKind,
il.indexTag,
il.indexTag32,
il.indexTagAddr,
il.indexPTagKind,
} {
if err := txn.Drop(dbi, true); err != nil {
return err
}
}
return nil
})
if err != nil {
return err
}
return il.lmdbEnv.Close()
}
func (b *MultiMmapManager) removeAllReferencesFromLayer(txn *lmdb.Txn, layerId uint16) error {
cursor, err := txn.OpenCursor(b.indexId)
if err != nil {
return fmt.Errorf("when opening cursor on %v: %w", b.indexId, err)
}
defer cursor.Close()
for {
idPrefix8, val, err := cursor.Get(nil, nil, lmdb.Next)
if lmdb.IsNotFound(err) {
break
}
if err != nil {
return fmt.Errorf("when moving the cursor: %w", err)
}
var zeroRefs bool
var update bool
needle := binary.BigEndian.AppendUint16(nil, layerId)
for s := 12; s < len(val); s += 2 {
if slices.Equal(val[s:s+2], needle) {
// swap delete
copy(val[s:s+2], val[len(val)-2:])
val = val[0 : len(val)-2]
update = true
// we must erase this event if its references reach zero
zeroRefs = len(val) == 12
break
}
}
if zeroRefs {
posb := val[0:12]
pos := positionFromBytes(posb)
if err := b.purge(txn, idPrefix8, pos); err != nil {
return fmt.Errorf("failed to purge unreferenced event %x: %w", idPrefix8, err)
}
} else if update {
if err := txn.Put(b.indexId, idPrefix8, val, 0); err != nil {
return fmt.Errorf("failed to put updated index+refs: %w", err)
}
}
}
return nil
}
func (b *MultiMmapManager) loadEvent(pos position, eventReceiver *nostr.Event) error {
return betterbinary.Unmarshal(b.mmapf[pos.start:pos.start+uint64(pos.size)], eventReceiver)
}
// getNextAvailableLayerId iterates through all existing layers to find a vacant id
func (b *MultiMmapManager) getNextAvailableLayerId(txn *lmdb.Txn) (uint16, error) {
cursor, err := txn.OpenCursor(b.knownLayers)
if err != nil {
return 0, fmt.Errorf("failed to open cursor: %w", err)
}
used := [1 << 16]bool{}
_, val, err := cursor.Get(nil, nil, lmdb.First)
for err == nil {
// something was found
used[binary.BigEndian.Uint16(val)] = true
// next
_, val, err = cursor.Get(nil, nil, lmdb.Next)
}
if !lmdb.IsNotFound(err) {
// a real error
return 0, err
}
// loop exited, get the first available
var id uint16
for num, isUsed := range used {
if !isUsed {
id = uint16(num)
break
}
}
return id, nil
}
func (b *MultiMmapManager) Close() {
b.lmdbEnv.Close()
for _, il := range b.layers {
il.Close()
}
}

386
eventstore/mmm/mmmm_test.go Normal file
View File

@@ -0,0 +1,386 @@
package mmm
import (
"context"
"encoding/binary"
"encoding/hex"
"fmt"
"os"
"testing"
"github.com/PowerDNS/lmdb-go/lmdb"
"github.com/nbd-wtf/go-nostr"
"github.com/rs/zerolog"
"github.com/stretchr/testify/require"
)
func TestMultiLayerIndexing(t *testing.T) {
// Create a temporary directory for the test
tmpDir := "/tmp/eventstore-mmm-test"
os.RemoveAll(tmpDir)
logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr})
// initialize MMM with three layers:
// 1. odd timestamps layer
// 2. even timestamps layer
// 3. all events layer
mmm := &MultiMmapManager{
Dir: tmpDir,
Logger: &logger,
}
err := mmm.Init()
require.NoError(t, err)
defer mmm.Close()
// create layers
err = mmm.EnsureLayer("odd", &IndexingLayer{
MaxLimit: 100,
ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool {
return evt.CreatedAt%2 == 1
},
})
require.NoError(t, err)
err = mmm.EnsureLayer("even", &IndexingLayer{
MaxLimit: 100,
ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool {
return evt.CreatedAt%2 == 0
},
})
require.NoError(t, err)
err = mmm.EnsureLayer("all", &IndexingLayer{
MaxLimit: 100,
ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool {
return true
},
})
require.NoError(t, err)
// create test events
ctx := context.Background()
baseTime := nostr.Timestamp(0)
sk := "945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb"
events := make([]*nostr.Event, 10)
for i := 0; i < 10; i++ {
evt := &nostr.Event{
CreatedAt: baseTime + nostr.Timestamp(i),
Kind: 1,
Tags: nostr.Tags{},
Content: "test content",
}
evt.Sign(sk)
events[i] = evt
stored, err := mmm.StoreGlobal(ctx, evt)
require.NoError(t, err)
require.True(t, stored)
}
{
// query odd layer
oddResults, err := mmm.layers[0].QueryEvents(ctx, nostr.Filter{
Kinds: []int{1},
})
require.NoError(t, err)
oddCount := 0
for evt := range oddResults {
require.Equal(t, evt.CreatedAt%2, nostr.Timestamp(1))
oddCount++
}
require.Equal(t, 5, oddCount)
}
{
// query even layer
evenResults, err := mmm.layers[1].QueryEvents(ctx, nostr.Filter{
Kinds: []int{1},
})
require.NoError(t, err)
evenCount := 0
for evt := range evenResults {
require.Equal(t, evt.CreatedAt%2, nostr.Timestamp(0))
evenCount++
}
require.Equal(t, 5, evenCount)
}
{
// query all layer
allResults, err := mmm.layers[2].QueryEvents(ctx, nostr.Filter{
Kinds: []int{1},
})
require.NoError(t, err)
allCount := 0
for range allResults {
allCount++
}
require.Equal(t, 10, allCount)
}
// delete some events
err = mmm.layers[0].DeleteEvent(ctx, events[1]) // odd timestamp
require.NoError(t, err)
err = mmm.layers[1].DeleteEvent(ctx, events[2]) // even timestamp
// verify deletions
{
oddResults, err := mmm.layers[0].QueryEvents(ctx, nostr.Filter{
Kinds: []int{1},
})
require.NoError(t, err)
oddCount := 0
for range oddResults {
oddCount++
}
require.Equal(t, 4, oddCount)
}
{
evenResults, err := mmm.layers[1].QueryEvents(ctx, nostr.Filter{
Kinds: []int{1},
})
require.NoError(t, err)
evenCount := 0
for range evenResults {
evenCount++
}
require.Equal(t, 4, evenCount)
}
{
allResults, err := mmm.layers[2].QueryEvents(ctx, nostr.Filter{
Kinds: []int{1},
})
require.NoError(t, err)
allCount := 0
for range allResults {
allCount++
}
require.Equal(t, 10, allCount)
}
// save events directly to layers regardless of timestamp
{
oddEvent := &nostr.Event{
CreatedAt: baseTime + 100, // even timestamp
Kind: 1,
Content: "forced odd",
}
oddEvent.Sign(sk)
err = mmm.layers[0].SaveEvent(ctx, oddEvent) // save even timestamp to odd layer
require.NoError(t, err)
// it is added to the odd il
oddResults, err := mmm.layers[0].QueryEvents(ctx, nostr.Filter{
Kinds: []int{1},
})
require.NoError(t, err)
oddCount := 0
for range oddResults {
oddCount++
}
require.Equal(t, 5, oddCount)
// it doesn't affect the event il
evenResults, err := mmm.layers[1].QueryEvents(ctx, nostr.Filter{
Kinds: []int{1},
})
require.NoError(t, err)
evenCount := 0
for range evenResults {
evenCount++
}
require.Equal(t, 4, evenCount)
}
// test replaceable events
for _, layer := range mmm.layers {
replaceable := &nostr.Event{
CreatedAt: baseTime + 0,
Kind: 0,
Content: fmt.Sprintf("first"),
}
replaceable.Sign(sk)
err := layer.ReplaceEvent(ctx, replaceable)
require.NoError(t, err)
}
// replace events alternating between layers
for i := range mmm.layers {
content := fmt.Sprintf("last %d", i)
newEvt := &nostr.Event{
CreatedAt: baseTime + 1000,
Kind: 0,
Content: content,
}
newEvt.Sign(sk)
layer := mmm.layers[i]
err = layer.ReplaceEvent(ctx, newEvt)
require.NoError(t, err)
// verify replacement in the layer that did it
results, err := layer.QueryEvents(ctx, nostr.Filter{
Kinds: []int{0},
})
require.NoError(t, err)
count := 0
for evt := range results {
require.Equal(t, content, evt.Content)
count++
}
require.Equal(t, 1, count)
// verify other layers still have the old version
for j := 0; j < 3; j++ {
if mmm.layers[j] == layer {
continue
}
results, err := mmm.layers[j].QueryEvents(ctx, nostr.Filter{
Kinds: []int{0},
})
require.NoError(t, err)
count := 0
for evt := range results {
if i < j {
require.Equal(t, "first", evt.Content)
} else {
require.Equal(t, evt.Content, fmt.Sprintf("last %d", j))
}
count++
}
require.Equal(t, 1, count, "%d/%d", i, j)
}
}
}
func TestLayerReferenceTracking(t *testing.T) {
// Create a temporary directory for the test
tmpDir, err := os.MkdirTemp("", "mmm-test-*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr})
// initialize MMM with three layers
mmm := &MultiMmapManager{
Dir: tmpDir,
Logger: &logger,
}
err = mmm.Init()
require.NoError(t, err)
defer mmm.Close()
// create three layers
err = mmm.EnsureLayer("layer1", &IndexingLayer{
MaxLimit: 100,
ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool { return true },
})
require.NoError(t, err)
err = mmm.EnsureLayer("layer2", &IndexingLayer{
MaxLimit: 100,
ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool { return true },
})
require.NoError(t, err)
err = mmm.EnsureLayer("layer3", &IndexingLayer{
MaxLimit: 100,
ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool { return true },
})
require.NoError(t, err)
err = mmm.EnsureLayer("layer4", &IndexingLayer{
MaxLimit: 100,
ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool { return true },
})
require.NoError(t, err)
// create test events
ctx := context.Background()
sk := "945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb"
evt1 := &nostr.Event{
CreatedAt: 1000,
Kind: 1,
Tags: nostr.Tags{},
Content: "event 1",
}
evt1.Sign(sk)
evt2 := &nostr.Event{
CreatedAt: 2000,
Kind: 1,
Tags: nostr.Tags{},
Content: "event 2",
}
evt2.Sign(sk)
// save evt1 to layer1
err = mmm.layers[0].SaveEvent(ctx, evt1)
require.NoError(t, err)
// save evt1 to layer2
err = mmm.layers[1].SaveEvent(ctx, evt1)
require.NoError(t, err)
// save evt1 to layer4
err = mmm.layers[0].SaveEvent(ctx, evt1)
require.NoError(t, err)
// delete evt1 from layer1
err = mmm.layers[0].DeleteEvent(ctx, evt1)
require.NoError(t, err)
// save evt2 to layer3
err = mmm.layers[2].SaveEvent(ctx, evt2)
require.NoError(t, err)
// save evt2 to layer4
err = mmm.layers[3].SaveEvent(ctx, evt2)
require.NoError(t, err)
// save evt2 to layer3 again
err = mmm.layers[2].SaveEvent(ctx, evt2)
require.NoError(t, err)
// delete evt1 from layer4
err = mmm.layers[3].DeleteEvent(ctx, evt1)
require.NoError(t, err)
// verify the state of the indexId database
err = mmm.lmdbEnv.View(func(txn *lmdb.Txn) error {
cursor, err := txn.OpenCursor(mmm.indexId)
if err != nil {
return err
}
defer cursor.Close()
count := 0
for k, v, err := cursor.Get(nil, nil, lmdb.First); err == nil; k, v, err = cursor.Get(nil, nil, lmdb.Next) {
count++
if hex.EncodeToString(k) == evt1.ID[:16] {
// evt1 should only reference layer2
require.Equal(t, 14, len(v), "evt1 should have one layer reference")
layerRef := binary.BigEndian.Uint16(v[12:14])
require.Equal(t, mmm.layers[1].id, layerRef, "evt1 should reference layer2")
} else if hex.EncodeToString(k) == evt2.ID[:16] {
// evt2 should references to layer3 and layer4
require.Equal(t, 16, len(v), "evt2 should have two layer references")
layer3Ref := binary.BigEndian.Uint16(v[12:14])
require.Equal(t, mmm.layers[2].id, layer3Ref, "evt2 should reference layer3")
layer4Ref := binary.BigEndian.Uint16(v[14:16])
require.Equal(t, mmm.layers[3].id, layer4Ref, "evt2 should reference layer4")
} else {
t.Errorf("unexpected event in indexId: %x", k)
}
}
require.Equal(t, 2, count, "should have exactly two events in indexId")
return nil
})
require.NoError(t, err)
}

27
eventstore/mmm/position.go Normal file
View File

@@ -0,0 +1,27 @@
package mmm
import (
"encoding/binary"
"fmt"
)
type position struct {
start uint64
size uint32
}
func (pos position) String() string {
return fmt.Sprintf("<%d|%d|%d>", pos.start, pos.size, pos.start+uint64(pos.size))
}
func positionFromBytes(posb []byte) position {
return position{
size: binary.BigEndian.Uint32(posb[0:4]),
start: binary.BigEndian.Uint64(posb[4:12]),
}
}
func bytesFromPosition(out []byte, pos position) {
binary.BigEndian.PutUint32(out[0:4], pos.size)
binary.BigEndian.PutUint64(out[4:12], pos.start)
}

36
eventstore/mmm/purge.go Normal file
View File

@@ -0,0 +1,36 @@
package mmm
import (
"bytes"
"fmt"
"os"
"github.com/PowerDNS/lmdb-go/lmdb"
)
func (b *MultiMmapManager) purge(txn *lmdb.Txn, idPrefix8 []byte, pos position) error {
b.Logger.Debug().Hex("event", idPrefix8).Stringer("pos", pos).Msg("purging")
// delete from index
if err := txn.Del(b.indexId, idPrefix8, nil); err != nil {
return err
}
// will add the current range to free ranges, which means it is "deleted" (or merge with existing)
isAtEnd := b.mergeNewFreeRange(pos)
if isAtEnd {
// when at the end, truncate the mmap
// [new_pos_to_be_freed][end_of_file] -> shrink file!
pos.size = 0 // so we don't try to add this some lines below
if err := os.Truncate(b.mmapfPath, int64(pos.start)); err != nil {
panic(fmt.Errorf("error decreasing %s: %w", b.mmapfPath, err))
}
b.mmapfEnd = pos.start
} else {
// this is for debugging -------------
copy(b.mmapf[pos.start:], bytes.Repeat([]byte{'!'}, int(pos.size)))
}
return b.saveFreeRanges(txn)
}

460
eventstore/mmm/query.go Normal file
View File

@@ -0,0 +1,460 @@
package mmm
import (
"bytes"
"context"
"encoding/binary"
"encoding/hex"
"fmt"
"log"
"slices"
"github.com/PowerDNS/lmdb-go/lmdb"
"github.com/fiatjaf/eventstore/internal"
"github.com/fiatjaf/eventstore/mmm/betterbinary"
"github.com/nbd-wtf/go-nostr"
)
// GetByID returns the event -- if found in this mmm -- and all the IndexingLayers it belongs to.
func (b *MultiMmapManager) GetByID(id string) (*nostr.Event, IndexingLayers) {
events := make(chan *nostr.Event)
presence := make(chan []uint16)
b.queryByIDs(events, []string{id}, presence)
for evt := range events {
p := <-presence
present := make([]*IndexingLayer, len(p))
for i, id := range p {
present[i] = b.layers.ByID(id)
}
return evt, present
}
return nil, nil
}
// queryByIDs emits the events of the given id to the given channel if they exist anywhere in this mmm.
// if presence is given it will also be used to emit slices of the ids of the IndexingLayers this event is stored in.
// it closes the channels when it ends.
func (b *MultiMmapManager) queryByIDs(ch chan *nostr.Event, ids []string, presence chan []uint16) {
go b.lmdbEnv.View(func(txn *lmdb.Txn) error {
txn.RawRead = true
defer close(ch)
if presence != nil {
defer close(presence)
}
for _, id := range ids {
if len(id) != 64 {
continue
}
idPrefix8, _ := hex.DecodeString(id[0 : 8*2])
val, err := txn.Get(b.indexId, idPrefix8)
if err == nil {
pos := positionFromBytes(val[0:12])
evt := &nostr.Event{}
if err := b.loadEvent(pos, evt); err != nil {
panic(fmt.Errorf("failed to decode event from %v: %w", pos, err))
}
ch <- evt
if presence != nil {
layers := make([]uint16, 0, (len(val)-12)/2)
for s := 12; s < len(val); s += 2 {
layers = append(layers, binary.BigEndian.Uint16(val[s:s+2]))
}
presence <- layers
}
}
}
return nil
})
}
func (il *IndexingLayer) QueryEvents(ctx context.Context, filter nostr.Filter) (chan *nostr.Event, error) {
ch := make(chan *nostr.Event)
if len(filter.IDs) > 0 {
il.mmmm.queryByIDs(ch, filter.IDs, nil)
return ch, nil
}
if filter.Search != "" {
close(ch)
return ch, nil
}
// max number of events we'll return
limit := il.MaxLimit / 4
if filter.Limit > 0 && filter.Limit < il.MaxLimit {
limit = filter.Limit
}
if tlimit := nostr.GetTheoreticalLimit(filter); tlimit == 0 {
close(ch)
return ch, nil
} else if tlimit > 0 {
limit = tlimit
}
go il.lmdbEnv.View(func(txn *lmdb.Txn) error {
txn.RawRead = true
defer close(ch)
results, err := il.query(txn, filter, limit)
for _, ie := range results {
ch <- ie.Event
}
return err
})
return ch, nil
}
func (il *IndexingLayer) query(txn *lmdb.Txn, filter nostr.Filter, limit int) ([]internal.IterEvent, error) {
queries, extraAuthors, extraKinds, extraTagKey, extraTagValues, since, err := il.prepareQueries(filter)
if err != nil {
return nil, err
}
iterators := make([]*iterator, len(queries))
exhausted := make([]bool, len(queries)) // indicates that a query won't be used anymore
results := make([][]internal.IterEvent, len(queries))
pulledPerQuery := make([]int, len(queries))
// these are kept updated so we never pull from the iterator that is at further distance
// (i.e. the one that has the oldest event among all)
// we will continue to pull from it as soon as some other iterator takes the position
oldest := internal.IterEvent{Q: -1}
secondPhase := false // after we have gathered enough events we will change the way we iterate
secondBatch := make([][]internal.IterEvent, 0, len(queries)+1)
secondPhaseParticipants := make([]int, 0, len(queries)+1)
// while merging results in the second phase we will alternate between these two lists
// to avoid having to create new lists all the time
var secondPhaseResultsA []internal.IterEvent
var secondPhaseResultsB []internal.IterEvent
var secondPhaseResultsToggle bool // this is just a dummy thing we use to keep track of the alternating
var secondPhaseHasResultsPending bool
remainingUnexhausted := len(queries) // when all queries are exhausted we can finally end this thing
batchSizePerQuery := internal.BatchSizePerNumberOfQueries(limit, remainingUnexhausted)
firstPhaseTotalPulled := 0
exhaust := func(q int) {
exhausted[q] = true
remainingUnexhausted--
if q == oldest.Q {
oldest = internal.IterEvent{Q: -1}
}
}
var firstPhaseResults []internal.IterEvent
for q := range queries {
cursor, err := txn.OpenCursor(queries[q].dbi)
if err != nil {
return nil, err
}
iterators[q] = &iterator{cursor: cursor}
defer cursor.Close()
iterators[q].seek(queries[q].startingPoint)
results[q] = make([]internal.IterEvent, 0, batchSizePerQuery*2)
}
// fmt.Println("queries", len(queries))
for c := 0; ; c++ {
batchSizePerQuery = internal.BatchSizePerNumberOfQueries(limit, remainingUnexhausted)
// fmt.Println(" iteration", c, "remaining", remainingUnexhausted, "batchsize", batchSizePerQuery)
// we will go through all the iterators in batches until we have pulled all the required results
for q, query := range queries {
if exhausted[q] {
continue
}
if oldest.Q == q && remainingUnexhausted > 1 {
continue
}
// fmt.Println(" query", q, unsafe.Pointer(&results[q]), hex.EncodeToString(query.prefix), len(results[q]))
it := iterators[q]
pulledThisIteration := 0
for {
// we already have a k and a v and an err from the cursor setup, so check and use these
if it.err != nil ||
len(it.key) != query.keySize ||
!bytes.HasPrefix(it.key, query.prefix) {
// either iteration has errored or we reached the end of this prefix
// fmt.Println(" reached end", it.key, query.keySize, query.prefix)
exhaust(q)
break
}
// "id" indexes don't contain a timestamp
if query.timestampSize == 4 {
createdAt := binary.BigEndian.Uint32(it.key[len(it.key)-4:])
if createdAt < since {
// fmt.Println(" reached since", createdAt, "<", since)
exhaust(q)
break
}
}
// fetch actual event
pos := positionFromBytes(it.posb)
bin := il.mmmm.mmapf[pos.start : pos.start+uint64(pos.size)]
// check it against pubkeys without decoding the entire thing
if extraAuthors != nil && !slices.Contains(extraAuthors, [32]byte(bin[39:71])) {
it.next()
continue
}
// check it against kinds without decoding the entire thing
if extraKinds != nil && !slices.Contains(extraKinds, [2]byte(bin[1:3])) {
it.next()
continue
}
// decode the entire thing (TODO: do a conditional decode while also checking the extra tag)
event := &nostr.Event{}
if err := betterbinary.Unmarshal(bin, event); err != nil {
log.Printf("mmm: value read error (id %x) on query prefix %x sp %x dbi %d: %s\n",
bin[0:32], query.prefix, query.startingPoint, query.dbi, err)
return nil, fmt.Errorf("event read error: %w", err)
}
// fmt.Println(" event", hex.EncodeToString(val[0:4]), "kind", binary.BigEndian.Uint16(val[132:134]), "author", hex.EncodeToString(val[32:36]), "ts", nostr.Timestamp(binary.BigEndian.Uint32(val[128:132])), hex.EncodeToString(it.key), it.valIdx)
// if there is still a tag to be checked, do it now
if extraTagValues != nil && !event.Tags.ContainsAny(extraTagKey, extraTagValues) {
it.next()
continue
}
// this event is good to be used
evt := internal.IterEvent{Event: event, Q: q}
//
//
if secondPhase {
// do the process described below at HIWAWVRTP.
// if we've reached here this means we've already passed the `since` check.
// now we have to eliminate the event currently at the `since` threshold.
nextThreshold := firstPhaseResults[len(firstPhaseResults)-2]
if oldest.Event == nil {
// fmt.Println(" b1", evt.ID[0:8])
// BRANCH WHEN WE DON'T HAVE THE OLDEST EVENT (BWWDHTOE)
// when we don't have the oldest set, we will keep the results
// and not change the cutting point -- it's bad, but hopefully not that bad.
results[q] = append(results[q], evt)
secondPhaseHasResultsPending = true
} else if nextThreshold.CreatedAt > oldest.CreatedAt {
// fmt.Println(" b2", nextThreshold.CreatedAt, ">", oldest.CreatedAt, evt.ID[0:8])
// one of the events we have stored is the actual next threshold
// eliminate last, update since with oldest
firstPhaseResults = firstPhaseResults[0 : len(firstPhaseResults)-1]
since = uint32(oldest.CreatedAt)
// fmt.Println(" new since", since, evt.ID[0:8])
// we null the oldest Event as we can't rely on it anymore
// (we'll fall under BWWDHTOE above) until we have a new oldest set.
oldest = internal.IterEvent{Q: -1}
// anything we got that would be above this won't trigger an update to
// the oldest anyway, because it will be discarded as being after the limit.
//
// finally
// add this to the results to be merged later
results[q] = append(results[q], evt)
secondPhaseHasResultsPending = true
} else if nextThreshold.CreatedAt < evt.CreatedAt {
// the next last event in the firstPhaseResults is the next threshold
// fmt.Println(" b3", nextThreshold.CreatedAt, "<", oldest.CreatedAt, evt.ID[0:8])
// eliminate last, update since with the antelast
firstPhaseResults = firstPhaseResults[0 : len(firstPhaseResults)-1]
since = uint32(nextThreshold.CreatedAt)
// fmt.Println(" new since", since)
// add this to the results to be merged later
results[q] = append(results[q], evt)
secondPhaseHasResultsPending = true
// update the oldest event
if evt.CreatedAt < oldest.CreatedAt {
oldest = evt
}
} else {
// fmt.Println(" b4", evt.ID[0:8])
// oops, _we_ are the next `since` threshold
firstPhaseResults[len(firstPhaseResults)-1] = evt
since = uint32(evt.CreatedAt)
// fmt.Println(" new since", since)
// do not add us to the results to be merged later
// as we're already inhabiting the firstPhaseResults slice
}
} else {
results[q] = append(results[q], evt)
firstPhaseTotalPulled++
// update the oldest event
if oldest.Event == nil || evt.CreatedAt < oldest.CreatedAt {
oldest = evt
}
}
pulledPerQuery[q]++
pulledThisIteration++
if pulledThisIteration > batchSizePerQuery {
// batch filled
it.next()
// fmt.Println(" filled", hex.EncodeToString(it.key), it.valIdx)
break
}
if pulledPerQuery[q] >= limit {
// batch filled + reached limit for this query (which is the global limit)
exhaust(q)
it.next()
break
}
it.next()
}
}
// we will do this check if we don't accumulated the requested number of events yet
// fmt.Println("oldest", oldest.Event, "from iter", oldest.Q)
if secondPhase && secondPhaseHasResultsPending && (oldest.Event == nil || remainingUnexhausted == 0) {
// fmt.Println("second phase aggregation!")
// when we are in the second phase we will aggressively aggregate results on every iteration
//
secondBatch = secondBatch[:0]
for s := 0; s < len(secondPhaseParticipants); s++ {
q := secondPhaseParticipants[s]
if len(results[q]) > 0 {
secondBatch = append(secondBatch, results[q])
}
if exhausted[q] {
secondPhaseParticipants = internal.SwapDelete(secondPhaseParticipants, s)
s--
}
}
// every time we get here we will alternate between these A and B lists
// combining everything we have into a new partial results list.
// after we've done that we can again set the oldest.
// fmt.Println(" xxx", secondPhaseResultsToggle)
if secondPhaseResultsToggle {
secondBatch = append(secondBatch, secondPhaseResultsB)
secondPhaseResultsA = internal.MergeSortMultiple(secondBatch, limit, secondPhaseResultsA)
oldest = secondPhaseResultsA[len(secondPhaseResultsA)-1]
// fmt.Println(" new aggregated a", len(secondPhaseResultsB))
} else {
secondBatch = append(secondBatch, secondPhaseResultsA)
secondPhaseResultsB = internal.MergeSortMultiple(secondBatch, limit, secondPhaseResultsB)
oldest = secondPhaseResultsB[len(secondPhaseResultsB)-1]
// fmt.Println(" new aggregated b", len(secondPhaseResultsB))
}
secondPhaseResultsToggle = !secondPhaseResultsToggle
since = uint32(oldest.CreatedAt)
// fmt.Println(" new since", since)
// reset the `results` list so we can keep using it
results = results[:len(queries)]
for _, q := range secondPhaseParticipants {
results[q] = results[q][:0]
}
} else if !secondPhase && firstPhaseTotalPulled >= limit && remainingUnexhausted > 0 {
// fmt.Println("have enough!", firstPhaseTotalPulled, "/", limit, "remaining", remainingUnexhausted)
// we will exclude this oldest number as it is not relevant anymore
// (we now want to keep track only of the oldest among the remaining iterators)
oldest = internal.IterEvent{Q: -1}
// HOW IT WORKS AFTER WE'VE REACHED THIS POINT (HIWAWVRTP)
// now we can combine the results we have and check what is our current oldest event.
// we also discard anything that is after the current cutting point (`limit`).
// so if we have [1,2,3], [10, 15, 20] and [7, 21, 49] but we only want 6 total
// we can just keep [1,2,3,7,10,15] and discard [20, 21, 49],
// and also adjust our `since` parameter to `15`, discarding anything we get after it
// and immediately declaring that iterator exhausted.
// also every time we get result that is more recent than this updated `since` we can
// keep it but also discard the previous since, moving the needle one back -- for example,
// if we get an `8` we can keep it and move the `since` parameter to `10`, discarding `15`
// in the process.
all := make([][]internal.IterEvent, len(results))
copy(all, results) // we have to use this otherwise internal.MergeSortMultiple will scramble our results slice
firstPhaseResults = internal.MergeSortMultiple(all, limit, nil)
oldest = firstPhaseResults[limit-1]
since = uint32(oldest.CreatedAt)
// fmt.Println("new since", since)
for q := range queries {
if exhausted[q] {
continue
}
// we also automatically exhaust any of the iterators that have already passed the
// cutting point (`since`)
if results[q][len(results[q])-1].CreatedAt < oldest.CreatedAt {
exhausted[q] = true
remainingUnexhausted--
continue
}
// for all the remaining iterators,
// since we have merged all the events in this `firstPhaseResults` slice, we can empty the
// current `results` slices and reuse them.
results[q] = results[q][:0]
// build this index of indexes with everybody who remains
secondPhaseParticipants = append(secondPhaseParticipants, q)
}
// we create these two lists and alternate between them so we don't have to create a
// a new one every time
secondPhaseResultsA = make([]internal.IterEvent, 0, limit*2)
secondPhaseResultsB = make([]internal.IterEvent, 0, limit*2)
// from now on we won't run this block anymore
secondPhase = true
}
// fmt.Println("remaining", remainingUnexhausted)
if remainingUnexhausted == 0 {
break
}
}
// fmt.Println("is secondPhase?", secondPhase)
var combinedResults []internal.IterEvent
if secondPhase {
// fmt.Println("ending second phase")
// when we reach this point either secondPhaseResultsA or secondPhaseResultsB will be full of stuff,
// the other will be empty
var secondPhaseResults []internal.IterEvent
// fmt.Println("xxx", secondPhaseResultsToggle, len(secondPhaseResultsA), len(secondPhaseResultsB))
if secondPhaseResultsToggle {
secondPhaseResults = secondPhaseResultsB
combinedResults = secondPhaseResultsA[0:limit] // reuse this
// fmt.Println(" using b", len(secondPhaseResultsA))
} else {
secondPhaseResults = secondPhaseResultsA
combinedResults = secondPhaseResultsB[0:limit] // reuse this
// fmt.Println(" using a", len(secondPhaseResultsA))
}
all := [][]internal.IterEvent{firstPhaseResults, secondPhaseResults}
combinedResults = internal.MergeSortMultiple(all, limit, combinedResults)
// fmt.Println("final combinedResults", len(combinedResults), cap(combinedResults), limit)
} else {
combinedResults = make([]internal.IterEvent, limit)
combinedResults = internal.MergeSortMultiple(results, limit, combinedResults)
}
return combinedResults, nil
}

202
eventstore/mmm/query_planner.go Normal file
View File

@@ -0,0 +1,202 @@
package mmm
import (
"encoding/binary"
"encoding/hex"
"fmt"
"github.com/PowerDNS/lmdb-go/lmdb"
"github.com/fiatjaf/eventstore/internal"
"github.com/nbd-wtf/go-nostr"
)
type query struct {
i int
dbi lmdb.DBI
prefix []byte
results chan *nostr.Event
keySize int
timestampSize int
startingPoint []byte
}
func (il *IndexingLayer) prepareQueries(filter nostr.Filter) (
queries []query,
extraAuthors [][32]byte,
extraKinds [][2]byte,
extraTagKey string,
extraTagValues []string,
since uint32,
err error,
) {
// we will apply this to every query we return
defer func() {
if queries == nil {
return
}
var until uint32 = 4294967295
if filter.Until != nil {
if fu := uint32(*filter.Until); fu < until {
until = fu + 1
}
}
for i, q := range queries {
sp := make([]byte, len(q.prefix))
sp = sp[0:len(q.prefix)]
copy(sp, q.prefix)
queries[i].startingPoint = binary.BigEndian.AppendUint32(sp, uint32(until))
queries[i].results = make(chan *nostr.Event, 12)
}
}()
// this is where we'll end the iteration
if filter.Since != nil {
if fs := uint32(*filter.Since); fs > since {
since = fs
}
}
if len(filter.Tags) > 0 {
// we will select ONE tag to query for and ONE extra tag to do further narrowing, if available
tagKey, tagValues, goodness := internal.ChooseNarrowestTag(filter)
// we won't use a tag index for this as long as we have something else to match with
if goodness < 2 && (len(filter.Authors) > 0 || len(filter.Kinds) > 0) {
goto pubkeyMatching
}
// only "p" tag has a goodness of 2, so
if goodness == 2 {
// this means we got a "p" tag, so we will use the ptag-kind index
i := 0
if filter.Kinds != nil {
queries = make([]query, len(tagValues)*len(filter.Kinds))
for _, value := range tagValues {
if len(value) != 64 {
return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
}
for _, kind := range filter.Kinds {
k := make([]byte, 8+2)
if _, err := hex.Decode(k[0:8], []byte(value[0:8*2])); err != nil {
return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
}
binary.BigEndian.PutUint16(k[8:8+2], uint16(kind))
queries[i] = query{i: i, dbi: il.indexPTagKind, prefix: k[0 : 8+2], keySize: 8 + 2 + 4, timestampSize: 4}
i++
}
}
} else {
// even if there are no kinds, in that case we will just return any kind and not care
queries = make([]query, len(tagValues))
for i, value := range tagValues {
if len(value) != 64 {
return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
}
k := make([]byte, 8)
if _, err := hex.Decode(k[0:8], []byte(value[0:8*2])); err != nil {
return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
}
queries[i] = query{i: i, dbi: il.indexPTagKind, prefix: k[0:8], keySize: 8 + 2 + 4, timestampSize: 4}
}
}
} else {
// otherwise we will use a plain tag index
queries = make([]query, len(tagValues))
for i, value := range tagValues {
// get key prefix (with full length) and offset where to write the created_at
dbi, k, offset := il.getTagIndexPrefix(value)
// remove the last parts part to get just the prefix we want here
prefix := k[0:offset]
queries[i] = query{i: i, dbi: dbi, prefix: prefix, keySize: len(prefix) + 4, timestampSize: 4}
i++
}
// add an extra kind filter if available (only do this on plain tag index, not on ptag-kind index)
if filter.Kinds != nil {
extraKinds = make([][2]byte, len(filter.Kinds))
for i, kind := range filter.Kinds {
binary.BigEndian.PutUint16(extraKinds[i][0:2], uint16(kind))
}
}
}
// add an extra author search if possible
if filter.Authors != nil {
extraAuthors = make([][32]byte, len(filter.Authors))
for i, pk := range filter.Authors {
hex.Decode(extraAuthors[i][:], []byte(pk))
}
}
// add an extra useless tag if available
filter.Tags = internal.CopyMapWithoutKey(filter.Tags, tagKey)
if len(filter.Tags) > 0 {
extraTagKey, extraTagValues, _ = internal.ChooseNarrowestTag(filter)
}
return queries, extraAuthors, extraKinds, extraTagKey, extraTagValues, since, nil
}
pubkeyMatching:
if len(filter.Authors) > 0 {
if len(filter.Kinds) == 0 {
// will use pubkey index
queries = make([]query, len(filter.Authors))
for i, pubkeyHex := range filter.Authors {
if len(pubkeyHex) != 64 {
return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
}
prefix := make([]byte, 8)
if _, err := hex.Decode(prefix[0:8], []byte(pubkeyHex[0:8*2])); err != nil {
return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
}
queries[i] = query{i: i, dbi: il.indexPubkey, prefix: prefix[0:8], keySize: 8 + 4, timestampSize: 4}
}
} else {
// will use pubkeyKind index
queries = make([]query, len(filter.Authors)*len(filter.Kinds))
i := 0
for _, pubkeyHex := range filter.Authors {
for _, kind := range filter.Kinds {
if len(pubkeyHex) != 64 {
return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
}
prefix := make([]byte, 8+2)
if _, err := hex.Decode(prefix[0:8], []byte(pubkeyHex[0:8*2])); err != nil {
return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
}
binary.BigEndian.PutUint16(prefix[8:8+2], uint16(kind))
queries[i] = query{i: i, dbi: il.indexPubkeyKind, prefix: prefix[0 : 8+2], keySize: 10 + 4, timestampSize: 4}
i++
}
}
}
// potentially with an extra useless tag filtering
extraTagKey, extraTagValues, _ = internal.ChooseNarrowestTag(filter)
return queries, nil, nil, extraTagKey, extraTagValues, since, nil
}
if len(filter.Kinds) > 0 {
// will use a kind index
queries = make([]query, len(filter.Kinds))
for i, kind := range filter.Kinds {
prefix := make([]byte, 2)
binary.BigEndian.PutUint16(prefix[0:2], uint16(kind))
queries[i] = query{i: i, dbi: il.indexKind, prefix: prefix[0:2], keySize: 2 + 4, timestampSize: 4}
}
// potentially with an extra useless tag filtering
tagKey, tagValues, _ := internal.ChooseNarrowestTag(filter)
return queries, nil, nil, tagKey, tagValues, since, nil
}
// if we got here our query will have nothing to filter with
queries = make([]query, 1)
prefix := make([]byte, 0)
queries[0] = query{i: 0, dbi: il.indexCreatedAt, prefix: prefix, keySize: 0 + 4, timestampSize: 4}
return queries, nil, nil, "", nil, since, nil
}

54
eventstore/mmm/replace.go Normal file
View File

@@ -0,0 +1,54 @@
package mmm
import (
"context"
"fmt"
"math"
"github.com/PowerDNS/lmdb-go/lmdb"
"github.com/fiatjaf/eventstore/internal"
"github.com/nbd-wtf/go-nostr"
)
func (il *IndexingLayer) ReplaceEvent(ctx context.Context, evt *nostr.Event) error {
// sanity checking
if evt.CreatedAt > math.MaxUint32 || evt.Kind > math.MaxUint16 {
return fmt.Errorf("event with values out of expected boundaries")
}
filter := nostr.Filter{Limit: 1, Kinds: []int{evt.Kind}, Authors: []string{evt.PubKey}}
if nostr.IsAddressableKind(evt.Kind) {
// when addressable, add the "d" tag to the filter
filter.Tags = nostr.TagMap{"d": []string{evt.Tags.GetD()}}
}
return il.mmmm.lmdbEnv.Update(func(mmmtxn *lmdb.Txn) error {
mmmtxn.RawRead = true
return il.lmdbEnv.Update(func(iltxn *lmdb.Txn) error {
// now we fetch the past events, whatever they are, delete them and then save the new
prevResults, err := il.query(iltxn, filter, 10) // in theory limit could be just 1 and this should work
if err != nil {
return fmt.Errorf("failed to query past events with %s: %w", filter, err)
}
shouldStore := true
for _, previous := range prevResults {
if internal.IsOlder(previous.Event, evt) {
if err := il.delete(mmmtxn, iltxn, previous.Event); err != nil {
return fmt.Errorf("failed to delete event %s for replacing: %w", previous.Event.ID, err)
}
} else {
// there is a newer event already stored, so we won't store this
shouldStore = false
}
}
if shouldStore {
_, err := il.mmmm.storeOn(mmmtxn, []*IndexingLayer{il}, []*lmdb.Txn{iltxn}, evt)
return err
}
return nil
})
})
}

234
eventstore/mmm/save.go Normal file
View File

@@ -0,0 +1,234 @@
package mmm
import (
"context"
"encoding/binary"
"encoding/hex"
"fmt"
"os"
"runtime"
"slices"
"syscall"
"unsafe"
"github.com/PowerDNS/lmdb-go/lmdb"
"github.com/fiatjaf/eventstore/mmm/betterbinary"
"github.com/nbd-wtf/go-nostr"
)
func (b *MultiMmapManager) StoreGlobal(ctx context.Context, evt *nostr.Event) (stored bool, err error) {
someoneWantsIt := false
b.mutex.Lock()
defer b.mutex.Unlock()
runtime.LockOSThread()
defer runtime.UnlockOSThread()
// do this just so it's cleaner, we're already locking the thread and the mutex anyway
mmmtxn, err := b.lmdbEnv.BeginTxn(nil, 0)
if err != nil {
return false, fmt.Errorf("failed to begin global transaction: %w", err)
}
mmmtxn.RawRead = true
iltxns := make([]*lmdb.Txn, 0, len(b.layers))
ils := make([]*IndexingLayer, 0, len(b.layers))
// ask if any of the indexing layers want this
for _, il := range b.layers {
if il.ShouldIndex != nil && il.ShouldIndex(ctx, evt) {
someoneWantsIt = true
iltxn, err := il.lmdbEnv.BeginTxn(nil, 0)
if err != nil {
mmmtxn.Abort()
for _, txn := range iltxns {
txn.Abort()
}
return false, fmt.Errorf("failed to start txn on %s: %w", il.name, err)
}
ils = append(ils, il)
iltxns = append(iltxns, iltxn)
}
}
if !someoneWantsIt {
// no one wants it
mmmtxn.Abort()
return false, fmt.Errorf("not wanted")
}
stored, err = b.storeOn(mmmtxn, ils, iltxns, evt)
if stored {
mmmtxn.Commit()
for _, txn := range iltxns {
txn.Commit()
}
} else {
mmmtxn.Abort()
for _, txn := range iltxns {
txn.Abort()
}
}
return stored, err
}
func (il *IndexingLayer) SaveEvent(ctx context.Context, evt *nostr.Event) error {
il.mmmm.mutex.Lock()
defer il.mmmm.mutex.Unlock()
runtime.LockOSThread()
defer runtime.UnlockOSThread()
// do this just so it's cleaner, we're already locking the thread and the mutex anyway
mmmtxn, err := il.mmmm.lmdbEnv.BeginTxn(nil, 0)
if err != nil {
return fmt.Errorf("failed to begin global transaction: %w", err)
}
mmmtxn.RawRead = true
iltxn, err := il.lmdbEnv.BeginTxn(nil, 0)
if err != nil {
mmmtxn.Abort()
return fmt.Errorf("failed to start txn on %s: %w", il.name, err)
}
if _, err := il.mmmm.storeOn(mmmtxn, []*IndexingLayer{il}, []*lmdb.Txn{iltxn}, evt); err != nil {
mmmtxn.Abort()
if iltxn != nil {
iltxn.Abort()
}
return err
}
mmmtxn.Commit()
iltxn.Commit()
return nil
}
func (b *MultiMmapManager) storeOn(
mmmtxn *lmdb.Txn,
ils []*IndexingLayer,
iltxns []*lmdb.Txn,
evt *nostr.Event,
) (stored bool, err error) {
// sanity checking
if evt.CreatedAt > maxuint32 || evt.Kind > maxuint16 {
return false, fmt.Errorf("event with values out of expected boundaries")
}
// check if we already have this id
idPrefix8, _ := hex.DecodeString(evt.ID[0 : 8*2])
val, err := mmmtxn.Get(b.indexId, idPrefix8)
if err == nil {
// we found the event, now check if it is already indexed by the layers that want to store it
for i := len(ils) - 1; i >= 0; i-- {
for s := 12; s < len(val); s += 2 {
ilid := binary.BigEndian.Uint16(val[s : s+2])
if ils[i].id == ilid {
// swap delete this il, but keep the deleted ones at the end
// (so the caller can successfully finalize the transactions)
ils[i], ils[len(ils)-1] = ils[len(ils)-1], ils[i]
ils = ils[0 : len(ils)-1]
iltxns[i], iltxns[len(iltxns)-1] = iltxns[len(iltxns)-1], iltxns[i]
iltxns = iltxns[0 : len(iltxns)-1]
break
}
}
}
} else if !lmdb.IsNotFound(err) {
// now if we got an error from lmdb we will only proceed if we get a NotFound -- for anything else we will error
return false, fmt.Errorf("error checking existence: %w", err)
}
// if all ils already have this event indexed (or no il was given) we can end here
if len(ils) == 0 {
return false, nil
}
// get event binary size
pos := position{
size: uint32(betterbinary.Measure(*evt)),
}
if pos.size >= 1<<16 {
return false, fmt.Errorf("event too large to store, max %d, got %d", 1<<16, pos.size)
}
// find a suitable place for this to be stored in
appendToMmap := true
for f, fr := range b.freeRanges {
if fr.size >= pos.size {
// found the smallest possible place that can fit this event
appendToMmap = false
pos.start = fr.start
// modify the free ranges we're keeping track of
// (i.e. delete the current and add a new freerange with the remaining space)
b.freeRanges = slices.Delete(b.freeRanges, f, f+1)
if pos.size != fr.size {
b.addNewFreeRange(position{
start: fr.start + uint64(pos.size),
size: fr.size - pos.size,
})
}
if err := b.saveFreeRanges(mmmtxn); err != nil {
return false, fmt.Errorf("failed to save modified free ranges: %w", err)
}
break
}
}
if appendToMmap {
// no free ranges found, so write to the end of the mmap file
pos.start = b.mmapfEnd
mmapfNewSize := int64(b.mmapfEnd) + int64(pos.size)
if err := os.Truncate(b.mmapfPath, mmapfNewSize); err != nil {
return false, fmt.Errorf("error increasing %s: %w", b.mmapfPath, err)
}
b.mmapfEnd = uint64(mmapfNewSize)
}
// write to the mmap
if err := betterbinary.Marshal(*evt, b.mmapf[pos.start:]); err != nil {
return false, fmt.Errorf("error marshaling to %d: %w", pos.start, err)
}
// prepare value to be saved in the id index (if we didn't have it already)
// val: [posb][layerIdRefs...]
if val == nil {
val = make([]byte, 12, 12+2*len(b.layers))
binary.BigEndian.PutUint32(val[0:4], pos.size)
binary.BigEndian.PutUint64(val[4:12], pos.start)
}
// each index that was reserved above for the different layers
for i, il := range ils {
iltxn := iltxns[i]
for k := range il.getIndexKeysForEvent(evt) {
if err := iltxn.Put(k.dbi, k.key, val[0:12] /* pos */, 0); err != nil {
b.Logger.Warn().Str("name", il.name).Msg("failed to index event on layer")
}
}
val = binary.BigEndian.AppendUint16(val, il.id)
}
// store the id index with the refcounts
if err := mmmtxn.Put(b.indexId, idPrefix8, val, 0); err != nil {
panic(fmt.Errorf("failed to store %x by id: %w", idPrefix8, err))
}
// msync
_, _, errno := syscall.Syscall(syscall.SYS_MSYNC,
uintptr(unsafe.Pointer(&b.mmapf[0])), uintptr(len(b.mmapf)), syscall.MS_SYNC)
if errno != 0 {
panic(fmt.Errorf("msync failed: %w", syscall.Errno(errno)))
}
return true, nil
}