bring in khatru and eventstore.

eventstore/bluge/bluge_test.go

@@ -0,0 +1,81 @@
+package bluge
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/fiatjaf/eventstore/badger"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBlugeFlow(t *testing.T) {
+	os.RemoveAll("/tmp/blugetest-badger")
+	os.RemoveAll("/tmp/blugetest-bluge")
+
+	bb := &badger.BadgerBackend{Path: "/tmp/blugetest-badger"}
+	bb.Init()
+	defer bb.Close()
+
+	bl := BlugeBackend{
+		Path:          "/tmp/blugetest-bluge",
+		RawEventStore: bb,
+	}
+	bl.Init()
+	defer bl.Close()
+
+	ctx := context.Background()
+
+	willDelete := make([]*nostr.Event, 0, 3)
+
+	for i, content := range []string{
+		"good morning mr paper maker",
+		"good night",
+		"I'll see you again in the paper house",
+		"tonight we dine in my house",
+		"the paper in this house if very good, mr",
+	} {
+		evt := &nostr.Event{Content: content, Tags: nostr.Tags{}}
+		evt.Sign("0000000000000000000000000000000000000000000000000000000000000001")
+
+		bb.SaveEvent(ctx, evt)
+		bl.SaveEvent(ctx, evt)
+
+		if i%2 == 0 {
+			willDelete = append(willDelete, evt)
+		}
+	}
+
+	{
+		ch, err := bl.QueryEvents(ctx, nostr.Filter{Search: "good"})
+		if err != nil {
+			t.Fatalf("QueryEvents error: %s", err)
+			return
+		}
+		n := 0
+		for range ch {
+			n++
+		}
+		assert.Equal(t, 3, n)
+	}
+
+	for _, evt := range willDelete {
+		bl.DeleteEvent(ctx, evt)
+	}
+
+	{
+		ch, err := bl.QueryEvents(ctx, nostr.Filter{Search: "good"})
+		if err != nil {
+			t.Fatalf("QueryEvents error: %s", err)
+			return
+		}
+		n := 0
+		for res := range ch {
+			n++
+			assert.Equal(t, res.Content, "good night")
+			assert.Equal(t, res.PubKey, "79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798")
+		}
+		assert.Equal(t, 1, n)
+	}
+}

eventstore/bluge/delete.go

@@ -0,0 +1,11 @@
+package bluge
+
+import (
+	"context"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *BlugeBackend) DeleteEvent(ctx context.Context, evt *nostr.Event) error {
+	return b.writer.Delete(eventIdentifier(evt.ID))
+}

eventstore/bluge/helpers.go

@@ -0,0 +1,23 @@
+package bluge
+
+import "encoding/hex"
+
+const (
+	contentField   = "c"
+	kindField      = "k"
+	createdAtField = "a"
+	pubkeyField    = "p"
+)
+
+type eventIdentifier string
+
+const idField = "i"
+
+func (id eventIdentifier) Field() string {
+	return idField
+}
+
+func (id eventIdentifier) Term() []byte {
+	v, _ := hex.DecodeString(string(id))
+	return v
+}

eventstore/bluge/lib.go

@@ -0,0 +1,52 @@
+package bluge
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/blugelabs/bluge"
+	"github.com/blugelabs/bluge/analysis/token"
+	"github.com/fiatjaf/eventstore"
+	"golang.org/x/text/unicode/norm"
+)
+
+var _ eventstore.Store = (*BlugeBackend)(nil)
+
+type BlugeBackend struct {
+	sync.Mutex
+	// Path is where the index will be saved
+	Path string
+
+	// RawEventStore is where we'll fetch the raw events from
+	// bluge will only store ids, so the actual events must be somewhere else
+	RawEventStore eventstore.Store
+
+	searchConfig bluge.Config
+	writer       *bluge.Writer
+}
+
+func (b *BlugeBackend) Close() {
+	defer b.writer.Close()
+}
+
+func (b *BlugeBackend) Init() error {
+	if b.Path == "" {
+		return fmt.Errorf("missing Path")
+	}
+	if b.RawEventStore == nil {
+		return fmt.Errorf("missing RawEventStore")
+	}
+
+	b.searchConfig = bluge.DefaultConfig(b.Path)
+	b.searchConfig.DefaultSearchAnalyzer.TokenFilters = append(b.searchConfig.DefaultSearchAnalyzer.TokenFilters,
+		token.NewUnicodeNormalizeFilter(norm.NFKC),
+	)
+
+	var err error
+	b.writer, err = bluge.OpenWriter(b.searchConfig)
+	if err != nil {
+		return fmt.Errorf("error opening writer: %w", err)
+	}
+
+	return nil
+}

eventstore/bluge/query.go

@@ -0,0 +1,117 @@
+package bluge
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"strconv"
+
+	"github.com/blugelabs/bluge"
+	"github.com/blugelabs/bluge/search"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *BlugeBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (chan *nostr.Event, error) {
+	ch := make(chan *nostr.Event)
+
+	if len(filter.Search) < 2 {
+		close(ch)
+		return ch, nil
+	}
+
+	reader, err := b.writer.Reader()
+	if err != nil {
+		close(ch)
+		return nil, fmt.Errorf("unable to open reader: %w", err)
+	}
+
+	searchQ := bluge.NewMatchQuery(filter.Search)
+	searchQ.SetField(contentField)
+	var q bluge.Query = searchQ
+
+	complicatedQuery := bluge.NewBooleanQuery().AddMust(searchQ)
+
+	if len(filter.Kinds) > 0 {
+		eitherKind := bluge.NewBooleanQuery()
+		eitherKind.SetMinShould(1)
+		for _, kind := range filter.Kinds {
+			kindQ := bluge.NewTermQuery(strconv.Itoa(kind))
+			kindQ.SetField(kindField)
+			eitherKind.AddShould(kindQ)
+		}
+		complicatedQuery.AddMust(eitherKind)
+		q = complicatedQuery
+	}
+
+	if len(filter.Authors) > 0 {
+		eitherPubkey := bluge.NewBooleanQuery()
+		eitherPubkey.SetMinShould(1)
+		for _, pubkey := range filter.Authors {
+			if len(pubkey) != 64 {
+				continue
+			}
+			pubkeyQ := bluge.NewTermQuery(pubkey[56:])
+			pubkeyQ.SetField(pubkeyField)
+			eitherPubkey.AddShould(pubkeyQ)
+		}
+		complicatedQuery.AddMust(eitherPubkey)
+		q = complicatedQuery
+	}
+
+	if filter.Since != nil || filter.Until != nil {
+		min := 0.0
+		if filter.Since != nil {
+			min = float64(*filter.Since)
+		}
+		max := float64(nostr.Now())
+		if filter.Until != nil {
+			max = float64(*filter.Until)
+		}
+		dateRangeQ := bluge.NewNumericRangeInclusiveQuery(min, max, true, true)
+		dateRangeQ.SetField(createdAtField)
+		complicatedQuery.AddMust(dateRangeQ)
+		q = complicatedQuery
+	}
+
+	limit := 40
+	if filter.Limit != 0 {
+		limit = filter.Limit
+		if filter.Limit > 150 {
+			limit = 150
+		}
+	}
+
+	req := bluge.NewTopNSearch(limit, q)
+
+	dmi, err := reader.Search(context.Background(), req)
+	if err != nil {
+		close(ch)
+		reader.Close()
+		return ch, fmt.Errorf("error executing search: %w", err)
+	}
+
+	go func() {
+		defer reader.Close()
+		defer close(ch)
+
+		var next *search.DocumentMatch
+		for next, err = dmi.Next(); next != nil; next, err = dmi.Next() {
+			next.VisitStoredFields(func(field string, value []byte) bool {
+				id := hex.EncodeToString(value)
+				rawch, err := b.RawEventStore.QueryEvents(ctx, nostr.Filter{IDs: []string{id}})
+				if err != nil {
+					return false
+				}
+				for evt := range rawch {
+					ch <- evt
+				}
+				return false
+			})
+		}
+		if err != nil {
+			return
+		}
+	}()
+
+	return ch, nil
+}

eventstore/bluge/replace.go

@@ -0,0 +1,44 @@
+package bluge
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/fiatjaf/eventstore/internal"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *BlugeBackend) ReplaceEvent(ctx context.Context, evt *nostr.Event) error {
+	b.Lock()
+	defer b.Unlock()
+
+	filter := nostr.Filter{Limit: 1, Kinds: []int{evt.Kind}, Authors: []string{evt.PubKey}}
+	if nostr.IsAddressableKind(evt.Kind) {
+		filter.Tags = nostr.TagMap{"d": []string{evt.Tags.GetD()}}
+	}
+
+	ch, err := b.QueryEvents(ctx, filter)
+	if err != nil {
+		return fmt.Errorf("failed to query before replacing: %w", err)
+	}
+
+	shouldStore := true
+	for previous := range ch {
+		if internal.IsOlder(previous, evt) {
+			if err := b.DeleteEvent(ctx, previous); err != nil {
+				return fmt.Errorf("failed to delete event for replacing: %w", err)
+			}
+		} else {
+			shouldStore = false
+		}
+	}
+
+	if shouldStore {
+		if err := b.SaveEvent(ctx, evt); err != nil && err != eventstore.ErrDupEvent {
+			return fmt.Errorf("failed to save: %w", err)
+		}
+	}
+
+	return nil
+}

eventstore/bluge/save.go

@@ -0,0 +1,28 @@
+package bluge
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"github.com/blugelabs/bluge"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *BlugeBackend) SaveEvent(ctx context.Context, evt *nostr.Event) error {
+	id := eventIdentifier(evt.ID)
+	doc := &bluge.Document{
+		bluge.NewKeywordFieldBytes(id.Field(), id.Term()).Sortable().StoreValue(),
+	}
+
+	doc.AddField(bluge.NewTextField(contentField, evt.Content))
+	doc.AddField(bluge.NewTextField(kindField, strconv.Itoa(evt.Kind)))
+	doc.AddField(bluge.NewTextField(pubkeyField, evt.PubKey[56:]))
+	doc.AddField(bluge.NewNumericField(createdAtField, float64(evt.CreatedAt)))
+
+	if err := b.writer.Update(doc.ID(), doc); err != nil {
+		return fmt.Errorf("failed to write '%s' document: %w", evt.ID, err)
+	}
+
+	return nil
+}