From 6cffcc3b47256cebb5d31d1625702cabe6d3c64c Mon Sep 17 00:00:00 2001 From: fiatjaf Date: Fri, 17 Jan 2025 18:21:19 -0300 Subject: [PATCH] sdk: make it so replaceable stuff is automatically reloaded from time to time. --- sdk/list.go | 55 +++++++++++++++++++++++++++++---- sdk/loader_helpers.go | 36 ++++++++++++++++++++++ sdk/metadata.go | 71 ++++++++++++++++++++++++++++++------------- sdk/set.go | 58 ++++++++++++++++++++++++++++++----- 4 files changed, 185 insertions(+), 35 deletions(-) create mode 100644 sdk/loader_helpers.go diff --git a/sdk/list.go b/sdk/list.go index 9b2d495..be09e72 100644 --- a/sdk/list.go +++ b/sdk/list.go @@ -60,27 +60,70 @@ func fetchGenericList[I TagItemWithValue]( events, _ := sys.StoreRelay.QuerySync(ctx, nostr.Filter{Kinds: []int{actualKind}, Authors: []string{pubkey}}) if len(events) != 0 { + // ok, we found something locally items := parseItemsFromEventTags(events[0], parseTag) v.Event = events[0] v.Items = items + + // but if we haven't tried fetching from the network recently we should do it + lastFetchKey := makeLastFetchKey(actualKind, pubkey) + lastFetchData, _ := sys.KVStore.Get(lastFetchKey) + if nostr.Now()-decodeTimestamp(lastFetchData) > 7*24*60*60 { + newV := tryFetchListFromNetwork(ctx, sys, pubkey, replaceableIndex, parseTag) + if newV != nil && newV.Event.CreatedAt > v.Event.CreatedAt { + v = *newV + } + + // even if we didn't find anything register this because we tried + // (and we still have the previous event in our local store) + sys.KVStore.Set(lastFetchKey, encodeTimestamp(nostr.Now())) + } + + // and finally save this to cache cache.SetWithTTL(pubkey, v, time.Hour*6) valueWasJustCached[lockIdx] = true + return v, true } - thunk := sys.replaceableLoaders[replaceableIndex].Load(ctx, pubkey) - evt, err := thunk() - if err == nil { - items := parseItemsFromEventTags(evt, parseTag) - v.Items = items - sys.StoreRelay.Publish(ctx, *evt) + if newV := tryFetchListFromNetwork(ctx, sys, pubkey, replaceableIndex, parseTag); newV != nil { + v = *newV + + // we'll only save this if we got something which means we found at least one event + lastFetchKey := makeLastFetchKey(actualKind, pubkey) + sys.KVStore.Set(lastFetchKey, encodeTimestamp(nostr.Now())) } + + // save cache even if we didn't get anything cache.SetWithTTL(pubkey, v, time.Hour*6) valueWasJustCached[lockIdx] = true return v, false } +func tryFetchListFromNetwork[I TagItemWithValue]( + ctx context.Context, + sys *System, + pubkey string, + replaceableIndex replaceableIndex, + parseTag func(nostr.Tag) (I, bool), +) *GenericList[I] { + thunk := sys.replaceableLoaders[replaceableIndex].Load(ctx, pubkey) + evt, err := thunk() + if err != nil { + return nil + } + + v := &GenericList[I]{ + PubKey: pubkey, + Event: evt, + Items: parseItemsFromEventTags(evt, parseTag), + } + sys.StoreRelay.Publish(ctx, *evt) + + return v +} + func parseItemsFromEventTags[I TagItemWithValue]( evt *nostr.Event, parseTag func(nostr.Tag) (I, bool), diff --git a/sdk/loader_helpers.go b/sdk/loader_helpers.go new file mode 100644 index 0000000..8837c3c --- /dev/null +++ b/sdk/loader_helpers.go @@ -0,0 +1,36 @@ +package sdk + +import ( + "encoding/binary" + "encoding/hex" + + "github.com/nbd-wtf/go-nostr" +) + +var kvStoreLastFetchPrefix = byte('f') + +func makeLastFetchKey(kind int, pubkey string) []byte { + buf := make([]byte, 1+5+32) + buf[0] = kvStoreLastFetchPrefix + binary.LittleEndian.PutUint32(buf[1:], uint32(kind)) + hex.Decode(buf[5:], []byte(pubkey)) + return buf +} + +// encodeTimestamp encodes a unix timestamp as 4 bytes +func encodeTimestamp(t nostr.Timestamp) []byte { + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, uint32(t)) + return b +} + +// decodeTimestamp decodes a 4-byte timestamp into unix seconds +func decodeTimestamp(b []byte) nostr.Timestamp { + return nostr.Timestamp(binary.BigEndian.Uint32(b)) +} + +// shouldRefreshFromNetwork checks if we should try fetching from network +func shouldRefreshFromNetwork(lastFetchData []byte) bool { + lastFetch := decodeTimestamp(lastFetchData) + return nostr.Now()-lastFetch > 7*24*60*60 +} diff --git a/sdk/metadata.go b/sdk/metadata.go index b0ef469..7165cfb 100644 --- a/sdk/metadata.go +++ b/sdk/metadata.go @@ -106,32 +106,42 @@ func (sys *System) FetchProfileMetadata(ctx context.Context, pubkey string) (pm res, _ := sys.StoreRelay.QuerySync(ctx, nostr.Filter{Kinds: []int{0}, Authors: []string{pubkey}}) if len(res) != 0 { - if m, err := ParseMetadata(res[0]); err == nil { - m.PubKey = pubkey - m.Event = res[0] - sys.MetadataCache.SetWithTTL(pubkey, m, time.Hour*6) - return m + // ok, we found something locally + pm, _ = ParseMetadata(res[0]) + pm.PubKey = pubkey + pm.Event = res[0] + + // but if we haven't tried fetching from the network recently we should do it + lastFetchKey := makeLastFetchKey(0, pubkey) + lastFetchData, _ := sys.KVStore.Get(lastFetchKey) + if nostr.Now()-decodeTimestamp(lastFetchData) > 7*24*60*60 { + newM := sys.tryFetchMetadataFromNetwork(ctx, pubkey) + if newM != nil && newM.Event.CreatedAt > pm.Event.CreatedAt { + pm = *newM + } + + // even if we didn't find anything register this because we tried + // (and we still have the previous event in our local store) + sys.KVStore.Set(lastFetchKey, encodeTimestamp(nostr.Now())) } - } - pm.PubKey = pubkey - - thunk0 := sys.replaceableLoaders[kind_0].Load(ctx, pubkey) - evt, err := thunk0() - if err == nil { - pm, _ = ParseMetadata(evt) - - // save on store even if the metadata json is malformed - if pm.Event != nil { - sys.StoreRelay.Publish(ctx, *pm.Event) - } - } - - // save on cache even if the metadata isn't found (unless the context was canceled) - if err == nil || err != context.Canceled { + // and finally save this to cache sys.MetadataCache.SetWithTTL(pubkey, pm, time.Hour*6) + + return pm } + if newM := sys.tryFetchMetadataFromNetwork(ctx, pubkey); newM != nil { + pm = *newM + + // we'll only save this if we got something which means we found at least one event + lastFetchKey := makeLastFetchKey(0, pubkey) + sys.KVStore.Set(lastFetchKey, encodeTimestamp(nostr.Now())) + } + + // save cache even if we didn't get anything + sys.MetadataCache.SetWithTTL(pubkey, pm, time.Hour*6) + return pm } @@ -159,6 +169,25 @@ func (sys *System) FetchUserEvents(ctx context.Context, filter nostr.Filter) (ma return results, nil } +func (sys *System) tryFetchMetadataFromNetwork(ctx context.Context, pubkey string) *ProfileMetadata { + thunk0 := sys.replaceableLoaders[kind_0].Load(ctx, pubkey) + evt, err := thunk0() + if err != nil { + return nil + } + + pm, err := ParseMetadata(evt) + if err != nil { + return nil + } + + pm.PubKey = pubkey + pm.Event = evt + sys.StoreRelay.Publish(ctx, *evt) + sys.MetadataCache.SetWithTTL(pubkey, pm, time.Hour*6) + return &pm +} + func ParseMetadata(event *nostr.Event) (meta ProfileMetadata, err error) { if event.Kind != 0 { err = fmt.Errorf("event %s is kind %d, not 0", event.ID, event.Kind) diff --git a/sdk/set.go b/sdk/set.go index 0b2f617..64b7820 100644 --- a/sdk/set.go +++ b/sdk/set.go @@ -49,29 +49,71 @@ func fetchGenericSets[I TagItemWithValue]( events, _ := sys.StoreRelay.QuerySync(ctx, nostr.Filter{Kinds: []int{actualKind}, Authors: []string{pubkey}}) if len(events) != 0 { + // ok, we found something locally sets := parseSetsFromEvents(events, parseTag) v.Events = events v.Sets = sets + + // but if we haven't tried fetching from the network recently we should do it + lastFetchKey := makeLastFetchKey(actualKind, pubkey) + lastFetchData, _ := sys.KVStore.Get(lastFetchKey) + if nostr.Now()-decodeTimestamp(lastFetchData) > 7*24*60*60 { + newV := tryFetchSetsFromNetwork(ctx, sys, pubkey, addressableIndex, parseTag) + + // unlike for lists, when fetching sets we will blindly trust whatever we get from the network + v = *newV + + // even if we didn't find anything register this because we tried + // (and we still have the previous event in our local store) + sys.KVStore.Set(lastFetchKey, encodeTimestamp(nostr.Now())) + } + + // and finally save this to cache cache.SetWithTTL(pubkey, v, time.Hour*6) valueWasJustCached[lockIdx] = true + return v, true } - thunk := sys.addressableLoaders[addressableIndex].Load(ctx, pubkey) - events, err := thunk() - if err == nil { - sets := parseSetsFromEvents(events, parseTag) - v.Sets = sets - for _, evt := range events { - sys.StoreRelay.Publish(ctx, *evt) - } + if newV := tryFetchSetsFromNetwork(ctx, sys, pubkey, addressableIndex, parseTag); newV != nil { + v = *newV + + // we'll only save this if we got something which means we found at least one event + lastFetchKey := makeLastFetchKey(actualKind, pubkey) + sys.KVStore.Set(lastFetchKey, encodeTimestamp(nostr.Now())) } + + // save cache even if we didn't get anything cache.SetWithTTL(pubkey, v, time.Hour*6) valueWasJustCached[lockIdx] = true return v, false } +func tryFetchSetsFromNetwork[I TagItemWithValue]( + ctx context.Context, + sys *System, + pubkey string, + addressableIndex addressableIndex, + parseTag func(nostr.Tag) (I, bool), +) *GenericSets[I] { + thunk := sys.addressableLoaders[addressableIndex].Load(ctx, pubkey) + events, err := thunk() + if err != nil { + return nil + } + + v := &GenericSets[I]{ + PubKey: pubkey, + Events: events, + Sets: parseSetsFromEvents(events, parseTag), + } + for _, evt := range events { + sys.StoreRelay.Publish(ctx, *evt) + } + return v +} + func parseSetsFromEvents[I TagItemWithValue]( events []*nostr.Event, parseTag func(nostr.Tag) (I, bool),