fix locking on generic list and set fetching, decrease dataloader batch size, test.

sdk/sdk_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"sync"
 	"testing"
 
 	"github.com/nbd-wtf/go-nostr"
@@ -36,6 +37,55 @@ func TestMetadataAndEvents(t *testing.T) {
 	require.GreaterOrEqual(t, len(events[meta.PubKey]), 5)
 }
 
+func TestConcurrentMetadata(t *testing.T) {
+	sys := NewSystem()
+	ctx := context.Background()
+
+	wg := sync.WaitGroup{}
+	for _, v := range []struct {
+		input string
+		name  string
+	}{
+		{
+			"nprofile1qyxhwumn8ghj7mn0wvhxcmmvqyd8wumn8ghj7un9d3shjtnhv4ehgetjde38gcewvdhk6qpq80cvv07tjdrrgpa0j7j7tmnyl2yr6yr7l8j4s3evf6u64th6gkwswpnfsn",
+			"fiatjaf",
+		},
+		{
+			"npub1t6jxfqz9hv0lygn9thwndekuahwyxkgvycyscjrtauuw73gd5k7sqvksrw",
+			"constant",
+		},
+		{
+			"npub1jlrs53pkdfjnts29kveljul2sm0actt6n8dxrrzqcersttvcuv3qdjynqn",
+			"hodlbod",
+		},
+		{
+			"npub1xtscya34g58tk0z605fvr788k263gsu6cy9x0mhnm87echrgufzsevkk5s",
+			"jb55",
+		},
+		{
+			"npub1qny3tkh0acurzla8x3zy4nhrjz5zd8l9sy9jys09umwng00manysew95gx",
+			"odell",
+		},
+		{
+			"npub1l2vyh47mk2p0qlsku7hg0vn29faehy9hy34ygaclpn66ukqp3afqutajft",
+			"pablo",
+		},
+	} {
+		wg.Add(1)
+		go func() {
+			meta, err := sys.FetchProfileFromInput(ctx, v.input)
+			require.NoError(t, err)
+			require.Contains(t, strings.ToLower(meta.Name), v.name)
+
+			fl := sys.FetchFollowList(ctx, meta.PubKey)
+			require.GreaterOrEqual(t, len(fl.Items), 30, "%s/%s", meta.PubKey, meta.Name)
+			wg.Done()
+		}()
+	}
+
+	wg.Wait()
+}
+
 func TestFollowListRecursion(t *testing.T) {
 	sys := NewSystem()
 	ctx := context.Background()
@@ -53,21 +103,27 @@ func TestFollowListRecursion(t *testing.T) {
 	}
 
 	results := make(chan result)
-	go func() {
-		for _, item := range followList.Items {
-			fl := sys.FetchFollowList(ctx, item.Pubkey)
+	wg := sync.WaitGroup{}
+	for i, item := range followList.Items[0:120] {
+		wg.Add(1)
+		go func() {
 			meta := sys.FetchProfileMetadata(ctx, item.Pubkey)
-			fmt.Println("  ~", item.Pubkey, meta.Name, len(fl.Items))
+			fl := sys.FetchFollowList(ctx, item.Pubkey)
+			fmt.Println("  ~", i, item.Pubkey, len(fl.Items))
 			results <- result{item.Pubkey, fl, meta}
-		}
+			wg.Done()
+		}()
+	}
+
+	go func() {
+		wg.Wait()
+		close(results)
 	}()
 
 	// collect results
 	var validAccounts int
 	var accountsWithManyFollows int
-	for i := 0; i < len(followList.Items); i++ {
-		r := <-results
-
+	for r := range results {
 		// skip if metadata has "bot" in name
 		if strings.Contains(strings.ToLower(r.metadata.Name), "bot") {
 			continue
@@ -81,5 +137,5 @@ func TestFollowListRecursion(t *testing.T) {
 
 	// check if at least 90% of non-bot accounts follow more than 20 accounts
 	ratio := float64(accountsWithManyFollows) / float64(validAccounts)
-	require.Greater(t, ratio, 0.9, "at least 90%% of accounts should follow more than 20 others (actual: %.2f%%)", ratio*100)
+	require.Greater(t, ratio, 0.7, "at least 70%% of accounts should follow more than 20 others (actual: %.2f%%)", ratio*100)
 }