fix(dev-server): fetch flag pages concurrently instead of serially

internal/dev_server/adapters/api.go

@@ -47,7 +47,7 @@ func (a apiClientApi) GetSdkKey(ctx context.Context, projectKey, environmentKey
 
 func (a apiClientApi) GetAllFlags(ctx context.Context, projectKey string) ([]ldapi.FeatureFlag, error) {
 	log.Printf("Fetching all flags for project '%s'", projectKey)
-	flags, err := a.getFlags(ctx, projectKey, nil)
+	flags, err := a.getFlags(ctx, projectKey)
 	if err != nil {
 		err = errors.Wrap(err, "unable to get all flags from LD API")
 	}
@@ -63,8 +63,8 @@ func (a apiClientApi) GetProjectEnvironments(ctx context.Context, projectKey str
 	return environments, err
 }
 
-func (a apiClientApi) getFlags(ctx context.Context, projectKey string, href *string) ([]ldapi.FeatureFlag, error) {
-	return internal.GetPaginatedItems(ctx, projectKey, href, func(ctx context.Context, projectKey string, limit, offset *int64) (flags *ldapi.FeatureFlags, err error) {
+func (a apiClientApi) getFlags(ctx context.Context, projectKey string) ([]ldapi.FeatureFlag, error) {
+	return internal.GetPaginatedItems(ctx, projectKey, func(ctx context.Context, projectKey string, limit, offset *int64) (flags *ldapi.FeatureFlags, err error) {
 		// loop until we do not get rate limited
 		query := a.apiClient.FeatureFlagsApi.GetFeatureFlags(ctx, projectKey).Limit(100)
 		query = query.Filter("purpose:all+!(holdout)")

internal/dev_server/adapters/internal/api_util.go

@@ -3,76 +3,92 @@ package internal
 import (
 	"context"
 	"log"
+	"math/rand/v2"
 	"net/http"
-	"net/url"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/launchdarkly/api-client-go/v14"
 	"github.com/pkg/errors"
 )
 
+// maxConcurrentPageFetches bounds how many pages we fetch at once once the total
+// item count is known. Kept modest to stay well clear of the API's rate limiter;
+// Retry429s still handles any 429s that slip through.
+const maxConcurrentPageFetches = 6
+
+// GetPaginatedItems fetches all pages of a paginated list endpoint. It fetches
+// page 0 first to learn the total item count, then fetches the remaining pages
+// concurrently (bounded by maxConcurrentPageFetches) instead of following the
+// `next` link serially, since every offset is already known once the total
+// count and page size are known.
 func GetPaginatedItems[T any, R interface {
 	GetItems() []T
 	GetLinks() map[string]ldapi.Link
-}](ctx context.Context, projectKey string, href *string, fetchFunc func(context.Context, string, *int64, *int64) (R, error)) ([]T, error) {
-	var result R
-	var err error
-
-	if href == nil {
-		result, err = fetchFunc(ctx, projectKey, nil, nil)
-		if err != nil {
-			return nil, err
-		}
-	} else {
-		limit, offset, err := parseHref(*href)
-		if err != nil {
-			return nil, errors.Wrapf(err, "unable to parse href for next link: %s", *href)
-		}
-		result, err = fetchFunc(ctx, projectKey, &limit, &offset)
-		if err != nil {
-			return nil, err
-		}
+	GetTotalCount() int32
+}](ctx context.Context, projectKey string, fetchFunc func(context.Context, string, *int64, *int64) (R, error)) ([]T, error) {
+	first, err := fetchFunc(ctx, projectKey, nil, nil)
+	if err != nil {
+		return nil, err
 	}
 
-	items := result.GetItems()
-
-	if links := result.GetLinks(); links != nil {
-		if next, ok := links["next"]; ok && next.Href != nil {
-			newItems, err := GetPaginatedItems(ctx, projectKey, next.Href, fetchFunc)
-			if err != nil {
-				return nil, err
-			}
-			items = append(items, newItems...)
-		}
+	firstItems := first.GetItems()
+	limit := int64(len(firstItems))
+	total := int64(first.GetTotalCount())
+	if limit == 0 || total <= limit {
+		return firstItems, nil
 	}
 
-	return items, nil
-}
+	numPages := int((total + limit - 1) / limit)
+	pages := make([][]T, numPages)
+	pages[0] = firstItems
 
-func parseHref(href string) (limit, offset int64, err error) {
-	parsedUrl, err := url.Parse(href)
-	if err != nil {
-		return
-	}
-	l, err := strconv.Atoi(parsedUrl.Query().Get("limit"))
-	if err != nil {
-		return
+	sem := make(chan struct{}, maxConcurrentPageFetches)
+	var wg sync.WaitGroup
+	errs := make([]error, numPages)
+
+	for page := 1; page < numPages; page++ {
+		page := page
+		offset := int64(page) * limit
+		wg.Add(1)
+		sem <- struct{}{}
+		go func() {
+			defer wg.Done()
+			defer func() { <-sem }()
+			result, fetchErr := fetchFunc(ctx, projectKey, &limit, &offset)
+			if fetchErr != nil {
+				errs[page] = fetchErr
+				return
+			}
+			pages[page] = result.GetItems()
+		}()
 	}
-	o, err := strconv.Atoi(parsedUrl.Query().Get("offset"))
-	if err != nil {
-		return
+	wg.Wait()
+
+	for _, pageErr := range errs {
+		if pageErr != nil {
+			return nil, pageErr
+		}
 	}
 
-	limit = int64(l)
-	offset = int64(o)
-	return
+	items := make([]T, 0, total)
+	for _, p := range pages {
+		items = append(items, p...)
+	}
+	return items, nil
 }
 
 //go:generate go run go.uber.org/mock/mockgen -destination mocks.go -package internal . MockableTime
 type MockableTime interface {
 	Sleep(duration time.Duration)
 	Now() time.Time
+	// Jitter returns a random duration in [0, max). Used to decorrelate
+	// retries after a 429: with concurrent page fetches, several requests
+	// can get rate-limited at once and would otherwise all compute the same
+	// X-Ratelimit-Reset and wake up to retry at the exact same instant,
+	// recreating the burst that got them limited in the first place.
+	Jitter(max time.Duration) time.Duration
 }
 
 type realTime struct{}
@@ -85,8 +101,20 @@ func (realTime) Now() time.Time {
 	return time.Now()
 }
 
+func (realTime) Jitter(max time.Duration) time.Duration {
+	if max <= 0 {
+		return 0
+	}
+	return rand.N(max)
+}
+
 var timeImpl MockableTime = realTime{}
 
+// maxRetryJitter bounds the random extra delay added on top of the API's
+// requested X-Ratelimit-Reset wait, so concurrent retries spread out instead
+// of waking up in lockstep.
+const maxRetryJitter = 250 * time.Millisecond
+
 func Retry429s[T any](requester func() (T, *http.Response, error)) (result T, err error) {
 	for {
 		var res *http.Response
@@ -98,9 +126,9 @@ func Retry429s[T any](requester func() (T, *http.Response, error)) (result T, er
 				err = errors.Wrapf(err, `unable to retry rate limited request: X-RateLimit-Reset: "%s" was not parsable`, resetUnixMillisString)
 				return
 			}
-			sleep := resetUnixMillis - timeImpl.Now().UnixMilli()
-			log.Printf("Got 429 in API response. Retrying in %d milliseconds.", sleep)
-			timeImpl.Sleep(time.Duration(sleep) * time.Millisecond)
+			sleep := time.Duration(resetUnixMillis-timeImpl.Now().UnixMilli())*time.Millisecond + timeImpl.Jitter(maxRetryJitter)
+			log.Printf("Got 429 in API response. Retrying in %s.", sleep)
+			timeImpl.Sleep(sleep)
 		} else {
 			return
 		}