From 42b1a662abcfd90baf3dcae401288b094b9827fd Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 20 Feb 2026 08:45:36 +0000 Subject: [PATCH] Add client/ and fetch/ sub-packages Promote the HTTP client, error types, and URL builder from internal/core into a public client/ package. Add a fetch/ package for streaming artifact downloads with retry, per-host circuit breaking, DNS caching, and URL resolution across ecosystems. Internal packages now use thin type aliases over client/ for backward compatibility. --- README.md | 120 ++++++++++++- client/client.go | 207 ++++++++++++++++++++++ client/errors.go | 52 ++++++ client/urls.go | 66 ++++++++ fetch/circuit_breaker.go | 137 +++++++++++++++ fetch/circuit_breaker_test.go | 209 +++++++++++++++++++++++ fetch/fetcher.go | 281 ++++++++++++++++++++++++++++++ fetch/fetcher_test.go | 311 ++++++++++++++++++++++++++++++++++ fetch/resolver.go | 194 +++++++++++++++++++++ fetch/resolver_test.go | 137 +++++++++++++++ go.mod | 6 + go.sum | 12 ++ internal/core/client.go | 277 ++---------------------------- internal/core/errors.go | 53 +----- registries.go | 90 +++++----- 15 files changed, 1798 insertions(+), 354 deletions(-) create mode 100644 client/client.go create mode 100644 client/errors.go create mode 100644 client/urls.go create mode 100644 fetch/circuit_breaker.go create mode 100644 fetch/circuit_breaker_test.go create mode 100644 fetch/fetcher.go create mode 100644 fetch/fetcher_test.go create mode 100644 fetch/resolver.go create mode 100644 fetch/resolver_test.go diff --git a/README.md b/README.md index e6687a4..f1e128f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # registries -Go library for fetching package metadata from registry APIs. Supports 25 ecosystems with a unified interface. +Go library for fetching package metadata from registry APIs. Supports 25 ecosystems with a unified interface. Also provides sub-packages for HTTP client usage (`client/`) and streaming artifact downloads (`fetch/`). ## Installation @@ -281,22 +281,130 @@ if err != nil { } ``` -## HTTP Client +## HTTP Client (`client/`) + +The `client` sub-package provides an HTTP client with retry logic, error types, and URL building. You can use it through the top-level `registries` package or import it directly. The default client includes: - 30 second timeout -- 5 retries with exponential backoff (50ms base) +- 5 retries with exponential backoff (50ms base, 10% jitter) - Automatic retry on 429 and 5xx responses -Custom client: +Custom client via the top-level package: ```go -client := registries.NewClient( +c := registries.NewClient( registries.WithTimeout(60 * time.Second), registries.WithMaxRetries(3), ) -pkg, err := registries.FetchPackageFromPURL(ctx, "pkg:npm/lodash", client) +pkg, err := registries.FetchPackageFromPURL(ctx, "pkg:npm/lodash", c) +``` + +Or import the sub-package directly: + +```go +import "github.com/git-pkgs/registries/client" + +c := client.NewClient( + client.WithTimeout(60 * time.Second), + client.WithMaxRetries(3), +) + +// JSON decoding +var data map[string]any +err := c.GetJSON(ctx, "https://registry.npmjs.org/lodash", &data) + +// Raw body +body, err := c.GetBody(ctx, "https://crates.io/api/v1/crates/serde") + +// HEAD request +statusCode, err := c.Head(ctx, "https://registry.npmjs.org/lodash") +``` + +## Artifact Downloads (`fetch/`) + +The `fetch` sub-package provides streaming artifact downloads with retry, circuit breaking, DNS caching, and URL resolution. + +### Fetching artifacts + +```go +import "github.com/git-pkgs/registries/fetch" + +f := fetch.NewFetcher( + fetch.WithMaxRetries(3), + fetch.WithBaseDelay(500 * time.Millisecond), + fetch.WithUserAgent("my-app/1.0"), +) + +artifact, err := f.Fetch(ctx, "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz") +if err != nil { + log.Fatal(err) +} +defer artifact.Body.Close() + +// artifact.Body is an io.ReadCloser +// artifact.Size is the content length (-1 if unknown) +// artifact.ContentType and artifact.ETag are also available +io.Copy(dst, artifact.Body) +``` + +The fetcher uses DNS caching (5-minute refresh), connection pooling, and a 5-minute timeout suited for large artifacts. It retries on rate limits and server errors with exponential backoff and jitter. + +### Authentication + +Pass a function that returns auth headers per URL: + +```go +f := fetch.NewFetcher( + fetch.WithAuthFunc(func(url string) (string, string) { + if strings.Contains(url, "npm.pkg.github.com") { + return "Authorization", "Bearer " + token + } + return "", "" + }), +) +``` + +### Circuit breaker + +Wrap a fetcher with per-host circuit breakers to avoid hammering a failing registry. The breaker trips after 5 consecutive failures and resets with exponential backoff (30s initial, 5min max). + +```go +f := fetch.NewFetcher() +cbf := fetch.NewCircuitBreakerFetcher(f) + +// Same interface as Fetcher +artifact, err := cbf.Fetch(ctx, url) + +// Check breaker states for health monitoring +states := cbf.GetBreakerState() +// map[string]string{"registry.npmjs.org": "closed", "crates.io": "open"} +``` + +### URL resolution + +The resolver maps ecosystem/name/version to download URLs and filenames. It uses each registry's `URLBuilder` when available, and falls back to hardcoded URL patterns for common ecosystems (npm, cargo, gem, golang, hex, pub, maven, nuget). For ecosystems with dynamic URLs (like PyPI), it fetches version metadata to find the download link. + +```go +resolver := fetch.NewResolver() + +// Register a registry for URL building and metadata fallback +reg, _ := registries.New("cargo", "", nil) +resolver.RegisterRegistry(reg) + +info, err := resolver.Resolve(ctx, "cargo", "serde", "1.0.0") +// info.URL = "https://static.crates.io/crates/serde/serde-1.0.0.crate" +// info.Filename = "serde-1.0.0.crate" +// info.Integrity may be populated for metadata-resolved URLs +``` + +The resolver also works without a registered registry for ecosystems with predictable URL patterns: + +```go +resolver := fetch.NewResolver() +info, _ := resolver.Resolve(ctx, "npm", "lodash", "4.17.21") +// info.URL = "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" ``` ## Private Registries diff --git a/client/client.go b/client/client.go new file mode 100644 index 0000000..234dee9 --- /dev/null +++ b/client/client.go @@ -0,0 +1,207 @@ +// Package client provides an HTTP client with retry logic for registry APIs. +package client + +import ( + "context" + "encoding/json" + "io" + "math" + "math/rand" + "net/http" + "strconv" + "time" +) + +// RateLimiter controls request pacing. +type RateLimiter interface { + Wait(ctx context.Context) error +} + +// Client is an HTTP client with retry logic for registry APIs. +type Client struct { + HTTPClient *http.Client + UserAgent string + MaxRetries int + BaseDelay time.Duration + RateLimiter RateLimiter +} + +// DefaultClient returns a client with sensible defaults. +func DefaultClient() *Client { + return &Client{ + HTTPClient: &http.Client{ + Timeout: 30 * time.Second, + }, + UserAgent: "registries", + MaxRetries: 5, + BaseDelay: 50 * time.Millisecond, + } +} + +// GetJSON fetches a URL and decodes the JSON response into v. +func (c *Client) GetJSON(ctx context.Context, url string, v any) error { + body, err := c.GetBody(ctx, url) + if err != nil { + return err + } + return json.Unmarshal(body, v) +} + +// GetBody fetches a URL and returns the response body. +func (c *Client) GetBody(ctx context.Context, url string) ([]byte, error) { + var lastErr error + + for attempt := 0; attempt <= c.MaxRetries; attempt++ { + if attempt > 0 { + delay := c.BaseDelay * time.Duration(math.Pow(2, float64(attempt-1))) + jitter := time.Duration(float64(delay) * (rand.Float64() * 0.1)) + delay += jitter + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(delay): + } + } + + if c.RateLimiter != nil { + if err := c.RateLimiter.Wait(ctx); err != nil { + return nil, err + } + } + + body, err := c.doRequest(ctx, url) + if err == nil { + return body, nil + } + + lastErr = err + + var httpErr *HTTPError + if ok := isHTTPError(err, &httpErr); ok { + if httpErr.StatusCode == 404 { + return nil, err + } + if httpErr.StatusCode == 429 || httpErr.StatusCode >= 500 { + continue + } + return nil, err + } + } + + return nil, lastErr +} + +func (c *Client) doRequest(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + req.Header.Set("User-Agent", c.UserAgent) + req.Header.Set("Accept", "application/json") + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode >= 400 { + httpErr := &HTTPError{ + StatusCode: resp.StatusCode, + URL: url, + Body: string(body), + } + if resp.StatusCode == 429 { + if retryAfter := resp.Header.Get("Retry-After"); retryAfter != "" { + if seconds, err := strconv.Atoi(retryAfter); err == nil { + return nil, &RateLimitError{RetryAfter: seconds} + } + } + } + return nil, httpErr + } + + return body, nil +} + +func isHTTPError(err error, target **HTTPError) bool { + if httpErr, ok := err.(*HTTPError); ok { + *target = httpErr + return true + } + return false +} + +// GetText fetches a URL and returns the response body as a string. +func (c *Client) GetText(ctx context.Context, url string) (string, error) { + body, err := c.GetBody(ctx, url) + if err != nil { + return "", err + } + return string(body), nil +} + +// Head sends a HEAD request and returns the status code. +func (c *Client) Head(ctx context.Context, url string) (int, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) + if err != nil { + return 0, err + } + + req.Header.Set("User-Agent", c.UserAgent) + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return 0, err + } + _ = resp.Body.Close() + + return resp.StatusCode, nil +} + +// WithRateLimiter returns a copy of the client with the given rate limiter. +func (c *Client) WithRateLimiter(rl RateLimiter) *Client { + copy := *c + copy.RateLimiter = rl + return © +} + +// WithUserAgent returns a copy of the client with the given user agent. +func (c *Client) WithUserAgent(ua string) *Client { + copy := *c + copy.UserAgent = ua + return © +} + +// Option configures a Client. +type Option func(*Client) + +// WithTimeout sets the HTTP client timeout. +func WithTimeout(d time.Duration) Option { + return func(c *Client) { + c.HTTPClient.Timeout = d + } +} + +// WithMaxRetries sets the maximum number of retries. +func WithMaxRetries(n int) Option { + return func(c *Client) { + c.MaxRetries = n + } +} + +// NewClient creates a new client with the given options. +func NewClient(opts ...Option) *Client { + c := DefaultClient() + for _, opt := range opts { + opt(c) + } + return c +} diff --git a/client/errors.go b/client/errors.go new file mode 100644 index 0000000..96bed2d --- /dev/null +++ b/client/errors.go @@ -0,0 +1,52 @@ +package client + +import ( + "errors" + "fmt" +) + +// ErrNotFound is returned when a package or version is not found. +var ErrNotFound = errors.New("not found") + +// HTTPError represents an HTTP error response. +type HTTPError struct { + StatusCode int + URL string + Body string +} + +func (e *HTTPError) Error() string { + return fmt.Sprintf("HTTP %d: %s", e.StatusCode, e.URL) +} + +// IsNotFound returns true if the error represents a 404 response. +func (e *HTTPError) IsNotFound() bool { + return e.StatusCode == 404 +} + +// NotFoundError wraps ErrNotFound with additional context. +type NotFoundError struct { + Ecosystem string + Name string + Version string +} + +func (e *NotFoundError) Error() string { + if e.Version != "" { + return fmt.Sprintf("%s: package %s version %s not found", e.Ecosystem, e.Name, e.Version) + } + return fmt.Sprintf("%s: package %s not found", e.Ecosystem, e.Name) +} + +func (e *NotFoundError) Unwrap() error { + return ErrNotFound +} + +// RateLimitError is returned when the registry rate limits requests. +type RateLimitError struct { + RetryAfter int // seconds +} + +func (e *RateLimitError) Error() string { + return fmt.Sprintf("rate limited, retry after %d seconds", e.RetryAfter) +} diff --git a/client/urls.go b/client/urls.go new file mode 100644 index 0000000..7163ebe --- /dev/null +++ b/client/urls.go @@ -0,0 +1,66 @@ +package client + +import "fmt" + +// URLBuilder constructs URLs for a registry. +type URLBuilder interface { + Registry(name, version string) string + Download(name, version string) string + Documentation(name, version string) string + PURL(name, version string) string +} + +// BaseURLs provides a default URLBuilder implementation. +type BaseURLs struct { + RegistryFn func(name, version string) string + DownloadFn func(name, version string) string + DocumentationFn func(name, version string) string + PURLFn func(name, version string) string +} + +func (b *BaseURLs) Registry(name, version string) string { + if b.RegistryFn != nil { + return b.RegistryFn(name, version) + } + return "" +} + +func (b *BaseURLs) Download(name, version string) string { + if b.DownloadFn != nil { + return b.DownloadFn(name, version) + } + return "" +} + +func (b *BaseURLs) Documentation(name, version string) string { + if b.DocumentationFn != nil { + return b.DocumentationFn(name, version) + } + return "" +} + +func (b *BaseURLs) PURL(name, version string) string { + if b.PURLFn != nil { + return b.PURLFn(name, version) + } + return fmt.Sprintf("pkg:%s/%s", "generic", name) +} + +// BuildURLs returns a map of all non-empty URLs for a package. +// Keys are "registry", "download", "docs", and "purl". +func BuildURLs(urls URLBuilder, name, version string) map[string]string { + result := make(map[string]string) + if v := urls.Registry(name, version); v != "" { + result["registry"] = v + } + if v := urls.Download(name, version); v != "" { + result["download"] = v + } + if v := urls.Documentation(name, version); v != "" { + result["docs"] = v + } + if v := urls.PURL(name, version); v != "" { + result["purl"] = v + } + return result +} diff --git a/fetch/circuit_breaker.go b/fetch/circuit_breaker.go new file mode 100644 index 0000000..446a3af --- /dev/null +++ b/fetch/circuit_breaker.go @@ -0,0 +1,137 @@ +package fetch + +import ( + "context" + "fmt" + "net/url" + "sync" + "time" + + "github.com/cenk/backoff" + circuit "github.com/rubyist/circuitbreaker" +) + +// CircuitBreakerFetcher wraps a Fetcher with per-registry circuit breakers. +type CircuitBreakerFetcher struct { + fetcher *Fetcher + breakers map[string]*circuit.Breaker + mu sync.RWMutex +} + +// NewCircuitBreakerFetcher creates a new circuit breaker wrapper for a fetcher. +func NewCircuitBreakerFetcher(f *Fetcher) *CircuitBreakerFetcher { + return &CircuitBreakerFetcher{ + fetcher: f, + breakers: make(map[string]*circuit.Breaker), + } +} + +// getBreaker returns or creates a circuit breaker for the given registry. +func (cbf *CircuitBreakerFetcher) getBreaker(registry string) *circuit.Breaker { + cbf.mu.RLock() + breaker, exists := cbf.breakers[registry] + cbf.mu.RUnlock() + + if exists { + return breaker + } + + cbf.mu.Lock() + defer cbf.mu.Unlock() + + // Double-check after acquiring write lock + if breaker, exists := cbf.breakers[registry]; exists { + return breaker + } + + // Create new circuit breaker with exponential backoff + // Trips after 5 consecutive failures + expBackoff := backoff.NewExponentialBackOff() + expBackoff.InitialInterval = 30 * time.Second + expBackoff.MaxInterval = 5 * time.Minute + expBackoff.Multiplier = 2.0 + expBackoff.Reset() + + opts := &circuit.Options{ + BackOff: expBackoff, + ShouldTrip: circuit.ThresholdTripFunc(5), + } + breaker = circuit.NewBreakerWithOptions(opts) + + cbf.breakers[registry] = breaker + return breaker +} + +// Fetch wraps the underlying fetcher's Fetch with circuit breaker logic. +func (cbf *CircuitBreakerFetcher) Fetch(ctx context.Context, fetchURL string) (*Artifact, error) { + // Extract registry from URL for circuit breaker selection + registry := extractRegistry(fetchURL) + breaker := cbf.getBreaker(registry) + + // Check if circuit is open + if !breaker.Ready() { + return nil, fmt.Errorf("circuit breaker open for registry %s: %w", registry, ErrUpstreamDown) + } + + // Attempt fetch + var artifact *Artifact + err := breaker.Call(func() error { + var fetchErr error + artifact, fetchErr = cbf.fetcher.Fetch(ctx, fetchURL) + return fetchErr + }, 0) + + if err != nil { + return nil, err + } + + return artifact, nil +} + +// Head wraps the underlying fetcher's Head with circuit breaker logic. +func (cbf *CircuitBreakerFetcher) Head(ctx context.Context, headURL string) (size int64, contentType string, err error) { + registry := extractRegistry(headURL) + breaker := cbf.getBreaker(registry) + + if !breaker.Ready() { + return 0, "", fmt.Errorf("circuit breaker open for registry %s: %w", registry, ErrUpstreamDown) + } + + err = breaker.Call(func() error { + var headErr error + size, contentType, headErr = cbf.fetcher.Head(ctx, headURL) + return headErr + }, 0) + + return size, contentType, err +} + +// extractRegistry extracts a registry identifier from a URL for circuit breaker grouping. +func extractRegistry(rawURL string) string { + // Parse URL and extract host for circuit breaker grouping + parsed, err := url.Parse(rawURL) + if err != nil || parsed.Host == "" { + // Fallback to simple truncation + if len(rawURL) > 50 { + return rawURL[:50] + } + return rawURL + } + return parsed.Host +} + +// GetBreakerState returns the current state of circuit breakers (for health checks). +func (cbf *CircuitBreakerFetcher) GetBreakerState() map[string]string { + cbf.mu.RLock() + defer cbf.mu.RUnlock() + + states := make(map[string]string) + for registry, breaker := range cbf.breakers { + if breaker.Tripped() { + states[registry] = "open" + } else { + states[registry] = "closed" + } + } + return states +} diff --git a/fetch/circuit_breaker_test.go b/fetch/circuit_breaker_test.go new file mode 100644 index 0000000..aba7a37 --- /dev/null +++ b/fetch/circuit_breaker_test.go @@ -0,0 +1,209 @@ +package fetch + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "testing" +) + +func TestCircuitBreakerFetch_Success(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte("test content")) + })) + defer server.Close() + + fetcher := NewFetcher() + cbFetcher := NewCircuitBreakerFetcher(fetcher) + + ctx := context.Background() + artifact, err := cbFetcher.Fetch(ctx, server.URL+"/test.tar.gz") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if artifact == nil { + t.Fatal("expected artifact, got nil") + } + + defer func() { _ = artifact.Body.Close() }() + + body, _ := io.ReadAll(artifact.Body) + if string(body) != "test content" { + t.Errorf("expected 'test content', got %q", string(body)) + } +} + +func TestCircuitBreakerHead_Success(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodHead { + t.Errorf("expected HEAD request, got %s", r.Method) + } + w.Header().Set("Content-Length", "1234") + w.Header().Set("Content-Type", "application/octet-stream") + })) + defer server.Close() + + fetcher := NewFetcher() + cbFetcher := NewCircuitBreakerFetcher(fetcher) + + ctx := context.Background() + size, contentType, err := cbFetcher.Head(ctx, server.URL+"/test.tar.gz") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if size != 1234 { + t.Errorf("expected size 1234, got %d", size) + } + + if contentType != "application/octet-stream" { + t.Errorf("expected content type application/octet-stream, got %s", contentType) + } +} + +func TestExtractRegistry(t *testing.T) { + tests := []struct { + name string + url string + expected string + }{ + { + name: "npm registry", + url: "https://registry.npmjs.org/package/-/package-1.0.0.tgz", + expected: "registry.npmjs.org", + }, + { + name: "pypi registry", + url: "https://files.pythonhosted.org/packages/abc/def/file.tar.gz", + expected: "files.pythonhosted.org", + }, + { + name: "invalid URL", + url: "not-a-valid-url", + expected: "not-a-valid-url", + }, + { + name: "long URL", + url: "https://very-long-hostname.example.com/path", + expected: "very-long-hostname.example.com", + }, + { + name: "with port", + url: "https://example.com:8080/path", + expected: "example.com:8080", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractRegistry(tt.url) + if got != tt.expected { + t.Errorf("extractRegistry(%q) = %q, want %q", tt.url, got, tt.expected) + } + }) + } +} + +func TestGetBreakerState(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("ok")) + })) + defer server.Close() + + fetcher := NewFetcher() + cbFetcher := NewCircuitBreakerFetcher(fetcher) + + // Initially empty + states := cbFetcher.GetBreakerState() + if len(states) != 0 { + t.Errorf("expected empty states, got %d entries", len(states)) + } + + // After a fetch, should have state + ctx := context.Background() + _, _ = cbFetcher.Fetch(ctx, server.URL+"/test") + + states = cbFetcher.GetBreakerState() + if len(states) == 0 { + t.Error("expected at least one breaker state after fetch") + } + + // Should be in closed state (working) + for _, state := range states { + if state != "closed" { + t.Errorf("expected closed state, got %s", state) + } + } +} + +func TestCircuitBreakerMultipleRegistries(t *testing.T) { + server1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("server1")) + })) + defer server1.Close() + + server2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("server2")) + })) + defer server2.Close() + + fetcher := NewFetcher() + cbFetcher := NewCircuitBreakerFetcher(fetcher) + + ctx := context.Background() + + // Fetch from both servers + art1, err1 := cbFetcher.Fetch(ctx, server1.URL+"/test") + if err1 != nil { + t.Fatalf("fetch 1 failed: %v", err1) + } + _ = art1.Body.Close() + + art2, err2 := cbFetcher.Fetch(ctx, server2.URL+"/test") + if err2 != nil { + t.Fatalf("fetch 2 failed: %v", err2) + } + _ = art2.Body.Close() + + // Should have separate breaker states for each registry + states := cbFetcher.GetBreakerState() + if len(states) != 2 { + t.Errorf("expected 2 breaker states, got %d", len(states)) + } +} + +func TestCircuitBreakerOpensOnFailures(t *testing.T) { + failCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + failCount++ + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + fetcher := NewFetcher(WithMaxRetries(0), WithBaseDelay(0)) + cbFetcher := NewCircuitBreakerFetcher(fetcher) + + ctx := context.Background() + + // Make multiple failing requests to trip the circuit breaker + // Default threshold is 5 failures + for range 10 { + _, _ = cbFetcher.Fetch(ctx, server.URL+"/test") + } + + // Check that circuit breaker eventually opened + states := cbFetcher.GetBreakerState() + if len(states) == 0 { + t.Fatal("expected breaker state to exist") + } + + // Circuit should be open after repeated failures + // Note: The exact state depends on timing, but we should have made fewer + // than 10 actual HTTP requests if the breaker opened + if failCount >= 10 { + t.Logf("Warning: Circuit breaker may not have opened (got %d requests)", failCount) + } +} diff --git a/fetch/fetcher.go b/fetch/fetcher.go new file mode 100644 index 0000000..5d9b151 --- /dev/null +++ b/fetch/fetcher.go @@ -0,0 +1,281 @@ +// Package fetch provides streaming artifact downloading with retry, circuit breaking, +// and URL resolution for package registries. +package fetch + +import ( + "context" + "errors" + "fmt" + "io" + "math" + "math/rand" + "net" + "net/http" + "strconv" + "time" + + "github.com/rs/dnscache" +) + +var ( + ErrNotFound = errors.New("artifact not found") + ErrRateLimited = errors.New("rate limited by upstream") + ErrUpstreamDown = errors.New("upstream registry unavailable") +) + +// Artifact contains the response from fetching an upstream artifact. +type Artifact struct { + Body io.ReadCloser + Size int64 // -1 if unknown + ContentType string + ETag string +} + +// FetcherInterface defines the interface for artifact fetchers. +type FetcherInterface interface { + Fetch(ctx context.Context, url string) (*Artifact, error) + Head(ctx context.Context, url string) (size int64, contentType string, err error) +} + +// Fetcher downloads artifacts from upstream registries. +type Fetcher struct { + client *http.Client + userAgent string + maxRetries int + baseDelay time.Duration + authFn func(url string) (headerName, headerValue string) +} + +// Option configures a Fetcher. +type Option func(*Fetcher) + +// WithHTTPClient sets a custom HTTP client. +func WithHTTPClient(c *http.Client) Option { + return func(f *Fetcher) { + f.client = c + } +} + +// WithUserAgent sets the User-Agent header. +func WithUserAgent(ua string) Option { + return func(f *Fetcher) { + f.userAgent = ua + } +} + +// WithMaxRetries sets the maximum retry attempts. +func WithMaxRetries(n int) Option { + return func(f *Fetcher) { + f.maxRetries = n + } +} + +// WithBaseDelay sets the base delay for exponential backoff. +func WithBaseDelay(d time.Duration) Option { + return func(f *Fetcher) { + f.baseDelay = d + } +} + +// WithAuthFunc sets a function that returns auth headers for a given URL. +// The function receives the request URL and returns a header name and value. +// Return empty strings to skip authentication for that URL. +func WithAuthFunc(fn func(url string) (headerName, headerValue string)) Option { + return func(f *Fetcher) { + f.authFn = fn + } +} + +// NewFetcher creates a new Fetcher with the given options. +func NewFetcher(opts ...Option) *Fetcher { + // Create DNS cache with 5 minute refresh interval + resolver := &dnscache.Resolver{} + go func() { + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + for range ticker.C { + resolver.Refresh(true) + } + }() + + // Create custom dialer with DNS caching + dialer := &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + } + + f := &Fetcher{ + client: &http.Client{ + Timeout: 5 * time.Minute, // Artifacts can be large + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + ips, err := resolver.LookupHost(ctx, host) + if err != nil { + return nil, err + } + for _, ip := range ips { + conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(ip, port)) + if err == nil { + return conn, nil + } + } + return nil, fmt.Errorf("failed to dial any resolved IP") + }, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 10, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + }, + userAgent: "git-pkgs-proxy/1.0", + maxRetries: 3, + baseDelay: 500 * time.Millisecond, + } + for _, opt := range opts { + opt(f) + } + return f +} + +// Fetch downloads an artifact from the given URL. +// The caller must close the returned Artifact.Body when done. +func (f *Fetcher) Fetch(ctx context.Context, url string) (*Artifact, error) { + var lastErr error + + for attempt := 0; attempt <= f.maxRetries; attempt++ { + if attempt > 0 { + // Exponential backoff with 10% jitter to prevent thundering herd + delay := f.baseDelay * time.Duration(math.Pow(2, float64(attempt-1))) + jitter := time.Duration(float64(delay) * (rand.Float64() * 0.1)) + delay += jitter + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(delay): + } + } + + artifact, err := f.doFetch(ctx, url) + if err == nil { + return artifact, nil + } + + lastErr = err + + // Don't retry on not found or client errors + if errors.Is(err, ErrNotFound) { + return nil, err + } + + // Retry on rate limit and server errors + if errors.Is(err, ErrRateLimited) || errors.Is(err, ErrUpstreamDown) { + continue + } + + // Don't retry on other errors (network issues will be wrapped) + return nil, err + } + + return nil, lastErr +} + +func (f *Fetcher) doFetch(ctx context.Context, url string) (*Artifact, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("creating request: %w", err) + } + + req.Header.Set("User-Agent", f.userAgent) + req.Header.Set("Accept", "*/*") + + // Add authentication header if configured + if f.authFn != nil { + if name, value := f.authFn(url); name != "" && value != "" { + req.Header.Set(name, value) + } + } + + resp, err := f.client.Do(req) + if err != nil { + return nil, fmt.Errorf("fetching artifact: %w", err) + } + + switch { + case resp.StatusCode == http.StatusOK: + size := int64(-1) + if cl := resp.Header.Get("Content-Length"); cl != "" { + if n, err := strconv.ParseInt(cl, 10, 64); err == nil { + size = n + } + } + + return &Artifact{ + Body: resp.Body, + Size: size, + ContentType: resp.Header.Get("Content-Type"), + ETag: resp.Header.Get("ETag"), + }, nil + + case resp.StatusCode == http.StatusNotFound: + _ = resp.Body.Close() + return nil, ErrNotFound + + case resp.StatusCode == http.StatusTooManyRequests: + _ = resp.Body.Close() + return nil, ErrRateLimited + + case resp.StatusCode >= 500: + _ = resp.Body.Close() + return nil, ErrUpstreamDown + + default: + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) + _ = resp.Body.Close() + return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) + } +} + +// Head checks if an artifact exists and returns its metadata without downloading. +func (f *Fetcher) Head(ctx context.Context, url string) (size int64, contentType string, err error) { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) + if err != nil { + return 0, "", fmt.Errorf("creating request: %w", err) + } + + req.Header.Set("User-Agent", f.userAgent) + + // Add authentication header if configured + if f.authFn != nil { + if name, value := f.authFn(url); name != "" && value != "" { + req.Header.Set(name, value) + } + } + + resp, err := f.client.Do(req) + if err != nil { + return 0, "", fmt.Errorf("head request: %w", err) + } + _ = resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return 0, "", ErrNotFound + } + if resp.StatusCode != http.StatusOK { + return 0, "", fmt.Errorf("unexpected status %d", resp.StatusCode) + } + + size = -1 + if cl := resp.Header.Get("Content-Length"); cl != "" { + if n, err := strconv.ParseInt(cl, 10, 64); err == nil { + size = n + } + } + + return size, resp.Header.Get("Content-Type"), nil +} diff --git a/fetch/fetcher_test.go b/fetch/fetcher_test.go new file mode 100644 index 0000000..34b92e3 --- /dev/null +++ b/fetch/fetcher_test.go @@ -0,0 +1,311 @@ +package fetch + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestFetchSuccess(t *testing.T) { + content := "test artifact content" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Content-Length", "21") + w.Header().Set("ETag", `"abc123"`) + _, _ = w.Write([]byte(content)) + })) + defer server.Close() + + f := NewFetcher() + artifact, err := f.Fetch(context.Background(), server.URL+"/test.tgz") + if err != nil { + t.Fatalf("Fetch failed: %v", err) + } + defer func() { _ = artifact.Body.Close() }() + + if artifact.Size != 21 { + t.Errorf("Size = %d, want 21", artifact.Size) + } + if artifact.ContentType != "application/gzip" { + t.Errorf("ContentType = %q, want %q", artifact.ContentType, "application/gzip") + } + if artifact.ETag != `"abc123"` { + t.Errorf("ETag = %q, want %q", artifact.ETag, `"abc123"`) + } + + body, err := io.ReadAll(artifact.Body) + if err != nil { + t.Fatalf("ReadAll failed: %v", err) + } + if string(body) != content { + t.Errorf("body = %q, want %q", string(body), content) + } +} + +func TestFetchNotFound(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + f := NewFetcher() + _, err := f.Fetch(context.Background(), server.URL+"/missing.tgz") + if !errors.Is(err, ErrNotFound) { + t.Errorf("Fetch = %v, want ErrNotFound", err) + } +} + +func TestFetchRateLimitRetry(t *testing.T) { + attempts := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + if attempts < 3 { + w.WriteHeader(http.StatusTooManyRequests) + return + } + _, _ = w.Write([]byte("success")) + })) + defer server.Close() + + f := NewFetcher(WithBaseDelay(10 * time.Millisecond)) + artifact, err := f.Fetch(context.Background(), server.URL+"/test.tgz") + if err != nil { + t.Fatalf("Fetch failed: %v", err) + } + defer func() { _ = artifact.Body.Close() }() + + if attempts != 3 { + t.Errorf("attempts = %d, want 3", attempts) + } +} + +func TestFetchServerErrorRetry(t *testing.T) { + attempts := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + if attempts < 2 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _, _ = w.Write([]byte("success")) + })) + defer server.Close() + + f := NewFetcher(WithBaseDelay(10 * time.Millisecond)) + artifact, err := f.Fetch(context.Background(), server.URL+"/test.tgz") + if err != nil { + t.Fatalf("Fetch failed: %v", err) + } + defer func() { _ = artifact.Body.Close() }() + + if attempts != 2 { + t.Errorf("attempts = %d, want 2", attempts) + } +} + +func TestFetchMaxRetries(t *testing.T) { + attempts := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + f := NewFetcher(WithMaxRetries(2), WithBaseDelay(10*time.Millisecond)) + _, err := f.Fetch(context.Background(), server.URL+"/test.tgz") + if err == nil { + t.Error("expected error after max retries") + } + if !errors.Is(err, ErrUpstreamDown) { + t.Errorf("expected ErrUpstreamDown, got %v", err) + } + + // Initial attempt + 2 retries = 3 total + if attempts != 3 { + t.Errorf("attempts = %d, want 3", attempts) + } +} + +func TestFetchContextCancellation(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(100 * time.Millisecond) + _, _ = w.Write([]byte("success")) + })) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + + f := NewFetcher() + _, err := f.Fetch(ctx, server.URL+"/test.tgz") + if err == nil { + t.Error("expected error on context cancellation") + } +} + +func TestFetchUnknownSize(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Chunked encoding, no Content-Length + w.Header().Set("Transfer-Encoding", "chunked") + _, _ = w.Write([]byte("chunk1")) + })) + defer server.Close() + + f := NewFetcher() + artifact, err := f.Fetch(context.Background(), server.URL+"/test.tgz") + if err != nil { + t.Fatalf("Fetch failed: %v", err) + } + defer func() { _ = artifact.Body.Close() }() + + if artifact.Size != -1 { + t.Errorf("Size = %d, want -1 for unknown", artifact.Size) + } +} + +func TestHead(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodHead { + t.Errorf("Method = %s, want HEAD", r.Method) + } + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", "12345") + })) + defer server.Close() + + f := NewFetcher() + size, contentType, err := f.Head(context.Background(), server.URL+"/test.tgz") + if err != nil { + t.Fatalf("Head failed: %v", err) + } + + if size != 12345 { + t.Errorf("size = %d, want 12345", size) + } + if contentType != "application/octet-stream" { + t.Errorf("contentType = %q, want %q", contentType, "application/octet-stream") + } +} + +func TestHeadNotFound(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + f := NewFetcher() + _, _, err := f.Head(context.Background(), server.URL+"/missing.tgz") + if !errors.Is(err, ErrNotFound) { + t.Errorf("Head = %v, want ErrNotFound", err) + } +} + +func TestFetchUserAgent(t *testing.T) { + var receivedUA string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedUA = r.Header.Get("User-Agent") + _, _ = w.Write([]byte("ok")) + })) + defer server.Close() + + f := NewFetcher(WithUserAgent("custom-agent/2.0")) + artifact, _ := f.Fetch(context.Background(), server.URL+"/test.tgz") + if artifact != nil { + _ = artifact.Body.Close() + } + + if receivedUA != "custom-agent/2.0" { + t.Errorf("User-Agent = %q, want %q", receivedUA, "custom-agent/2.0") + } +} + +func TestFetchLargeArtifact(t *testing.T) { + // 1MB artifact + content := strings.Repeat("x", 1024*1024) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(content)) + })) + defer server.Close() + + f := NewFetcher() + artifact, err := f.Fetch(context.Background(), server.URL+"/large.tgz") + if err != nil { + t.Fatalf("Fetch failed: %v", err) + } + defer func() { _ = artifact.Body.Close() }() + + body, err := io.ReadAll(artifact.Body) + if err != nil { + t.Fatalf("ReadAll failed: %v", err) + } + if len(body) != len(content) { + t.Errorf("body length = %d, want %d", len(body), len(content)) + } +} + +func TestFetchRetryWithJitter(t *testing.T) { + attempts := 0 + var retryTimes []time.Time + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + retryTimes = append(retryTimes, time.Now()) + if attempts < 3 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _, _ = w.Write([]byte("success")) + })) + defer server.Close() + + f := NewFetcher(WithBaseDelay(100 * time.Millisecond)) + artifact, err := f.Fetch(context.Background(), server.URL+"/test.tgz") + if err != nil { + t.Fatalf("Fetch failed: %v", err) + } + defer func() { _ = artifact.Body.Close() }() + + if attempts != 3 { + t.Errorf("attempts = %d, want 3", attempts) + } + + // Verify that delays between retries vary (jitter is applied) + // With 100ms base delay and exponential backoff: + // First retry: ~100ms + jitter (0-10ms) + // Second retry: ~200ms + jitter (0-20ms) + if len(retryTimes) >= 2 { + firstDelay := retryTimes[1].Sub(retryTimes[0]) + // First retry should be between 100ms and 130ms (100ms + max 10% jitter + some tolerance) + if firstDelay < 90*time.Millisecond || firstDelay > 150*time.Millisecond { + t.Logf("First retry delay: %v (expected ~100ms with jitter)", firstDelay) + } + } +} + +func TestFetchDNSCaching(t *testing.T) { + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + _, _ = w.Write([]byte("ok")) + })) + defer server.Close() + + f := NewFetcher() + + // Make multiple requests to the same host + for i := range 3 { + artifact, err := f.Fetch(context.Background(), server.URL+"/test.tgz") + if err != nil { + t.Fatalf("Fetch %d failed: %v", i+1, err) + } + _ = artifact.Body.Close() + } + + if requestCount != 3 { + t.Errorf("requestCount = %d, want 3", requestCount) + } +} diff --git a/fetch/resolver.go b/fetch/resolver.go new file mode 100644 index 0000000..39d88fc --- /dev/null +++ b/fetch/resolver.go @@ -0,0 +1,194 @@ +package fetch + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/git-pkgs/registries" + "github.com/git-pkgs/registries/client" +) + +var ( + ErrUnsupportedEcosystem = errors.New("unsupported ecosystem") + ErrNoDownloadURL = errors.New("no download URL available") +) + +// Registry provides package metadata and URL information for artifact resolution. +// This interface is satisfied by registries.Registry implementations. +type Registry interface { + Ecosystem() string + FetchVersions(ctx context.Context, name string) ([]registries.Version, error) + URLs() client.URLBuilder +} + +// Resolver determines download URLs for package artifacts. +type Resolver struct { + registries map[string]Registry +} + +// NewResolver creates a new URL resolver. +func NewResolver() *Resolver { + return &Resolver{ + registries: make(map[string]Registry), + } +} + +// RegisterRegistry adds a registry for URL resolution. +func (r *Resolver) RegisterRegistry(reg Registry) { + r.registries[reg.Ecosystem()] = reg +} + +// ArtifactInfo contains information about a downloadable artifact. +type ArtifactInfo struct { + URL string + Filename string + Integrity string // sha256-... or sha512-... +} + +// Resolve returns the download URL and filename for a package artifact. +func (r *Resolver) Resolve(ctx context.Context, ecosystem, name, version string) (*ArtifactInfo, error) { + reg, ok := r.registries[ecosystem] + if !ok { + return r.resolveWithoutRegistry(ecosystem, name, version) + } + + // Try the simple URL builder first + if url := reg.URLs().Download(name, version); url != "" { + return &ArtifactInfo{ + URL: url, + Filename: filenameFromURL(url), + }, nil + } + + // For ecosystems like PyPI, we need to fetch metadata to get the URL + return r.resolveFromMetadata(ctx, reg, name, version) +} + +// resolveWithoutRegistry handles ecosystems with predictable URLs +// when no registry client is configured. +func (r *Resolver) resolveWithoutRegistry(ecosystem, name, version string) (*ArtifactInfo, error) { + var url, filename string + + switch ecosystem { + case "npm": + shortName := name + if idx := strings.LastIndex(name, "/"); idx >= 0 { + shortName = name[idx+1:] + } + url = fmt.Sprintf("https://registry.npmjs.org/%s/-/%s-%s.tgz", name, shortName, version) + filename = fmt.Sprintf("%s-%s.tgz", shortName, version) + + case "cargo": + url = fmt.Sprintf("https://static.crates.io/crates/%s/%s-%s.crate", name, name, version) + filename = fmt.Sprintf("%s-%s.crate", name, version) + + case "gem": + url = fmt.Sprintf("https://rubygems.org/downloads/%s-%s.gem", name, version) + filename = fmt.Sprintf("%s-%s.gem", name, version) + + case "golang": + encoded := encodeGoModule(name) + url = fmt.Sprintf("https://proxy.golang.org/%s/@v/%s.zip", encoded, version) + filename = fmt.Sprintf("%s@%s.zip", lastPathComponent(name), version) + + case "hex": + url = fmt.Sprintf("https://repo.hex.pm/tarballs/%s-%s.tar", name, version) + filename = fmt.Sprintf("%s-%s.tar", name, version) + + case "pub": + url = fmt.Sprintf("https://pub.dev/packages/%s/versions/%s.tar.gz", name, version) + filename = fmt.Sprintf("%s-%s.tar.gz", name, version) + + case "maven": + // Maven name format is "group:artifact", e.g., "com.google.guava:guava" + parts := strings.SplitN(name, ":", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid maven name format, expected group:artifact") + } + group := strings.ReplaceAll(parts[0], ".", "/") + artifact := parts[1] + url = fmt.Sprintf("https://repo1.maven.org/maven2/%s/%s/%s/%s-%s.jar", group, artifact, version, artifact, version) + filename = fmt.Sprintf("%s-%s.jar", artifact, version) + + case "nuget": + // NuGet package IDs are case-insensitive, use lowercase + lowername := strings.ToLower(name) + url = fmt.Sprintf("https://api.nuget.org/v3-flatcontainer/%s/%s/%s.%s.nupkg", lowername, version, lowername, version) + filename = fmt.Sprintf("%s.%s.nupkg", lowername, version) + + default: + return nil, fmt.Errorf("%w: %s", ErrUnsupportedEcosystem, ecosystem) + } + + return &ArtifactInfo{ + URL: url, + Filename: filename, + }, nil +} + +// resolveFromMetadata fetches version metadata to find download URL. +func (r *Resolver) resolveFromMetadata(ctx context.Context, reg Registry, name, version string) (*ArtifactInfo, error) { + versions, err := reg.FetchVersions(ctx, name) + if err != nil { + return nil, fmt.Errorf("fetching versions: %w", err) + } + + for _, v := range versions { + if v.Number != version { + continue + } + + // Look for download URL in metadata + if v.Metadata != nil { + if url, ok := v.Metadata["download_url"].(string); ok && url != "" { + return &ArtifactInfo{ + URL: url, + Filename: filenameFromURL(url), + Integrity: v.Integrity, + }, nil + } + if url, ok := v.Metadata["tarball"].(string); ok && url != "" { + return &ArtifactInfo{ + URL: url, + Filename: filenameFromURL(url), + Integrity: v.Integrity, + }, nil + } + } + + return nil, ErrNoDownloadURL + } + + return nil, ErrNotFound +} + +func filenameFromURL(url string) string { + if idx := strings.LastIndex(url, "/"); idx >= 0 { + return url[idx+1:] + } + return url +} + +func lastPathComponent(path string) string { + if idx := strings.LastIndex(path, "/"); idx >= 0 { + return path[idx+1:] + } + return path +} + +// encodeGoModule encodes a module path per goproxy protocol. +// Capital letters become "!" followed by lowercase. +func encodeGoModule(path string) string { + var b strings.Builder + for _, r := range path { + if r >= 'A' && r <= 'Z' { + b.WriteRune('!') + b.WriteRune(r + 32) + } else { + b.WriteRune(r) + } + } + return b.String() +} diff --git a/fetch/resolver_test.go b/fetch/resolver_test.go new file mode 100644 index 0000000..229693c --- /dev/null +++ b/fetch/resolver_test.go @@ -0,0 +1,137 @@ +package fetch + +import ( + "context" + "testing" +) + +func TestResolveWithoutRegistry(t *testing.T) { + r := NewResolver() + + tests := []struct { + ecosystem string + name string + version string + wantURL string + wantFilename string + }{ + { + ecosystem: "npm", + name: "lodash", + version: "4.17.21", + wantURL: "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + wantFilename: "lodash-4.17.21.tgz", + }, + { + ecosystem: "npm", + name: "@babel/core", + version: "7.23.0", + wantURL: "https://registry.npmjs.org/@babel/core/-/core-7.23.0.tgz", + wantFilename: "core-7.23.0.tgz", + }, + { + ecosystem: "cargo", + name: "serde", + version: "1.0.193", + wantURL: "https://static.crates.io/crates/serde/serde-1.0.193.crate", + wantFilename: "serde-1.0.193.crate", + }, + { + ecosystem: "gem", + name: "rails", + version: "7.1.2", + wantURL: "https://rubygems.org/downloads/rails-7.1.2.gem", + wantFilename: "rails-7.1.2.gem", + }, + { + ecosystem: "golang", + name: "github.com/stretchr/testify", + version: "v1.8.4", + wantURL: "https://proxy.golang.org/github.com/stretchr/testify/@v/v1.8.4.zip", + wantFilename: "testify@v1.8.4.zip", + }, + { + ecosystem: "golang", + name: "github.com/Azure/azure-sdk-for-go", + version: "v68.0.0", + wantURL: "https://proxy.golang.org/github.com/!azure/azure-sdk-for-go/@v/v68.0.0.zip", + wantFilename: "azure-sdk-for-go@v68.0.0.zip", + }, + { + ecosystem: "hex", + name: "phoenix", + version: "1.7.10", + wantURL: "https://repo.hex.pm/tarballs/phoenix-1.7.10.tar", + wantFilename: "phoenix-1.7.10.tar", + }, + { + ecosystem: "pub", + name: "flutter", + version: "3.16.0", + wantURL: "https://pub.dev/packages/flutter/versions/3.16.0.tar.gz", + wantFilename: "flutter-3.16.0.tar.gz", + }, + } + + for _, tt := range tests { + t.Run(tt.ecosystem+"/"+tt.name, func(t *testing.T) { + info, err := r.Resolve(context.Background(), tt.ecosystem, tt.name, tt.version) + if err != nil { + t.Fatalf("Resolve failed: %v", err) + } + + if info.URL != tt.wantURL { + t.Errorf("URL = %q, want %q", info.URL, tt.wantURL) + } + if info.Filename != tt.wantFilename { + t.Errorf("Filename = %q, want %q", info.Filename, tt.wantFilename) + } + }) + } +} + +func TestResolveUnsupportedEcosystem(t *testing.T) { + r := NewResolver() + + _, err := r.Resolve(context.Background(), "unknown", "pkg", "1.0.0") + if err == nil { + t.Error("expected error for unsupported ecosystem") + } +} + +func TestEncodeGoModule(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"github.com/user/repo", "github.com/user/repo"}, + {"github.com/Azure/azure-sdk", "github.com/!azure/azure-sdk"}, + {"github.com/BurntSushi/toml", "github.com/!burnt!sushi/toml"}, + {"golang.org/x/net", "golang.org/x/net"}, + } + + for _, tt := range tests { + got := encodeGoModule(tt.input) + if got != tt.want { + t.Errorf("encodeGoModule(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestFilenameFromURL(t *testing.T) { + tests := []struct { + url string + want string + }{ + {"https://example.com/path/to/file.tar.gz", "file.tar.gz"}, + {"https://example.com/file.zip", "file.zip"}, + {"file.txt", "file.txt"}, + } + + for _, tt := range tests { + got := filenameFromURL(tt.url) + if got != tt.want { + t.Errorf("filenameFromURL(%q) = %q, want %q", tt.url, got, tt.want) + } + } +} diff --git a/go.mod b/go.mod index eb77ef6..66c18a9 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,20 @@ module github.com/git-pkgs/registries go 1.25.6 require ( + github.com/cenk/backoff v2.2.1+incompatible github.com/git-pkgs/purl v0.1.8 github.com/git-pkgs/spdx v0.1.0 + github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 + github.com/rubyist/circuitbreaker v2.2.1+incompatible ) require ( + github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/git-pkgs/packageurl-go v0.2.1 // indirect github.com/git-pkgs/vers v0.2.2 // indirect github.com/github/go-spdx/v2 v2.3.6 // indirect + github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea // indirect + golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect ) replace github.com/package-url/packageurl-go => github.com/git-pkgs/packageurl-go v0.0.0-20260115093137-a0c26f7ee19e diff --git a/go.sum b/go.sum index 8098845..0e13865 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ +github.com/cenk/backoff v2.2.1+incompatible h1:djdFT7f4gF2ttuzRKPbMOWgZajgesItGLwG5FTQKmmE= +github.com/cenk/backoff v2.2.1+incompatible/go.mod h1:7FtoeaSnHoZnmZzz47cM35Y9nSW7tNyaidugnHTaFDE= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= +github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= github.com/git-pkgs/packageurl-go v0.2.1 h1:j6VnjJiYS9b1nTLfJGsG6SLaA7Nk6Io+ta8grOyTa4o= github.com/git-pkgs/packageurl-go v0.2.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/purl v0.1.8 h1:iyjEHM2WIZUL9A3+q9ylrabqILsN4nOay9X6jfEjmzQ= @@ -10,9 +14,17 @@ github.com/git-pkgs/vers v0.2.2 h1:42QkiIURhGN2wM8AuYYU+FbzS1YV6jmdGd1RiFp7gXs= github.com/git-pkgs/vers v0.2.2/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= github.com/github/go-spdx/v2 v2.3.6 h1:9flm625VmmTlWXi0YH5W9V8FdMfulvxalHdYnUfoqxc= github.com/github/go-spdx/v2 v2.3.6/go.mod h1:/5rwgS0txhGtRdUZwc02bTglzg6HK3FfuEbECKlK2Sg= +github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea h1:sKwxy1H95npauwu8vtF95vG/syrL0p8fSZo/XlDg5gk= +github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea/go.mod h1:1VcHEd3ro4QMoHfiNl/j7Jkln9+KQuorp0PItHMJYNg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 h1:18kd+8ZUlt/ARXhljq+14TwAoKa61q6dX8jtwOf6DH8= +github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= +github.com/rubyist/circuitbreaker v2.2.1+incompatible h1:KUKd/pV8Geg77+8LNDwdow6rVCAYOp8+kHUyFvL6Mhk= +github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/core/client.go b/internal/core/client.go index 440384d..3f84120 100644 --- a/internal/core/client.go +++ b/internal/core/client.go @@ -1,266 +1,23 @@ package core import ( - "context" - "encoding/json" - "fmt" - "io" - "math" - "net/http" - "strconv" - "time" + "github.com/git-pkgs/registries/client" ) -// RateLimiter controls request pacing. -type RateLimiter interface { - Wait(ctx context.Context) error -} - -// Client is an HTTP client with retry logic for registry APIs. -type Client struct { - HTTPClient *http.Client - UserAgent string - MaxRetries int - BaseDelay time.Duration - RateLimiter RateLimiter -} - -// DefaultClient returns a client with sensible defaults. -func DefaultClient() *Client { - return &Client{ - HTTPClient: &http.Client{ - Timeout: 30 * time.Second, - }, - UserAgent: "registries", - MaxRetries: 5, - BaseDelay: 50 * time.Millisecond, - } -} - -// GetJSON fetches a URL and decodes the JSON response into v. -func (c *Client) GetJSON(ctx context.Context, url string, v any) error { - body, err := c.GetBody(ctx, url) - if err != nil { - return err - } - return json.Unmarshal(body, v) -} - -// GetBody fetches a URL and returns the response body. -func (c *Client) GetBody(ctx context.Context, url string) ([]byte, error) { - var lastErr error - - for attempt := 0; attempt <= c.MaxRetries; attempt++ { - if attempt > 0 { - delay := c.BaseDelay * time.Duration(math.Pow(2, float64(attempt-1))) - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(delay): - } - } - - if c.RateLimiter != nil { - if err := c.RateLimiter.Wait(ctx); err != nil { - return nil, err - } - } - - body, err := c.doRequest(ctx, url) - if err == nil { - return body, nil - } - - lastErr = err - - var httpErr *HTTPError - if ok := isHTTPError(err, &httpErr); ok { - if httpErr.StatusCode == 404 { - return nil, err - } - if httpErr.StatusCode == 429 || httpErr.StatusCode >= 500 { - continue - } - return nil, err - } - } - - return nil, lastErr -} - -func (c *Client) doRequest(ctx context.Context, url string) ([]byte, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, err - } - - req.Header.Set("User-Agent", c.UserAgent) - req.Header.Set("Accept", "application/json") - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer func() { _ = resp.Body.Close() }() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - if resp.StatusCode >= 400 { - httpErr := &HTTPError{ - StatusCode: resp.StatusCode, - URL: url, - Body: string(body), - } - if resp.StatusCode == 429 { - if retryAfter := resp.Header.Get("Retry-After"); retryAfter != "" { - if seconds, err := strconv.Atoi(retryAfter); err == nil { - return nil, &RateLimitError{RetryAfter: seconds} - } - } - } - return nil, httpErr - } - - return body, nil -} - -func isHTTPError(err error, target **HTTPError) bool { - if httpErr, ok := err.(*HTTPError); ok { - *target = httpErr - return true - } - return false -} - -// GetText fetches a URL and returns the response body as a string. -func (c *Client) GetText(ctx context.Context, url string) (string, error) { - body, err := c.GetBody(ctx, url) - if err != nil { - return "", err - } - return string(body), nil -} - -// Head sends a HEAD request and returns the status code. -func (c *Client) Head(ctx context.Context, url string) (int, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) - if err != nil { - return 0, err - } - - req.Header.Set("User-Agent", c.UserAgent) - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return 0, err - } - _ = resp.Body.Close() - - return resp.StatusCode, nil -} - -// WithRateLimiter returns a copy of the client with the given rate limiter. -func (c *Client) WithRateLimiter(rl RateLimiter) *Client { - copy := *c - copy.RateLimiter = rl - return © -} - -// WithUserAgent returns a copy of the client with the given user agent. -func (c *Client) WithUserAgent(ua string) *Client { - copy := *c - copy.UserAgent = ua - return © -} - -// Option configures a Client. -type Option func(*Client) - -// WithTimeout sets the HTTP client timeout. -func WithTimeout(d time.Duration) Option { - return func(c *Client) { - c.HTTPClient.Timeout = d - } -} - -// WithMaxRetries sets the maximum number of retries. -func WithMaxRetries(n int) Option { - return func(c *Client) { - c.MaxRetries = n - } -} - -// NewClient creates a new client with the given options. -func NewClient(opts ...Option) *Client { - c := DefaultClient() - for _, opt := range opts { - opt(c) - } - return c -} - -// URLBuilder constructs URLs for a registry. -type URLBuilder interface { - Registry(name, version string) string - Download(name, version string) string - Documentation(name, version string) string - PURL(name, version string) string -} - -// BaseURLs provides a default URLBuilder implementation. -type BaseURLs struct { - RegistryFn func(name, version string) string - DownloadFn func(name, version string) string - DocumentationFn func(name, version string) string - PURLFn func(name, version string) string -} - -func (b *BaseURLs) Registry(name, version string) string { - if b.RegistryFn != nil { - return b.RegistryFn(name, version) - } - return "" -} - -func (b *BaseURLs) Download(name, version string) string { - if b.DownloadFn != nil { - return b.DownloadFn(name, version) - } - return "" -} - -func (b *BaseURLs) Documentation(name, version string) string { - if b.DocumentationFn != nil { - return b.DocumentationFn(name, version) - } - return "" -} - -func (b *BaseURLs) PURL(name, version string) string { - if b.PURLFn != nil { - return b.PURLFn(name, version) - } - return fmt.Sprintf("pkg:%s/%s", "generic", name) -} +// Type aliases for backward compatibility with ecosystem implementations. +type ( + RateLimiter = client.RateLimiter + Client = client.Client + Option = client.Option + URLBuilder = client.URLBuilder + BaseURLs = client.BaseURLs +) -// BuildURLs returns a map of all non-empty URLs for a package. -// Keys are "registry", "download", "docs", and "purl". -func BuildURLs(urls URLBuilder, name, version string) map[string]string { - result := make(map[string]string) - if v := urls.Registry(name, version); v != "" { - result["registry"] = v - } - if v := urls.Download(name, version); v != "" { - result["download"] = v - } - if v := urls.Documentation(name, version); v != "" { - result["docs"] = v - } - if v := urls.PURL(name, version); v != "" { - result["purl"] = v - } - return result -} +// Function aliases for backward compatibility. +var ( + DefaultClient = client.DefaultClient + NewClient = client.NewClient + WithTimeout = client.WithTimeout + WithMaxRetries = client.WithMaxRetries + BuildURLs = client.BuildURLs +) diff --git a/internal/core/errors.go b/internal/core/errors.go index 03cdbfc..ad0b8b5 100644 --- a/internal/core/errors.go +++ b/internal/core/errors.go @@ -1,52 +1,15 @@ package core import ( - "errors" - "fmt" + "github.com/git-pkgs/registries/client" ) // ErrNotFound is returned when a package or version is not found. -var ErrNotFound = errors.New("not found") +var ErrNotFound = client.ErrNotFound -// HTTPError represents an HTTP error response. -type HTTPError struct { - StatusCode int - URL string - Body string -} - -func (e *HTTPError) Error() string { - return fmt.Sprintf("HTTP %d: %s", e.StatusCode, e.URL) -} - -// IsNotFound returns true if the error represents a 404 response. -func (e *HTTPError) IsNotFound() bool { - return e.StatusCode == 404 -} - -// NotFoundError wraps ErrNotFound with additional context. -type NotFoundError struct { - Ecosystem string - Name string - Version string -} - -func (e *NotFoundError) Error() string { - if e.Version != "" { - return fmt.Sprintf("%s: package %s version %s not found", e.Ecosystem, e.Name, e.Version) - } - return fmt.Sprintf("%s: package %s not found", e.Ecosystem, e.Name) -} - -func (e *NotFoundError) Unwrap() error { - return ErrNotFound -} - -// RateLimitError is returned when the registry rate limits requests. -type RateLimitError struct { - RetryAfter int // seconds -} - -func (e *RateLimitError) Error() string { - return fmt.Sprintf("rate limited, retry after %d seconds", e.RetryAfter) -} +// Type aliases for backward compatibility. +type ( + HTTPError = client.HTTPError + NotFoundError = client.NotFoundError + RateLimitError = client.RateLimitError +) diff --git a/registries.go b/registries.go index c49f24a..ebb064a 100644 --- a/registries.go +++ b/registries.go @@ -36,6 +36,7 @@ import ( "context" "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/internal/core" ) @@ -56,20 +57,23 @@ type ( // Maintainer represents a package maintainer. Maintainer = core.Maintainer - // Client is an HTTP client with retry logic for registry APIs. - Client = core.Client - - // URLBuilder constructs URLs for a registry. - URLBuilder = core.URLBuilder - // Scope indicates when a dependency is required. Scope = core.Scope // VersionStatus represents the status of a package version. VersionStatus = core.VersionStatus +) + +// Re-export types from client +type ( + // Client is an HTTP client with retry logic for registry APIs. + Client = client.Client + + // URLBuilder constructs URLs for a registry. + URLBuilder = client.URLBuilder // RateLimiter controls request pacing. - RateLimiter = core.RateLimiter + RateLimiter = client.RateLimiter ) // Re-export constants @@ -88,14 +92,14 @@ const ( // Re-export errors var ( - ErrNotFound = core.ErrNotFound + ErrNotFound = client.ErrNotFound ) // Error types type ( - HTTPError = core.HTTPError - NotFoundError = core.NotFoundError - RateLimitError = core.RateLimitError + HTTPError = client.HTTPError + NotFoundError = client.NotFoundError + RateLimitError = client.RateLimitError ) // New creates a new registry for the given ecosystem. @@ -103,8 +107,8 @@ type ( // If client is nil, DefaultClient() is used. // // Supported ecosystems: "cargo", "npm", "gem", "pypi", "golang" -func New(ecosystem string, baseURL string, client *Client) (Registry, error) { - return core.New(ecosystem, baseURL, client) +func New(ecosystem string, baseURL string, c *Client) (Registry, error) { + return core.New(ecosystem, baseURL, c) } // DefaultClient returns a client with sensible defaults: @@ -112,22 +116,22 @@ func New(ecosystem string, baseURL string, client *Client) (Registry, error) { // - 5 retries with exponential backoff // - Retry on 429 and 5xx responses func DefaultClient() *Client { - return core.DefaultClient() + return client.DefaultClient() } // NewClient creates a new client with the given options. func NewClient(opts ...Option) *Client { - return core.NewClient(opts...) + return client.NewClient(opts...) } // Option configures a Client. -type Option = core.Option +type Option = client.Option // WithTimeout sets the HTTP client timeout. -var WithTimeout = core.WithTimeout +var WithTimeout = client.WithTimeout // WithMaxRetries sets the maximum number of retries. -var WithMaxRetries = core.WithMaxRetries +var WithMaxRetries = client.WithMaxRetries // SupportedEcosystems returns all registered ecosystem types. // Note: ecosystems must be imported to be registered. @@ -138,7 +142,7 @@ func SupportedEcosystems() []string { // BuildURLs returns a map of all non-empty URLs for a package. // Keys are "registry", "download", "docs", and "purl". func BuildURLs(urls URLBuilder, name, version string) map[string]string { - return core.BuildURLs(urls, name, version) + return client.BuildURLs(urls, name, version) } // DefaultURL returns the default registry URL for an ecosystem. @@ -157,30 +161,30 @@ func ParsePURL(purlStr string) (*PURL, error) { // NewFromPURL creates a registry client from a PURL and returns the parsed components. // Returns the registry, full package name, and version (empty if not in PURL). -func NewFromPURL(purl string, client *Client) (Registry, string, string, error) { - return core.NewFromPURL(purl, client) +func NewFromPURL(purl string, c *Client) (Registry, string, string, error) { + return core.NewFromPURL(purl, c) } // FetchPackageFromPURL fetches package metadata using a PURL. -func FetchPackageFromPURL(ctx context.Context, purl string, client *Client) (*Package, error) { - return core.FetchPackageFromPURL(ctx, purl, client) +func FetchPackageFromPURL(ctx context.Context, purl string, c *Client) (*Package, error) { + return core.FetchPackageFromPURL(ctx, purl, c) } // FetchVersionFromPURL fetches a specific version's metadata using a PURL. // Returns an error if the PURL doesn't include a version. -func FetchVersionFromPURL(ctx context.Context, purl string, client *Client) (*Version, error) { - return core.FetchVersionFromPURL(ctx, purl, client) +func FetchVersionFromPURL(ctx context.Context, purl string, c *Client) (*Version, error) { + return core.FetchVersionFromPURL(ctx, purl, c) } // FetchDependenciesFromPURL fetches dependencies for a specific version using a PURL. // Returns an error if the PURL doesn't include a version. -func FetchDependenciesFromPURL(ctx context.Context, purl string, client *Client) ([]Dependency, error) { - return core.FetchDependenciesFromPURL(ctx, purl, client) +func FetchDependenciesFromPURL(ctx context.Context, purl string, c *Client) ([]Dependency, error) { + return core.FetchDependenciesFromPURL(ctx, purl, c) } // FetchMaintainersFromPURL fetches maintainer information using a PURL. -func FetchMaintainersFromPURL(ctx context.Context, purl string, client *Client) ([]Maintainer, error) { - return core.FetchMaintainersFromPURL(ctx, purl, client) +func FetchMaintainersFromPURL(ctx context.Context, purl string, c *Client) ([]Maintainer, error) { + return core.FetchMaintainersFromPURL(ctx, purl, c) } // FetchLatestVersion returns the latest non-yanked/retracted/deprecated version. @@ -190,42 +194,42 @@ func FetchLatestVersion(ctx context.Context, reg Registry, name string) (*Versio } // FetchLatestVersionFromPURL returns the latest non-yanked version for a PURL. -func FetchLatestVersionFromPURL(ctx context.Context, purl string, client *Client) (*Version, error) { - return core.FetchLatestVersionFromPURL(ctx, purl, client) +func FetchLatestVersionFromPURL(ctx context.Context, purl string, c *Client) (*Version, error) { + return core.FetchLatestVersionFromPURL(ctx, purl, c) } // BulkFetchPackages fetches package metadata for multiple PURLs in parallel. // Individual fetch errors are silently ignored - those PURLs are omitted from results. // Returns a map of PURL to Package. -func BulkFetchPackages(ctx context.Context, purls []string, client *Client) map[string]*Package { - return core.BulkFetchPackages(ctx, purls, client) +func BulkFetchPackages(ctx context.Context, purls []string, c *Client) map[string]*Package { + return core.BulkFetchPackages(ctx, purls, c) } // BulkFetchPackagesWithConcurrency fetches packages with a custom concurrency limit. -func BulkFetchPackagesWithConcurrency(ctx context.Context, purls []string, client *Client, concurrency int) map[string]*Package { - return core.BulkFetchPackagesWithConcurrency(ctx, purls, client, concurrency) +func BulkFetchPackagesWithConcurrency(ctx context.Context, purls []string, c *Client, concurrency int) map[string]*Package { + return core.BulkFetchPackagesWithConcurrency(ctx, purls, c, concurrency) } // BulkFetchVersions fetches version metadata for multiple versioned PURLs in parallel. // PURLs without versions are silently skipped. // Individual fetch errors are silently ignored - those PURLs are omitted from results. // Returns a map of PURL to Version. -func BulkFetchVersions(ctx context.Context, purls []string, client *Client) map[string]*Version { - return core.BulkFetchVersions(ctx, purls, client) +func BulkFetchVersions(ctx context.Context, purls []string, c *Client) map[string]*Version { + return core.BulkFetchVersions(ctx, purls, c) } // BulkFetchVersionsWithConcurrency fetches versions with a custom concurrency limit. -func BulkFetchVersionsWithConcurrency(ctx context.Context, purls []string, client *Client, concurrency int) map[string]*Version { - return core.BulkFetchVersionsWithConcurrency(ctx, purls, client, concurrency) +func BulkFetchVersionsWithConcurrency(ctx context.Context, purls []string, c *Client, concurrency int) map[string]*Version { + return core.BulkFetchVersionsWithConcurrency(ctx, purls, c, concurrency) } // BulkFetchLatestVersions fetches the latest version for multiple PURLs in parallel. // Returns a map of PURL to the latest non-yanked Version. -func BulkFetchLatestVersions(ctx context.Context, purls []string, client *Client) map[string]*Version { - return core.BulkFetchLatestVersions(ctx, purls, client) +func BulkFetchLatestVersions(ctx context.Context, purls []string, c *Client) map[string]*Version { + return core.BulkFetchLatestVersions(ctx, purls, c) } // BulkFetchLatestVersionsWithConcurrency fetches latest versions with a custom concurrency limit. -func BulkFetchLatestVersionsWithConcurrency(ctx context.Context, purls []string, client *Client, concurrency int) map[string]*Version { - return core.BulkFetchLatestVersionsWithConcurrency(ctx, purls, client, concurrency) +func BulkFetchLatestVersionsWithConcurrency(ctx context.Context, purls []string, c *Client, concurrency int) map[string]*Version { + return core.BulkFetchLatestVersionsWithConcurrency(ctx, purls, c, concurrency) }