From 75712e9f9e842cbe60719ad853cd5310cab16ee3 Mon Sep 17 00:00:00 2001 From: frozenhelium Date: Wed, 8 Jul 2026 16:49:37 +0545 Subject: [PATCH 1/3] feat(source): add XLSX-from-URL mock mode for QA instances - add TRANSLATION_SOURCE=api|url with per-mode config validation - add URLSource fetching the XLSX from a plain HTTP(S) URL - cap downloads at 50MB for both sources - record pull outcome; /status gains source, last_pull_error, last_import_rows - redact signed-URL credentials from logs, errors, and /status - helm: TRANSLATION_SOURCE in values; alpha-1 switches to url mode - regenerate helm snapshots (helm 3.15.4) and openapi.json - pass new vars through docker-compose; document mock mode in README - tests for url source, config matrix, status fields, redaction --- .env.example | 3 + README.md | 32 ++++- config.go | 43 +++++-- config_test.go | 84 ++++++++++++ db.go | 8 +- docker-compose.yml | 8 +- handlers.go | 47 ++++--- helm/snapshots/alpha-1.yaml | 9 +- helm/snapshots/production.yaml | 1 + helm/snapshots/staging.yaml | 1 + helm/tests/alpha-1.yaml | 16 ++- helm/values.yaml | 4 + main.go | 13 +- openapi.json | 9 ++ source.go | 121 +++++++++++++++++ source_test.go | 229 +++++++++++++++++++++++++++++++++ sync_pull.go | 74 ++++++----- sync_pull_test.go | 27 ++-- 18 files changed, 632 insertions(+), 97 deletions(-) create mode 100644 source.go create mode 100644 source_test.go diff --git a/.env.example b/.env.example index 195b2dd..b0a8afe 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,9 @@ TRANSLATION_BASE_URL=https://example.com TRANSLATION_APPLICATION_ID=your-application-id TRANSLATION_API_KEY=your-api-key +# Mock mode: pull the XLSX from a plain URL instead of the API. +# TRANSLATION_SOURCE=url +# TRANSLATION_XLSX_URL=https://example.com/translations.xlsx PULL_INTERVAL=10m HTTP_TIMEOUT=30s INITIAL_PULL_DEADLINE=45s diff --git a/README.md b/README.md index 32c51d2..70885d9 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ It periodically downloads an XLSX export from a translation service, stores the ## Features - XLSX import from external translation service +- Mock mode: pull the XLSX from a plain URL instead of the API (`TRANSLATION_SOURCE=url`) - Periodic background sync (full replace per import; the XLSX is the source of truth) - SQLite-backed cache - Fetch translations by page(s) + language @@ -101,14 +102,37 @@ other header names (e.g. `Namespace`) are rejected. `/strings` responses carry an `ETag` derived from the last import hash and `Cache-Control: public, max-age=60`. Requests with a matching `If-None-Match` get `304 Not Modified`. +## Mock mode (`TRANSLATION_SOURCE=url`) + +For QA/alpha instances the service can pull the XLSX from any plain HTTP(S) URL +instead of the IFRC translation API: + +```env +TRANSLATION_SOURCE=url +TRANSLATION_XLSX_URL=https://example.com/ifrc-go/translations.xlsx +``` + +- The file must be in the exact format the translation service exports + (see "XLSX format" above; `Namespace` headers are rejected). +- Public URLs, Azure Blob SAS URLs, and S3 presigned URLs all work; no + auth headers are sent. +- The regular pull loop applies: updates to the hosted file show up within + `PULL_INTERVAL` (alpha uses `1m`); unchanged files are skipped by hash. +- Check `GET /status` to debug a broken file: it reports `source`, + `last_pull_error` (cleared on success), and `last_import_rows`. +- In `url` mode the `TRANSLATION_BASE_URL`, `TRANSLATION_APPLICATION_ID`, + and `TRANSLATION_API_KEY` variables are ignored. + ## Environment Variables | Variable | Required | Description | |----------|----------|------------| -| `TRANSLATION_BASE_URL` | Yes | Base URL of translation service | -| `TRANSLATION_APPLICATION_ID` | Yes | Translation application ID | -| `TRANSLATION_API_KEY` | Yes | Sent as `X-API-KEY` header | +| `TRANSLATION_SOURCE` | No | `api` (default) or `url` (mock mode) | +| `TRANSLATION_BASE_URL` | api mode | Base URL of translation service | +| `TRANSLATION_APPLICATION_ID` | api mode | Translation application ID | +| `TRANSLATION_API_KEY` | api mode | Sent as `X-API-KEY` header | +| `TRANSLATION_XLSX_URL` | url mode | HTTP(S) URL of the mock XLSX file | | `SQLITE_PATH` | No | Default: `/data/cacheppuccino.db` | | `PULL_INTERVAL` | No | Default: `10m` | | `HTTP_TIMEOUT` | No | Default: `30s` | @@ -182,6 +206,8 @@ go run . --schema - Metadata table stores: - `last_pull_rfc3339` - `last_xlsx_sha256` + - `last_pull_error` + - `last_import_rows` ## Development diff --git a/config.go b/config.go index b149366..0abef4d 100644 --- a/config.go +++ b/config.go @@ -3,17 +3,27 @@ package main import ( "errors" "fmt" + "net/url" "os" "slices" "time" ) +// Translation source kinds: the real IFRC API, or a plain XLSX URL used +// to mock the service on QA/alpha instances. +const ( + sourceAPI = "api" + sourceURL = "url" +) + type Config struct { ListenAddr string SQLitePath string + TranslationSource string TranslationBaseURL string TranslationApplicationID string TranslationAPIKey string + TranslationXLSXURL string HTTPTimeout time.Duration PullInterval time.Duration InitialPullDeadline time.Duration @@ -28,14 +38,6 @@ var validLogLevels = []string{"debug", "info", "warn", "error"} func LoadConfig() (Config, error) { var errs []error - requireEnv := func(k string) string { - v := os.Getenv(k) - if v == "" { - errs = append(errs, fmt.Errorf("missing required env: %s", k)) - } - return v - } - envDuration := func(k string, def time.Duration) time.Duration { v := os.Getenv(k) if v == "" { @@ -52,15 +54,34 @@ func LoadConfig() (Config, error) { cfg := Config{ ListenAddr: env("LISTEN_ADDR", ":8080"), SQLitePath: env("SQLITE_PATH", "/data/cacheppuccino.db"), - TranslationBaseURL: requireEnv("TRANSLATION_BASE_URL"), - TranslationApplicationID: requireEnv("TRANSLATION_APPLICATION_ID"), - TranslationAPIKey: requireEnv("TRANSLATION_API_KEY"), + TranslationSource: env("TRANSLATION_SOURCE", sourceAPI), + TranslationBaseURL: os.Getenv("TRANSLATION_BASE_URL"), + TranslationApplicationID: os.Getenv("TRANSLATION_APPLICATION_ID"), + TranslationAPIKey: os.Getenv("TRANSLATION_API_KEY"), + TranslationXLSXURL: os.Getenv("TRANSLATION_XLSX_URL"), HTTPTimeout: envDuration("HTTP_TIMEOUT", 30*time.Second), PullInterval: envDuration("PULL_INTERVAL", 10*time.Minute), InitialPullDeadline: envDuration("INITIAL_PULL_DEADLINE", 45*time.Second), LogLevel: env("LOG_LEVEL", "info"), } + switch cfg.TranslationSource { + case sourceAPI: + for _, k := range []string{"TRANSLATION_BASE_URL", "TRANSLATION_APPLICATION_ID", "TRANSLATION_API_KEY"} { + if os.Getenv(k) == "" { + errs = append(errs, fmt.Errorf("missing required env: %s (required when TRANSLATION_SOURCE=api)", k)) + } + } + case sourceURL: + if cfg.TranslationXLSXURL == "" { + errs = append(errs, errors.New("missing required env: TRANSLATION_XLSX_URL (required when TRANSLATION_SOURCE=url)")) + } else if u, err := url.Parse(cfg.TranslationXLSXURL); err != nil || (u.Scheme != "http" && u.Scheme != "https") { + errs = append(errs, fmt.Errorf("invalid TRANSLATION_XLSX_URL: %q (must be an http(s) URL)", cfg.TranslationXLSXURL)) + } + default: + errs = append(errs, fmt.Errorf("invalid TRANSLATION_SOURCE: %q (valid: api, url)", cfg.TranslationSource)) + } + if cfg.HTTPTimeout <= 0 { errs = append(errs, fmt.Errorf("HTTP_TIMEOUT must be positive, got %s", cfg.HTTPTimeout)) } diff --git a/config_test.go b/config_test.go index 7e76bf1..303acc6 100644 --- a/config_test.go +++ b/config_test.go @@ -9,9 +9,11 @@ import ( // ctAllConfigEnvVars is every env var LoadConfig reads. Each test sets all of // them explicitly ("" simulates unset) to shield tests from the host env. var ctAllConfigEnvVars = []string{ + "TRANSLATION_SOURCE", "TRANSLATION_BASE_URL", "TRANSLATION_APPLICATION_ID", "TRANSLATION_API_KEY", + "TRANSLATION_XLSX_URL", "LISTEN_ADDR", "SQLITE_PATH", "HTTP_TIMEOUT", @@ -73,6 +75,88 @@ func TestLoadConfigDefaults(t *testing.T) { if got, want := cfg.LogLevel, "info"; got != want { t.Errorf("LogLevel = %q, want %q", got, want) } + if got, want := cfg.TranslationSource, "api"; got != want { + t.Errorf("TranslationSource = %q, want %q", got, want) + } +} + +func TestLoadConfigSourceMatrix(t *testing.T) { + tests := []struct { + name string + env map[string]string + wantErr bool + wantInError []string + }{ + { + name: "url mode requires only the xlsx url", + env: map[string]string{ + "TRANSLATION_SOURCE": "url", + "TRANSLATION_XLSX_URL": "https://files.example.com/translations.xlsx", + }, + }, + { + name: "url mode without xlsx url fails", + env: map[string]string{"TRANSLATION_SOURCE": "url"}, + wantErr: true, + wantInError: []string{"TRANSLATION_XLSX_URL"}, + }, + { + name: "url mode rejects non-http url", + env: map[string]string{ + "TRANSLATION_SOURCE": "url", + "TRANSLATION_XLSX_URL": "ftp://files.example.com/translations.xlsx", + }, + wantErr: true, + wantInError: []string{"TRANSLATION_XLSX_URL", "http(s)"}, + }, + { + name: "invalid source value fails", + env: map[string]string{"TRANSLATION_SOURCE": "s3"}, + wantErr: true, + wantInError: []string{"TRANSLATION_SOURCE", `"s3"`}, + }, + { + name: "api mode still requires the api trio", + env: map[string]string{ + "TRANSLATION_SOURCE": "api", + "TRANSLATION_XLSX_URL": "https://files.example.com/translations.xlsx", + }, + wantErr: true, + wantInError: []string{ + "TRANSLATION_BASE_URL", + "TRANSLATION_APPLICATION_ID", + "TRANSLATION_API_KEY", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctSetConfigEnv(t, tt.env) + + cfg, err := LoadConfig() + if tt.wantErr { + if err == nil { + t.Fatal("LoadConfig() error = nil, want error") + } + for _, want := range tt.wantInError { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not contain %q", err.Error(), want) + } + } + return + } + if err != nil { + t.Fatalf("LoadConfig() error = %v, want nil", err) + } + if cfg.TranslationSource != tt.env["TRANSLATION_SOURCE"] { + t.Errorf("TranslationSource = %q, want %q", cfg.TranslationSource, tt.env["TRANSLATION_SOURCE"]) + } + if cfg.TranslationXLSXURL != tt.env["TRANSLATION_XLSX_URL"] { + t.Errorf("TranslationXLSXURL = %q, want %q", cfg.TranslationXLSXURL, tt.env["TRANSLATION_XLSX_URL"]) + } + }) + } } func TestLoadConfigMissingRequired(t *testing.T) { diff --git a/db.go b/db.go index 849ffc7..b722505 100644 --- a/db.go +++ b/db.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "time" @@ -16,8 +17,10 @@ import ( ) const ( - metaKeyLastPull = "last_pull_rfc3339" - metaKeyLastHash = "last_xlsx_sha256" + metaKeyLastPull = "last_pull_rfc3339" + metaKeyLastHash = "last_xlsx_sha256" + metaKeyLastPullError = "last_pull_error" + metaKeyLastImportRows = "last_import_rows" ) type DB struct { @@ -160,6 +163,7 @@ func (db *DB) ReplaceImport(ctx context.Context, rows []StringRow, hash string, meta := []MetaModel{ {K: metaKeyLastHash, V: hash}, {K: metaKeyLastPull, V: pulledAt.UTC().Format(time.RFC3339)}, + {K: metaKeyLastImportRows, V: strconv.Itoa(len(models))}, } for _, m := range meta { if err := upsertMetaTx(ctx, tx, m); err != nil { diff --git a/docker-compose.yml b/docker-compose.yml index 73e7ce8..4e45d64 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,9 +8,11 @@ services: - "8080:8080" # user: "0:0" environment: - TRANSLATION_BASE_URL: ${TRANSLATION_BASE_URL} - TRANSLATION_APPLICATION_ID: ${TRANSLATION_APPLICATION_ID} - TRANSLATION_API_KEY: ${TRANSLATION_API_KEY} + TRANSLATION_SOURCE: ${TRANSLATION_SOURCE:-api} + TRANSLATION_BASE_URL: ${TRANSLATION_BASE_URL:-} + TRANSLATION_APPLICATION_ID: ${TRANSLATION_APPLICATION_ID:-} + TRANSLATION_API_KEY: ${TRANSLATION_API_KEY:-} + TRANSLATION_XLSX_URL: ${TRANSLATION_XLSX_URL:-} SQLITE_PATH: "/data/cacheppuccino.db" SQLITE_TMP_DIR: "/data/tmp" PULL_INTERVAL: ${PULL_INTERVAL:-10m} diff --git a/handlers.go b/handlers.go index 990cbac..89f9263 100644 --- a/handlers.go +++ b/handlers.go @@ -3,6 +3,7 @@ package main import ( "log/slog" "net/http" + "strconv" "strings" "github.com/rs/cors" @@ -12,6 +13,7 @@ type Server struct { db *DB ready *ReadyState logger *slog.Logger + source string } type StringsResponse struct { @@ -29,10 +31,13 @@ type ReadyResponse struct { } type StatusResponse struct { - LastPull string `json:"last_pull"` - LastHash string `json:"last_hash"` - Ready bool `json:"ready"` - Version string `json:"version"` + Source string `json:"source"` + LastPull string `json:"last_pull"` + LastHash string `json:"last_hash"` + LastPullError string `json:"last_pull_error"` + LastImportRows int `json:"last_import_rows"` + Ready bool `json:"ready"` + Version string `json:"version"` } func (s *Server) routes() http.Handler { @@ -78,22 +83,32 @@ func (s *Server) handleReadyz(w http.ResponseWriter, r *http.Request) { } func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { - lastPull, _, err := s.db.GetMeta(r.Context(), metaKeyLastPull) - if err != nil { - writeErr(w, http.StatusInternalServerError, "internal_error", "failed to read status", nil) - return + meta := map[string]string{ + metaKeyLastPull: "", + metaKeyLastHash: "", + metaKeyLastPullError: "", + metaKeyLastImportRows: "", } - lastHash, _, err := s.db.GetMeta(r.Context(), metaKeyLastHash) - if err != nil { - writeErr(w, http.StatusInternalServerError, "internal_error", "failed to read status", nil) - return + for k := range meta { + v, _, err := s.db.GetMeta(r.Context(), k) + if err != nil { + writeErr(w, http.StatusInternalServerError, "internal_error", "failed to read status", nil) + return + } + meta[k] = v } + // Absent or malformed count reads as 0. + importRows, _ := strconv.Atoi(meta[metaKeyLastImportRows]) + writeOK(w, http.StatusOK, StatusResponse{ - LastPull: lastPull, - LastHash: lastHash, - Ready: s.ready.IsReady(), - Version: version, + Source: s.source, + LastPull: meta[metaKeyLastPull], + LastHash: meta[metaKeyLastHash], + LastPullError: meta[metaKeyLastPullError], + LastImportRows: importRows, + Ready: s.ready.IsReady(), + Version: version, }) } diff --git a/helm/snapshots/alpha-1.yaml b/helm/snapshots/alpha-1.yaml index dd1e840..b3723dd 100644 --- a/helm/snapshots/alpha-1.yaml +++ b/helm/snapshots/alpha-1.yaml @@ -24,7 +24,6 @@ metadata: app.kubernetes.io/managed-by: Helm type: Opaque stringData: - TRANSLATION_API_KEY: "dummy" --- # Source: cacheppuccino-helm/templates/configmap.yaml apiVersion: v1 @@ -43,9 +42,11 @@ data: INITIAL_PULL_DEADLINE: "2m" LISTEN_ADDR: ":8080" LOG_LEVEL: "info" - PULL_INTERVAL: "10m" - TRANSLATION_APPLICATION_ID: "18" - TRANSLATION_BASE_URL: "https://ifrc-translationapi.azurewebsites.net" + PULL_INTERVAL: "1m" + TRANSLATION_APPLICATION_ID: "" + TRANSLATION_BASE_URL: "" + TRANSLATION_SOURCE: "url" + TRANSLATION_XLSX_URL: "https://example.com/ifrc-go/translations.xlsx" --- # Source: cacheppuccino-helm/templates/pvc.yaml apiVersion: v1 diff --git a/helm/snapshots/production.yaml b/helm/snapshots/production.yaml index 4981a2d..87230dd 100644 --- a/helm/snapshots/production.yaml +++ b/helm/snapshots/production.yaml @@ -33,6 +33,7 @@ data: PULL_INTERVAL: "10m" TRANSLATION_APPLICATION_ID: "18" TRANSLATION_BASE_URL: "https://ifrc-test-translationapi.azurewebsites.net" + TRANSLATION_SOURCE: "api" --- # Source: cacheppuccino-helm/templates/pvc.yaml apiVersion: v1 diff --git a/helm/snapshots/staging.yaml b/helm/snapshots/staging.yaml index f96afcc..de185e7 100644 --- a/helm/snapshots/staging.yaml +++ b/helm/snapshots/staging.yaml @@ -33,6 +33,7 @@ data: PULL_INTERVAL: "10m" TRANSLATION_APPLICATION_ID: "18" TRANSLATION_BASE_URL: "https://ifrc-test-translationapi.azurewebsites.net" + TRANSLATION_SOURCE: "api" --- # Source: cacheppuccino-helm/templates/pvc.yaml apiVersion: v1 diff --git a/helm/tests/alpha-1.yaml b/helm/tests/alpha-1.yaml index b76c4c8..c9b0b8a 100644 --- a/helm/tests/alpha-1.yaml +++ b/helm/tests/alpha-1.yaml @@ -1,4 +1,8 @@ # Values for alpha instance +# +# Alpha runs in mock mode for QA of the GO web-app: strings come from a +# QA-hosted XLSX file (same format as the translation service export) +# instead of the real IFRC translation API. ingress: enabled: true host: cacheppuccino-alpha-1.ifrc-go.dev.togglecorp.com @@ -19,11 +23,11 @@ sqlite: env: LISTEN_ADDR: ":8080" LOG_LEVEL: "info" - TRANSLATION_BASE_URL: "https://ifrc-translationapi.azurewebsites.net" - TRANSLATION_APPLICATION_ID: "18" - PULL_INTERVAL: "10m" + TRANSLATION_SOURCE: "url" + # TODO: point at the QA-hosted XLSX (any public or signed HTTPS URL). + TRANSLATION_XLSX_URL: "https://example.com/ifrc-go/translations.xlsx" + # Short interval so QA edits show up within a minute (unchanged files + # are skipped by content hash, so frequent polling is cheap). + PULL_INTERVAL: "1m" HTTP_TIMEOUT: "1m" INITIAL_PULL_DEADLINE: "2m" - -secrets: - TRANSLATION_API_KEY: "dummy" diff --git a/helm/values.yaml b/helm/values.yaml index 913d33a..ece2f3c 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -77,6 +77,10 @@ sqlite: env: LISTEN_ADDR: ":8080" LOG_LEVEL: "info" + # api: pull from the IFRC translation service (needs TRANSLATION_BASE_URL, + # TRANSLATION_APPLICATION_ID and the TRANSLATION_API_KEY secret). + # url: pull a plain XLSX file from TRANSLATION_XLSX_URL (QA/alpha mock). + TRANSLATION_SOURCE: "api" TRANSLATION_BASE_URL: "" TRANSLATION_APPLICATION_ID: "" PULL_INTERVAL: "10m" diff --git a/main.go b/main.go index 416286e..b05ecbb 100644 --- a/main.go +++ b/main.go @@ -57,7 +57,12 @@ func main() { } defer func() { _ = db.Close() }() - client := NewTranslationClient(cfg) + var source XLSXSource + if cfg.TranslationSource == sourceURL { + source = NewURLSource(cfg) + } else { + source = NewTranslationClient(cfg) + } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() @@ -72,11 +77,12 @@ func main() { } ready.SetReady(hasData) - srv := &Server{db: db, ready: ready, logger: logger} + srv := &Server{db: db, ready: ready, logger: logger, source: source.Name()} logger.Info("cacheppuccino starting", slog.String("version", version), slog.String("addr", cfg.ListenAddr), + slog.String("source", source.Name()), slog.Bool("has_data", hasData), slog.String("pull_interval", cfg.PullInterval.String()), slog.String("http_timeout", cfg.HTTPTimeout.String()), @@ -107,10 +113,9 @@ func main() { StartPeriodicPuller( ctx, db, - client, + source, cfg.PullInterval, cfg.InitialPullDeadline, - cfg.TranslationApplicationID, ready, logger, ) diff --git a/openapi.json b/openapi.json index fe809b8..dcb16a9 100644 --- a/openapi.json +++ b/openapi.json @@ -111,12 +111,21 @@ "last_hash": { "type": "string" }, + "last_import_rows": { + "type": "integer" + }, "last_pull": { "type": "string" }, + "last_pull_error": { + "type": "string" + }, "ready": { "type": "boolean" }, + "source": { + "type": "string" + }, "version": { "type": "string" } diff --git a/source.go b/source.go new file mode 100644 index 0000000..277357a --- /dev/null +++ b/source.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "strings" +) + +// maxXLSXBytes caps downloads so a wrong URL (a huge file or an endless +// stream) fails loudly instead of exhausting memory. +const maxXLSXBytes = 50 << 20 + +// XLSXSource provides the translation XLSX export bytes. +type XLSXSource interface { + // Name identifies the source kind ("api" or "url") for /status and logs. + Name() string + Fetch(ctx context.Context, logger *slog.Logger) ([]byte, error) +} + +// URLSource fetches the XLSX from a plain HTTP(S) URL. It mocks the +// translation service on QA/alpha instances (TRANSLATION_SOURCE=url): +// QA hosts a file in the same format the real service exports, and the +// regular pull loop picks up changes. +// +// The URL may carry credentials in its query string (SAS/presigned URLs), +// so logs and returned errors only ever use the redacted form: pull errors +// end up on the unauthenticated /status endpoint. +type URLSource struct { + url string + redactedURL string + http *http.Client +} + +func NewURLSource(cfg Config) *URLSource { + return &URLSource{ + url: cfg.TranslationXLSXURL, + redactedURL: redactURL(cfg.TranslationXLSXURL), + http: &http.Client{ + Timeout: cfg.HTTPTimeout, + }, + } +} + +func (s *URLSource) Name() string { return "url" } + +func (s *URLSource) Fetch(ctx context.Context, logger *slog.Logger) ([]byte, error) { + logger.Info("pull: requesting xlsx", slog.String("url", s.redactedURL)) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url, nil) + if err != nil { + // The wrapped error would embed the raw URL; keep it out. + return nil, fmt.Errorf("xlsx request failed: invalid URL %s", s.redactedURL) + } + + resp, err := s.http.Do(req) + if err != nil { + // Transport errors (*url.Error) embed the full URL including any + // query credentials; rebuild the message around the redacted URL. + var ue *url.Error + if errors.As(err, &ue) { + return nil, fmt.Errorf("xlsx request failed: %s %s: %w", ue.Op, s.redactedURL, ue.Err) + } + return nil, fmt.Errorf("xlsx request failed: GET %s", s.redactedURL) + } + defer resp.Body.Close() + + if resp.StatusCode/100 != 2 { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<10)) + // Error bodies from blob stores can echo signature parameters. + return nil, fmt.Errorf("download failed: %s: %s", resp.Status, s.redactSecrets(string(b))) + } + + return readAllLimited(resp.Body, maxXLSXBytes) +} + +// redactURL strips query, fragment, and userinfo (where SAS tokens and +// presigned signatures live) so the URL is safe for logs and /status. +func redactURL(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return "" + } + u.RawQuery = "" + u.Fragment = "" + u.User = nil + return u.String() +} + +// redactSecrets masks the URL's query parameter values in msg, in case an +// upstream error body echoes them back. +func (s *URLSource) redactSecrets(msg string) string { + u, err := url.Parse(s.url) + if err != nil { + return msg + } + for _, values := range u.Query() { + for _, v := range values { + if len(v) >= 8 { + msg = strings.ReplaceAll(msg, v, "***") + } + } + } + return msg +} + +// readAllLimited reads r fully, erroring if it exceeds max bytes. +func readAllLimited(r io.Reader, max int64) ([]byte, error) { + b, err := io.ReadAll(io.LimitReader(r, max+1)) + if err != nil { + return nil, err + } + if int64(len(b)) > max { + return nil, fmt.Errorf("response exceeds %d bytes", max) + } + return b, nil +} diff --git a/source_test.go b/source_test.go new file mode 100644 index 0000000..381bd09 --- /dev/null +++ b/source_test.go @@ -0,0 +1,229 @@ +package main + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" +) + +func srcLogger() *slog.Logger { + return slog.New(slog.NewJSONHandler(io.Discard, nil)) +} + +func srcURLSource(rawURL string) *URLSource { + return NewURLSource(Config{ + TranslationXLSXURL: rawURL, + HTTPTimeout: 5 * time.Second, + }) +} + +func TestURLSourceFetch(t *testing.T) { + t.Run("plain GET, no auth header", func(t *testing.T) { + var gotPath, gotKey string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotKey = r.Header.Get("X-API-KEY") + _, _ = w.Write([]byte("xlsx-bytes")) + })) + defer srv.Close() + + src := srcURLSource(srv.URL + "/files/translations.xlsx") + body, err := src.Fetch(context.Background(), srcLogger()) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if string(body) != "xlsx-bytes" { + t.Errorf("body = %q, want %q", body, "xlsx-bytes") + } + if gotPath != "/files/translations.xlsx" { + t.Errorf("path = %q, want /files/translations.xlsx", gotPath) + } + if gotKey != "" { + t.Errorf("X-API-KEY sent (%q), want none", gotKey) + } + if src.Name() != "url" { + t.Errorf("Name() = %q, want url", src.Name()) + } + }) + + t.Run("non-2xx fails with status and body excerpt", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "no such blob", http.StatusNotFound) + })) + defer srv.Close() + + _, err := srcURLSource(srv.URL).Fetch(context.Background(), srcLogger()) + if err == nil { + t.Fatal("Fetch on 404: want error, got nil") + } + if !strings.Contains(err.Error(), "404") || !strings.Contains(err.Error(), "no such blob") { + t.Errorf("error %q should mention status and body", err.Error()) + } + }) + + t.Run("transport error redacts query credentials", func(t *testing.T) { + // Errors end up on the public /status endpoint; a SAS/presigned + // token in the query must never appear there. + src := srcURLSource("http://127.0.0.1:1/translations.xlsx?sv=2022-11-02&sig=SUPERSECRETSAS") + _, err := src.Fetch(context.Background(), srcLogger()) + if err == nil { + t.Fatal("Fetch on unreachable host: want error, got nil") + } + if strings.Contains(err.Error(), "SUPERSECRETSAS") { + t.Errorf("error %q leaks the query credential", err.Error()) + } + if !strings.Contains(err.Error(), "/translations.xlsx") { + t.Errorf("error %q should keep host/path for diagnosis", err.Error()) + } + }) + + t.Run("error body echoing query credentials is masked", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "signature mismatch for sig=SUPERSECRETSAS", http.StatusForbidden) + })) + defer srv.Close() + + src := srcURLSource(srv.URL + "/translations.xlsx?sig=SUPERSECRETSAS") + _, err := src.Fetch(context.Background(), srcLogger()) + if err == nil { + t.Fatal("Fetch on 403: want error, got nil") + } + if strings.Contains(err.Error(), "SUPERSECRETSAS") { + t.Errorf("error %q leaks the credential echoed by the body", err.Error()) + } + if !strings.Contains(err.Error(), "403") { + t.Errorf("error %q should keep the status", err.Error()) + } + }) +} + +func TestRedactURL(t *testing.T) { + tests := []struct { + in string + want string + }{ + {"https://blob.example.com/c/translations.xlsx?sv=1&sig=SECRET", "https://blob.example.com/c/translations.xlsx"}, + {"https://files.example.com/translations.xlsx", "https://files.example.com/translations.xlsx"}, + {"https://user:pass@files.example.com/t.xlsx#frag", "https://files.example.com/t.xlsx"}, + } + for _, tt := range tests { + if got := redactURL(tt.in); got != tt.want { + t.Errorf("redactURL(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +func TestReadAllLimited(t *testing.T) { + tests := []struct { + name string + input string + max int64 + wantErr bool + }{ + {name: "under limit", input: "12345", max: 10}, + {name: "exactly at limit", input: "1234567890", max: 10}, + {name: "over limit", input: "12345678901", max: 10, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := readAllLimited(strings.NewReader(tt.input), tt.max) + if tt.wantErr { + if err == nil { + t.Fatalf("readAllLimited = %q, want error", got) + } + return + } + if err != nil { + t.Fatalf("readAllLimited: %v", err) + } + if string(got) != tt.input { + t.Errorf("readAllLimited = %q, want %q", got, tt.input) + } + }) + } +} + +// TestStatusReportsSourceAndPullOutcome covers the /status additions for QA +// self-diagnosis: source kind, last pull error, and last import row count. +func TestStatusReportsSourceAndPullOutcome(t *testing.T) { + ctx := context.Background() + + db, err := OpenDB(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("OpenDB: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + ready := &ReadyState{} + ready.SetReady(true) + srv := &Server{db: db, ready: ready, logger: srcLogger(), source: "url"} + ts := httptest.NewServer(srv.routes()) + defer ts.Close() + + getStatus := func() StatusResponse { + t.Helper() + resp, err := http.Get(ts.URL + "/status") + if err != nil { + t.Fatalf("GET /status: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET /status = %d, want 200", resp.StatusCode) + } + var envelope struct { + Ok bool `json:"ok"` + Data StatusResponse `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&envelope); err != nil { + t.Fatalf("decode /status: %v", err) + } + return envelope.Data + } + + // Fresh DB: source is reported, counters empty. + st := getStatus() + if st.Source != "url" { + t.Errorf("source = %q, want url", st.Source) + } + if st.LastPullError != "" || st.LastImportRows != 0 { + t.Errorf("fresh status = %+v, want empty error and 0 rows", st) + } + + // After an import: row count present. + rows := []StringRow{ + {Page: "home", Key: "a", Lang: "en", Value: "A", UpdatedAt: time.Now()}, + {Page: "home", Key: "b", Lang: "en", Value: "B", UpdatedAt: time.Now()}, + } + if err := db.ReplaceImport(ctx, rows, "hash-1", time.Now()); err != nil { + t.Fatalf("ReplaceImport: %v", err) + } + st = getStatus() + if st.LastImportRows != 2 { + t.Errorf("last_import_rows = %d, want 2", st.LastImportRows) + } + if st.LastHash != "hash-1" { + t.Errorf("last_hash = %q, want hash-1", st.LastHash) + } + + // A recorded pull failure surfaces, then clears. + if err := db.SetMeta(ctx, metaKeyLastPullError, "download failed: 404"); err != nil { + t.Fatalf("SetMeta: %v", err) + } + if st = getStatus(); st.LastPullError != "download failed: 404" { + t.Errorf("last_pull_error = %q, want the recorded error", st.LastPullError) + } + if err := db.SetMeta(ctx, metaKeyLastPullError, ""); err != nil { + t.Fatalf("SetMeta: %v", err) + } + if st = getStatus(); st.LastPullError != "" { + t.Errorf("last_pull_error = %q, want cleared", st.LastPullError) + } +} diff --git a/sync_pull.go b/sync_pull.go index 676c39a..70c0f8d 100644 --- a/sync_pull.go +++ b/sync_pull.go @@ -9,28 +9,29 @@ import ( "time" ) +// TranslationClient fetches the XLSX export from the IFRC translation API. type TranslationClient struct { - baseURL string - apiKey string - http *http.Client + baseURL string + applicationID string + apiKey string + http *http.Client } func NewTranslationClient(cfg Config) *TranslationClient { return &TranslationClient{ - baseURL: cfg.TranslationBaseURL, - apiKey: cfg.TranslationAPIKey, + baseURL: cfg.TranslationBaseURL, + applicationID: cfg.TranslationApplicationID, + apiKey: cfg.TranslationAPIKey, http: &http.Client{ Timeout: cfg.HTTPTimeout, }, } } -func (c *TranslationClient) DownloadXLSX( - ctx context.Context, - applicationID string, - logger *slog.Logger, -) ([]byte, error) { - url := fmt.Sprintf("%s/api/Application/%s/Translation/export", c.baseURL, applicationID) +func (c *TranslationClient) Name() string { return "api" } + +func (c *TranslationClient) Fetch(ctx context.Context, logger *slog.Logger) ([]byte, error) { + url := fmt.Sprintf("%s/api/Application/%s/Translation/export", c.baseURL, c.applicationID) logger.Info("pull: requesting export", slog.String("url", url)) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) @@ -53,7 +54,7 @@ func (c *TranslationClient) DownloadXLSX( return nil, fmt.Errorf("download failed: %s: %s", resp.Status, string(b)) } - return io.ReadAll(resp.Body) + return readAllLimited(resp.Body, maxXLSXBytes) } type PullResult struct { @@ -65,13 +66,12 @@ type PullResult struct { func PullOnce( ctx context.Context, db *DB, - client *TranslationClient, - applicationID string, + source XLSXSource, logger *slog.Logger, ) (PullResult, error) { t0 := time.Now() - xlsx, err := client.DownloadXLSX(ctx, applicationID, logger) + xlsx, err := source.Fetch(ctx, logger) if err != nil { return PullResult{}, err } @@ -111,17 +111,35 @@ func PullOnce( func StartPeriodicPuller( ctx context.Context, db *DB, - client *TranslationClient, + source XLSXSource, interval time.Duration, initialDeadline time.Duration, - applicationID string, ready *ReadyState, logger *slog.Logger, ) { jitterMax := time.Duration(float64(interval) * 0.10) ticker := time.NewTicker(interval) - logPull := func(kind string, res PullResult) { + // runPull records the outcome in meta so /status can report the most + // recent pull error without access to pod logs (QA has none). + runPull := func(pullCtx context.Context, kind string) { + res, err := PullOnce(pullCtx, db, source, logger) + if err != nil { + // Skip recording on shutdown; the parent ctx is the process ctx. + if ctx.Err() == nil { + if merr := db.SetMeta(ctx, metaKeyLastPullError, err.Error()); merr != nil { + logger.Warn("pull: failed to record pull error", "err", merr) + } + } + logger.Warn(kind+" pull failed; serving cached data if any", "err", err) + return + } + + if merr := db.SetMeta(ctx, metaKeyLastPullError, ""); merr != nil { + logger.Warn("pull: failed to clear pull error", "err", merr) + } + ready.SetReady(true) + if res.Skipped { logger.Info(kind+" pull unchanged", slog.String("hash", res.Hash)) } else { @@ -134,7 +152,7 @@ func StartPeriodicPuller( // Immediate pull on startup (async; does not block HTTP server startup) func() { - logger.Info("initial pull started") + logger.Info("initial pull started", slog.String("source", source.Name())) pullCtx := ctx cancel := func() {} @@ -143,14 +161,7 @@ func StartPeriodicPuller( } defer cancel() - res, err := PullOnce(pullCtx, db, client, applicationID, logger) - if err != nil { - logger.Warn("initial pull failed; service stays unready unless it already has data", "err", err) - return - } - - ready.SetReady(true) - logPull("initial", res) + runPull(pullCtx, "initial") }() for { @@ -169,14 +180,7 @@ func StartPeriodicPuller( } } - res, err := PullOnce(ctx, db, client, applicationID, logger) - if err != nil { - logger.Warn("periodic pull failed", "err", err) - continue - } - - ready.SetReady(true) - logPull("periodic", res) + runPull(ctx, "periodic") } } }() diff --git a/sync_pull_test.go b/sync_pull_test.go index 618a5c8..fdf1f57 100644 --- a/sync_pull_test.go +++ b/sync_pull_test.go @@ -35,9 +35,10 @@ func stOpenDB(t *testing.T) *DB { func stClient(baseURL, apiKey string) *TranslationClient { return NewTranslationClient(Config{ - TranslationBaseURL: baseURL, - TranslationAPIKey: apiKey, - HTTPTimeout: 5 * time.Second, + TranslationBaseURL: baseURL, + TranslationApplicationID: "app-1", + TranslationAPIKey: apiKey, + HTTPTimeout: 5 * time.Second, }) } @@ -118,9 +119,9 @@ func TestTranslationClientDownloadXLSXRequestShape(t *testing.T) { defer srv.Close() client := stClient(srv.URL, tc.apiKey) - body, err := client.DownloadXLSX(context.Background(), "app-1", stLogger()) + body, err := client.Fetch(context.Background(), stLogger()) if err != nil { - t.Fatalf("DownloadXLSX: %v", err) + t.Fatalf("Fetch: %v", err) } if string(body) != "body-bytes" { t.Errorf("body = %q, want %q", body, "body-bytes") @@ -178,7 +179,7 @@ func TestPullOnceFirstThenSkipThenReplace(t *testing.T) { client := stClient(srv.URL, "secret") // First pull: full import. - res1, err := PullOnce(ctx, db, client, "app-1", logger) + res1, err := PullOnce(ctx, db, client, logger) if err != nil { t.Fatalf("first PullOnce: %v", err) } @@ -234,7 +235,7 @@ func TestPullOnceFirstThenSkipThenReplace(t *testing.T) { t.Fatalf("SetMeta: %v", err) } - res2, err := PullOnce(ctx, db, client, "app-1", logger) + res2, err := PullOnce(ctx, db, client, logger) if err != nil { t.Fatalf("second PullOnce: %v", err) } @@ -269,7 +270,7 @@ func TestPullOnceFirstThenSkipThenReplace(t *testing.T) { // Third pull with changed payload: full replacement, removed row gone. setPayload(payload2) - res3, err := PullOnce(ctx, db, client, "app-1", logger) + res3, err := PullOnce(ctx, db, client, logger) if err != nil { t.Fatalf("third PullOnce: %v", err) } @@ -331,7 +332,7 @@ func TestPullOnceUpstreamHTTPError(t *testing.T) { client := stClient(srv.URL, "secret") - res, err := PullOnce(ctx, db, client, "app-1", logger) + res, err := PullOnce(ctx, db, client, logger) if err != nil { t.Fatalf("seed PullOnce: %v", err) } @@ -340,7 +341,7 @@ func TestPullOnceUpstreamHTTPError(t *testing.T) { fail = true mu.Unlock() - _, err = PullOnce(ctx, db, client, "app-1", logger) + _, err = PullOnce(ctx, db, client, logger) if err == nil { t.Fatal("PullOnce on upstream 500: want error, got nil") } @@ -386,7 +387,7 @@ func TestPullOnceInvalidXLSXBody(t *testing.T) { client := stClient(srv.URL, "secret") - res, err := PullOnce(ctx, db, client, "app-1", logger) + res, err := PullOnce(ctx, db, client, logger) if err != nil { t.Fatalf("seed PullOnce: %v", err) } @@ -395,7 +396,7 @@ func TestPullOnceInvalidXLSXBody(t *testing.T) { junk = true mu.Unlock() - _, err = PullOnce(ctx, db, client, "app-1", logger) + _, err = PullOnce(ctx, db, client, logger) if err == nil { t.Fatal("PullOnce on junk body: want parse error, got nil") } @@ -433,7 +434,7 @@ func TestPullOnceContextCancelled(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := PullOnce(ctx, db, client, "app-1", stLogger()) + _, err := PullOnce(ctx, db, client, stLogger()) if err == nil { t.Fatal("PullOnce with cancelled context: want error, got nil") } From 4f08e6da92515e3b10ecb38f586665b775508e13 Mon Sep 17 00:00:00 2001 From: frozenhelium Date: Wed, 8 Jul 2026 17:30:23 +0545 Subject: [PATCH 2/3] refactor(naming): make identifiers clear and consistent - rename TranslationClient to APISource, pairing with URLSource - rename MetaModel.K/V to Key/Value (columns unchanged via bun tags) - rename PullResult.Skipped to Unchanged (the pull runs, import is skipped) - rename Server.source to sourceName, handleGetStrings to handleStrings - rename metaKeyLastHash to metaKeyLastXLSXHash, mirroring the stored key - rename writeSchemaFile to writeOpenAPISpecFile, env to envOr - fix limit param shadowing the max builtin in readAllLimited - name pull timers by phase (fetchStart/parseStart/importStart) - clarify locals: bunDB, sqlDB, jitter, prevHash, metaErr, urlErr, slogLevel - consolidate duplicated test helpers into helpers_test.go - drop cryptic test helper prefixes (xt/dbt/ht/ct/st/src) --- .gitignore | 3 + config.go | 10 ++-- config_test.go | 44 +++++++-------- db.go | 28 ++++----- db_test.go | 127 +++++++++++++++++++---------------------- handlers.go | 20 +++---- handlers_test.go | 119 ++++++++++++++++++-------------------- helpers_test.go | 87 ++++++++++++++++++++++++++++ main.go | 20 +++---- models.go | 4 +- source.go | 14 ++--- source_test.go | 32 ++++------- sync_pull.go | 52 ++++++++--------- sync_pull_test.go | 141 +++++++++++++++------------------------------- xlsx_test.go | 75 +++++------------------- 15 files changed, 370 insertions(+), 406 deletions(-) create mode 100644 helpers_test.go diff --git a/.gitignore b/.gitignore index 6b1edb5..14f0512 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,6 @@ data/ # ----------------------------- tmp/ vendor/ + +# build output +/cacheppuccino diff --git a/config.go b/config.go index 0abef4d..703ffd2 100644 --- a/config.go +++ b/config.go @@ -52,9 +52,9 @@ func LoadConfig() (Config, error) { } cfg := Config{ - ListenAddr: env("LISTEN_ADDR", ":8080"), - SQLitePath: env("SQLITE_PATH", "/data/cacheppuccino.db"), - TranslationSource: env("TRANSLATION_SOURCE", sourceAPI), + ListenAddr: envOr("LISTEN_ADDR", ":8080"), + SQLitePath: envOr("SQLITE_PATH", "/data/cacheppuccino.db"), + TranslationSource: envOr("TRANSLATION_SOURCE", sourceAPI), TranslationBaseURL: os.Getenv("TRANSLATION_BASE_URL"), TranslationApplicationID: os.Getenv("TRANSLATION_APPLICATION_ID"), TranslationAPIKey: os.Getenv("TRANSLATION_API_KEY"), @@ -62,7 +62,7 @@ func LoadConfig() (Config, error) { HTTPTimeout: envDuration("HTTP_TIMEOUT", 30*time.Second), PullInterval: envDuration("PULL_INTERVAL", 10*time.Minute), InitialPullDeadline: envDuration("INITIAL_PULL_DEADLINE", 45*time.Second), - LogLevel: env("LOG_LEVEL", "info"), + LogLevel: envOr("LOG_LEVEL", "info"), } switch cfg.TranslationSource { @@ -103,7 +103,7 @@ func LoadConfig() (Config, error) { return cfg, nil } -func env(k, def string) string { +func envOr(k, def string) string { v := os.Getenv(k) if v == "" { return def diff --git a/config_test.go b/config_test.go index 303acc6..1ce3eaa 100644 --- a/config_test.go +++ b/config_test.go @@ -6,9 +6,9 @@ import ( "time" ) -// ctAllConfigEnvVars is every env var LoadConfig reads. Each test sets all of +// allConfigEnvVars is every env var LoadConfig reads. Each test sets all of // them explicitly ("" simulates unset) to shield tests from the host env. -var ctAllConfigEnvVars = []string{ +var allConfigEnvVars = []string{ "TRANSLATION_SOURCE", "TRANSLATION_BASE_URL", "TRANSLATION_APPLICATION_ID", @@ -22,17 +22,17 @@ var ctAllConfigEnvVars = []string{ "LOG_LEVEL", } -// ctSetConfigEnv sets every config env var, using values from overrides and +// setConfigEnv sets every config env var, using values from overrides and // "" for anything not listed there. -func ctSetConfigEnv(t *testing.T, overrides map[string]string) { +func setConfigEnv(t *testing.T, overrides map[string]string) { t.Helper() - for _, k := range ctAllConfigEnvVars { + for _, k := range allConfigEnvVars { t.Setenv(k, overrides[k]) } } -// ctRequiredEnv sets only the three required vars to placeholder values. -func ctRequiredEnv() map[string]string { +// requiredAPIEnv sets only the three required vars to placeholder values. +func requiredAPIEnv() map[string]string { return map[string]string{ "TRANSLATION_BASE_URL": "https://translate.example.com", "TRANSLATION_APPLICATION_ID": "app-id", @@ -41,7 +41,7 @@ func ctRequiredEnv() map[string]string { } func TestLoadConfigDefaults(t *testing.T) { - ctSetConfigEnv(t, ctRequiredEnv()) + setConfigEnv(t, requiredAPIEnv()) cfg, err := LoadConfig() if err != nil { @@ -132,7 +132,7 @@ func TestLoadConfigSourceMatrix(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctSetConfigEnv(t, tt.env) + setConfigEnv(t, tt.env) cfg, err := LoadConfig() if tt.wantErr { @@ -168,9 +168,9 @@ func TestLoadConfigMissingRequired(t *testing.T) { for _, missing := range required { t.Run(missing, func(t *testing.T) { - env := ctRequiredEnv() + env := requiredAPIEnv() env[missing] = "" - ctSetConfigEnv(t, env) + setConfigEnv(t, env) _, err := LoadConfig() if err == nil { @@ -183,7 +183,7 @@ func TestLoadConfigMissingRequired(t *testing.T) { } t.Run("all missing", func(t *testing.T) { - ctSetConfigEnv(t, nil) + setConfigEnv(t, nil) _, err := LoadConfig() if err == nil { @@ -247,11 +247,11 @@ func TestLoadConfigInvalidValues(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - env := ctRequiredEnv() + env := requiredAPIEnv() for k, v := range tt.overrides { env[k] = v } - ctSetConfigEnv(t, env) + setConfigEnv(t, env) _, err := LoadConfig() if err == nil { @@ -269,11 +269,11 @@ func TestLoadConfigInvalidValues(t *testing.T) { func TestLoadConfigMultipleProblems(t *testing.T) { // Missing one required var plus a bad duration plus a bad log level: // all must be reported in the single joined error. - env := ctRequiredEnv() + env := requiredAPIEnv() env["TRANSLATION_API_KEY"] = "" env["PULL_INTERVAL"] = "bogus" env["LOG_LEVEL"] = "loud" - ctSetConfigEnv(t, env) + setConfigEnv(t, env) _, err := LoadConfig() if err == nil { @@ -292,9 +292,9 @@ func TestLoadConfigMultipleProblems(t *testing.T) { func TestLoadConfigZeroInitialPullDeadline(t *testing.T) { // Zero means "no deadline" and must be accepted. - env := ctRequiredEnv() + env := requiredAPIEnv() env["INITIAL_PULL_DEADLINE"] = "0s" - ctSetConfigEnv(t, env) + setConfigEnv(t, env) cfg, err := LoadConfig() if err != nil { @@ -306,14 +306,14 @@ func TestLoadConfigZeroInitialPullDeadline(t *testing.T) { } func TestLoadConfigValidOverrides(t *testing.T) { - env := ctRequiredEnv() + env := requiredAPIEnv() env["LISTEN_ADDR"] = "127.0.0.1:9999" env["SQLITE_PATH"] = "/tmp/other.db" env["HTTP_TIMEOUT"] = "5s" env["PULL_INTERVAL"] = "1h30m" env["INITIAL_PULL_DEADLINE"] = "250ms" env["LOG_LEVEL"] = "debug" - ctSetConfigEnv(t, env) + setConfigEnv(t, env) cfg, err := LoadConfig() if err != nil { @@ -343,9 +343,9 @@ func TestLoadConfigValidOverrides(t *testing.T) { func TestLoadConfigValidLogLevels(t *testing.T) { for _, level := range []string{"debug", "info", "warn", "error"} { t.Run(level, func(t *testing.T) { - env := ctRequiredEnv() + env := requiredAPIEnv() env["LOG_LEVEL"] = level - ctSetConfigEnv(t, env) + setConfigEnv(t, env) cfg, err := LoadConfig() if err != nil { diff --git a/db.go b/db.go index b722505..4da997a 100644 --- a/db.go +++ b/db.go @@ -18,7 +18,7 @@ import ( const ( metaKeyLastPull = "last_pull_rfc3339" - metaKeyLastHash = "last_xlsx_sha256" + metaKeyLastXLSXHash = "last_xlsx_sha256" metaKeyLastPullError = "last_pull_error" metaKeyLastImportRows = "last_import_rows" ) @@ -34,15 +34,15 @@ func OpenDB(path string) (*DB, error) { return nil, err } - sqldb, err := sql.Open(sqliteshim.ShimName, "file:"+path+"?mode=rwc") + sqlDB, err := sql.Open(sqliteshim.ShimName, "file:"+path+"?mode=rwc") if err != nil { return nil, err } // Single connection: pragmas below stick for the process lifetime, // and SQLite sees one writer, so no busy contention between our own queries. - sqldb.SetMaxOpenConns(1) - sqldb.SetConnMaxLifetime(0) + sqlDB.SetMaxOpenConns(1) + sqlDB.SetConnMaxLifetime(0) // Set via Exec (not DSN params) so behavior is identical across the // cgo (mattn) and pure-Go (modernc) drivers sqliteshim may pick. @@ -53,17 +53,17 @@ func OpenDB(path string) (*DB, error) { "PRAGMA temp_store = MEMORY", } for _, p := range pragmas { - if _, err := sqldb.Exec(p); err != nil { - _ = sqldb.Close() + if _, err := sqlDB.Exec(p); err != nil { + _ = sqlDB.Close() return nil, fmt.Errorf("%s: %w", p, err) } } - bdb := bun.NewDB(sqldb, sqlitedialect.New()) + bunDB := bun.NewDB(sqlDB, sqlitedialect.New()) - db := &DB{sql: sqldb, bun: bdb} + db := &DB{sql: sqlDB, bun: bunDB} if err := db.migrate(context.Background()); err != nil { - _ = sqldb.Close() + _ = sqlDB.Close() return nil, err } @@ -161,9 +161,9 @@ func (db *DB) ReplaceImport(ctx context.Context, rows []StringRow, hash string, } meta := []MetaModel{ - {K: metaKeyLastHash, V: hash}, - {K: metaKeyLastPull, V: pulledAt.UTC().Format(time.RFC3339)}, - {K: metaKeyLastImportRows, V: strconv.Itoa(len(models))}, + {Key: metaKeyLastXLSXHash, Value: hash}, + {Key: metaKeyLastPull, Value: pulledAt.UTC().Format(time.RFC3339)}, + {Key: metaKeyLastImportRows, Value: strconv.Itoa(len(models))}, } for _, m := range meta { if err := upsertMetaTx(ctx, tx, m); err != nil { @@ -228,7 +228,7 @@ func (db *DB) GetStringsByPagesLang(ctx context.Context, pages []string, lang st } func (db *DB) SetMeta(ctx context.Context, k, v string) error { - m := &MetaModel{K: k, V: v} + m := &MetaModel{Key: k, Value: v} _, err := db.bun.NewInsert(). Model(m). @@ -254,5 +254,5 @@ func (db *DB) GetMeta(ctx context.Context, k string) (string, bool, error) { return "", false, err } - return m.V, true, nil + return m.Value, true, nil } diff --git a/db_test.go b/db_test.go index 2c07d6e..e0fbd09 100644 --- a/db_test.go +++ b/db_test.go @@ -10,31 +10,20 @@ import ( "time" ) -var dbtUpdatedAt = time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) +var fixedUpdatedAt = time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) -// dbtOpen opens a fresh DB under t.TempDir() and closes it on cleanup. -func dbtOpen(t *testing.T) *DB { - t.Helper() - db, err := OpenDB(filepath.Join(t.TempDir(), "test.db")) - if err != nil { - t.Fatalf("OpenDB: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - return db -} - -func dbtRow(page, key, lang, value string) StringRow { - return StringRow{Page: page, Key: key, Lang: lang, Value: value, UpdatedAt: dbtUpdatedAt} +func newStringRow(page, key, lang, value string) StringRow { + return StringRow{Page: page, Key: key, Lang: lang, Value: value, UpdatedAt: fixedUpdatedAt} } -func dbtMustImport(t *testing.T, db *DB, rows []StringRow, hash string, pulledAt time.Time) { +func mustImport(t *testing.T, db *DB, rows []StringRow, hash string, pulledAt time.Time) { t.Helper() if err := db.ReplaceImport(context.Background(), rows, hash, pulledAt); err != nil { t.Fatalf("ReplaceImport: %v", err) } } -func dbtCountStrings(t *testing.T, db *DB) int { +func countStrings(t *testing.T, db *DB) int { t.Helper() var n int if err := db.sql.QueryRow("SELECT COUNT(*) FROM strings").Scan(&n); err != nil { @@ -43,8 +32,8 @@ func dbtCountStrings(t *testing.T, db *DB) int { return n } -// dbtMeta fetches a meta key and fails the test if the key is absent. -func dbtMeta(t *testing.T, db *DB, key string) string { +// mustMeta fetches a meta key and fails the test if the key is absent. +func mustMeta(t *testing.T, db *DB, key string) string { t.Helper() v, ok, err := db.GetMeta(context.Background(), key) if err != nil { @@ -56,10 +45,10 @@ func dbtMeta(t *testing.T, db *DB, key string) string { return v } -// dbtPullTime parses the stored last-pull meta value as RFC3339. -func dbtPullTime(t *testing.T, db *DB) time.Time { +// lastPullTime parses the stored last-pull meta value as RFC3339. +func lastPullTime(t *testing.T, db *DB) time.Time { t.Helper() - raw := dbtMeta(t, db, metaKeyLastPull) + raw := mustMeta(t, db, metaKeyLastPull) parsed, err := time.Parse(time.RFC3339, raw) if err != nil { t.Fatalf("meta %q = %q is not RFC3339: %v", metaKeyLastPull, raw, err) @@ -81,7 +70,7 @@ func TestOpenDBCreatesParentDirs(t *testing.T) { } func TestOpenDBEnablesWAL(t *testing.T) { - db := dbtOpen(t) + db := openTestDB(t) var mode string if err := db.sql.QueryRow("PRAGMA journal_mode").Scan(&mode); err != nil { @@ -93,7 +82,7 @@ func TestOpenDBEnablesWAL(t *testing.T) { } func TestHasStrings(t *testing.T) { - db := dbtOpen(t) + db := openTestDB(t) ctx := context.Background() has, err := db.HasStrings(ctx) @@ -104,7 +93,7 @@ func TestHasStrings(t *testing.T) { t.Fatal("HasStrings on fresh DB = true, want false") } - dbtMustImport(t, db, []StringRow{dbtRow("home", "title", "en", "Home")}, "h1", time.Now()) + mustImport(t, db, []StringRow{newStringRow("home", "title", "en", "Home")}, "h1", time.Now()) has, err = db.HasStrings(ctx) if err != nil { @@ -116,18 +105,18 @@ func TestHasStrings(t *testing.T) { } func TestReplaceImportBasics(t *testing.T) { - db := dbtOpen(t) + db := openTestDB(t) ctx := context.Background() hash := "deadbeefcafe" pulledAt := time.Date(2026, 7, 8, 10, 30, 0, 0, time.FixedZone("NPT", 5*3600+45*60)) rows := []StringRow{ - dbtRow("home", "title", "en", "Home"), - dbtRow("home", "subtitle", "en", "Welcome"), - dbtRow("about", "title", "en", "About"), + newStringRow("home", "title", "en", "Home"), + newStringRow("home", "subtitle", "en", "Welcome"), + newStringRow("about", "title", "en", "About"), } - dbtMustImport(t, db, rows, hash, pulledAt) + mustImport(t, db, rows, hash, pulledAt) got, cleaned, err := db.GetStringsByPagesLang(ctx, []string{"home", "about"}, "en") if err != nil { @@ -144,31 +133,31 @@ func TestReplaceImportBasics(t *testing.T) { t.Errorf("cleaned = %v, want %v", cleaned, wantCleaned) } - if gotHash := dbtMeta(t, db, metaKeyLastHash); gotHash != hash { - t.Errorf("meta %q = %q, want %q", metaKeyLastHash, gotHash, hash) + if gotHash := mustMeta(t, db, metaKeyLastXLSXHash); gotHash != hash { + t.Errorf("meta %q = %q, want %q", metaKeyLastXLSXHash, gotHash, hash) } - if gotPull := dbtPullTime(t, db); !gotPull.Equal(pulledAt) { + if gotPull := lastPullTime(t, db); !gotPull.Equal(pulledAt) { t.Errorf("meta %q = %v, want time equal to %v", metaKeyLastPull, gotPull, pulledAt) } } func TestReplaceImportFullReplace(t *testing.T) { - db := dbtOpen(t) + db := openTestDB(t) ctx := context.Background() first := []StringRow{ - dbtRow("old", "title", "en", "Old title"), - dbtRow("old", "body", "en", "Old body"), + newStringRow("old", "title", "en", "Old title"), + newStringRow("old", "body", "en", "Old body"), } - dbtMustImport(t, db, first, "hash-1", time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + mustImport(t, db, first, "hash-1", time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) second := []StringRow{ - dbtRow("new", "title", "en", "New title"), + newStringRow("new", "title", "en", "New title"), } pulledAt2 := time.Date(2026, 2, 2, 0, 0, 0, 0, time.UTC) - dbtMustImport(t, db, second, "hash-2", pulledAt2) + mustImport(t, db, second, "hash-2", pulledAt2) - if n := dbtCountStrings(t, db); n != 1 { + if n := countStrings(t, db); n != 1 { t.Errorf("row count after second import = %d, want 1", n) } @@ -183,28 +172,28 @@ func TestReplaceImportFullReplace(t *testing.T) { t.Errorf("got[\"new\"] = %v, want map with only new title", got["new"]) } - if gotHash := dbtMeta(t, db, metaKeyLastHash); gotHash != "hash-2" { - t.Errorf("meta %q = %q, want %q", metaKeyLastHash, gotHash, "hash-2") + if gotHash := mustMeta(t, db, metaKeyLastXLSXHash); gotHash != "hash-2" { + t.Errorf("meta %q = %q, want %q", metaKeyLastXLSXHash, gotHash, "hash-2") } - if gotPull := dbtPullTime(t, db); !gotPull.Equal(pulledAt2) { + if gotPull := lastPullTime(t, db); !gotPull.Equal(pulledAt2) { t.Errorf("meta %q = %v, want time equal to %v", metaKeyLastPull, gotPull, pulledAt2) } } func TestReplaceImportDuplicateRowsLastWins(t *testing.T) { - db := dbtOpen(t) + db := openTestDB(t) ctx := context.Background() rows := []StringRow{ - dbtRow("home", "title", "en", "first"), - dbtRow("home", "title", "en", "second"), - dbtRow("home", "title", "en", "third"), + newStringRow("home", "title", "en", "first"), + newStringRow("home", "title", "en", "second"), + newStringRow("home", "title", "en", "third"), } if err := db.ReplaceImport(ctx, rows, "h", time.Now()); err != nil { t.Fatalf("ReplaceImport with duplicate (page,key,lang) rows: %v", err) } - if n := dbtCountStrings(t, db); n != 1 { + if n := countStrings(t, db); n != 1 { t.Errorf("row count = %d, want 1", n) } @@ -218,17 +207,17 @@ func TestReplaceImportDuplicateRowsLastWins(t *testing.T) { } func TestReplaceImportChunking(t *testing.T) { - db := dbtOpen(t) + db := openTestDB(t) // > 400 rows spans several ~189-row insert chunks. const total = 450 rows := make([]StringRow, 0, total) for i := range total { - rows = append(rows, dbtRow("bulk", fmt.Sprintf("key-%03d", i), "en", fmt.Sprintf("value-%03d", i))) + rows = append(rows, newStringRow("bulk", fmt.Sprintf("key-%03d", i), "en", fmt.Sprintf("value-%03d", i))) } - dbtMustImport(t, db, rows, "bulk-hash", time.Now()) + mustImport(t, db, rows, "bulk-hash", time.Now()) - if n := dbtCountStrings(t, db); n != total { + if n := countStrings(t, db); n != total { t.Fatalf("row count = %d, want %d", n, total) } @@ -245,10 +234,10 @@ func TestReplaceImportChunking(t *testing.T) { } func TestReplaceImportEmptyWipesTableAndSetsMeta(t *testing.T) { - db := dbtOpen(t) + db := openTestDB(t) ctx := context.Background() - dbtMustImport(t, db, []StringRow{dbtRow("home", "title", "en", "Home")}, "hash-1", time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + mustImport(t, db, []StringRow{newStringRow("home", "title", "en", "Home")}, "hash-1", time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) pulledAt2 := time.Date(2026, 3, 3, 6, 7, 8, 0, time.UTC) if err := db.ReplaceImport(ctx, nil, "hash-empty", pulledAt2); err != nil { @@ -263,21 +252,21 @@ func TestReplaceImportEmptyWipesTableAndSetsMeta(t *testing.T) { t.Error("HasStrings after empty import = true, want false") } - if gotHash := dbtMeta(t, db, metaKeyLastHash); gotHash != "hash-empty" { - t.Errorf("meta %q = %q, want %q", metaKeyLastHash, gotHash, "hash-empty") + if gotHash := mustMeta(t, db, metaKeyLastXLSXHash); gotHash != "hash-empty" { + t.Errorf("meta %q = %q, want %q", metaKeyLastXLSXHash, gotHash, "hash-empty") } - if gotPull := dbtPullTime(t, db); !gotPull.Equal(pulledAt2) { + if gotPull := lastPullTime(t, db); !gotPull.Equal(pulledAt2) { t.Errorf("meta %q = %v, want time equal to %v", metaKeyLastPull, gotPull, pulledAt2) } } func TestGetStringsByPagesLangPageCleaning(t *testing.T) { - db := dbtOpen(t) + db := openTestDB(t) ctx := context.Background() - dbtMustImport(t, db, []StringRow{ - dbtRow("home", "title", "en", "Home"), - dbtRow("about", "title", "en", "About"), + mustImport(t, db, []StringRow{ + newStringRow("home", "title", "en", "Home"), + newStringRow("about", "title", "en", "About"), }, "h", time.Now()) cases := []struct { @@ -322,9 +311,9 @@ func TestGetStringsByPagesLangPageCleaning(t *testing.T) { } func TestGetStringsByPagesLangUnknownPageEmptyMap(t *testing.T) { - db := dbtOpen(t) + db := openTestDB(t) - dbtMustImport(t, db, []StringRow{dbtRow("home", "title", "en", "Home")}, "h", time.Now()) + mustImport(t, db, []StringRow{newStringRow("home", "title", "en", "Home")}, "h", time.Now()) got, _, err := db.GetStringsByPagesLang(context.Background(), []string{"missing"}, "en") if err != nil { @@ -343,13 +332,13 @@ func TestGetStringsByPagesLangUnknownPageEmptyMap(t *testing.T) { } func TestGetStringsByPagesLangFiltersByLang(t *testing.T) { - db := dbtOpen(t) + db := openTestDB(t) ctx := context.Background() - dbtMustImport(t, db, []StringRow{ - dbtRow("home", "title", "en", "Home"), - dbtRow("home", "title", "fr", "Accueil"), - dbtRow("home", "greeting", "fr", "Bonjour"), + mustImport(t, db, []StringRow{ + newStringRow("home", "title", "en", "Home"), + newStringRow("home", "title", "fr", "Accueil"), + newStringRow("home", "greeting", "fr", "Bonjour"), }, "h", time.Now()) gotFR, _, err := db.GetStringsByPagesLang(ctx, []string{"home"}, "fr") @@ -372,7 +361,7 @@ func TestGetStringsByPagesLangFiltersByLang(t *testing.T) { } func TestGetMetaMissingKey(t *testing.T) { - db := dbtOpen(t) + db := openTestDB(t) v, ok, err := db.GetMeta(context.Background(), "no-such-key") if err != nil { @@ -387,7 +376,7 @@ func TestGetMetaMissingKey(t *testing.T) { } func TestSetMetaOverwrites(t *testing.T) { - db := dbtOpen(t) + db := openTestDB(t) ctx := context.Background() if err := db.SetMeta(ctx, "k1", "v1"); err != nil { diff --git a/handlers.go b/handlers.go index 89f9263..e5380c2 100644 --- a/handlers.go +++ b/handlers.go @@ -10,10 +10,10 @@ import ( ) type Server struct { - db *DB - ready *ReadyState - logger *slog.Logger - source string + db *DB + ready *ReadyState + logger *slog.Logger + sourceName string } type StringsResponse struct { @@ -42,7 +42,7 @@ type StatusResponse struct { func (s *Server) routes() http.Handler { mux := http.NewServeMux() - mux.HandleFunc("GET /strings", s.handleGetStrings) + mux.HandleFunc("GET /strings", s.handleStrings) mux.HandleFunc("GET /healthz", s.handleHealthz) mux.HandleFunc("GET /readyz", s.handleReadyz) mux.HandleFunc("GET /status", s.handleStatus) @@ -85,7 +85,7 @@ func (s *Server) handleReadyz(w http.ResponseWriter, r *http.Request) { func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { meta := map[string]string{ metaKeyLastPull: "", - metaKeyLastHash: "", + metaKeyLastXLSXHash: "", metaKeyLastPullError: "", metaKeyLastImportRows: "", } @@ -102,9 +102,9 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { importRows, _ := strconv.Atoi(meta[metaKeyLastImportRows]) writeOK(w, http.StatusOK, StatusResponse{ - Source: s.source, + Source: s.sourceName, LastPull: meta[metaKeyLastPull], - LastHash: meta[metaKeyLastHash], + LastHash: meta[metaKeyLastXLSXHash], LastPullError: meta[metaKeyLastPullError], LastImportRows: importRows, Ready: s.ready.IsReady(), @@ -112,7 +112,7 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { }) } -func (s *Server) handleGetStrings(w http.ResponseWriter, r *http.Request) { +func (s *Server) handleStrings(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() // Language codes are case-insensitive (BCP 47); import stores lowercase. @@ -159,7 +159,7 @@ func (s *Server) handleGetStrings(w http.ResponseWriter, r *http.Request) { // so the import hash is a valid ETag for every /strings URL. // Only 200/304 responses carry the caching headers. var etag string - if hash, ok, err := s.db.GetMeta(r.Context(), metaKeyLastHash); err == nil && ok { + if hash, ok, err := s.db.GetMeta(r.Context(), metaKeyLastXLSXHash); err == nil && ok { etag = `"` + hash + `"` if etagMatches(r.Header.Get("If-None-Match"), etag) { setCacheHeaders(w, etag) diff --git a/handlers_test.go b/handlers_test.go index 8855f81..d59c260 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -3,56 +3,47 @@ package main import ( "context" "encoding/json" - "io" - "log/slog" "net/http" "net/http/httptest" - "path/filepath" "reflect" "strings" "testing" "time" ) -const htTestHash = "51b23fc6f6c1de1c69a9b0f0f8a06f21c2e4a89f3a2e2b9d7c6a5e4d3c2b1a09" +const seedHash = "51b23fc6f6c1de1c69a9b0f0f8a06f21c2e4a89f3a2e2b9d7c6a5e4d3c2b1a09" -type htAPIError struct { +type apiError struct { Code string `json:"code"` Message string `json:"message"` Details map[string]string `json:"details"` } -type htEnvelope struct { +type responseEnvelope struct { Ok bool `json:"ok"` Data json.RawMessage `json:"data"` - Error *htAPIError `json:"error"` + Error *apiError `json:"error"` } -type htStringsData struct { +type stringsData struct { Lang string `json:"lang"` Pages []string `json:"pages"` Strings map[string]map[string]string `json:"strings"` } -func htNewServer(t *testing.T) *Server { +func newTestServer(t *testing.T) *Server { t.Helper() - db, err := OpenDB(filepath.Join(t.TempDir(), "test.db")) - if err != nil { - t.Fatalf("OpenDB: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - return &Server{ - db: db, + db: openTestDB(t), ready: &ReadyState{}, - logger: slog.New(slog.NewJSONHandler(io.Discard, nil)), + logger: discardLogger(), } } -// htSeedFrench imports a small fixture of French strings and returns the +// seedFrench imports a small fixture of French strings and returns the // import hash used as the ETag source. -func htSeedFrench(t *testing.T, srv *Server) string { +func seedFrench(t *testing.T, srv *Server) string { t.Helper() rows := []StringRow{ @@ -60,13 +51,13 @@ func htSeedFrench(t *testing.T, srv *Server) string { {Page: "a", Key: "welcome", Lang: "fr", Value: "bienvenue", UpdatedAt: time.Now()}, {Page: "b", Key: "bye", Lang: "fr", Value: "au revoir", UpdatedAt: time.Now()}, } - if err := srv.db.ReplaceImport(context.Background(), rows, htTestHash, time.Now()); err != nil { + if err := srv.db.ReplaceImport(context.Background(), rows, seedHash, time.Now()); err != nil { t.Fatalf("ReplaceImport: %v", err) } - return htTestHash + return seedHash } -func htGet(t *testing.T, h http.Handler, target string, header map[string]string) *httptest.ResponseRecorder { +func doGet(t *testing.T, h http.Handler, target string, header map[string]string) *httptest.ResponseRecorder { t.Helper() req := httptest.NewRequest(http.MethodGet, target, nil) @@ -78,17 +69,17 @@ func htGet(t *testing.T, h http.Handler, target string, header map[string]string return rec } -func htDecodeEnvelope(t *testing.T, rec *httptest.ResponseRecorder) htEnvelope { +func decodeEnvelope(t *testing.T, rec *httptest.ResponseRecorder) responseEnvelope { t.Helper() - var env htEnvelope + var env responseEnvelope if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil { t.Fatalf("decode envelope: %v (body: %q)", err, rec.Body.String()) } return env } -func htDecodeData(t *testing.T, env htEnvelope, out any) { +func decodeData(t *testing.T, env responseEnvelope, out any) { t.Helper() if env.Data == nil { @@ -100,8 +91,8 @@ func htDecodeData(t *testing.T, env htEnvelope, out any) { } func TestHealthz(t *testing.T) { - srv := htNewServer(t) - rec := htGet(t, srv.routes(), "/healthz", nil) + srv := newTestServer(t) + rec := doGet(t, srv.routes(), "/healthz", nil) if rec.Code != http.StatusOK { t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) @@ -115,7 +106,7 @@ func TestHealthz(t *testing.T) { } func TestReadyz(t *testing.T) { - srv := htNewServer(t) + srv := newTestServer(t) h := srv.routes() // Always 200 while the process is up, even before any servable data @@ -123,18 +114,18 @@ func TestReadyz(t *testing.T) { for _, ready := range []bool{false, true} { srv.ready.SetReady(ready) - rec := htGet(t, h, "/readyz", nil) + rec := doGet(t, h, "/readyz", nil) if rec.Code != http.StatusOK { t.Fatalf("ready=%v: status = %d, want %d", ready, rec.Code, http.StatusOK) } - env := htDecodeEnvelope(t, rec) + env := decodeEnvelope(t, rec) if !env.Ok { t.Errorf("ready=%v: ok = false, want true", ready) } var data struct { Status string `json:"status"` } - htDecodeData(t, env, &data) + decodeData(t, env, &data) if data.Status != "ready" { t.Errorf("ready=%v: status = %q, want %q", ready, data.Status, "ready") } @@ -142,20 +133,20 @@ func TestReadyz(t *testing.T) { } func TestStatus(t *testing.T) { - srv := htNewServer(t) + srv := newTestServer(t) h := srv.routes() - rec := htGet(t, h, "/status", nil) + rec := doGet(t, h, "/status", nil) if rec.Code != http.StatusOK { t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) } - env := htDecodeEnvelope(t, rec) + env := decodeEnvelope(t, rec) if !env.Ok { t.Fatalf("ok = false, want true") } var fields map[string]any - htDecodeData(t, env, &fields) + decodeData(t, env, &fields) for _, k := range []string{"last_pull", "last_hash", "ready", "version"} { if _, present := fields[k]; !present { t.Errorf("data missing field %q", k) @@ -163,7 +154,7 @@ func TestStatus(t *testing.T) { } var data StatusResponse - htDecodeData(t, env, &data) + decodeData(t, env, &data) if data.Version != "dev" { t.Errorf("version = %q, want %q", data.Version, "dev") } @@ -174,14 +165,14 @@ func TestStatus(t *testing.T) { t.Errorf("last_hash = %q, want empty before import", data.LastHash) } - hash := htSeedFrench(t, srv) + hash := seedFrench(t, srv) srv.ready.SetReady(true) - rec = htGet(t, h, "/status", nil) + rec = doGet(t, h, "/status", nil) if rec.Code != http.StatusOK { t.Fatalf("after import: status = %d, want %d", rec.Code, http.StatusOK) } - htDecodeData(t, htDecodeEnvelope(t, rec), &data) + decodeData(t, decodeEnvelope(t, rec), &data) if data.LastHash != hash { t.Errorf("last_hash = %q, want %q", data.LastHash, hash) } @@ -194,7 +185,7 @@ func TestStatus(t *testing.T) { } func TestGetStringsValidation(t *testing.T) { - srv := htNewServer(t) + srv := newTestServer(t) h := srv.routes() tests := []struct { @@ -216,11 +207,11 @@ func TestGetStringsValidation(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - rec := htGet(t, h, tc.target, nil) + rec := doGet(t, h, tc.target, nil) if rec.Code != http.StatusBadRequest { t.Fatalf("status = %d, want %d", rec.Code, http.StatusBadRequest) } - env := htDecodeEnvelope(t, rec) + env := decodeEnvelope(t, rec) if env.Ok { t.Errorf("ok = true, want false") } @@ -238,16 +229,16 @@ func TestGetStringsValidation(t *testing.T) { } func TestGetStringsLangNormalization(t *testing.T) { - srv := htNewServer(t) - htSeedFrench(t, srv) + srv := newTestServer(t) + seedFrench(t, srv) - rec := htGet(t, srv.routes(), "/strings?lang=FR&page=a", nil) + rec := doGet(t, srv.routes(), "/strings?lang=FR&page=a", nil) if rec.Code != http.StatusOK { t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) } - var data htStringsData - htDecodeData(t, htDecodeEnvelope(t, rec), &data) + var data stringsData + decodeData(t, decodeEnvelope(t, rec), &data) if data.Lang != "fr" { t.Errorf("lang = %q, want %q", data.Lang, "fr") } @@ -257,8 +248,8 @@ func TestGetStringsLangNormalization(t *testing.T) { } func TestGetStringsPagesParsing(t *testing.T) { - srv := htNewServer(t) - htSeedFrench(t, srv) + srv := newTestServer(t) + seedFrench(t, srv) h := srv.routes() tests := []struct { @@ -276,13 +267,13 @@ func TestGetStringsPagesParsing(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - rec := htGet(t, h, "/strings?lang=fr&"+tc.query, nil) + rec := doGet(t, h, "/strings?lang=fr&"+tc.query, nil) if rec.Code != http.StatusOK { t.Fatalf("status = %d, want %d (body: %s)", rec.Code, http.StatusOK, rec.Body.String()) } - var data htStringsData - htDecodeData(t, htDecodeEnvelope(t, rec), &data) + var data stringsData + decodeData(t, decodeEnvelope(t, rec), &data) if !reflect.DeepEqual(data.Pages, tc.wantPages) { t.Errorf("pages = %v, want %v", data.Pages, tc.wantPages) } @@ -299,16 +290,16 @@ func TestGetStringsPagesParsing(t *testing.T) { } func TestGetStringsUnknownPage(t *testing.T) { - srv := htNewServer(t) - htSeedFrench(t, srv) + srv := newTestServer(t) + seedFrench(t, srv) - rec := htGet(t, srv.routes(), "/strings?lang=fr&page=a&page=nope", nil) + rec := doGet(t, srv.routes(), "/strings?lang=fr&page=a&page=nope", nil) if rec.Code != http.StatusOK { t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) } - var data htStringsData - htDecodeData(t, htDecodeEnvelope(t, rec), &data) + var data stringsData + decodeData(t, decodeEnvelope(t, rec), &data) unknown, present := data.Strings["nope"] if !present { t.Fatalf("strings missing unknown page %q", "nope") @@ -322,9 +313,9 @@ func TestGetStringsUnknownPage(t *testing.T) { } func TestGetStringsNoETagBeforeImport(t *testing.T) { - srv := htNewServer(t) + srv := newTestServer(t) - rec := htGet(t, srv.routes(), "/strings?lang=fr&page=a", nil) + rec := doGet(t, srv.routes(), "/strings?lang=fr&page=a", nil) if rec.Code != http.StatusOK { t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) } @@ -337,8 +328,8 @@ func TestGetStringsNoETagBeforeImport(t *testing.T) { } func TestGetStringsETag(t *testing.T) { - srv := htNewServer(t) - hash := htSeedFrench(t, srv) + srv := newTestServer(t) + hash := seedFrench(t, srv) h := srv.routes() etag := `"` + hash + `"` @@ -361,7 +352,7 @@ func TestGetStringsETag(t *testing.T) { if tc.ifNoneMatch != "" { header["If-None-Match"] = tc.ifNoneMatch } - rec := htGet(t, h, "/strings?lang=fr&page=a", header) + rec := doGet(t, h, "/strings?lang=fr&page=a", header) if rec.Code != tc.wantStatus { t.Fatalf("status = %d, want %d", rec.Code, tc.wantStatus) @@ -381,8 +372,8 @@ func TestGetStringsETag(t *testing.T) { return } - var data htStringsData - htDecodeData(t, htDecodeEnvelope(t, rec), &data) + var data stringsData + decodeData(t, decodeEnvelope(t, rec), &data) if got := data.Strings["a"]["hello"]; got != "bonjour" { t.Errorf("strings.a.hello = %q, want %q", got, "bonjour") } diff --git a/helpers_test.go b/helpers_test.go new file mode 100644 index 0000000..74b92eb --- /dev/null +++ b/helpers_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "io" + "log/slog" + "path/filepath" + "testing" + + "github.com/xuri/excelize/v2" +) + +// discardLogger returns a logger that swallows all output. +func discardLogger() *slog.Logger { + return slog.New(slog.NewJSONHandler(io.Discard, nil)) +} + +// openTestDB opens a fresh SQLite database under t.TempDir() and closes it +// on cleanup. +func openTestDB(t *testing.T) *DB { + t.Helper() + db, err := OpenDB(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("OpenDB: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return db +} + +// xlsxSheet describes one sheet of an in-memory test workbook. +type xlsxSheet struct { + name string + rows [][]any +} + +// buildXLSX builds an in-memory workbook. The first sheet replaces the +// default "Sheet1"; subsequent sheets are appended. +func buildXLSX(t *testing.T, sheets []xlsxSheet) []byte { + t.Helper() + + f := excelize.NewFile() + defer func() { _ = f.Close() }() + + for i, sheet := range sheets { + if i == 0 { + if sheet.name != "Sheet1" { + if err := f.SetSheetName("Sheet1", sheet.name); err != nil { + t.Fatalf("SetSheetName: %v", err) + } + } + } else { + if _, err := f.NewSheet(sheet.name); err != nil { + t.Fatalf("NewSheet: %v", err) + } + } + for rowIndex, row := range sheet.rows { + cell, err := excelize.CoordinatesToCellName(1, rowIndex+1) + if err != nil { + t.Fatalf("CoordinatesToCellName: %v", err) + } + if err := f.SetSheetRow(sheet.name, cell, &row); err != nil { + t.Fatalf("SetSheetRow: %v", err) + } + } + } + + buf, err := f.WriteToBuffer() + if err != nil { + t.Fatalf("WriteToBuffer: %v", err) + } + return buf.Bytes() +} + +// translationsXLSX builds a workbook with a single "Translations" sheet; +// rows[0] is the header. +func translationsXLSX(t *testing.T, rows [][]string) []byte { + t.Helper() + + anyRows := make([][]any, 0, len(rows)) + for _, row := range rows { + anyRow := make([]any, len(row)) + for i, cell := range row { + anyRow[i] = cell + } + anyRows = append(anyRows, anyRow) + } + return buildXLSX(t, []xlsxSheet{{name: "Translations", rows: anyRows}}) +} diff --git a/main.go b/main.go index b05ecbb..fc4ce42 100644 --- a/main.go +++ b/main.go @@ -24,7 +24,7 @@ func main() { flag.Parse() if *schema { - if err := writeSchemaFile("openapi.json"); err != nil { + if err := writeOpenAPISpecFile("openapi.json"); err != nil { fmt.Fprintln(os.Stderr, err.Error()) os.Exit(1) } @@ -61,7 +61,7 @@ func main() { if cfg.TranslationSource == sourceURL { source = NewURLSource(cfg) } else { - source = NewTranslationClient(cfg) + source = NewAPISource(cfg) } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) @@ -77,7 +77,7 @@ func main() { } ready.SetReady(hasData) - srv := &Server{db: db, ready: ready, logger: logger, source: source.Name()} + srv := &Server{db: db, ready: ready, logger: logger, sourceName: source.Name()} logger.Info("cacheppuccino starting", slog.String("version", version), @@ -133,7 +133,7 @@ func main() { } } -func writeSchemaFile(path string) error { +func writeOpenAPISpecFile(path string) error { spec, err := buildOpenAPISpec("/") if err != nil { return err @@ -148,19 +148,19 @@ func writeSchemaFile(path string) error { } func newLogger(level string) *slog.Logger { - var lvl slog.Level + var slogLevel slog.Level switch level { case "debug": - lvl = slog.LevelDebug + slogLevel = slog.LevelDebug case "warn": - lvl = slog.LevelWarn + slogLevel = slog.LevelWarn case "error": - lvl = slog.LevelError + slogLevel = slog.LevelError default: - lvl = slog.LevelInfo + slogLevel = slog.LevelInfo } - h := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: lvl}) + h := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slogLevel}) return slog.New(h) } diff --git a/models.go b/models.go index 1c3878b..a203399 100644 --- a/models.go +++ b/models.go @@ -19,6 +19,6 @@ type StringModel struct { type MetaModel struct { bun.BaseModel `bun:"table:meta"` - K string `bun:"k,pk,notnull"` - V string `bun:"v,notnull"` + Key string `bun:"k,pk,notnull"` + Value string `bun:"v,notnull"` } diff --git a/source.go b/source.go index 277357a..9b6956e 100644 --- a/source.go +++ b/source.go @@ -61,9 +61,9 @@ func (s *URLSource) Fetch(ctx context.Context, logger *slog.Logger) ([]byte, err if err != nil { // Transport errors (*url.Error) embed the full URL including any // query credentials; rebuild the message around the redacted URL. - var ue *url.Error - if errors.As(err, &ue) { - return nil, fmt.Errorf("xlsx request failed: %s %s: %w", ue.Op, s.redactedURL, ue.Err) + var urlErr *url.Error + if errors.As(err, &urlErr) { + return nil, fmt.Errorf("xlsx request failed: %s %s: %w", urlErr.Op, s.redactedURL, urlErr.Err) } return nil, fmt.Errorf("xlsx request failed: GET %s", s.redactedURL) } @@ -109,13 +109,13 @@ func (s *URLSource) redactSecrets(msg string) string { } // readAllLimited reads r fully, erroring if it exceeds max bytes. -func readAllLimited(r io.Reader, max int64) ([]byte, error) { - b, err := io.ReadAll(io.LimitReader(r, max+1)) +func readAllLimited(r io.Reader, limit int64) ([]byte, error) { + b, err := io.ReadAll(io.LimitReader(r, limit+1)) if err != nil { return nil, err } - if int64(len(b)) > max { - return nil, fmt.Errorf("response exceeds %d bytes", max) + if int64(len(b)) > limit { + return nil, fmt.Errorf("response exceeds %d bytes", limit) } return b, nil } diff --git a/source_test.go b/source_test.go index 381bd09..63c7660 100644 --- a/source_test.go +++ b/source_test.go @@ -3,21 +3,14 @@ package main import ( "context" "encoding/json" - "io" - "log/slog" "net/http" "net/http/httptest" - "path/filepath" "strings" "testing" "time" ) -func srcLogger() *slog.Logger { - return slog.New(slog.NewJSONHandler(io.Discard, nil)) -} - -func srcURLSource(rawURL string) *URLSource { +func testURLSource(rawURL string) *URLSource { return NewURLSource(Config{ TranslationXLSXURL: rawURL, HTTPTimeout: 5 * time.Second, @@ -34,8 +27,8 @@ func TestURLSourceFetch(t *testing.T) { })) defer srv.Close() - src := srcURLSource(srv.URL + "/files/translations.xlsx") - body, err := src.Fetch(context.Background(), srcLogger()) + src := testURLSource(srv.URL + "/files/translations.xlsx") + body, err := src.Fetch(context.Background(), discardLogger()) if err != nil { t.Fatalf("Fetch: %v", err) } @@ -59,7 +52,7 @@ func TestURLSourceFetch(t *testing.T) { })) defer srv.Close() - _, err := srcURLSource(srv.URL).Fetch(context.Background(), srcLogger()) + _, err := testURLSource(srv.URL).Fetch(context.Background(), discardLogger()) if err == nil { t.Fatal("Fetch on 404: want error, got nil") } @@ -71,8 +64,8 @@ func TestURLSourceFetch(t *testing.T) { t.Run("transport error redacts query credentials", func(t *testing.T) { // Errors end up on the public /status endpoint; a SAS/presigned // token in the query must never appear there. - src := srcURLSource("http://127.0.0.1:1/translations.xlsx?sv=2022-11-02&sig=SUPERSECRETSAS") - _, err := src.Fetch(context.Background(), srcLogger()) + src := testURLSource("http://127.0.0.1:1/translations.xlsx?sv=2022-11-02&sig=SUPERSECRETSAS") + _, err := src.Fetch(context.Background(), discardLogger()) if err == nil { t.Fatal("Fetch on unreachable host: want error, got nil") } @@ -90,8 +83,8 @@ func TestURLSourceFetch(t *testing.T) { })) defer srv.Close() - src := srcURLSource(srv.URL + "/translations.xlsx?sig=SUPERSECRETSAS") - _, err := src.Fetch(context.Background(), srcLogger()) + src := testURLSource(srv.URL + "/translations.xlsx?sig=SUPERSECRETSAS") + _, err := src.Fetch(context.Background(), discardLogger()) if err == nil { t.Fatal("Fetch on 403: want error, got nil") } @@ -156,15 +149,10 @@ func TestReadAllLimited(t *testing.T) { func TestStatusReportsSourceAndPullOutcome(t *testing.T) { ctx := context.Background() - db, err := OpenDB(filepath.Join(t.TempDir(), "test.db")) - if err != nil { - t.Fatalf("OpenDB: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - + db := openTestDB(t) ready := &ReadyState{} ready.SetReady(true) - srv := &Server{db: db, ready: ready, logger: srcLogger(), source: "url"} + srv := &Server{db: db, ready: ready, logger: discardLogger(), sourceName: "url"} ts := httptest.NewServer(srv.routes()) defer ts.Close() diff --git a/sync_pull.go b/sync_pull.go index 70c0f8d..6f28f49 100644 --- a/sync_pull.go +++ b/sync_pull.go @@ -9,16 +9,16 @@ import ( "time" ) -// TranslationClient fetches the XLSX export from the IFRC translation API. -type TranslationClient struct { +// APISource fetches the XLSX export from the IFRC translation API. +type APISource struct { baseURL string applicationID string apiKey string http *http.Client } -func NewTranslationClient(cfg Config) *TranslationClient { - return &TranslationClient{ +func NewAPISource(cfg Config) *APISource { + return &APISource{ baseURL: cfg.TranslationBaseURL, applicationID: cfg.TranslationApplicationID, apiKey: cfg.TranslationAPIKey, @@ -28,9 +28,9 @@ func NewTranslationClient(cfg Config) *TranslationClient { } } -func (c *TranslationClient) Name() string { return "api" } +func (c *APISource) Name() string { return "api" } -func (c *TranslationClient) Fetch(ctx context.Context, logger *slog.Logger) ([]byte, error) { +func (c *APISource) Fetch(ctx context.Context, logger *slog.Logger) ([]byte, error) { url := fmt.Sprintf("%s/api/Application/%s/Translation/export", c.baseURL, c.applicationID) logger.Info("pull: requesting export", slog.String("url", url)) @@ -58,9 +58,9 @@ func (c *TranslationClient) Fetch(ctx context.Context, logger *slog.Logger) ([]b } type PullResult struct { - Skipped bool - Hash string - Rows int + Unchanged bool + Hash string + Rows int } func PullOnce( @@ -69,43 +69,43 @@ func PullOnce( source XLSXSource, logger *slog.Logger, ) (PullResult, error) { - t0 := time.Now() + fetchStart := time.Now() xlsx, err := source.Fetch(ctx, logger) if err != nil { return PullResult{}, err } - logger.Info("pull: download done", slog.Duration("dur", time.Since(t0)), slog.Int("bytes", len(xlsx))) + logger.Info("pull: download done", slog.Duration("dur", time.Since(fetchStart)), slog.Int("bytes", len(xlsx))) hash := HashBytes(xlsx) - prev, ok, err := db.GetMeta(ctx, metaKeyLastHash) + prevHash, ok, err := db.GetMeta(ctx, metaKeyLastXLSXHash) if err != nil { return PullResult{}, err } - if ok && prev == hash { + if ok && prevHash == hash { // Content unchanged; still record that a pull succeeded. if err := db.SetMeta(ctx, metaKeyLastPull, time.Now().UTC().Format(time.RFC3339)); err != nil { return PullResult{}, err } - return PullResult{Skipped: true, Hash: hash, Rows: 0}, nil + return PullResult{Unchanged: true, Hash: hash, Rows: 0}, nil } - t1 := time.Now() + parseStart := time.Now() rows, err := ParseXLSX(xlsx) if err != nil { return PullResult{}, err } - logger.Info("pull: parse done", slog.Duration("dur", time.Since(t1)), slog.Int("rows", len(rows))) + logger.Info("pull: parse done", slog.Duration("dur", time.Since(parseStart)), slog.Int("rows", len(rows))) - t2 := time.Now() + importStart := time.Now() if err := db.ReplaceImport(ctx, rows, hash, time.Now()); err != nil { return PullResult{}, err } - logger.Info("pull: import done", slog.Duration("dur", time.Since(t2))) + logger.Info("pull: import done", slog.Duration("dur", time.Since(importStart))) - return PullResult{Skipped: false, Hash: hash, Rows: len(rows)}, nil + return PullResult{Unchanged: false, Hash: hash, Rows: len(rows)}, nil } func StartPeriodicPuller( @@ -127,20 +127,20 @@ func StartPeriodicPuller( if err != nil { // Skip recording on shutdown; the parent ctx is the process ctx. if ctx.Err() == nil { - if merr := db.SetMeta(ctx, metaKeyLastPullError, err.Error()); merr != nil { - logger.Warn("pull: failed to record pull error", "err", merr) + if metaErr := db.SetMeta(ctx, metaKeyLastPullError, err.Error()); metaErr != nil { + logger.Warn("pull: failed to record pull error", "err", metaErr) } } logger.Warn(kind+" pull failed; serving cached data if any", "err", err) return } - if merr := db.SetMeta(ctx, metaKeyLastPullError, ""); merr != nil { - logger.Warn("pull: failed to clear pull error", "err", merr) + if metaErr := db.SetMeta(ctx, metaKeyLastPullError, ""); metaErr != nil { + logger.Warn("pull: failed to clear pull error", "err", metaErr) } ready.SetReady(true) - if res.Skipped { + if res.Unchanged { logger.Info(kind+" pull unchanged", slog.String("hash", res.Hash)) } else { logger.Info(kind+" pull imported", slog.Int("rows", res.Rows), slog.String("hash", res.Hash)) @@ -170,8 +170,8 @@ func StartPeriodicPuller( return case <-ticker.C: if jitterMax > 0 { - sleep := time.Duration(time.Now().UnixNano() % int64(jitterMax+1)) - timer := time.NewTimer(sleep) + jitter := time.Duration(time.Now().UnixNano() % int64(jitterMax+1)) + timer := time.NewTimer(jitter) select { case <-ctx.Done(): timer.Stop() diff --git a/sync_pull_test.go b/sync_pull_test.go index fdf1f57..1c8d5ba 100644 --- a/sync_pull_test.go +++ b/sync_pull_test.go @@ -3,38 +3,17 @@ package main import ( "context" "errors" - "io" - "log/slog" "net/http" "net/http/httptest" - "path/filepath" "reflect" "strings" "sync" "testing" "time" - - "github.com/xuri/excelize/v2" ) -const stSheetName = "Translations" - -func stLogger() *slog.Logger { - return slog.New(slog.NewJSONHandler(io.Discard, nil)) -} - -func stOpenDB(t *testing.T) *DB { - t.Helper() - db, err := OpenDB(filepath.Join(t.TempDir(), "test.db")) - if err != nil { - t.Fatalf("OpenDB: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - return db -} - -func stClient(baseURL, apiKey string) *TranslationClient { - return NewTranslationClient(Config{ +func testAPISource(baseURL, apiKey string) *APISource { + return NewAPISource(Config{ TranslationBaseURL: baseURL, TranslationApplicationID: "app-1", TranslationAPIKey: apiKey, @@ -42,38 +21,10 @@ func stClient(baseURL, apiKey string) *TranslationClient { }) } -// stXLSX builds an in-memory workbook whose "Translations" sheet holds the -// given rows; rows[0] is the header. -func stXLSX(t *testing.T, rows [][]string) []byte { - t.Helper() - - f := excelize.NewFile() - defer func() { _ = f.Close() }() - - if err := f.SetSheetName("Sheet1", stSheetName); err != nil { - t.Fatalf("SetSheetName: %v", err) - } - for i := range rows { - cell, err := excelize.CoordinatesToCellName(1, i+1) - if err != nil { - t.Fatalf("CoordinatesToCellName: %v", err) - } - if err := f.SetSheetRow(stSheetName, cell, &rows[i]); err != nil { - t.Fatalf("SetSheetRow: %v", err) - } - } - - buf, err := f.WriteToBuffer() - if err != nil { - t.Fatalf("WriteToBuffer: %v", err) - } - return buf.Bytes() -} - -// stSeedPayload has 3 data rows x 2 languages = 6 imported StringRows. -func stSeedPayload(t *testing.T) []byte { +// seedXLSX has 3 data rows x 2 languages = 6 imported StringRows. +func seedXLSX(t *testing.T) []byte { t.Helper() - return stXLSX(t, [][]string{ + return translationsXLSX(t, [][]string{ {"page", "key", "en", "fr"}, {"home", "greeting", "Hello", "Bonjour"}, {"home", "farewell", "Goodbye", "Au revoir"}, @@ -81,14 +32,14 @@ func stSeedPayload(t *testing.T) []byte { }) } -func stSeedWantEN() map[string]map[string]string { +func seedWantEN() map[string]map[string]string { return map[string]map[string]string{ "home": {"greeting": "Hello", "farewell": "Goodbye"}, "about": {"title": "About us"}, } } -func TestTranslationClientDownloadXLSXRequestShape(t *testing.T) { +func TestAPISourceDownloadXLSXRequestShape(t *testing.T) { cases := []struct { name string apiKey string @@ -118,8 +69,8 @@ func TestTranslationClientDownloadXLSXRequestShape(t *testing.T) { })) defer srv.Close() - client := stClient(srv.URL, tc.apiKey) - body, err := client.Fetch(context.Background(), stLogger()) + client := testAPISource(srv.URL, tc.apiKey) + body, err := client.Fetch(context.Background(), discardLogger()) if err != nil { t.Fatalf("Fetch: %v", err) } @@ -148,12 +99,12 @@ func TestTranslationClientDownloadXLSXRequestShape(t *testing.T) { func TestPullOnceFirstThenSkipThenReplace(t *testing.T) { ctx := context.Background() - db := stOpenDB(t) - logger := stLogger() + db := openTestDB(t) + logger := discardLogger() - payload1 := stSeedPayload(t) + payload1 := seedXLSX(t) // v2: "home"/"farewell" removed, "contact"/"email" added. - payload2 := stXLSX(t, [][]string{ + payload2 := translationsXLSX(t, [][]string{ {"page", "key", "en", "fr"}, {"home", "greeting", "Hello", "Bonjour"}, {"about", "title", "About us", "A propos"}, @@ -176,15 +127,15 @@ func TestPullOnceFirstThenSkipThenReplace(t *testing.T) { })) defer srv.Close() - client := stClient(srv.URL, "secret") + client := testAPISource(srv.URL, "secret") // First pull: full import. res1, err := PullOnce(ctx, db, client, logger) if err != nil { t.Fatalf("first PullOnce: %v", err) } - if res1.Skipped { - t.Error("first pull: Skipped = true, want false") + if res1.Unchanged { + t.Error("first pull: Unchanged = true, want false") } if res1.Rows != 6 { t.Errorf("first pull: Rows = %d, want 6", res1.Rows) @@ -197,7 +148,7 @@ func TestPullOnceFirstThenSkipThenReplace(t *testing.T) { if err != nil { t.Fatalf("GetStringsByPagesLang(en): %v", err) } - if want := stSeedWantEN(); !reflect.DeepEqual(gotEN, want) { + if want := seedWantEN(); !reflect.DeepEqual(gotEN, want) { t.Errorf("en strings after first pull = %v, want %v", gotEN, want) } gotFR, _, err := db.GetStringsByPagesLang(ctx, []string{"home"}, "fr") @@ -211,12 +162,12 @@ func TestPullOnceFirstThenSkipThenReplace(t *testing.T) { t.Errorf("fr strings after first pull = %v, want %v", gotFR, wantFR) } - hashMeta, ok, err := db.GetMeta(ctx, metaKeyLastHash) + hashMeta, ok, err := db.GetMeta(ctx, metaKeyLastXLSXHash) if err != nil { - t.Fatalf("GetMeta(%s): %v", metaKeyLastHash, err) + t.Fatalf("GetMeta(%s): %v", metaKeyLastXLSXHash, err) } if !ok || hashMeta != res1.Hash { - t.Errorf("meta %s = %q (ok=%v), want %q", metaKeyLastHash, hashMeta, ok, res1.Hash) + t.Errorf("meta %s = %q (ok=%v), want %q", metaKeyLastXLSXHash, hashMeta, ok, res1.Hash) } pullMeta, ok, err := db.GetMeta(ctx, metaKeyLastPull) if err != nil { @@ -239,8 +190,8 @@ func TestPullOnceFirstThenSkipThenReplace(t *testing.T) { if err != nil { t.Fatalf("second PullOnce: %v", err) } - if !res2.Skipped { - t.Error("second pull: Skipped = false, want true") + if !res2.Unchanged { + t.Error("second pull: Unchanged = false, want true") } if res2.Rows != 0 { t.Errorf("second pull: Rows = %d, want 0", res2.Rows) @@ -274,8 +225,8 @@ func TestPullOnceFirstThenSkipThenReplace(t *testing.T) { if err != nil { t.Fatalf("third PullOnce: %v", err) } - if res3.Skipped { - t.Error("third pull: Skipped = true, want false") + if res3.Unchanged { + t.Error("third pull: Unchanged = true, want false") } if res3.Rows != 6 { t.Errorf("third pull: Rows = %d, want 6", res3.Rows) @@ -300,20 +251,20 @@ func TestPullOnceFirstThenSkipThenReplace(t *testing.T) { t.Errorf("en strings after third pull = %v, want %v (removed row must be gone)", gotEN3, wantEN3) } - hashMeta3, ok, err := db.GetMeta(ctx, metaKeyLastHash) + hashMeta3, ok, err := db.GetMeta(ctx, metaKeyLastXLSXHash) if err != nil { - t.Fatalf("GetMeta(%s): %v", metaKeyLastHash, err) + t.Fatalf("GetMeta(%s): %v", metaKeyLastXLSXHash, err) } if !ok || hashMeta3 != res3.Hash { - t.Errorf("meta %s = %q (ok=%v), want %q", metaKeyLastHash, hashMeta3, ok, res3.Hash) + t.Errorf("meta %s = %q (ok=%v), want %q", metaKeyLastXLSXHash, hashMeta3, ok, res3.Hash) } } func TestPullOnceUpstreamHTTPError(t *testing.T) { ctx := context.Background() - db := stOpenDB(t) - logger := stLogger() - payload := stSeedPayload(t) + db := openTestDB(t) + logger := discardLogger() + payload := seedXLSX(t) var mu sync.Mutex fail := false @@ -330,7 +281,7 @@ func TestPullOnceUpstreamHTTPError(t *testing.T) { })) defer srv.Close() - client := stClient(srv.URL, "secret") + client := testAPISource(srv.URL, "secret") res, err := PullOnce(ctx, db, client, logger) if err != nil { @@ -349,27 +300,27 @@ func TestPullOnceUpstreamHTTPError(t *testing.T) { t.Errorf("error = %q, want it to mention status 500", err) } - hashMeta, ok, err := db.GetMeta(ctx, metaKeyLastHash) + hashMeta, ok, err := db.GetMeta(ctx, metaKeyLastXLSXHash) if err != nil { - t.Fatalf("GetMeta(%s): %v", metaKeyLastHash, err) + t.Fatalf("GetMeta(%s): %v", metaKeyLastXLSXHash, err) } if !ok || hashMeta != res.Hash { - t.Errorf("meta %s = %q (ok=%v), want unchanged %q", metaKeyLastHash, hashMeta, ok, res.Hash) + t.Errorf("meta %s = %q (ok=%v), want unchanged %q", metaKeyLastXLSXHash, hashMeta, ok, res.Hash) } gotEN, _, err := db.GetStringsByPagesLang(ctx, []string{"home", "about"}, "en") if err != nil { t.Fatalf("GetStringsByPagesLang: %v", err) } - if want := stSeedWantEN(); !reflect.DeepEqual(gotEN, want) { + if want := seedWantEN(); !reflect.DeepEqual(gotEN, want) { t.Errorf("en strings after failed pull = %v, want unchanged %v", gotEN, want) } } func TestPullOnceInvalidXLSXBody(t *testing.T) { ctx := context.Background() - db := stOpenDB(t) - logger := stLogger() - payload := stSeedPayload(t) + db := openTestDB(t) + logger := discardLogger() + payload := seedXLSX(t) var mu sync.Mutex junk := false @@ -385,7 +336,7 @@ func TestPullOnceInvalidXLSXBody(t *testing.T) { })) defer srv.Close() - client := stClient(srv.URL, "secret") + client := testAPISource(srv.URL, "secret") res, err := PullOnce(ctx, db, client, logger) if err != nil { @@ -404,24 +355,24 @@ func TestPullOnceInvalidXLSXBody(t *testing.T) { t.Errorf("error = %q, want a parse error, not a download error", err) } - hashMeta, ok, err := db.GetMeta(ctx, metaKeyLastHash) + hashMeta, ok, err := db.GetMeta(ctx, metaKeyLastXLSXHash) if err != nil { - t.Fatalf("GetMeta(%s): %v", metaKeyLastHash, err) + t.Fatalf("GetMeta(%s): %v", metaKeyLastXLSXHash, err) } if !ok || hashMeta != res.Hash { - t.Errorf("meta %s = %q (ok=%v), want unchanged %q", metaKeyLastHash, hashMeta, ok, res.Hash) + t.Errorf("meta %s = %q (ok=%v), want unchanged %q", metaKeyLastXLSXHash, hashMeta, ok, res.Hash) } gotEN, _, err := db.GetStringsByPagesLang(ctx, []string{"home", "about"}, "en") if err != nil { t.Fatalf("GetStringsByPagesLang: %v", err) } - if want := stSeedWantEN(); !reflect.DeepEqual(gotEN, want) { + if want := seedWantEN(); !reflect.DeepEqual(gotEN, want) { t.Errorf("en strings after failed pull = %v, want unchanged %v", gotEN, want) } } func TestPullOnceContextCancelled(t *testing.T) { - db := stOpenDB(t) + db := openTestDB(t) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { time.Sleep(50 * time.Millisecond) @@ -429,12 +380,12 @@ func TestPullOnceContextCancelled(t *testing.T) { })) defer srv.Close() - client := stClient(srv.URL, "secret") + client := testAPISource(srv.URL, "secret") ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := PullOnce(ctx, db, client, stLogger()) + _, err := PullOnce(ctx, db, client, discardLogger()) if err == nil { t.Fatal("PullOnce with cancelled context: want error, got nil") } diff --git a/xlsx_test.go b/xlsx_test.go index b4a1780..067371d 100644 --- a/xlsx_test.go +++ b/xlsx_test.go @@ -4,65 +4,20 @@ import ( "regexp" "testing" "time" - - "github.com/xuri/excelize/v2" ) -type xtSheet struct { - name string - rows [][]any -} - -// xtBuildXLSX builds an in-memory workbook. The first sheet in sheets -// replaces the default "Sheet1"; subsequent sheets are appended. -func xtBuildXLSX(t *testing.T, sheets []xtSheet) []byte { - t.Helper() - - f := excelize.NewFile() - defer func() { _ = f.Close() }() - - for i, s := range sheets { - if i == 0 { - if s.name != "Sheet1" { - if err := f.SetSheetName("Sheet1", s.name); err != nil { - t.Fatalf("SetSheetName: %v", err) - } - } - } else { - if _, err := f.NewSheet(s.name); err != nil { - t.Fatalf("NewSheet: %v", err) - } - } - for r, row := range s.rows { - cell, err := excelize.CoordinatesToCellName(1, r+1) - if err != nil { - t.Fatalf("CoordinatesToCellName: %v", err) - } - if err := f.SetSheetRow(s.name, cell, &row); err != nil { - t.Fatalf("SetSheetRow: %v", err) - } - } - } - - buf, err := f.WriteToBuffer() - if err != nil { - t.Fatalf("WriteToBuffer: %v", err) - } - return buf.Bytes() -} - -// xtRowEq compares everything except UpdatedAt. -func xtRowEq(a, b StringRow) bool { +// rowContentEq compares everything except UpdatedAt. +func rowContentEq(a, b StringRow) bool { return a.Page == b.Page && a.Key == b.Key && a.Lang == b.Lang && a.Value == b.Value } -func xtAssertRows(t *testing.T, got, want []StringRow) { +func assertRows(t *testing.T, got, want []StringRow) { t.Helper() if len(got) != len(want) { t.Fatalf("got %d rows, want %d\ngot: %+v", len(got), len(want), got) } for i := range want { - if !xtRowEq(got[i], want[i]) { + if !rowContentEq(got[i], want[i]) { t.Errorf("row %d: got %+v, want %+v", i, got[i], want[i]) } } @@ -71,7 +26,7 @@ func xtAssertRows(t *testing.T, got, want []StringRow) { func TestParseXLSXHappyPath(t *testing.T) { // A decoy first sheet with an invalid header ensures the // "Translations" sheet is preferred over the first sheet. - data := xtBuildXLSX(t, []xtSheet{ + data := buildXLSX(t, []xlsxSheet{ { name: "Decoy", rows: [][]any{{"Namespace", "Key", "en"}, {"x", "y", "z"}}, @@ -99,7 +54,7 @@ func TestParseXLSXHappyPath(t *testing.T) { {Page: "home", Key: "subtitle", Lang: "en", Value: "World"}, {Page: "home", Key: "subtitle", Lang: "es", Value: "Mundo"}, } - xtAssertRows(t, got, want) + assertRows(t, got, want) for i, r := range got { if r.UpdatedAt.IsZero() { @@ -119,7 +74,7 @@ func TestParseXLSXHappyPath(t *testing.T) { func TestParseXLSXSheetFallback(t *testing.T) { // No "Translations" sheet: the first sheet is parsed instead. - data := xtBuildXLSX(t, []xtSheet{ + data := buildXLSX(t, []xlsxSheet{ { name: "Sheet1", rows: [][]any{ @@ -133,7 +88,7 @@ func TestParseXLSXSheetFallback(t *testing.T) { if err != nil { t.Fatalf("ParseXLSX: %v", err) } - xtAssertRows(t, got, []StringRow{ + assertRows(t, got, []StringRow{ {Page: "about", Key: "heading", Lang: "en", Value: "About us"}, }) } @@ -152,7 +107,7 @@ func TestParseXLSXHeaderValidation(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - data := xtBuildXLSX(t, []xtSheet{ + data := buildXLSX(t, []xlsxSheet{ { name: "Translations", rows: [][]any{tt.header, {"p1", "k1", "v1"}}, @@ -168,7 +123,7 @@ func TestParseXLSXHeaderValidation(t *testing.T) { if err != nil { t.Fatalf("ParseXLSX: %v", err) } - xtAssertRows(t, got, []StringRow{ + assertRows(t, got, []StringRow{ {Page: "p1", Key: "k1", Lang: "en", Value: "v1"}, }) }) @@ -176,7 +131,7 @@ func TestParseXLSXHeaderValidation(t *testing.T) { } func TestParseXLSXRowFiltering(t *testing.T) { - data := xtBuildXLSX(t, []xtSheet{ + data := buildXLSX(t, []xlsxSheet{ { name: "Translations", rows: [][]any{ @@ -195,14 +150,14 @@ func TestParseXLSXRowFiltering(t *testing.T) { if err != nil { t.Fatalf("ParseXLSX: %v", err) } - xtAssertRows(t, got, []StringRow{ + assertRows(t, got, []StringRow{ {Page: "p3", Key: "k3", Lang: "fr", Value: "salut"}, {Page: "p4", Key: "k4", Lang: "en", Value: "hello"}, }) } func TestParseXLSXEmptySheet(t *testing.T) { - data := xtBuildXLSX(t, []xtSheet{{name: "Translations"}}) + data := buildXLSX(t, []xlsxSheet{{name: "Translations"}}) got, err := ParseXLSX(data) if err != nil { @@ -219,7 +174,7 @@ func TestParseXLSXInvalidInput(t *testing.T) { } } -var xtHexRe = regexp.MustCompile(`^[0-9a-f]{64}$`) +var sha256HexRe = regexp.MustCompile(`^[0-9a-f]{64}$`) func TestHashBytes(t *testing.T) { tests := []struct { @@ -241,7 +196,7 @@ func TestHashBytes(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := HashBytes(tt.input) - if !xtHexRe.MatchString(got) { + if !sha256HexRe.MatchString(got) { t.Errorf("HashBytes(%q) = %q, not 64 lowercase hex chars", tt.input, got) } if got != tt.want { From e340cfd41a5bf806188f4b3e0f7d85db7d393762 Mon Sep 17 00:00:00 2001 From: frozenhelium Date: Wed, 8 Jul 2026 17:57:53 +0545 Subject: [PATCH 3/3] refactor(http): extract isHTTPSuccess helper for 2xx checks --- main.go | 2 +- source.go | 7 ++++++- source_test.go | 21 +++++++++++++++++++++ sync_pull.go | 2 +- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index fc4ce42..4f849cf 100644 --- a/main.go +++ b/main.go @@ -189,7 +189,7 @@ func doHealthcheck(url string) error { } defer resp.Body.Close() - if resp.StatusCode/100 != 2 { + if !isHTTPSuccess(resp.StatusCode) { return fmt.Errorf("healthcheck failed: %s", resp.Status) } return nil diff --git a/source.go b/source.go index 9b6956e..d5fc106 100644 --- a/source.go +++ b/source.go @@ -69,7 +69,7 @@ func (s *URLSource) Fetch(ctx context.Context, logger *slog.Logger) ([]byte, err } defer resp.Body.Close() - if resp.StatusCode/100 != 2 { + if !isHTTPSuccess(resp.StatusCode) { b, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<10)) // Error bodies from blob stores can echo signature parameters. return nil, fmt.Errorf("download failed: %s: %s", resp.Status, s.redactSecrets(string(b))) @@ -108,6 +108,11 @@ func (s *URLSource) redactSecrets(msg string) string { return msg } +// isHTTPSuccess reports whether the status code is in the 2xx range. +func isHTTPSuccess(statusCode int) bool { + return statusCode >= 200 && statusCode < 300 +} + // readAllLimited reads r fully, erroring if it exceeds max bytes. func readAllLimited(r io.Reader, limit int64) ([]byte, error) { b, err := io.ReadAll(io.LimitReader(r, limit+1)) diff --git a/source_test.go b/source_test.go index 63c7660..1e1d7d6 100644 --- a/source_test.go +++ b/source_test.go @@ -113,6 +113,27 @@ func TestRedactURL(t *testing.T) { } } +func TestIsHTTPSuccess(t *testing.T) { + tests := []struct { + statusCode int + want bool + }{ + {199, false}, + {200, true}, + {204, true}, + {299, true}, + {300, false}, + {304, false}, + {404, false}, + {500, false}, + } + for _, tt := range tests { + if got := isHTTPSuccess(tt.statusCode); got != tt.want { + t.Errorf("isHTTPSuccess(%d) = %v, want %v", tt.statusCode, got, tt.want) + } + } +} + func TestReadAllLimited(t *testing.T) { tests := []struct { name string diff --git a/sync_pull.go b/sync_pull.go index 6f28f49..63472d2 100644 --- a/sync_pull.go +++ b/sync_pull.go @@ -49,7 +49,7 @@ func (c *APISource) Fetch(ctx context.Context, logger *slog.Logger) ([]byte, err } defer resp.Body.Close() - if resp.StatusCode/100 != 2 { + if !isHTTPSuccess(resp.StatusCode) { b, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<10)) return nil, fmt.Errorf("download failed: %s: %s", resp.Status, string(b)) }