From 3bbea6b3def7a097bfbfd647f973e442bf187547 Mon Sep 17 00:00:00 2001 From: Peter Lord Date: Fri, 10 Jul 2026 00:34:32 -0700 Subject: [PATCH] Run the snapshot rebuild before charts prewarm, not after MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The instrumentation (#583) showed the leader logs "refresh cycle: starting" and then nothing — it hangs in the very first cycle step, prewarm_charts(), and never reaches the entity-stats snapshot rebuild. prewarm_charts loads the full ~740k-run frame from Mongo, which under current DB load runs long enough to starve the rebuild indefinitely. That's why prod was frozen serving a stale snapshot (v10): the snapshot rebuild never ran. PR #567 moved prewarm_charts to the FRONT of the cycle so /charts warmed fast after a deploy. That let it block the snapshot rebuild. Move it back to the END: the snapshot rebuild (tier list / Codex Score / metrics — the critical product data) runs first and can't be starved. /charts warms a bit later after a deploy as the trade-off, which is far better than the snapshot never rebuilding. --- backend/app/routers/runs.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/backend/app/routers/runs.py b/backend/app/routers/runs.py index 4a09ea7f..12237072 100644 --- a/backend/app/routers/runs.py +++ b/backend/app/routers/runs.py @@ -1187,20 +1187,12 @@ def _run_refresh_cycle() -> None: _cyc0 = time.time() logger.info("refresh cycle: starting (leader)") - # Warm the /charts page FIRST so the default no-filter view of each chart - # is in Redis within seconds of a deploy, instead of waiting behind the - # entity-stats snapshot walk below (which can run for minutes). Frame - # charts warm immediately; blob charts that still need the snapshot come - # back "building" and re-warm on the next cycle. One worker, shared cache. - try: - if app_cache.enabled(): - from .charts import prewarm_charts - - prewarm_charts() - except Exception: - logger.warning("charts prewarm failed", exc_info=True) - logger.info("refresh cycle: charts prewarm done at %.1fs", time.time() - _cyc0) - + # The entity-stats snapshot rebuild below is the critical product data + # (tier list / Codex Score / metrics), so it runs FIRST and nothing may + # starve it. Charts prewarm — which loads the full ~740k-run frame from + # Mongo and can run long under DB load — is deferred to the END of the + # cycle. This reverses PR #567's ordering: putting prewarm first let it + # block the snapshot rebuild indefinitely, so the snapshot never advanced. try: refresh_stats_summary() except Exception: @@ -1251,6 +1243,19 @@ def _run_refresh_cycle() -> None: except Exception: logger.warning("entity-scores cache warm failed", exc_info=True) + # Charts prewarm LAST — it loads the full run frame from Mongo and can run + # for a while under load, so it must never delay the snapshot rebuild + # above. Trade-off: /charts warms a bit later after a deploy; the snapshot + # actually completing is the priority. + try: + if app_cache.enabled(): + from .charts import prewarm_charts + + prewarm_charts() + except Exception: + logger.warning("charts prewarm failed", exc_info=True) + logger.info("refresh cycle: charts prewarm done at %.1fs", time.time() - _cyc0) + threading.Thread(target=_loop, daemon=True, name="stats-refresher").start()