From 6ff1aecce687c1a15d4938020ee2ca48b9ca52dc Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 3 Nov 2026 07:29:29 +0000 Subject: [PATCH 01/14] Prompt rewrites fit cards with less free graphics memory, wait for memory other programs hold... Prompt rewrites fit cards with less free graphics memory, wait for memory other programs hold, and record memory readings when they fail Prompt rewrite places the model by the free graphics memory it needs, waits and starts again on its own when other programs hold that memory, names up to three of those programs on the page, and on a memory failure hands the card's total and free memory, the placement floors and the release result to the feedback form. The feedback quote leaves out the program names and matches the line the page shows. --- freevideo_engine/gpu_holders.py | 144 +++++++++++++++++++++++++ freevideo_engine/prompt_vlm/load.py | 18 +++- freevideo_engine/prompt_vlm/service.py | 142 ++++++++++++++++++++---- freevideo_engine/prompt_vlm/worker.py | 115 ++++++++++++++++---- freevideo_engine/resident_process.py | 11 +- web/issue_button.js | 7 +- web/prompt_enhance.css | 1 + web/prompt_enhance.js | 42 ++++++-- 8 files changed, 420 insertions(+), 60 deletions(-) create mode 100644 freevideo_engine/gpu_holders.py diff --git a/freevideo_engine/gpu_holders.py b/freevideo_engine/gpu_holders.py new file mode 100644 index 00000000..04408ad9 --- /dev/null +++ b/freevideo_engine/gpu_holders.py @@ -0,0 +1,144 @@ +"""Which other programs hold memory on the GPU, so the page can say what to close. + +Bytes come from NVML; Windows (WDDM) usually reports none per process, so +there they come from the "GPU Process Memory" performance counters that Task +Manager shows. Those are summed over every adapter the process uses. FreeVideo's +own processes and the desktop's system components are left out: the user +cannot close them. Names are shown on the page only, never written to reports. +""" +import ctypes as C +import os +from pathlib import Path +import re +import sys + +MiB = 2**20 +# Smaller holders do not free enough to matter. +FLOOR = 128 * MiB +LIMIT = 3 +# Desktop compositors and shells on Linux; Windows ones are found by location. +LINUX_DESKTOP = {'Xorg', 'Xwayland', 'gnome-shell', 'kwin_x11', 'kwin_wayland', 'plasmashell', 'cinnamon', 'mutter'} + + +def counter_bytes(): + """{pid: dedicated GPU bytes} from Windows performance counters, or None.""" + from ctypes import wintypes as W + + class Value(C.Structure): + _fields_ = [('status', W.DWORD), ('value', C.c_longlong)] + + class Item(C.Structure): + _fields_ = [('name', C.c_wchar_p), ('value', Value)] + + pdh = C.WinDLL('pdh') + query, counter = C.c_void_p(), C.c_void_p() + if pdh.PdhOpenQueryW(None, None, C.byref(query)): + return None + try: + if pdh.PdhAddEnglishCounterW(query, r'\GPU Process Memory(*)\Dedicated Usage', None, C.byref(counter)): + return None + if pdh.PdhCollectQueryData(query): + return None + large = 0x400 + for _ in range(3): + # Instances come and go between asking the size and reading them. + size, count = W.DWORD(0), W.DWORD(0) + pdh.PdhGetFormattedCounterArrayW(counter, large, C.byref(size), C.byref(count), None) + buffer = C.create_string_buffer(size.value + 4096) + size = W.DWORD(len(buffer)) + if pdh.PdhGetFormattedCounterArrayW(counter, large, C.byref(size), C.byref(count), buffer) == 0: + break + else: + return None + found = {} + for item in C.cast(buffer, C.POINTER(Item))[:count.value]: + match = re.match(r'pid_(\d+)_', item.name or '') + if match and item.value.status == 0 and item.value.value > 0: + pid = int(match.group(1)) + found[pid] = found.get(pid, 0) + item.value.value + return found + finally: + pdh.PdhCloseQuery(query) + + +def description(path): + """The executable's own product description (what Task Manager lists), or None.""" + from ctypes import wintypes as W + version = C.WinDLL('version') + size = version.GetFileVersionInfoSizeW(path, None) + if not size: + return None + data = C.create_string_buffer(size) + if not version.GetFileVersionInfoW(path, 0, size, data): + return None + pointer, length = C.c_void_p(), W.UINT() + if not version.VerQueryValueW(data, r'\VarFileInfo\Translation', C.byref(pointer), C.byref(length)) or length.value < 4: + return None + language, page = C.cast(pointer, C.POINTER(W.WORD * 2)).contents + key = r'\StringFileInfo\%04x%04x\FileDescription' % (language, page) + if not version.VerQueryValueW(data, key, C.byref(pointer), C.byref(length)) or length.value < 2: + return None + return C.wstring_at(pointer, length.value).rstrip('\0').strip() or None + + +def label(process): + """The name the user knows the program by, or None for a system component.""" + import psutil + try: + name = process.name() + except psutil.Error: + return None + if sys.platform != 'win32': + return None if name in LINUX_DESKTOP else name + try: + path = process.exe() + except psutil.Error: + return None # Protected system processes do not show their location. + system = Path(os.environ.get('SystemRoot', r'C:\Windows')).resolve() + if not path or Path(path).resolve().is_relative_to(system): + return None + try: + return description(path) or Path(name).stem + except OSError: + return Path(name).stem + + +def holders(device): + """[{'name', 'bytes'}] of other programs on the GPU, largest first; None when they cannot be listed. + + Programs of unknown size are left out: on Windows every desktop program + draws on the GPU, and naming them all would not say what to close.""" + import psutil + from .background_room import Gpu, family + gpu = Gpu(device) + try: + listed = gpu.processes() + finally: + gpu.close() + if listed is None: + return None + if sys.platform == 'win32' and all(size is None for size in listed.values()): + try: + measured = counter_bytes() + except (OSError, AttributeError): + measured = None + if measured: + listed = {pid: measured.get(pid) for pid in set(listed) | set(measured)} + try: + own = family(os.getpid()) + except OSError: + own = {os.getpid()} + found = {} + for pid, size in listed.items(): + if pid in own or size is None: + continue + try: + name = label(psutil.Process(pid)) + except psutil.Error: + continue + if name: + # One program often runs as several processes; list it once. + found[name] = found.get(name, 0) + size + rows = sorted((dict(name=name, bytes=size) for name, size in found.items() if size >= FLOOR), + key=lambda row: -row['bytes']) + return rows[:LIMIT] diff --git a/freevideo_engine/prompt_vlm/load.py b/freevideo_engine/prompt_vlm/load.py index 9119c961..9f66d576 100644 --- a/freevideo_engine/prompt_vlm/load.py +++ b/freevideo_engine/prompt_vlm/load.py @@ -36,11 +36,12 @@ def stored(folder): return rows -def resident(folder, torch, device, precision, keep=None): +def resident(folder, torch, device, precision, keep=None, vision_on_host=False): """Returns the model on `device`, with int8 projections if precision == 'int8'. With `keep`, layers from that index on hold their int8 rows in host memory - and are streamed to the GPU as they run.""" + and are streamed to the GPU as they run. With vision_on_host, the vision + tower stays in host memory and runs on the CPU.""" from accelerate import init_empty_weights from torch import nn from transformers import AutoConfig, GenerationConfig, Qwen3VLForConditionalGeneration @@ -65,6 +66,7 @@ def resident(folder, torch, device, precision, keep=None): readers = {} def load(prefix): + target = 'cpu' if vision_on_host and prefix.startswith('model.visual.') else device groups = {} for name, shard in shards.items(): if name.startswith(prefix): @@ -77,7 +79,7 @@ def load(prefix): module = model.get_submodule(parent) # The dtype Transformers would load it in (the skeleton's). dtype = getattr(module, leaf).dtype - setattr(module, leaf, nn.Parameter(tensor.to(device=device, dtype=dtype), requires_grad=False)) + setattr(module, leaf, nn.Parameter(tensor.to(device=target, dtype=dtype), requires_grad=False)) try: layers = 'model.language_model.layers.' @@ -101,7 +103,17 @@ def each_layer(layer, merge): reader.close() if tied: model.lm_head.weight = model.model.language_model.embed_tokens.weight + if vision_on_host: + visual = model.model.visual + del model.model.visual model = model.to(device) + if vision_on_host: + # Registered last, so the model's first parameter (its device) stays on the GPU. + model.model.visual = visual + move = lambda value: value.to('cpu') if torch.is_tensor(value) else value + visual.register_forward_pre_hook( + lambda module, args, kwargs: (tuple(map(move, args)), {k: move(v) for k, v in kwargs.items()}), + with_kwargs=True) if keep is not None: model.streamer = int8.Streamer(torch, list(model.model.language_model.layers)[keep:]) torch.cuda.empty_cache() # Merge and quantize temporaries; KV space comes next. diff --git a/freevideo_engine/prompt_vlm/service.py b/freevideo_engine/prompt_vlm/service.py index 430b1b45..d2cccc3f 100644 --- a/freevideo_engine/prompt_vlm/service.py +++ b/freevideo_engine/prompt_vlm/service.py @@ -14,6 +14,10 @@ from . import assets, rules +# How long a rewrite that does not fit waits for other programs to free the +# GPU (the page names them) before it fails; the user can cancel meanwhile. +GPU_WAIT_SECONDS = 300 + class Worker: """One pipe owner across idle imports and the subsequent rewrite.""" @@ -70,6 +74,30 @@ def close(self): stream.close() +def release_resident_models(env, descriptor): + """Release the generation worker's idle models before the rewrite model loads. + + Returns the readings for the rewrite report: a state, how long the release took and the + card's free memory around it (numbers and states only, never paths).""" + from .. import resident_process + release = dict(idle_release='no_resident_process') + if not (env.get(resident_process.ENV) and Path(env[resident_process.ENV]).is_file()): + return release + before, releasing = resident_process.gpu_memory(None), time.monotonic() + release['idle_release'] = 'released' + try: + resident_process.release_idle_cache(env, descriptor) + resident_process.OWNER.record(action='released', reason='prompt_model') + except resident_process.SessionRestarted: + release['idle_release'] = 'already_exited' # That worker already exited, which is the release we need. + env.pop(resident_process.ENV, None) + after = resident_process.gpu_memory(None) + release.update(idle_release_seconds=time.monotonic() - releasing, + idle_release_gpu_free_before_bytes=before[1] - before[0] if before else None, + idle_release_gpu_free_after_bytes=after[1] - after[0] if after else None) + return release + + class Service: def __init__(self, installation, input_directory): self.installation = installation @@ -91,6 +119,7 @@ def __init__(self, installation, input_directory): self.closed = False self.request_epoch = None self.active_worker = None + self.release_readings = None atexit.register(self.close) def prestart(self, root, machine, *, generating=False): @@ -420,7 +449,7 @@ def work(self, root, machine, action, value): else: with tempfile.TemporaryDirectory(prefix='freevideo-prompt-', ignore_cleanup_errors=True) as frames: self.video_frames(value, frames) - self.infer(root, machine, descriptor, value) + self.infer(root, machine, descriptor, value, wait=action == 'rewrite') except InterruptedError: self.progress(dict(phase='cancelled')) except BlockingIOError: @@ -444,7 +473,7 @@ def work(self, root, machine, action, value): # Output rejections contain condition IDs, counts, typed media labels and retry flags only. report = {key: item for key, item in self.state.items() if key in ('phase', 'error', 'exception', 'error_details', 'timings', 'diagnostics', - 'resources', 'cleanup', 'worker_returncode', 'output_rejection')} + 'resources', 'cleanup', 'worker_returncode', 'output_rejection', 'gpu_wait')} try: spec = assets.catalog() except (OSError, ValueError): @@ -465,20 +494,90 @@ def work(self, root, machine, action, value): except OSError: pass - def infer(self, root, machine, descriptor, value): - # A claimed idle worker has no inherited lease. Keep the parent's lease - # through bounded cleanup, including failures before request submission. + def infer(self, root, machine, descriptor, value, *, wait=False): + """Run the worker; when the model does not fit the GPU, name the programs + holding it and, for a rewrite, wait for them to free enough and run again.""" + until = time.monotonic() + GPU_WAIT_SECONDS + waited = dict(waits=0, seconds=0., freed=0) + self.release_readings = None + while True: + # A claimed idle worker has no inherited lease. Keep the parent's lease + # through bounded cleanup, including failures before request submission. + try: + self._infer(root, machine, descriptor, value) + finally: + session = self.active_worker + if session is not None: + from .. import processes + processes.stop(session.process, grace=5) + session.process.wait() + session.close() + self.active_worker = None + self.process = None + if self.state.get('phase') != 'failed' or self.state.get('error') != 'out_of_memory': + return + stats = self.state.get('diagnostics') or {} + needed, free = stats.get('gpu_free_needed_bytes'), stats.get('gpu_allocator_budget_bytes') + short = needed - free if type(needed) is int and type(free) is int and needed > free else None + freed = False + if wait and short is not None: + waiting = time.monotonic() + try: + freed = self.wait_for_gpu(machine, short, until) + finally: + # Counts and seconds only: the programs' names stay on the page. + waited.update(waits=waited['waits'] + 1, seconds=waited['seconds'] + time.monotonic() - waiting, + freed=waited['freed'] + freed) + self.progress(dict(gpu_wait=dict(waited))) + if not freed: + self.progress(dict(phase='failed', error='out_of_memory', gpu_holders=self.gpu_holders(machine))) + return + with self.mutex: + for key in ('error', 'exception', 'error_details', 'gpu_holders', 'gpu_short_bytes'): + self.state.pop(key, None) + self.state['phase'] = 'loading' + + @staticmethod + def gpu_holders(machine): + """Other programs holding GPU memory, for the page only; None when unknown.""" + from ..gpu_holders import holders try: - self._infer(root, machine, descriptor, value) + return holders(machine.get('gpu_uuid')) + except Exception: + return None + + def wait_for_gpu(self, machine, short, until): + """True once the GPU has `short` more bytes free than now; False at `until` + or when free memory cannot be read. Raises InterruptedError on cancel.""" + from ..background_room import Gpu + try: + gpu = Gpu(machine.get('gpu_uuid')) + except Exception: + return False + try: + reading = gpu.memory() + if reading is None: + return False + # The worker measured its shortfall against what it could use; ask + # for that much more than is free now, after it has exited. + target = reading[1] - reading[0] + short + listed = 0. + while time.monotonic() < until: + self.check() + reading = gpu.memory() + if reading is None: + return False + free = reading[1] - reading[0] + if free >= target: + return True + if time.monotonic() - listed >= 2: + listed = time.monotonic() + self.progress(dict(phase='waiting_gpu', gpu_short_bytes=target - free, + gpu_holders=self.gpu_holders(machine))) + self.stop.wait(1) + return False finally: - session = self.active_worker - if session is not None: - from .. import processes - processes.stop(session.process, grace=5) - session.process.wait() - session.close() - self.active_worker = None - self.process = None + gpu.close() def _infer(self, root, machine, descriptor, value): from .. import processes @@ -490,13 +589,12 @@ def _infer(self, root, machine, descriptor, value): owner = resident_process.OWNER if owner.process is not None and owner.process.poll() is None and owner.endpoint: env[resident_process.ENV] = str(owner.endpoint) - if env.get(resident_process.ENV) and Path(env[resident_process.ENV]).is_file(): - try: - resident_process.release_idle_cache(env, descriptor) - resident_process.OWNER.record(action='released', reason='prompt_model') - except resident_process.SessionRestarted: - pass # That worker already exited, which is the release we need. - env.pop(resident_process.ENV, None) + release = release_resident_models(env, descriptor) + # A run again after waiting has nothing left to release; the report keeps the first attempt's. + if release['idle_release'] != 'no_resident_process': + self.release_readings = release + elif self.release_readings: + release = self.release_readings expired = None with self.mutex: session, self.prewarm = self.prewarm, None @@ -517,7 +615,7 @@ def _infer(self, root, machine, descriptor, value): diagnostics = dict(prewarm=prewarm, prewarm_reason=reason if prewarm == 'cold' else 'none', prewarm_age_seconds=time.monotonic() - session.started if session else None, prewarm_import_seconds=session.import_seconds if session else None, - request_to_first_output_seconds=None) + request_to_first_output_seconds=None, **release) footprint = dict(prewarm_rss_bytes=None, prewarm_private_bytes=None) if session is not None: try: diff --git a/freevideo_engine/prompt_vlm/worker.py b/freevideo_engine/prompt_vlm/worker.py index 078078e8..64542ccd 100644 --- a/freevideo_engine/prompt_vlm/worker.py +++ b/freevideo_engine/prompt_vlm/worker.py @@ -137,9 +137,10 @@ def gpu_snapshot(stats): torch = sys.modules.get('torch') if torch is not None and torch.cuda.is_initialized(): try: + free, total = torch.cuda.mem_get_info() stats.update(gpu_peak_allocated_bytes=torch.cuda.max_memory_allocated(), gpu_peak_reserved_bytes=torch.cuda.max_memory_reserved(), - gpu_free_bytes=torch.cuda.mem_get_info()[0], + gpu_free_bytes=free, gpu_total_bytes=total, gpu_allocator_ooms=torch.cuda.memory_stats().get('num_ooms')) except (RuntimeError, OSError): stats['gpu_sample_unavailable'] = True @@ -220,10 +221,13 @@ def tokenize(processor, messages, max_pixels=512 * 32 * 32): OUTPUT_TOKENS = 1800 -def kv_positions(input_tokens): +def kv_positions(input_tokens, answer_tokens=None): """Cache positions reserved for a request: the input and its rewrite, then a correction retry that resends both with a short instruction and writes another. - A retry that would need more is skipped when the model is held on the GPU.""" + A retry that would need more is skipped when the model is held on the GPU. + With answer_tokens, only one answer of that length is reserved (no retry).""" + if answer_tokens is not None: + return input_tokens + answer_tokens + 256 return input_tokens + 2 * OUTPUT_TOKENS + 256 @@ -235,10 +239,10 @@ def reserved_tokens(stats): return tokens -def kv_bytes(stored, input_tokens): +def kv_bytes(stored, input_tokens, answer_tokens=None): keys = [row for name, row in stored.items() if name.endswith('self_attn.k_proj.weight')] per_token = 2 * len(keys) * (keys[0]['shape'][0] if keys else 0) * 2 # K and V, bf16 - return per_token * kv_positions(input_tokens) + return per_token * kv_positions(input_tokens, answer_tokens) def retry_allowed(tokens, stats): @@ -260,6 +264,61 @@ def placement(weights, int8_saving, kv, allocator_budget): return 'bf16' if weights <= room else 'int8' if weights - int8_saving <= room else None +def gpu_plan(weights, stored, free, input_tokens, answer_tokens, text_only): + """The GPU placement for `free` allocator bytes, or None when none fits. + + Best first: bf16 or int8 resident, then int8 with the later layers streamed + from host memory. A text-only request keeps the vision tower in host memory + when that gives a better tier, and when nothing else fits reserves the cache + for one answer (no correction retry). A request with images runs the vision + tower on the CPU only when nothing fits otherwise, and keeps the retry: a + rewrite that misses a picture reference fails without it.""" + from . import int8 + vision = sum(row['bytes'] for name, row in stored.items() if name.startswith('model.visual.')) + saving = int8.saved_bytes(stored) + def fit(answer, on_host): + kv = kv_bytes(stored, input_tokens, answer) + size = weights - vision if on_host else weights + plan = dict(kv=kv, positions=kv_positions(input_tokens, answer), vision_on_host=on_host, + vision_bytes=vision if on_host else 0, keep=None, host_bytes=0) + precision = placement(size, saving, kv, free) + if precision: + return dict(plan, precision=precision) + streamed = int8.resident_layers(size, stored, free - kv - WORKING_VRAM) + if streamed: + return dict(plan, precision='int8', keep=streamed[0], host_bytes=streamed[1]) + return None + if kv_positions(input_tokens, answer_tokens) >= kv_positions(input_tokens): + answer_tokens = None + if text_only: + # Host memory for the vision tower only when that buys a better tier. + rank = lambda plan: (plan['precision'] == 'bf16', plan['keep'] is None, plan['keep'] or 0) + whole, apart = fit(None, False), fit(None, True) + if whole and (not apart or rank(whole) >= rank(apart)): + return whole + options = ((None, True), (answer_tokens, True)) + else: + options = ((None, False), (None, True)) + for answer, on_host in dict.fromkeys(options): + plan = fit(answer, on_host) + if plan: + return plan + return None + + +def gpu_needed(weights, stored, free, input_tokens, answer_tokens, text_only): + """The fewest allocator bytes, to 32 MiB, for which gpu_plan finds a placement.""" + step = 32 * 2**20 + low, high = free // step, -(-(weights + 2 * kv_bytes(stored, input_tokens) + WORKING_VRAM) // step) + while low + 1 < high: + middle = (low + high) // 2 + if gpu_plan(weights, stored, middle * step, input_tokens, answer_tokens, text_only): + high = middle + else: + low = middle + return high * step + + def load_model(folder, torch, stats, checkpoint): """Weights, FreeToken operators and GPU placement; returns (model, device).""" from . import core, int8 @@ -272,32 +331,43 @@ def load_model(folder, torch, stats, checkpoint): keep = None host_bytes = 0 weight_budget = 0 + vision_bytes = 0 if torch.cuda.is_available(): from .. import gpu_budget - free, _ = torch.cuda.mem_get_info() - stats.update(gpu_name=torch.cuda.get_device_name(), gpu_free_before_load_bytes=free, + free, total = torch.cuda.mem_get_info() + stats.update(gpu_name=torch.cuda.get_device_name(), gpu_free_before_load_bytes=free, gpu_total_bytes=total, gpu_compute_capability=list(torch.cuda.get_device_capability())) admission = gpu_budget.configure(torch, free, reserve_bytes=256 * 2**20) free = min(free, admission.get('effective_allocator_limit_bytes') or free) # When the layers do not all fit, keep activation/KV space and the # desktop outside the budget for the layers that stay on the GPU. weight_budget = min(9 * 2**30, max(0, free - 2 * 2**30)) - kv = kv_bytes(shapes, reserved_tokens(stats)) - stats.update(gpu_allocator_budget_bytes=free, gpu_weight_budget_bytes=weight_budget, gpu_kv_reserve_bytes=kv) - resident = placement(weights, int8.saved_bytes(shapes), kv, free) - if not resident: - # Too big even as int8: keep what fits on the GPU and stream the other - # layers' int8 rows from host memory, a layer ahead of use. - plan = int8.resident_layers(weights, shapes, free - kv - WORKING_VRAM) - if plan: - resident, (keep, host_bytes) = 'int8', plan + tokens = reserved_tokens(stats) + plan = gpu_plan(weights, shapes, free, tokens, stats.get('answer_tokens', OUTPUT_TOKENS), + not stats.get('image_count')) + positions = plan['positions'] if plan else kv_positions(tokens) + stats.update(gpu_allocator_budget_bytes=free, gpu_weight_budget_bytes=weight_budget, + gpu_kv_reserve_bytes=plan['kv'] if plan else kv_bytes(shapes, tokens)) + if plan: + resident, keep, host_bytes = plan['precision'], plan['keep'], plan['host_bytes'] + vision_bytes = plan['vision_bytes'] + stats.update(vision_device='cpu' if plan['vision_on_host'] else 'gpu', + kv_reserve='one_answer' if positions < kv_positions(tokens) else 'with_retry') stats.update(streamed_layers=len(int8.layer_bytes(shapes)) - keep if keep is not None else 0, streamed_host_bytes=host_bytes) if not resident and weight_budget <= 2 * 2**30: # Not even the streaming tier fits: the model would run on the CPU at # about 0.2 tokens/s and hit the request deadline, so say so at once. + # The service waits for this much to become free before trying again. + answer = stats.get('answer_tokens', OUTPUT_TOKENS) + stats['gpu_free_needed_bytes'] = gpu_needed(weights, shapes, free, tokens, answer, not stats.get('image_count')) + # The report names the figure for a text-only request and for one with images; this + # request's own kind may fit at the free memory shown, so these start from zero. + stats.update(placement_refused='gpu_free_below_floor', + gpu_free_needed_text_only_bytes=gpu_needed(weights, shapes, 0, tokens, answer, True), + gpu_free_needed_with_images_bytes=gpu_needed(weights, shapes, 0, tokens, answer, False)) raise ValueError('out_of_memory') - stats['ram_required_bytes'] = (RESIDENT_RAM + host_bytes) if resident else MAPPED_RAM + stats['ram_required_bytes'] = (RESIDENT_RAM + host_bytes + vision_bytes) if resident else MAPPED_RAM available = stats['available_memory_at_start_bytes' if resident else 'available_memory_before_load_bytes'] if available < stats['ram_required_bytes']: raise ValueError('ram_space') @@ -307,9 +377,9 @@ def load_model(folder, torch, stats, checkpoint): loading = time.monotonic() if resident: from .load import resident as load_resident - model = load_resident(folder, torch, 'cuda:%d' % card, resident, keep) + model = load_resident(folder, torch, 'cuda:%d' % card, resident, keep, vision_on_host=bool(vision_bytes)) mapping, device = {'': card}, 'cuda:%d' % card - stats.update(weights=resident, kv_positions=kv_positions(reserved_tokens(stats))) + stats.update(weights=resident, kv_positions=positions) else: import psutil from transformers import Qwen3VLForConditionalGeneration @@ -509,7 +579,7 @@ def split(folder, value, stats): from .rules import SPLIT_RETRY, split_instruction, split_user_text, validate_split count = len(value['seconds']) limit = min(2 * OUTPUT_TOKENS - 200, 600 + 300 * count) # Within the cache reserved for a rewrite and its retry. - stats.update(stage='imports', pid=os.getpid(), mode='split', part_count=count, dtype='bfloat16', + stats.update(stage='imports', pid=os.getpid(), mode='split', part_count=count, answer_tokens=limit, dtype='bfloat16', attention='sdpa', kv_cache='dynamic', backend='transformers', accelerations=[], persistent_model=False) torch, checkpoint = begin(stats) @@ -574,7 +644,7 @@ def ask(folder, value, stats): questions = value['questions'] stats.update(stage='imports', pid=os.getpid(), mode='ask', question_count=len(questions), image_count=sum(len(q['images']) for q in questions), format=value['format'], - max_tokens=value['max_tokens'], custom_system=bool(value.get('system')), + max_tokens=value['max_tokens'], answer_tokens=value['max_tokens'], custom_system=bool(value.get('system')), dtype='bfloat16', attention='sdpa', kv_cache='dynamic', backend='transformers', accelerations=[], persistent_model=False) torch, checkpoint = begin(stats) @@ -636,7 +706,8 @@ def on_token(timing): {'role': 'user', 'content': ASK_JSON_RETRY}] answer['retried'] = True retry_inputs = tokenize(processor, retry, pixels) - if retry_inputs.input_ids.shape[-1] > 6500: + if retry_inputs.input_ids.shape[-1] > 6500 or (stats.get('kv_reserve') == 'one_answer' and + retry_inputs.input_ids.shape[-1] + value['max_tokens'] > stats['kv_positions']): answer['error'] = 'invalid_json' else: ids, _ = generate(torch, model, retry_inputs.to(device), value['max_tokens'], diff --git a/freevideo_engine/resident_process.py b/freevideo_engine/resident_process.py index a6420fde..60497fae 100644 --- a/freevideo_engine/resident_process.py +++ b/freevideo_engine/resident_process.py @@ -628,19 +628,24 @@ def held_summary(decisions): return {'held': sorted(held), 'evicted': evicted} -def gpu_used(gpu): +def gpu_memory(gpu): + """(used, total) bytes of the whole device, other programs included; None when unreadable.""" try: from .background_room import Gpu device = Gpu(gpu) try: - memory = device.memory() + return device.memory() or None finally: device.close() - return memory[0] if memory else None except (OSError, RuntimeError, AttributeError, ValueError, ImportError): return None +def gpu_used(gpu): + memory = gpu_memory(gpu) + return memory[0] if memory else None + + def gpu_own(gpu): """Card bytes held by this process tree (the kept worker and its leases), or None if not listed.""" try: diff --git a/web/issue_button.js b/web/issue_button.js index e43af07a..ecaedb69 100644 --- a/web/issue_button.js +++ b/web/issue_button.js @@ -9,9 +9,10 @@ const ISSUE_FORM = 'https://github.com/FlashML-org/FreeVideo/issues/new'; /** * Title, text and summary for the prefilled form. `where` is [English, Chinese] words for the place - * ("Export", "导出"); `message` is the sentence the user sees. + * ("Export", "导出"); `message` is the sentence the user sees; `facts` is an optional flat object of numbers and + * states (never text or paths) that travels in the summary. */ -export function issueDraft({where, message, kind = null}, t = en => en) { +export function issueDraft({where, message, kind = null, facts = null}, t = en => en) { const place = Array.isArray(where) ? where : [String(where || 'FreeVideo'), String(where || 'FreeVideo')]; const clean = createErrorReport({exception_message: String(message || '')}).message; const lines = clean.split(/\r?\n/).filter(line => line.trim()); @@ -20,7 +21,7 @@ export function issueDraft({where, message, kind = null}, t = en => en) { const text = [t('FreeVideo showed an error.', 'FreeVideo 显示了一个错误。'), `Where: ${place[0]}`, `Error: ${first.trim().slice(0, MESSAGE)}`, ...(kind ? [`Kind: ${kind}`] : [])].join('\n'); const tail = clean.length <= TAIL ? clean : clean.slice(0, 500) + '\n...\n' + clean.slice(-(TAIL - 505)); - return {title: `[Feedback] ${t(place[0], place[1])}`, text, summary: JSON.stringify({where: place[0], kind, tail})}; + return {title: `[Feedback] ${t(place[0], place[1])}`, text, summary: JSON.stringify(facts && Object.keys(facts).length ? {where: place[0], kind, tail, facts} : {where: place[0], kind, tail})}; } // Browser-only modules, loaded as the button appears so the click opens the page at once diff --git a/web/prompt_enhance.css b/web/prompt_enhance.css index 4c243910..419baffb 100644 --- a/web/prompt_enhance.css +++ b/web/prompt_enhance.css @@ -18,6 +18,7 @@ .fv-enhance-status:is([data-state=progress],[data-state=error],[data-state=note]){display:block} .fv-enhance-status[data-state=error]{color:var(--fv-danger,#ff8a80);background:color-mix(in srgb,var(--fv-danger,#ff8a80) 7%,transparent)} .fv-enhance-status .fv-issue-button{min-height:26px;margin-top:7px;padding:3px 9px;font-size:11px} +.fv-enhance-status .fv-issue-button:not([hidden]){display:flex;width:fit-content} .fv-enhance-meter{position:relative;height:3px;margin-top:7px;border-radius:2px;background:#ffffff12;overflow:hidden} .fv-enhance-meter>i{position:absolute;inset:0 auto 0 0;width:0;border-radius:inherit;background:var(--fv-accent,#6db8fa);transition:width .4s ease-out} .fv-enhance-meter[data-indeterminate=true]>i{width:32%;animation:fv-enhance-sweep 1.4s ease-in-out infinite} diff --git a/web/prompt_enhance.js b/web/prompt_enhance.js index 6e433f7d..db3c5366 100644 --- a/web/prompt_enhance.js +++ b/web/prompt_enhance.js @@ -113,13 +113,16 @@ export function createPromptEnhancer({node, input, editor, api, context, setText const meter = el('div', null, 'fv-enhance-meter'), fill = el('i'); meter.setAttribute('role', 'progressbar'); meter.setAttribute('aria-valuemin', '0'); meter.setAttribute('aria-valuemax', '100'); // The shared "Report a problem" button, handed in by the page. - const report = issueButton ? issueButton(t, () => ({where: ['Prompt enhancement', '提示词增强'], message: statusText.textContent})) : document.createElement('span'); + // Feedback quotes the line without the names of the user's other programs. + let quoted = ''; + const report = issueButton ? issueButton(t, () => ({where: ['Prompt enhancement', '提示词增强'], message: quoted, facts: memoryFacts})) : document.createElement('span'); report.hidden = true; meter.append(fill); status.append(statusText, meter, report); // One place sets what the status line says and how it looks. - function show(text, state = '', fraction = null) { + function show(text, state = '', fraction = null, quote = text) { if (downloadOnly) { publishModel({text: text || ''}); return; } statusText.textContent = text || ''; + quoted = quote || ''; status.dataset.state = text ? state : ''; report.hidden = !text || state !== 'error'; meter.hidden = state !== 'progress'; @@ -163,6 +166,13 @@ export function createPromptEnhancer({node, input, editor, api, context, setText if (versions.value.enabled) void prestart.request(); // Available RAM the model needs; the server knows the usual case for this card. let ramBytes = 11 * 2 ** 30; + // Memory readings of a failed rewrite for the feedback form: numbers and states only. + let memoryFacts = null; + const MEMORY_FACTS = ['gpu_total_bytes', 'gpu_free_before_load_bytes', 'gpu_allocator_budget_bytes', 'gpu_free_needed_text_only_bytes', 'gpu_free_needed_with_images_bytes', + 'placement_refused', 'gpu_free_bytes', 'gpu_allocator_ooms', 'idle_release', 'idle_release_seconds', + 'idle_release_gpu_free_before_bytes', 'idle_release_gpu_free_after_bytes']; + const factsOf = diagnostics => Object.fromEntries(MEMORY_FACTS.filter(key => ['number', 'string'].includes(typeof diagnostics?.[key])) + .map(key => [key, diagnostics[key]])); const gib = value => String(Math.round(value / 2 ** 30)); // Mentions are counted as the character descriptions they expand to, so the count can exceed what the box shows. const lengthOf = (error, english) => { @@ -171,6 +181,9 @@ export function createPromptEnhancer({node, input, editor, api, context, setText : `提示词${error.expanded ? '连同角色描述' : ''}共 ${number} 字符`; }; // Unknown codes keep the server's own reason, without paths, so the user can quote it. + // Other programs holding graphics memory, as the server measured them; no-break spaces keep name, size and unit on one line. + const holderNames = holders => (Array.isArray(holders) ? holders : []).slice(0, 3) + .map(row => `${String(row?.name || '').slice(0, 60)}\u00a0${((row?.bytes || 0) / 2 ** 30).toFixed(1)}\u00a0GiB`).join(t(', ', '、')); const reasonOf = error => String(error?.message || '').replace(/[A-Za-z]:[\\/][^\s'"]*|\/[^\s'"]+\/[^\s'"]*/g, '').replace(/\s+/g, ' ').trim().slice(0, 120); // A dropped request names a browser extension only when the error carries evidence of one. const networkText = error => { @@ -195,7 +208,10 @@ export function createPromptEnhancer({node, input, editor, api, context, setText : t('Enter a prompt of up to 12,000 characters.', '请输入不超过 12,000 字符的提示词。'), invalid_output: t('The rewrite was incomplete. Retry or use the original.', '改写结果不完整，请重试或使用原文。'), context_length: t('The scene is too long for this model. Shorten it or use the original.', '内容超出改写模型容量，请精简或使用原文。'), - out_of_memory: t('Not enough memory to rewrite. Close other GPU apps or use the original.', '改写时内存不足，请关闭其他显卡程序或使用原文。'), + out_of_memory: holderNames(error?.holders) + ? t(`Not enough graphics memory to rewrite. In use: ${holderNames(error.holders)}. Close the listed programs and rewrite again, or use the original.`, + `显存不足，无法改写。正在占用：${holderNames(error.holders)}。关闭上述程序后可重新改写，或使用原文。`) + : t('Not enough memory to rewrite. Close other GPU apps or use the original.', '改写时内存不足，请关闭其他显卡程序或使用原文。'), timeout: t('Rewriting took too long. Retry or use the original.', '改写超时，请重试或使用原文。'), stale: t('The prompt or references changed. Rewrite again to use the current scene.', '原文或素材已更改，可按当前内容重新改写。'), cancelled: t('Cancelled. Your original prompt is retained.', '已取消，原文已保留。'), @@ -215,7 +231,10 @@ export function createPromptEnhancer({node, input, editor, api, context, setText : t('Rewriting supports prompts of up to 12,000 characters.', '改写只支持不超过 12,000 字符的提示词。'), invalid_output: t('The rewrite was incomplete.', '改写结果不完整。'), context_length: t('The scene is too long for the rewrite model.', '内容超出改写模型容量。'), - out_of_memory: t('Not enough memory to rewrite. Close other GPU apps before rewriting again.', '改写时内存不足，关闭其他显卡程序后可再改写。'), + out_of_memory: holderNames(error?.holders) + ? t(`Not enough graphics memory to rewrite. In use: ${holderNames(error.holders)}. Close the listed programs before rewriting again.`, + `显存不足，无法改写。正在占用：${holderNames(error.holders)}。关闭上述程序后可重新改写。`) + : t('Not enough memory to rewrite. Close other GPU apps before rewriting again.', '改写时内存不足，关闭其他显卡程序后可再改写。'), timeout: t('Rewriting took too long.', '改写超时。'), unsupported_media: errorText(error), too_many_images: errorText(error), stale: errorText(error), connected_media: errorText(error), audio_media: errorText(error), unreadable_video: errorText(error), @@ -295,11 +314,19 @@ export function createPromptEnhancer({node, input, editor, api, context, setText onReport(row); } if (row.phase === 'complete') return row; - if (row.phase === 'failed') throw Object.assign(new Error(row.error || 'rewrite_failed'), {ramBytes: row.diagnostics?.ram_required_bytes}); + if (row.phase === 'failed') { + memoryFacts = row.error === 'out_of_memory' ? factsOf(row.diagnostics) : null; + throw Object.assign(new Error(row.error || 'rewrite_failed'), {ramBytes: row.diagnostics?.ram_required_bytes, holders: row.gpu_holders}); + } if (row.phase === 'cancelled') throw new Error(action === 'install' ? 'download_failed' : 'cancelled'); if (disposed) throw new Error('cancelled'); if (row.phase === 'download') { showDownload(row.done || 0, row.total || 0); + } else if (row.phase === 'waiting_gpu') { + // The rewrite starts by itself once enough graphics memory is free. + const short = Math.max(0.1, (row.gpu_short_bytes || 0) / 2 ** 30).toFixed(1), names = holderNames(row.gpu_holders); + show(names ? t(`Waiting for ${short}\u00a0GiB more graphics memory to rewrite. In use: ${names}.`, `显存不足，再空出 ${short}\u00a0GiB 后自动改写。正在占用：${names}。`) + : t(`Waiting for ${short}\u00a0GiB more graphics memory to rewrite.`, `显存不足，再空出 ${short}\u00a0GiB 后自动改写。`), 'progress'); } else if (action !== 'install') { // The ellipsis says "still going" until the seconds start counting. const [en, zh] = row.phase === 'rewriting' ? ['Enhancing the prompt', '正在改写提示词'] : ['Preparing enhancement', '正在准备提示词增强']; @@ -369,7 +396,7 @@ export function createPromptEnhancer({node, input, editor, api, context, setText waiting = true; show(errorText({message: 'prompt_required'}), 'note'); input.focus(); throw new Error('prompt_required'); } - cancelling = false; + cancelling = false; memoryFacts = null; const execute = async () => { if (disposed || cancelling) throw new Error('cancelled'); versions.value.useOriginal = false; @@ -391,7 +418,8 @@ export function createPromptEnhancer({node, input, editor, api, context, setText apply(); show(''); }; running = Promise.resolve().then(execute).catch(error => { - if (!disposed) show(errorText(error), error?.message === 'cancelled' ? '' : error?.message === 'stale' ? 'note' : 'error'); + if (!disposed) show(errorText(error), error?.message === 'cancelled' ? '' : error?.message === 'stale' ? 'note' : 'error', null, + errorText({name: error?.name, message: error?.message, ramBytes: error?.ramBytes, characters: error?.characters, expanded: error?.expanded})); throw error; }).finally(() => { running = null; cancelling = false; install.hidden = true; render(); }); render(); From c0c5a8878e8b82c78bba4bd6fbf66a858336fb8a Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 4 Nov 2026 20:55:56 +0000 Subject: [PATCH 02/14] Setup or Repair stopped by a running job waits for it and continues by itself The launcher refuses to start setup or repair while ComfyUI is generating, but the refusal appeared as a generic failure with no next step. It now keeps the reviewed plan, shows that a video is still generating, and installs it once the queue is empty. Going back or stopping cancels the wait, and after three automatic tries a card names the button to use. A job met after the engine's own install keeps the card and Retry installation, which inspects again. --- freevideo_engine/comfy_launcher_runtime.py | 4 ++- freevideo_engine/failure_details.py | 6 ++++ freevideo_engine/launcher/FButton.qml | 10 +++--- freevideo_engine/launcher/Main.qml | 36 +++++++++++++--------- freevideo_engine/launcher_copy.py | 2 ++ freevideo_engine/launcher_session.py | 35 +++++++++++++++++++-- 6 files changed, 71 insertions(+), 22 deletions(-) diff --git a/freevideo_engine/comfy_launcher_runtime.py b/freevideo_engine/comfy_launcher_runtime.py index dd444694..b8118d5c 100644 --- a/freevideo_engine/comfy_launcher_runtime.py +++ b/freevideo_engine/comfy_launcher_runtime.py @@ -502,6 +502,8 @@ def launcher_comfy(selected, info): ENGINE_UPDATE_WAITS = 'ComfyUI is running a job, so the engine update waits' +# Setup or Repair never starts beside a job; the launcher continues it when the queue is empty. +SETUP_WAITS_FOR_JOB = 'ComfyUI is running a job. Try again after it finishes; the job is left running.' def _refused(error): @@ -1229,7 +1231,7 @@ def _stop_server_for_setup(self, url): worker. Stop our own idle server; connecting after setup starts it again. A running job is never stopped, in a server another program started either.""" if queue_busy(loopback_url(self.server_address or url)): - raise RuntimeError('ComfyUI is running a job. Try again after it finishes; the job is left running.') + raise RuntimeError(SETUP_WAITS_FOR_JOB) if self.owns_server(): self.stop_owned_server() diff --git a/freevideo_engine/failure_details.py b/freevideo_engine/failure_details.py index 0e41438a..b4359135 100644 --- a/freevideo_engine/failure_details.py +++ b/freevideo_engine/failure_details.py @@ -652,6 +652,12 @@ def _launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_re ('A video is still generating', '还有视频正在生成'), ('Click Update now again when this video has finished. The video being generated is not interrupted.', '这个视频生成完成后，再点击“立即更新”。正在生成的视频不会被打断。')), + # Setup or Repair never starts beside a job (a server another program started either), and Repair + # runs into the same check, so the card offers Retry installation only. + (r'ComfyUI is running a job\. Try again after it finishes', 'setup-busy', + ('A video is still generating', '还有视频正在生成'), + ('Setup stopped so that the video being generated is not interrupted. When it has finished, click "Retry installation" and confirm the installation again. Completed steps are kept.', + '安装已停止，以免打断正在生成的视频。视频生成完成后，请点击“重试安装”，并再次确认安装。已完成的步骤会保留。')), # Moving the engine's editable install to the new source failed and was undone (engine_binding). (r'Editable install failed for|Editable verification failed for|The engine.s uv is missing', 'update-unfinished', ('The update did not finish', '这次更新没有完成'), diff --git a/freevideo_engine/launcher/FButton.qml b/freevideo_engine/launcher/FButton.qml index e5217e24..b7c81454 100644 --- a/freevideo_engine/launcher/FButton.qml +++ b/freevideo_engine/launcher/FButton.qml @@ -7,6 +7,8 @@ Button { property bool selected: false property bool danger: false property bool singleLine: false + // A disabled button that is the only statement of what is happening: full strength, not a call to act. + property bool waiting: false // A text link: accent text, no frame. property bool link: false implicitHeight: Math.max(theme.height, contentItem.implicitHeight + 14) @@ -15,10 +17,10 @@ Button { topPadding: 7; bottomPadding: 7 font.pixelSize: theme.body; font.weight: primary ? Font.DemiBold : Font.Medium hoverEnabled: true - opacity: enabled ? 1 : 0.42 + opacity: enabled || waiting ? 1 : 0.42 contentItem: Text { text: control.text; font: control.font - color: control.primary ? theme.bg : control.danger ? theme.danger : control.selected ? theme.accent + color: control.waiting ? theme.text : control.primary ? theme.bg : control.danger ? theme.danger : control.selected ? theme.accent : control.link ? (control.hovered ? theme.accentHover : theme.accent) : control.flat && !control.hovered ? theme.muted : theme.text horizontalAlignment: Text.AlignHCenter; verticalAlignment: Text.AlignVCenter @@ -28,10 +30,10 @@ Button { } background: Rectangle { radius: theme.radiusSm - color: control.primary ? (control.down ? theme.accentDim : control.hovered ? theme.accentHover : theme.accent) : + color: control.waiting ? theme.raised : control.primary ? (control.down ? theme.accentDim : control.hovered ? theme.accentHover : theme.accent) : control.selected ? theme.accentSubtle : control.down ? theme.sheen : control.hovered ? theme.hover : control.flat ? "transparent" : theme.raised - border.width: control.activeFocus && control.visualFocus ? 2 : control.primary || control.flat ? 0 : 1 + border.width: control.activeFocus && control.visualFocus ? 2 : (control.primary && !control.waiting) || control.flat ? 0 : 1 border.color: control.activeFocus && control.visualFocus ? theme.accentHover : control.selected ? theme.accentDim : theme.border Behavior on color { ColorAnimation { duration: 110 } } } diff --git a/freevideo_engine/launcher/Main.qml b/freevideo_engine/launcher/Main.qml index 06beb9d5..154194b4 100644 --- a/freevideo_engine/launcher/Main.qml +++ b/freevideo_engine/launcher/Main.qml @@ -32,6 +32,7 @@ ApplicationWindow { property bool releaseNotesOpen: false property bool licensesOpen: false property string previousStatus: "" + property bool previousJobWait: false property string previousReview: "" property bool compact: width < 1000 property bool narrow: width < 640 @@ -137,6 +138,8 @@ ApplicationWindow { // The label before diskLine() when the line states the disk rather than a problem. function diskLabel() { return disk && !disk.problem && !disk.short ? t("Disk:", "所在硬盘：") : "" } // On a Mac, a disk problem is fixed by choosing another disk, not by reporting a fault. + // Not a fault: a video is still generating, and the one way on is Retry installation. + readonly property bool notAFault: s.failure.kind === "setup-busy" || s.failure.kind === "update-busy" readonly property bool diskFailure: !!disk && (s.failure.kind || "").indexOf("disk") === 0 readonly property bool updateOffered: s.update.engine || (!!s.update.candidate && ["available", "downloading", "ready", "error", "cancelled"].indexOf(s.update.status) >= 0) // An engine update needs no download; offer it as the way to launch. @@ -231,6 +234,8 @@ ApplicationWindow { if (s.update.candidate) return t("Current version ", "当前版本 ") + releaseVersion(currentRelease) + t(". Updating keeps your models and settings and takes about a minute.", "。更新会保留模型和设置，约需 1 分钟。") return t("Installed engine ", "已安装引擎 ") + engineVersion + t(". Updating takes about a minute and keeps your models and settings.", "。更新约需 1 分钟，模型和设置都会保留。") } + // A narrow window may wrap the waiting sentence; "自动继续" is not split in the middle. + function keepTogether(text) { return String(text).replace("自动继续", "自\u2060动\u2060继\u2060续") } function primaryText() { if (s.retry_kind) return retryText() if (needsRuntime) return t("Choose environment package", "选择运行环境包") @@ -254,7 +259,10 @@ ApplicationWindow { if (previousPage) pageEnter.restart() } previousPage = s.page - if (s.status !== previousStatus || s.review_id !== previousReview) accepted = false + // A setup that waits for a video keeps the user's consent; leaving the wait asks again. + if (s.status !== previousStatus || s.review_id !== previousReview) accepted = !!s.setup_job_wait + else if (previousJobWait && !s.setup_job_wait) accepted = false + previousJobWait = !!s.setup_job_wait previousStatus = s.status; previousReview = s.review_id if (s.error !== previousError) { errorDetailsOpen = false @@ -457,7 +465,7 @@ ApplicationWindow { FText { objectName: "repairUpdatesEngine"; visible: !!s.update.with_repair; Layout.fillWidth: true; color: theme.text text: t("Repair also updates the engine to " + releaseVersion(currentRelease) + ".", "修复时会一并更新到新版引擎 " + releaseVersion(currentRelease) + "。") } Flow { - Layout.fillWidth: true; spacing: 8 + visible: !notAFault; Layout.fillWidth: true; spacing: 8 FButton { objectName: "moveToDisk"; visible: diskFailure && s.form.new_comfy && !!bestDisk; enabled: !s.busy; primary: true text: bestDisk ? t("Install on \"" + bestDisk.name + "\"", "改装到“" + bestDisk.name + "”") : "" onClicked: backend.useDisk(bestDisk.path) } @@ -469,13 +477,13 @@ ApplicationWindow { FButton { objectName: "repairCardButton"; primary: true; visible: !!s.repair_offered; text: t("Repair", "修复"); onClicked: backend.action("repair", false) } FButton { objectName: "latestLauncherLink"; visible: s.failure.kind === "older-launcher"; flat: true text: t("Download the latest ↗", "下载最新版 ↗"); onClicked: backend.link("https://github.com/FlashML-org/FreeVideo/releases/latest") } - FButton { objectName: "copyError"; visible: !diskFailure; text: t("Copy details", "复制详情"); onClicked: backend.copy(s.error) } - FButton { objectName: "showError"; text: errorDetailsOpen ? t("Hide details", "收起详情") : t("Show details", "查看详情"); flat: true; onClicked: errorDetailsOpen = !errorDetailsOpen } - FButton { objectName: "reportProblem"; visible: !diskFailure && s.failure.kind !== "port-taken"; text: t("Report a problem", "提交问题"); onClicked: backend.reportProblem(s.error, s.failure.title || "") } - FButton { objectName: "exportError"; visible: !diskFailure; text: t("Export report", "导出报告"); enabled: s.report.status !== "running"; onClicked: backend.exportReport() } + FButton { objectName: "copyError"; visible: !diskFailure && !notAFault; text: t("Copy details", "复制详情"); onClicked: backend.copy(s.error) } + FButton { objectName: "showError"; visible: !notAFault; text: errorDetailsOpen ? t("Hide details", "收起详情") : t("Show details", "查看详情"); flat: true; onClicked: errorDetailsOpen = !errorDetailsOpen } + FButton { objectName: "reportProblem"; visible: !diskFailure && !notAFault && s.failure.kind !== "port-taken"; text: t("Report a problem", "提交问题"); onClicked: backend.reportProblem(s.error, s.failure.title || "") } + FButton { objectName: "exportError"; visible: !diskFailure && !notAFault; text: t("Export report", "导出报告"); enabled: s.report.status !== "running"; onClicked: backend.exportReport() } } ScrollView { - visible: errorDetailsOpen; Layout.fillWidth: true; Layout.preferredHeight: Math.min(160, errorText.implicitHeight+10); clip: true + visible: errorDetailsOpen && !notAFault; Layout.fillWidth: true; Layout.preferredHeight: Math.min(160, errorText.implicitHeight+10); clip: true TextArea { id: errorText; objectName: "failureDetails"; text: s.error; readOnly: true; selectByMouse: true; wrapMode: Text.Wrap; color: theme.muted; font.family: theme.mono; font.pixelSize: theme.micro; background: null; textFormat: TextEdit.PlainText } } } @@ -791,7 +799,7 @@ ApplicationWindow { } } } - FCheck { objectName: "installConsent"; text: s.consent; checked: accepted; onToggled: accepted = checked; Layout.fillWidth: true } + FCheck { objectName: "installConsent"; text: s.consent; checked: accepted; enabled: !s.setup_job_wait; onToggled: accepted = checked; Layout.fillWidth: true } FButton { objectName: "planLicensesButton"; text: t("View licenses", "查看许可证"); flat: true; implicitHeight: theme.heightSm - 2; leftPadding: 32; font.pixelSize: theme.micro + 1; onClicked: licensesOpen = true } } } @@ -957,7 +965,7 @@ ApplicationWindow { FText { objectName: "oldVersions"; visible: !!s.old_versions; text: "✓ " + s.old_versions; color: theme.success; font.pixelSize: theme.micro; Layout.fillWidth: true } FText { objectName: "copiedModels"; visible: !!s.copied_models; text: s.copied_models || ""; color: theme.muted; font.pixelSize: theme.micro; wrapMode: Text.WordWrap; Layout.fillWidth: true } FText { objectName: "archivesDone"; visible: !!s.archives_toast; text: s.archives_toast || ""; color: theme.success; font.pixelSize: theme.micro; Layout.fillWidth: true } - FCheck { visible: s.needs_consent; text: s.consent; checked: accepted; onToggled: accepted = checked; Layout.fillWidth: true } + FCheck { visible: s.needs_consent; text: s.consent; checked: accepted; enabled: !s.setup_job_wait; onToggled: accepted = checked; Layout.fillWidth: true } FButton { objectName: "bundleLicensesButton"; visible: s.needs_consent; text: t("View licenses", "查看许可证"); flat: true; implicitHeight: theme.heightSm - 2; leftPadding: 32; font.pixelSize: theme.micro + 1; onClicked: licensesOpen = true } } FCard { @@ -1020,7 +1028,7 @@ ApplicationWindow { } Rectangle { id: footer; objectName: "footer" - height: win.narrow ? (narrowActions.twoRows ? 112 : 64) : shortWindow ? 64 : 76 + height: win.narrow ? (narrowActions.twoRows ? 112 : s.setup_job_wait ? 84 : 64) : shortWindow ? 64 : 76 anchors.bottom: parent.bottom; width: parent.width; color: theme.canvas Rectangle { height: 1; width: parent.width; color: theme.border } RowLayout { @@ -1031,7 +1039,7 @@ ApplicationWindow { FButton { visible: s.busy && !s.cleanup.busy; text: t("Pause", "暂停"); onClicked: backend.action("stop", false) } FButton { objectName: win.narrow ? "" : "launchInstalledButton"; visible: updateFirst && s.retry_kind !== "launch"; flat: true; text: t("Launch current version", "启动当前版本"); onClicked: backend.action("primary", accepted) } FButton { - objectName: win.narrow ? "" : "primaryButton"; primary: !s.repair_offered && !(s.retry_kind === "launch" && engineUpdate.visible) && !manualFailure; implicitWidth: Math.max(160, contentItem.implicitWidth+40); implicitHeight: theme.heightLg + objectName: win.narrow ? "" : "primaryButton"; primary: !s.repair_offered && !(s.retry_kind === "launch" && engineUpdate.visible) && !manualFailure; implicitWidth: Math.max(160, contentItem.implicitWidth+40); implicitHeight: theme.heightLg; waiting: !!s.setup_job_wait text: s.setup_pending ? s.setup_wait_text : s.busy ? t("Working…", "正在处理…") : primaryText() enabled: !s.busy && !s.setup_pending && !(s.page === "launcher" && s.needs_consent && !accepted) && !(s.page === "progress" && s.status === "review" && !s.retry_kind && (!accepted || !!s.error)) @@ -1070,12 +1078,12 @@ ApplicationWindow { anchors.right: parent.right; y: narrowActions.twoRows ? parent.height - height - 10 : (parent.height - height) / 2 primary: !s.repair_offered && !(s.retry_kind === "launch" && engineUpdate.visible) && !manualFailure leftPadding: 12; rightPadding: 12 - singleLine: true - implicitWidth: Math.max(120, contentItem.implicitWidth + leftPadding + rightPadding); implicitHeight: theme.heightLg + singleLine: !s.setup_job_wait; waiting: !!s.setup_job_wait + implicitWidth: Math.max(120, contentItem.implicitWidth + leftPadding + rightPadding); implicitHeight: s.setup_job_wait ? Math.max(theme.heightLg, contentItem.implicitHeight + 18) : theme.heightLg width: Math.min(implicitWidth, Math.max(0, narrowActions.width - (narrowActions.twoRows ? 0 : (backButton.visible ? backButton.width + 8 : 0) + (pauseButton.visible ? pauseButton.width + 8 : 0) + (launchInstalled.visible ? launchInstalled.width + 8 : 0)))) - text: s.setup_pending ? s.setup_wait_text : s.busy ? t("Working…", "正在处理…") : primaryText() + text: s.setup_pending ? keepTogether(s.setup_wait_text) : s.busy ? t("Working…", "正在处理…") : primaryText() enabled: !s.busy && !s.setup_pending && !(s.page === "launcher" && s.needs_consent && !accepted) && !(s.page === "progress" && s.status === "review" && !s.retry_kind && (!accepted || !!s.error)) onClicked: { if (s.retry_kind) { backend.action("retry", accepted); return } if (needsRuntime) { backend.browseRuntimePackage(); return } if (needsPackages) { backend.browsePackages(); return } if (updateFirst) { backend.update(""); return } backend.action(s.page === "launcher" && s.status === "open" ? "browser" : "primary", accepted) } diff --git a/freevideo_engine/launcher_copy.py b/freevideo_engine/launcher_copy.py index 940e711f..9cc9d1f2 100644 --- a/freevideo_engine/launcher_copy.py +++ b/freevideo_engine/launcher_copy.py @@ -25,6 +25,8 @@ '已记下，处理完当前任务后自动导入'), 'setup_queued': ('Models are being prepared in the background. Setup will continue when they are ready.', '后台正在准备模型，好了会接着继续'), + 'setup_job': ('A video is still generating. Setup will continue when it has finished.', + '还有视频正在生成，完成后将自动继续'), 'update_queued': ('The update will start after the current task finishes.', '当前任务完成后会自动更新'), } diff --git a/freevideo_engine/launcher_session.py b/freevideo_engine/launcher_session.py index 7eca332b..66b25595 100644 --- a/freevideo_engine/launcher_session.py +++ b/freevideo_engine/launcher_session.py @@ -14,8 +14,8 @@ from . import __version__ from . import disk_space -from .comfy_launcher_runtime import (ENGINE_UPDATE_WAITS, Controller, layout, local_url, loopback_url, managed_frontend, - new_layout, queue_busy) +from .comfy_launcher_runtime import (ENGINE_UPDATE_WAITS, SETUP_WAITS_FOR_JOB, Controller, layout, local_url, loopback_url, + managed_frontend, new_layout, queue_busy) from .desktop_runtime import launcher_root, materialize_source from .download_settings import Probe, read as download_preferences, speed_text from .launcher_settings import Store, default_language @@ -94,6 +94,8 @@ class Session: update_waiting = update_restarting = False engine_updating = engine_autoinstall = reload_expected = resume_pages = False browser_wait = queue_state = queue_thread = bridge = None + # Setup that stopped for a running job and continues by itself once the queue is empty. + setup_job_wait, setup_job_tries = False, 0 foreign_state = foreign_thread = None clients_polled = queue_polled = foreign_polled = receipt_polled = resume_until = resume_polled = bridge_polled = 0. bridge_pending = False @@ -550,6 +552,11 @@ def action(self, name, accepted=False, *, retry=None): self.pending_action = None if self.action_notice == 'setup_queued': self.action_notice = '' + self.setup_job_wait = False + if name != 'stop': + self.setup_job_tries = 0 + if name in ('back', 'setup', 'launcher'): + self.setup_job_wait = False setup_action = (name == 'primary' and self.page != 'launcher' or name == 'retry' and self.retry_kind() != 'launch') if setup_action and self.task_busy(): @@ -1793,6 +1800,12 @@ def sync_lan_from_store(self): if saved['lan_access'] != (self.form.get('lan_access') is True): self.edit('lan_access', saved['lan_access']) + def setup_replayable(self): + """Whether the accepted plan can still be installed as it is. The engine's own install consumes + the plan, so a job met after it (the ComfyUI environment step of a first installation) leaves + Retry installation, which inspects again.""" + return bool((self.controller.selection or {}).get('ready')) or getattr(self.controller, 'section', None) == 'engine' + def tick(self): if isinstance(self.controller, Controller) and not self.controller.busy: self.controller.poll_access_proxy() @@ -1850,6 +1863,21 @@ def tick(self): # Nothing new to download or approve: deploy, then restart ComfyUI. self.controller.run('install', True) row, busy = self.controller.state, self.controller.busy + if (not busy and row.get('status') == 'failed' and row.get('action') == 'install' + and str(row.get('error') or '').startswith(SETUP_WAITS_FOR_JOB) and self.setup_job_tries < 3 + and self.setup_replayable()): + # A video is generating, so setup stopped before it changed anything. Keep the plan the user + # accepted and install it when the queue is empty, instead of a failure card. + self.setup_job_tries += 1 + self.setup_job_wait = True + self.queue_state = None + self.controller.state = row = dict(row, status='review', action='inspect', error=None, exception=None) + if self.setup_job_wait and row.get('status') != 'review': + self.setup_job_wait = False + if self.setup_job_wait and not self.task_busy() and self._queue_busy(any_server=True) is False: + self.setup_job_wait = False + self.controller.run('install', True) + row, busy = self.controller.state, self.controller.busy if (self.engine_updating and not busy and row.get('status') == 'failed' and str(row.get('error') or '').startswith(ENGINE_UPDATE_WAITS)): # A video started after the check: keep waiting for it instead of showing a failure. @@ -2404,7 +2432,8 @@ def snapshot(self): disk=disk, decimal_sizes=sys.platform == 'darwin', zh=self.language.startswith('zh'), form=dict(self.form), page=self.page, status=row.get('status', 'idle'), busy=busy, working=working, action_notice=self.t(*ACTION_COPY[self.action_notice]) if self.action_notice else '', - setup_pending=bool(self.pending_action), setup_wait_text=self.t(*ACTION_COPY['setup_queued']), + setup_pending=bool(self.pending_action) or self.setup_job_wait, setup_job_wait=self.setup_job_wait, + setup_wait_text=self.t(*ACTION_COPY['setup_job' if self.setup_job_wait else 'setup_queued']), can_manage=self.can_manage(), busy_reason=self.t(*ACTION_COPY['busy']), cleanup=cleanup, can_cleanup=self.can_manage() and not (self.updater and self.updater.busy), From 2b92459501f8207602e903e7d4cec906dcad3fbf Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 4 Nov 2026 02:25:30 +0000 Subject: [PATCH 03/14] Autopilot header: Feedback leaves after the phone entry when the actions column is narrow, so... Autopilot header: Feedback leaves after the phone entry when the actions column is narrow, so the running bar keeps its full content --- web/agent_connect.css | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/agent_connect.css b/web/agent_connect.css index 73d65384..9c79d91c 100644 --- a/web/agent_connect.css +++ b/web/agent_connect.css @@ -54,6 +54,8 @@ @container fv-autopilot-actions (max-width:425px){:lang(en) .fv-autopilot-now-go>em{display:none}} @container fv-autopilot-actions (max-width:289px){:lang(zh) .fv-autopilot-now-go>span{display:none}} @container fv-autopilot-actions (max-width:322px){:lang(en) .fv-autopilot-now-go>span{display:none}} +/* Feedback retreats after the phone entry, so the running bar keeps its full content. */ +@container fv-autopilot-actions (max-width:260px){.fv-header:has(.fv-autopilot-now:not([hidden])) .fv-feedback-entry:not(.fv-guide-entry){display:none}} @media(max-width:1050px){.fv-autopilot-now-name{max-width:120px}} @media(max-width:825px){.fv-studio.fv-autopilot .fv-header:has(.fv-autopilot-now:not([hidden])){height:auto;min-height:98px;grid-template-columns:auto minmax(0,1fr);padding-block:10px}.fv-autopilot .fv-header:has(.fv-autopilot-now:not([hidden]))>.fv-view-switch{grid-column:1/-1;grid-row:2;justify-self:center}.fv-autopilot .fv-header:has(.fv-autopilot-now:not([hidden]))>.fv-header-actions{grid-column:2;grid-row:1}} @media(max-width:480px){.fv-studio.fv-autopilot .fv-header{grid-template-columns:auto minmax(0,1fr)}.fv-autopilot-now{font-size:12.5px}.fv-autopilot-now-name{display:none}.fv-autopilot .fv-header-actions>.fv-quiet{flex:none;white-space:nowrap}.fv-studio .fv-autopilot-now-go{padding:5px 2px 5px 10px}.fv-studio .fv-autopilot-lamp{width:30px;height:30px;padding:0;justify-content:center}.fv-autopilot-lamp>span{display:none}.fv-agent-connect-body{padding:22px}} From 9bacb3302a0c0774df26c06de9b2f1bccd5cff00 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 4 Nov 2026 22:01:38 +0000 Subject: [PATCH 04/14] Bridge: wait up to 20 s for the device lease and name the busy lease in the failure report The bridge asked for the device lease with a 3 s wait, so a start right behind the previous request's warm-up failed with BlockingIOError(11) about 4 s in and no detail. The wait now covers the warm-up's yield window; if it runs out the report carries the failure frames and the busy lease's name (no path) and holders. Adds a unit test and a repro script. --- freevideo_engine/comfy_bridge.py | 16 ++++++++++++++- freevideo_engine/locking.py | 32 +++++++++++++++++++++++++++++- freevideo_engine/support_report.py | 2 +- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.py index 1606790a..31dcf634 100644 --- a/freevideo_engine/comfy_bridge.py +++ b/freevideo_engine/comfy_bridge.py @@ -22,6 +22,11 @@ from .monitoring import save +# Longer than the idle encoder's yield window (YIELD_SECONDS plus its join slack), so a speculative +# warm-up of the previous request on the same device releases the lease before the wait runs out. +DEVICE_LEASE_WAIT_SECONDS = 20. + + @contextlib.contextmanager def _device_generation_lease(environment): """An executor of a multi-GPU pool holds its own device's lease, not the shared installation lock. @@ -38,7 +43,7 @@ def _device_generation_lease(environment): from .gpu_pool import child_lease_environment, generation_lease from .locking import LOCK_ENV device = SimpleNamespace(uuid=uuid) - with generation_lease(device, wait_seconds=3) as descriptor: + with generation_lease(device, wait_seconds=DEVICE_LEASE_WAIT_SECONDS) as descriptor: os.set_inheritable(descriptor, True) environment.update(child_lease_environment(device), **{LOCK_ENV: str(descriptor)}) try: @@ -1710,6 +1715,15 @@ def store_result(check): except BaseException as error: cancelled = isinstance(error, KeyboardInterrupt) or type(error).__name__ in ('InterruptProcessingException', 'CancelledError') state.update(status='cancelled' if cancelled else 'failed', error=repr(error)) + if not cancelled: + try: + from .locking import busy_lock_details, failure_frames + state['traceback'] = failure_frames(error) + busy = busy_lock_details(error) + if busy: + state['busy_lock'] = busy + except Exception: + pass try: if cancelled: # Windows may terminate the CLI before its finally block runs. diff --git a/freevideo_engine/locking.py b/freevideo_engine/locking.py index 2acd4a47..3cb0d13a 100644 --- a/freevideo_engine/locking.py +++ b/freevideo_engine/locking.py @@ -254,6 +254,34 @@ def device_lock_path(device, path=None): return base.with_name(base.stem + '-gpu-' + token + base.suffix) +def name_busy_lock(error, path, shared, wait_seconds): + """Say which lease a refused acquisition wanted; the bare errno names nothing.""" + path = Path(path) + token = path.stem.partition('-gpu-')[2] + error.busy_lock = dict(lock=('device lease ' + path.name.replace(token, token[:8]) if token + else 'installation lease ' + path.name), + shared=bool(shared), waited_seconds=round(wait_seconds, 3)) + error.busy_lock_path = path + + +def busy_lock_details(error): + """Name, wait and current holders of the lease an acquisition was refused, else {}.""" + details = dict(getattr(error, 'busy_lock', None) or {}) + if details: + try: + details['holders'] = describe_lock_holders([error.busy_lock_path]) + except Exception: + pass + return details + + +def failure_frames(error, limit=12): + """The traceback as 'file:line in function' rows: no source text, no folders, no values.""" + import traceback + return ['%s:%d in %s' % (Path(frame.filename).name, frame.lineno, frame.name) + for frame in traceback.extract_tb(error.__traceback__)[-limit:]] + + @contextmanager def runtime_lock(path=None, *, inherit=True, shared=False, wait_seconds=0): """Take a lease, optionally waiting a bounded time for acquisition only.""" @@ -294,13 +322,15 @@ def runtime_lock(path=None, *, inherit=True, shared=False, wait_seconds=0): import fcntl fcntl.flock(stream, (fcntl.LOCK_SH if shared else fcntl.LOCK_EX) | fcntl.LOCK_NB) break - except BlockingIOError: + except BlockingIOError as error: remaining = deadline - time.monotonic() if remaining <= 0: + name_busy_lock(error, path, shared, wait_seconds) raise time.sleep(min(0.075, remaining)) # A delayed wakeup must not start another attempt after budget. if time.monotonic() >= deadline: + name_busy_lock(error, path, shared, wait_seconds) raise # The body is outside the retry loop: a failed request must never rerun. yield stream.fileno() diff --git a/freevideo_engine/support_report.py b/freevideo_engine/support_report.py index ddd114f8..fecab5b0 100644 --- a/freevideo_engine/support_report.py +++ b/freevideo_engine/support_report.py @@ -501,7 +501,7 @@ def count(value): encoding=request.get('encoding_resources') or dict(ram=read_json('.encoding.memory.json')), scope='Keep working RAM, PSS/RSS, commit and whole-GPU/allocator observations separate. ' 'Cached encoder receipts do not describe this request’s peak.'), - bridge={k:bridge[k] for k in ('status', 'error', 'bridge_seconds', 'bridge_wall_seconds', 'sampling_cache_install', 'encoder_prewarm') if k in bridge}, + bridge={k:bridge[k] for k in ('status', 'error', 'bridge_seconds', 'bridge_wall_seconds', 'sampling_cache_install', 'encoder_prewarm', 'traceback', 'busy_lock') if k in bridge}, collection_notes=notes, log_tails={}) stalled = agent_video_stalls(bridge.get('agent_generation_jobs') or agent_poll.get('agent_generation_jobs')) From 10ebf68c343c952c06bee90cefb319e068b5d552 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 4 Nov 2026 21:17:43 +0000 Subject: [PATCH 05/14] Voice-only video references: small, low-frame-rate picture; audio-only mp4 judged by streams --- freevideo_engine/agent/jobs.py | 5 +++-- freevideo_engine/agent/media.py | 6 +++--- freevideo_engine/agent/schema.py | 3 +++ freevideo_engine/comfy_assets.py | 28 ++++++++++++++++++++++++++-- freevideo_engine/media_encoding.py | 25 ++++++++++++++++++++----- freevideo_engine/media_request.py | 16 +++++++++++++++- 6 files changed, 70 insertions(+), 13 deletions(-) diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py index 906d04a3..cff93a7a 100644 --- a/freevideo_engine/agent/jobs.py +++ b/freevideo_engine/agent/jobs.py @@ -1655,8 +1655,9 @@ def resolve(): role=role)) for i, item in enumerate(spec['media'].get('references', []), 1): asset = dict(file=self.media_file(row, item, 'reference-%02d' % i, target=target), role='reference') - if 'early_steps' in item: - asset['early_steps'] = item['early_steps'] + for key in ('early_steps', 'voice_only'): + if key in item: + asset[key] = item[key] assets.append(asset) return assets assets = resolve() diff --git a/freevideo_engine/agent/media.py b/freevideo_engine/agent/media.py index a274a13f..502ca4f8 100644 --- a/freevideo_engine/agent/media.py +++ b/freevideo_engine/agent/media.py @@ -5,7 +5,7 @@ from pathlib import Path import uuid -from ..comfy_assets import media_kind +from ..comfy_assets import media_kind_of FPS = 24 MAX_SAMPLES = 64 @@ -20,7 +20,7 @@ def inspect(path): path = _file(path) - kind = media_kind(path.name) + kind = media_kind_of(path) info = dict(path=str(path), name=path.name, kind=kind, bytes=path.stat().st_size) if kind == 'image': from PIL import Image @@ -726,7 +726,7 @@ def _file(path, kind=None): path = Path(path) if not path.is_file(): raise ValueError('No such media file: %s' % path.name) - if kind and media_kind(path.name) != kind: + if kind and media_kind_of(path) != kind: raise ValueError('%s is not a %s file' % (path.name, kind)) return path diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index ff7bab3a..5318d9f4 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -138,6 +138,9 @@ def media(image=False, *, local=False, clip=False, video=False, early=False): properties['early_steps'] = number(1, 32, integer=True, description= 'A reference video only: it guides the first pass for this many steps and is dropped afterwards, ' 'including from the second pass. Omitted: the video guides every step.') + properties['voice_only'] = {'type': 'boolean', 'description': + 'A reference video used only for its voice: the picture is encoded small and at a low frame rate, ' + 'the sound is unchanged. Not with early_steps.'} if clip: properties.pop('frame') properties.pop('edit') diff --git a/freevideo_engine/comfy_assets.py b/freevideo_engine/comfy_assets.py index 515b9341..a9bc04e8 100644 --- a/freevideo_engine/comfy_assets.py +++ b/freevideo_engine/comfy_assets.py @@ -65,6 +65,25 @@ def media_kind(filename): raise ValueError('Choose a supported image, video or audio file') +def media_kind_of(path): + """The kind of an existing file: the extension, except that a video container without a picture + stream but with sound (an audio-only mp4) is audio. An unreadable file keeps its extension's kind.""" + kind = media_kind(Path(path).name) + if kind != 'video': + return kind + try: + import av + import av.stream + attached = getattr(getattr(av.stream, 'Disposition', None), 'attached_pic', 0) + with av.open(str(path)) as container: + if (container.streams.audio + and all(attached and stream.disposition & attached for stream in container.streams.video)): + return 'audio' + except Exception: + pass + return kind + + def resolve_assets(serialized, input_directory): serialized = optional_text(serialized, '[]') if not isinstance(serialized, str) or len(serialized) > 128 * 1024: @@ -75,7 +94,7 @@ def resolve_assets(serialized, input_directory): root = Path(input_directory).resolve() result = {} for row in rows: - if not isinstance(row, dict) or set(row) - {'file', 'role', 'enabled', 'cast', 'early_steps', 'sound', 'guide', 'keep_source_audio', 'depth', + if not isinstance(row, dict) or set(row) - {'file', 'role', 'enabled', 'cast', 'early_steps', 'sound', 'guide', 'voice_only', 'keep_source_audio', 'depth', 'use', 'range', 'segment'}: raise ValueError('Invalid media item') if 'depth' in row: @@ -105,7 +124,7 @@ def resolve_assets(serialized, input_directory): raise ValueError('Media link points outside ComfyUI input') from None if not path.is_file(): raise ValueError('Upload or select the missing media file: ' + name) - kind = media_kind(name) + kind = media_kind_of(path) from .original_audio import fields audio_fields = fields(dict(row, kind=kind)) if role in ('first', 'last'): @@ -136,6 +155,11 @@ def resolve_assets(serialized, input_directory): validate_guide(dict(row, kind=kind)) if row['guide']: reference['guide'] = True + if 'voice_only' in row: + from .media_request import validate_voice_only + validate_voice_only(dict(row, kind=kind)) + if row['voice_only']: + reference['voice_only'] = True if 'keep_source_audio' in row: from .media_request import validate_keep_source_audio validate_keep_source_audio(dict(row, kind=kind), path) diff --git a/freevideo_engine/media_encoding.py b/freevideo_engine/media_encoding.py index 2c7255bc..2e34f3c3 100644 --- a/freevideo_engine/media_encoding.py +++ b/freevideo_engine/media_encoding.py @@ -14,6 +14,7 @@ REFERENCE_FRAMES = 360 # 15 s at 24 fps, the longest H3 generation +VOICE_ONLY_CANVAS = {'width': 256, 'height': 144} # a clip kept only for its voice is encoded this small def reference_frames(canvas): @@ -178,6 +179,8 @@ def prepare(media, canvas, directory): ref['early_steps'] = row['early_steps'] if row.get('guide'): ref['guide'] = True + if row.get('voice_only'): + ref['voice_only'] = True numbers[kind] += 1 longer, length, used = False, None, limit / 24. target = directory / ('reference-%02d.npy' % index) @@ -195,7 +198,8 @@ def prepare(media, canvas, directory): if not container.streams.video: raise ValueError('Reference video has no video stream') stream = container.streams.video[0] - width, height = reference_size(stream.width, stream.height, canvas) + voice_only = bool(row.get('voice_only')) + width, height = reference_size(stream.width, stream.height, VOICE_ONLY_CANVAS if voice_only else canvas) length = clip_seconds(container, stream) pixels = np.lib.format.open_memmap(target, mode='w+', dtype=np.uint8, shape=(limit, height, width, 3)) origin = None @@ -220,12 +224,23 @@ def prepare(media, canvas, directory): count += 1 previous = image del previous + step = 1 + if voice_only: + # Every third frame (8 fps), or every second (12 fps) when that would leave too few frames. + step = next((n for n in (3, 2) if count // n >= 22), 1) + kept = np.array(pixels[:count:step]) + del pixels + pixels, count = kept, len(kept) count = count - (count - 5) % 17 if count < 22: raise ValueError('The reference video is too short; it needs to be at least about 1 s long.') - pixels.flush() - ref.update(pixels=str(target), frames=count, normalized_fps=24, width=width, height=height, crop_mode='video_24fps') - used = count / 24. + if voice_only: + np.save(target, pixels[:count], allow_pickle=False) + else: + pixels.flush() + ref.update(pixels=str(target), frames=count, normalized_fps=24 // step, width=width, height=height, + crop_mode='voice_%dfps' % (24 // step) if voice_only else 'video_24fps') + used = count * step / 24. # The clip's own sound is a reference too, unless the request turned it off. soundtrack, longer_audio, _ = (None, False, None) if row.get('sound') is False or row.get('guide') else \ _audio(path, directory / ('reference-%02d-audio.npy' % index), limit / 24.) @@ -233,7 +248,7 @@ def prepare(media, canvas, directory): if soundtrack: ref['audio'] = soundtrack ref_items.append({'type': 'audio'}) - selected = list(range(0, count, 12)) + selected = list(range(0, count, 12 // step)) sampled = torch.from_numpy(np.array(pixels[selected])).float().div_(255) if not row.get('guide'): # A guide only supplies motion latents; it is not described to the diff --git a/freevideo_engine/media_request.py b/freevideo_engine/media_request.py index 12eda471..4bc1dd02 100644 --- a/freevideo_engine/media_request.py +++ b/freevideo_engine/media_request.py @@ -55,7 +55,7 @@ def file_info(name): raise ValueError('Invalid ' + field + ' entry') if field != 'references' and 'early_steps' in entry: raise ValueError('early_steps is only allowed on video references') - allowed = ({'path', 'kind', 'early_steps', 'sound', 'guide', 'keep_source_audio', 'use', 'range', 'segment'} if field == 'references' + allowed = ({'path', 'kind', 'early_steps', 'sound', 'guide', 'voice_only', 'keep_source_audio', 'use', 'range', 'segment'} if field == 'references' else {'path', 'strength', 'alpha'}) if set(entry) - allowed: raise ValueError('Unknown ' + field + ' fields') @@ -77,6 +77,10 @@ def file_info(name): # keep exactly the same rows and cache key as before. if entry['guide']: row['guide'] = True + if 'voice_only' in entry: + validate_voice_only(entry) + if entry['voice_only']: + row['voice_only'] = True if 'keep_source_audio' in entry: validate_keep_source_audio(entry, row['path']) if entry['keep_source_audio']: @@ -149,6 +153,16 @@ def validate_guide(row): raise ValueError('A guide clip cannot use early_steps') +def validate_voice_only(row): + """A voice-only video supplies its sound; its picture is kept small and slow because only the voice is wanted.""" + if row.get('kind') != 'video': + raise ValueError('voice_only is only allowed on video references') + if type(row['voice_only']) is not bool: + raise ValueError('Video reference voice_only must be true or false') + if row['voice_only'] and (row.get('guide') or row.get('sound') is False or 'early_steps' in row): + raise ValueError('A voice-only clip cannot be a guide, silent or use early_steps') + + def validate_keep_source_audio(row, path): """Keeping the guide's sound needs a guide clip that has an audio stream.""" if type(row['keep_source_audio']) is not bool: From 5bfa754b0f126458d7db9248401d4fe8dd38699f Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 4 Nov 2026 21:46:18 +0000 Subject: [PATCH 06/14] Agent: automatic QA on finished takes, withdraw other takes when one is chosen, two takes per... Agent: automatic QA on finished takes, withdraw other takes when one is chosen, two takes per shot by default Each finished take of a built-in director production is checked for size, duration, black and frozen picture, audio and (with a transcriber) dialogue; the result is stored on the take and the judge evaluates takes as they arrive and never picks one that failed. Choosing a take cancels the other queued or running takes of the shot. --- freevideo_engine/agent/builtin_director.py | 28 +++- freevideo_engine/agent/production.py | 22 ++- freevideo_engine/agent/productions.py | 43 +++++- freevideo_engine/agent/take_qa.py | 153 +++++++++++++++++++++ 4 files changed, 236 insertions(+), 10 deletions(-) create mode 100644 freevideo_engine/agent/take_qa.py diff --git a/freevideo_engine/agent/builtin_director.py b/freevideo_engine/agent/builtin_director.py index f530ad54..3879ba83 100644 --- a/freevideo_engine/agent/builtin_director.py +++ b/freevideo_engine/agent/builtin_director.py @@ -26,7 +26,7 @@ import urllib.request from urllib.parse import urlsplit -from . import drama, spec +from . import drama, spec, take_qa @dataclass(frozen=True) @@ -367,17 +367,27 @@ def judge(shot_view, sheets, *, ask): shot = shot_view if shot.get('chosen_by') == 'user' or shot.get('chosen') is not None: return Decision('keep', shot['id'], '保留已选定的版本。') - if shot.get('state') in ('waiting', 'shooting'): - return Decision('wait', shot['id'], '等待这一镜的各版完成。') + shooting = shot.get('state') in ('waiting', 'shooting') uncertain = Decision('ask_user', shot['id'], '拿不准，请挑一版。') - takes = _selectable(shot) + finished = _selectable(shot) + unchecked = any('qa' in t and t['qa'] is None for t in finished) + if shooting and (not finished or any(t.get('qa') is None for t in finished)) or unchecked: + return Decision('wait', shot['id'], '等待这一镜的各版完成并质检。') + # A take that failed the picture or sound check is never chosen; a shot still shooting is judged as takes arrive. + takes = [t for t in finished if (t.get('qa') or {}).get('ok', True)] if not takes: - return uncertain + if shooting: + return Decision('wait', shot['id'], '已完成的版本没有通过质检，等待其余版本。') + if not finished or _retaken(shot): + return uncertain + faults = '；'.join(sorted({p for t in finished for p in t['qa']['problems']})) + return Decision('reshoot', shot['id'], '各版都没有通过质检。', + note=('上一次的问题：' + faults)[:300], takes=_take_count(shot)) try: rows = sheets['takes'] images = {row['take']: row['image'] for row in rows} numbers = [t['take'] for t in takes] - if len(images) != len(rows) or set(images) != set(numbers): + if len(images) != len(rows) or not set(numbers) <= set(images): return uncertain previous = sheets.get('previous') batch_size = 15 if previous is not None else 16 @@ -391,9 +401,11 @@ def judge(shot_view, sheets, *, ask): ('ratings',)) prompt = ('评价这一镜的每个版本：0–10 分，reason 用不超过30字的中文指出具体优点或问题' '（多余的字、人物变形、和描述不符等）；note 给下一次生成能用的具体修正。' - 'captions 表示有叠加字幕/文字，场景内真实招牌不算。不评价声音。' + 'captions 表示有叠加字幕/文字，场景内真实招牌不算。声音由自动质检评价，这里不评价。' '\n镜头描述：' + _dump({k: shot[k] for k in ('screenplay', 'characters', 'style', 'captions') if k in shot}) + '\n图片顺序对应版号：' + _dump(batch) + + ''.join('\n版%d的%s' % (t['take'], take_qa.summary(t.get('qa'))) for t in takes + if t['take'] in batch and t.get('qa')) + ('；最后一张是上一镜选定版的末帧，仅供接续参考。' if previous is not None else '。')) reply = _ask(ask, prompt, [images[n] for n in batch] + ([previous] if previous is not None else []), schema) if reply is None or sorted(r['take'] for r in reply['ratings']) != sorted(batch): @@ -405,6 +417,8 @@ def judge(shot_view, sheets, *, ask): if row['captions'] and not shot.get('captions', False): row['score'] = min(row['score'], 3) best = max(ratings, key=lambda r: (r['score'], -r['take'])) + if best['score'] < 7 and shooting: + return Decision('wait', shot['id'], '已完成的版本不够好，等待其余版本。') if best['score'] >= 7: return Decision('choose', shot['id'], best['reason'], take=best['take']) if _retaken(shot): diff --git a/freevideo_engine/agent/production.py b/freevideo_engine/agent/production.py index ca20154c..ce05deba 100644 --- a/freevideo_engine/agent/production.py +++ b/freevideo_engine/agent/production.py @@ -118,11 +118,12 @@ def plan(request, *, fields=True): problems.extend(dict(path='/screenplay' + p['path'], message=_rule(p)) for p in error.problems) if fields: _fields(play, problems) - count = request.get('takes_per_shot', 1) - count_ok = _count(count, '/takes_per_shot', problems) director = request.get('director', 'agent') if director not in ('agent', 'builtin'): problems.append(dict(path='/director', message='must be "agent" or "builtin"')) + # The built-in director judges takes as they arrive and withdraws the rest once one is chosen, so two cost little. + count = request.get('takes_per_shot', 2 if director == 'builtin' else 1) + count_ok = _count(count, '/takes_per_shot', problems) rows = play.get('shots', []) if isinstance(play, dict) else [] total = 0 if isinstance(rows, list): @@ -530,6 +531,22 @@ def resume(state, *, by, now): _commit(state, draft, now) +def _withdraw_others(row, chosen): + """Stop the shot's other unfinished takes and unsent manual batches once one is chosen. + + Finished takes stay, so the choice can still change; automatic reconnects are not ours to drop. + """ + for other in row['takes']: + if other['take'] == chosen or other['stale'] or other['cancel_requested']: + continue + if other['state'] == 'paused': + other['state'] = 'cancelled' + other['cancel_requested'] = True + elif other['state'] in ACTIVE: + other['cancel_requested'] = True + row['pending'] = [g for g in row['pending'] if g[0].get('automatic')] + + def choose(state, shot, take, *, by, reason=None, cascade=True, now): """Choose a current take, optionally reconnecting descendants automatically.""" _editable(state, by) @@ -550,6 +567,7 @@ def choose(state, shot, take, *, by, reason=None, cascade=True, now): _commit(state, draft, now) return row['chosen'], row['chosen_by'] = take, by + _withdraw_others(row, take) _refresh(draft) if by != 'user': affected = [dict(shot=old['id']) for old, child in zip(state['shots'], draft['shots']) diff --git a/freevideo_engine/agent/productions.py b/freevideo_engine/agent/productions.py index 68345481..9722bda6 100644 --- a/freevideo_engine/agent/productions.py +++ b/freevideo_engine/agent/productions.py @@ -21,7 +21,7 @@ import time import uuid -from . import drama, production, schema, spec, errors, operation_receipts +from . import drama, production, schema, spec, errors, operation_receipts, take_qa from .jobs import DONE, JobError, iso, parts_problems from .forecast import generation_estimate @@ -173,6 +173,9 @@ def __init__(self, jobs): self._waiter_lock = threading.Lock() # gate(action, *, by, estimate_seconds, refs, question) raises when the user must approve first. self.gate = None + # transcribe(path) -> heard text, optional; without it the QA does not compare dialogue. + self.transcribe = None + self._inspecting = set() self.load() jobs.parent_allowed = self.allows @@ -649,6 +652,7 @@ def contextual(error, join=False): for take, saved in zip(shot['takes'], internal['takes']): take['error'] = contextual(saved.get('error')) take['task'] = tasks.get(take['job']) + take['qa'] = deepcopy(record.get('qa', {}).get(take['job'])) if value['join'] is not None: ref = record['state']['current_join'] join = next(j for j in record['state']['joins'] if j['ref'] == ref) @@ -884,6 +888,43 @@ def _advance(self, identity): self._bump(draft) # Wake since/wait readers when the elapsed reminder appears or clears. if draft != record: self._save(draft) + self._inspect_takes(identity) + + def _inspect_takes(self, identity): + """Start the picture and sound check of each finished take that has none; it runs off the lock. + + Only the built-in director reads the result, so films run by an agent are left alone. + """ + record = self._record(identity) + if record['state']['director'] != 'builtin': + return + for shot in record['state']['shots']: + for take in shot['takes']: + job = take['job'] + if (take['state'] != 'succeeded' or take['stale'] or not take['library_id'] + or job in record.get('qa', {}) or (identity, job) in self._inspecting): + continue + self._inspecting.add((identity, job)) + lines = [row['line'] for row in shot.get('lines') or [] if row.get('line')] + threading.Thread(target=self._inspect, daemon=True, args=( + identity, job, take['library_id'], shot.get('seconds'), lines)).start() + + def _inspect(self, identity, job, library, seconds, lines): + try: + with self.jobs.cpu: + result = take_qa.check(self.jobs.port.library_video(library), seconds=seconds, + lines=lines, transcribe=self.transcribe) + except Exception as error: # noqa: BLE001 - a failed check must not block the film; the judge sees it as unchecked + logging.getLogger(__name__).warning('A take could not be checked (%s)', type(error).__name__) + result = dict(error=type(error).__name__, problems=[], notes=[], ok=True) + with self.changed: + self._inspecting.discard((identity, job)) + if identity not in self.records: + return + draft = deepcopy(self._record(identity)) + draft.setdefault('qa', {})[job] = result + self._bump(draft) + self._save(draft) def _execute(self, identity, action): """Carry out one action; True when it changed a child, a receipt or a tombstone.""" diff --git a/freevideo_engine/agent/take_qa.py b/freevideo_engine/agent/take_qa.py new file mode 100644 index 00000000..0e2f7e21 --- /dev/null +++ b/freevideo_engine/agent/take_qa.py @@ -0,0 +1,153 @@ +"""Automatic checks on a finished take: picture, sound and dialogue. + +The result is plain data stored next to the take. `problems` lists what makes a take unusable, `notes` +what is only worth a look; the director's judge drops takes with problems and mentions the notes. +""" +import difflib +import re + +import numpy as np + +from . import media + +DURATION_TOLERANCE = 1.5 # s, against the shot's planned length +BLACK_LUMA = 26 # 0-255 mean luma of a black frame +BLACK_SECONDS = 0.3 +BLACK_SPREAD = 10.0 # a dark but detailed scene is not a black frame +FREEZE_DIFF = 0.05 # 0-255 mean change between consecutive frames +FREEZE_SECONDS = 1.5 +SILENT_DB = -45.0 # whole-take RMS below this is near silence +GAP_DB = -50.0 # 50 ms windows below this are a gap +GAP_SECONDS = 2.0 +GAP_EDGE = 0.5 # a gap counts as "in the middle" this far from both ends +SPEECH_DB = -35.0 +CLIP_PEAK = 0.999 +LINE_MATCH = 0.6 +ANALYSIS_WIDTH = 64 +SAMPLE_RATE = 16000 +WINDOW = 0.05 + + +def _words(text): + return re.findall(r"[\w']+", str(text).lower()) + + +def _pictures(path): + """Mean luma and a small gray frame for every video frame.""" + means, small, times = [], [], [] + with media._reading(media._file(path)) as container: + stream = media._video_stream(container) + for frame in container.decode(stream): + height = max(2, round(ANALYSIS_WIDTH * frame.height / frame.width)) + gray = frame.reformat(width=ANALYSIS_WIDTH, height=height, format='gray').to_ndarray().astype(np.float32) + means.append(float(gray.mean())) + small.append(gray) + times.append(float(frame.time) if frame.time is not None else len(times) / 24.0) + return means, small, times + + +def _samples(path): + import av + with media._reading(media._file(path)) as container: + stream = next(iter(container.streams.audio), None) + if stream is None: + return np.zeros(0, np.float32) + resampler = av.AudioResampler(format='flt', layout='mono', rate=SAMPLE_RATE) + chunks = [] + for frame in container.decode(stream): + for out in resampler.resample(frame): + chunks.append(out.to_ndarray().reshape(-1)) + for out in resampler.resample(None): + chunks.append(out.to_ndarray().reshape(-1)) + return np.concatenate(chunks).astype(np.float32) if chunks else np.zeros(0, np.float32) + + +def _runs(flags, times, minimum): + """Spans (start, end) where `flags` stays true for at least `minimum` seconds.""" + spans, start = [], None + for index, flag in enumerate(flags): + if flag and start is None: + start = index + if start is not None and (not flag or index == len(flags) - 1): + last = index if flag else index - 1 + if times[last] - times[start] >= minimum: + spans.append((round(times[start], 2), round(times[last], 2))) + start = None + return spans + + +def _db(value): + return float(20 * np.log10(max(float(value), 1e-9))) + + +def check(path, *, seconds=None, size=None, lines=None, transcribe=None): + """Inspect one take. `seconds` is the planned length, `size` the expected (width, height), + `lines` the spoken lines the shot should contain and `transcribe(path)` returns the heard text.""" + problems, notes = [], [] + info = media.inspect(path) + if size and (info['width'], info['height']) != tuple(size): + problems.append('size %dx%d, expected %dx%d' % (info['width'], info['height'], *size)) + if seconds and abs(info['seconds'] - seconds) > DURATION_TOLERANCE: + problems.append('duration %.1f s, planned %.1f s' % (info['seconds'], seconds)) + result = dict(width=info['width'], height=info['height'], seconds=round(info['seconds'], 2), audio=info['has_audio']) + + means, small, times = _pictures(path) + black = _runs([m < BLACK_LUMA and float(g.std()) < BLACK_SPREAD for m, g in zip(means, small)], times, BLACK_SECONDS) if means else [] + flags = [False] * len(small) + for index in range(1, len(small)): + if float(np.abs(small[index] - small[index - 1]).mean()) <= FREEZE_DIFF: + flags[index] = flags[index - 1] = True + frozen = _runs(flags, times, FREEZE_SECONDS) + if black: + problems.append('black picture at %s' % ', '.join('%.1f-%.1f s' % s for s in black)) + if frozen: + problems.append('frozen picture at %s' % ', '.join('%.1f-%.1f s' % s for s in frozen)) + result.update(black=black, frozen=frozen) + + if not info['has_audio']: + problems.append('no audio track') + else: + samples = _samples(path) + if samples.size: + rms, peak = float(np.sqrt(np.mean(samples ** 2))), float(np.abs(samples).max()) + result.update(rms_db=round(_db(rms), 1), peak=round(peak, 3)) + if _db(rms) < SILENT_DB: + problems.append('audio nearly silent (%.0f dBFS)' % _db(rms)) + elif peak >= CLIP_PEAK: + notes.append('audio clips') + step = int(SAMPLE_RATE * WINDOW) + count = samples.size // step + levels = [_db(np.sqrt(np.mean(samples[i * step:(i + 1) * step] ** 2))) for i in range(count)] + moments = [i * WINDOW for i in range(count)] + if any(level > SPEECH_DB for level in levels): + gaps = [(a, b) for a, b in _runs([level < GAP_DB for level in levels], moments, GAP_SECONDS) + if a > GAP_EDGE and b < moments[-1] - GAP_EDGE] + if gaps: + notes.append('silent gap at %s' % ', '.join('%.1f-%.1f s' % g for g in gaps)) + result['gaps'] = gaps + + if lines and transcribe is not None and info['has_audio']: + heard = _words(transcribe(path)) + missing = [] + for line in lines: + wanted = _words(line) + if not wanted: + continue + best = max([difflib.SequenceMatcher(None, wanted, heard[i:i + len(wanted) + 2]).ratio() + for i in range(max(1, len(heard) - len(wanted) + 3))] or [0.0]) + if best < LINE_MATCH: + missing.append(line) + result['heard'] = ' '.join(heard) + if missing: + problems.append('line not heard: %s' % ' | '.join(missing)) + return dict(result, problems=problems, notes=notes, ok=not problems) + + +def summary(qa): + """One line for the judge's prompt.""" + if not qa: + return '' + if qa.get('error'): + return 'QA unavailable' + parts = qa['problems'] + qa['notes'] + return 'QA: ' + ('; '.join(parts) if parts else 'clean') From 9e63a91105cd63928c4dfcce65ce48b90a1439a8 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Thu, 5 Nov 2026 01:42:58 +0000 Subject: [PATCH 07/14] Take QA: a take without a library video passes as unchecked, Chinese lines match by character... Take QA: a take without a library video passes as unchecked, Chinese lines match by character, built-in extra takes are not counted twice in the estimate --- freevideo_engine/agent/productions.py | 15 ++++++++++++++- freevideo_engine/agent/take_qa.py | 7 ++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/freevideo_engine/agent/productions.py b/freevideo_engine/agent/productions.py index 9722bda6..3c9f7d25 100644 --- a/freevideo_engine/agent/productions.py +++ b/freevideo_engine/agent/productions.py @@ -517,11 +517,17 @@ def fraction(row, status): return 1. value = (self.jobs.progress(row) or {}).get('fraction') if row and status == 'running' else None return min(1., max(0., value)) if type(value) in (int, float) and math.isfinite(value) else 0. + waiting = set() # built-in shots whose first live take is still going: the others are withdrawn once it is chosen for shot, take in self._takes(state): child = children.get(take['job']) status = child['status'] if child and take['state'] != 'paused' else take['state'] if take['stale'] or status == 'cancelled' or (take['cancel_requested'] and status != 'succeeded'): continue + if state['director'] == 'builtin': + if shot['id'] in waiting: + continue + if status not in ('succeeded', 'failed', 'interrupted'): + waiting.add(shot['id']) part = fraction(child, status) completed += status in ('succeeded', 'failed', 'interrupted') fractions.append(part) @@ -901,9 +907,16 @@ def _inspect_takes(self, identity): for shot in record['state']['shots']: for take in shot['takes']: job = take['job'] - if (take['state'] != 'succeeded' or take['stale'] or not take['library_id'] + if (take['state'] != 'succeeded' or take['stale'] or job in record.get('qa', {}) or (identity, job) in self._inspecting): continue + if not take['library_id']: + # Nothing to open: pass it as unchecked, or the judge would wait on it forever. + draft = deepcopy(self._record(identity)) + draft.setdefault('qa', {})[job] = dict(error='NoVideo', problems=[], notes=[], ok=True) + self._bump(draft) + self._save(draft) + continue self._inspecting.add((identity, job)) lines = [row['line'] for row in shot.get('lines') or [] if row.get('line')] threading.Thread(target=self._inspect, daemon=True, args=( diff --git a/freevideo_engine/agent/take_qa.py b/freevideo_engine/agent/take_qa.py index 0e2f7e21..57a420d8 100644 --- a/freevideo_engine/agent/take_qa.py +++ b/freevideo_engine/agent/take_qa.py @@ -28,8 +28,13 @@ WINDOW = 0.05 +_CJK = '\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af' +_WORD = re.compile("[%s]|(?:(?![%s])[\\w'])+" % (_CJK, _CJK)) + + def _words(text): - return re.findall(r"[\w']+", str(text).lower()) + """Words of a line; Chinese, Japanese and Korean have no spaces, so each of their characters is one.""" + return _WORD.findall(str(text).lower()) def _pictures(path): From e3826025d4f610317cd573dc5c1cbec9ba921c1f Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Thu, 5 Nov 2026 02:33:30 +0000 Subject: [PATCH 08/14] Settings: the three Getting started buttons keep one line each at phone width and the row wraps --- web/setup.css | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/web/setup.css b/web/setup.css index f363cbe7..1bcb8070 100644 --- a/web/setup.css +++ b/web/setup.css @@ -21,9 +21,10 @@ .fv-setup-head .fv-close{min-height:34px;width:34px;padding:0;font-size:20px;line-height:1;background:transparent;border-color:transparent;color:var(--fv-muted)} .fv-setup-head .fv-close:hover{background:var(--fv-raised);color:var(--fv-ink)} .fv-setup h3{font-size:var(--fv-body);font-weight:var(--fv-semibold);margin:0 0 12px} -.fv-setup .fv-guide-replay-row{display:flex;align-items:center;justify-content:space-between;gap:16px} +.fv-setup .fv-guide-replay-row{display:flex;flex-wrap:wrap;align-items:center;justify-content:space-between;gap:12px 16px} .fv-setup .fv-guide-replay-row h3{margin:0} -.fv-setup .fv-guide-replay-actions{display:flex;align-items:center;gap:16px} +.fv-setup .fv-guide-replay-actions{display:flex;flex-wrap:wrap;align-items:center;gap:8px 16px} +.fv-setup .fv-guide-replay-actions button{white-space:nowrap} .fv-setup .fv-guide-howtos{display:flex;flex-direction:column;align-items:stretch;gap:4px;margin-top:10px;width:100%} .fv-setup .fv-guide-howtos [role=menuitem]{width:100%;justify-content:flex-start;text-align:left} .fv-setup .fv-guide-howtos[hidden]{display:none} From 53774f2b1aad96e2684fa6e618b3d58cd297950e Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Thu, 5 Nov 2026 02:58:22 +0000 Subject: [PATCH 09/14] ComfyUI host: hold PyAV to the version the media code runs on, for new and existing environments --- freevideo_engine/comfy_host.py | 6 ++++-- freevideo_engine/comfy_service.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/freevideo_engine/comfy_host.py b/freevideo_engine/comfy_host.py index 10b44d95..770643ce 100644 --- a/freevideo_engine/comfy_host.py +++ b/freevideo_engine/comfy_host.py @@ -20,8 +20,10 @@ from .terminal_ui import LogProgress, TerminalUI # FreeVideo's nodes and Agent routes run in the ComfyUI process; its requirements -# do not name what only they use (the editor rules engine for project and image edits). -NODE_PACKAGES = ('quickjs==1.19.4',) +# do not name what only they use (the editor rules engine for project and image edits), +# and accept a range of PyAV: the version our media code was measured on is held here, +# because a newer one reads and writes different audio (it matches the engine's constraints). +NODE_PACKAGES = ('quickjs==1.19.4', 'av==18.1.0') def prepare(root, comfy, run=None, download=False, repair=False): diff --git a/freevideo_engine/comfy_service.py b/freevideo_engine/comfy_service.py index 5e9e48b1..44c57120 100644 --- a/freevideo_engine/comfy_service.py +++ b/freevideo_engine/comfy_service.py @@ -39,7 +39,7 @@ WRAPPER_MARK = '# FreeVideo command, written by FreeVideo setup.' DESKTOP_ID = 'freevideo.desktop' LOOPBACK = ('127.0.0.1', 'localhost', '::1') -NODE_PACKAGES_NOTE = ('The Agent editing tools could not be added now; everything else works. ' +NODE_PACKAGES_NOTE = ('The packages for the Agent editing tools could not be updated now; everything else works. ' 'FreeVideo will try again later.') COMMANDS = ('open', 'stop', 'status', 'restart', 'server') From 6bce493249b58189801c99f986eb6c4c138596bf Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Thu, 5 Nov 2026 03:11:53 +0000 Subject: [PATCH 10/14] Voice references: measure speech in the first six seconds and report it on upload --- freevideo_engine/agent/comfy.py | 9 ++++++++ freevideo_engine/agent/routes.py | 7 +++++++ freevideo_engine/agent/schema.py | 1 + freevideo_engine/voice_window.py | 36 ++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+) create mode 100644 freevideo_engine/voice_window.py diff --git a/freevideo_engine/agent/comfy.py b/freevideo_engine/agent/comfy.py index b27f6317..fe25b2f7 100644 --- a/freevideo_engine/agent/comfy.py +++ b/freevideo_engine/agent/comfy.py @@ -60,6 +60,15 @@ def output_path(self, relative, create=False): def output_relative(self, path): return Path(path).resolve().relative_to(self.output_root).as_posix() + def voice_window(self, path, seconds): + """Speech in the first seconds of an uploaded sound, or None without the speech model.""" + from ..comfy_bridge import installation + from ..image_gen import image_pack + from ..speech import MODEL_FILE + from ..voice_window import inspect + model = image_pack.base(installation()[1]['model_root']) / MODEL_FILE + return inspect(path, seconds, model, self.output_root / 'FreeVideo' / '.editor' / 'speech') + def library_video(self, identity): from ..comfy_library import _video, _report try: diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py index 7ef65d92..1df729cd 100644 --- a/freevideo_engine/agent/routes.py +++ b/freevideo_engine/agent/routes.py @@ -1889,6 +1889,13 @@ def digest(): os.replace(target, out) info['name'] = identity + suffix info['original_name'] = name + if info['kind'] in ('audio', 'video') and info.get('has_audio', info['kind'] == 'audio'): + try: + voice = await asyncio.to_thread(port.voice_window, out, info['seconds']) + except Exception: # noqa: BLE001 - the check is advice; an upload never fails for it + voice = None + if voice is not None: + info['voice'] = voice return reply(dict(**assets.view(relative), **info), 201) @routes.get(PREFIX + '/media/{kind}/{file:.+}/file') diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index 5318d9f4..1adc468e 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -169,6 +169,7 @@ def common(): def generation_media(*, local=False, enhance=False): return obj({'first': nullable(media(True, local=local)), 'last': nullable(media(True, local=local)), 'references': nullable(array(media(enhance, local=local, early=not enhance), high=8 if enhance else 32))}, + description='A first or last frame request does not take a voice reference.', allOf=[{'if': {'required': ['references'], 'properties': {'references': {'type': 'array', 'minItems': 1}}}, 'then': {'properties': {'first': {'type': 'null'}, 'last': {'type': 'null'}}}}]) diff --git a/freevideo_engine/voice_window.py b/freevideo_engine/voice_window.py new file mode 100644 index 00000000..7eaa8b44 --- /dev/null +++ b/freevideo_engine/voice_window.py @@ -0,0 +1,36 @@ +"""How much of a voice reference's first seconds is speech. + +The engine reads the first WINDOW seconds of a reference sound. A window with little speech, or one that opens with +music, gives a weak voice, so the caller can ask for another clip before spending a generation on it. +""" +from __future__ import annotations + +WINDOW = 6. +MIN_SPEECH = 4. + + +def measure(spans, duration, window=WINDOW): + """Speech seconds inside [0, window] from VAD spans, and when speech first starts (None without speech).""" + end = min(window, duration) + speech = sum(max(0., min(b, end) - max(a, 0.)) for a, b in spans) + first = next((max(a, 0.) for a, b in spans if b > 0 and a < end), None) + return round(speech, 2), None if first is None else round(first, 2), round(end, 2) + + +def check(spans, duration): + """{'window', 'speech_seconds', 'first_speech', 'usable', 'reason'}; reason is None when usable.""" + speech, first, window = measure(spans, duration) + need = min(MIN_SPEECH, window) + # A clip that opens with music has the same effect as a sparse one: the window holds too little speech. + reason = 'too_little_speech' if speech < need else None + return dict(window=window, speech_seconds=speech, first_speech=first, usable=reason is None, reason=reason) + + +def inspect(path, duration, model_path, cache): + """check() for a sound or video file, or None when the speech model is not installed or the file cannot be read.""" + from .speech import SpeechError, speech_spans + try: + spans = speech_spans(path, model_path, cache) + return check(spans, duration) + except SpeechError: + return None From 2553bf33b6e0e3ecc939abea9362678ef398962b Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Thu, 5 Nov 2026 03:15:38 +0000 Subject: [PATCH 11/14] Launcher: a setup check that ended without output says so in a sentence, and the download list... Launcher: a setup check that ended without output says so in a sentence, and the download list actions stack wherever one row cannot hold every label The check's summary line stays in the copied details and the report but no longer shows on the failure card or under the download list. The download list's buttons form one full-width column at phone width or wherever the row would wrap a label, and keep their own width in the row otherwise. --- freevideo_engine/failure_details.py | 8 ++++++++ freevideo_engine/launcher/Main.qml | 24 ++++++++++++++---------- freevideo_engine/launcher_session.py | 2 ++ 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/freevideo_engine/failure_details.py b/freevideo_engine/failure_details.py index b4359135..b361d86f 100644 --- a/freevideo_engine/failure_details.py +++ b/freevideo_engine/failure_details.py @@ -864,6 +864,11 @@ def _launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_re # Validation messages are already actionable. Show the cause, not a stack # trace or a guessed hardware diagnosis, and leave long details expandable. lines = [line.strip() for line in text.splitlines() if line.strip()] + if lines and all(re.fullmatch(INSPECTION_SUMMARY, line) for line in lines): + # The check printed nothing but its own summary line (once per place that kept it); that line + # is for the report, not the card. The next step goes under it, as for an unexpected failure. + return dict(title='暂时未能完成' if zh else 'Couldn’t complete this step', + detail=CHECK_UNFINISHED[zh] + '\n' + UNKNOWN_NEXT_STEP[zh], action='', kind='unknown') causes = [line for line in lines if re.match(r'^(?:[\w.]+(?:Error|Exception):|ERROR[: ]|Failed step:|Setup failed:)', line)] detail = causes[-1] if causes else next((line for line in reversed(lines) if not line.startswith(('File ', 'Traceback', '^', 'All files retained.', 'Existing files are retained.', 'Rerun the same setup command'))), '') @@ -889,6 +894,9 @@ def _launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_re return dict(title=title, detail=detail, action='', kind='unknown') +# comfy_setup.inspection_failed: the first line of a failed check. +INSPECTION_SUMMARY = r'Setup inspection failed(?: \([^()\r\n]*(?:\([^()\r\n]*\)[^()\r\n]*)*\))?' +CHECK_UNFINISHED = ('The check did not finish.', '这次检查没有完成。') UNKNOWN_NEXT_STEP = ('Try again. If it fails again, export the report, then use "Report a problem" and attach it.', '请重试。仍然失败时，请先导出报告，再点“提交问题”并附上报告。') diff --git a/freevideo_engine/launcher/Main.qml b/freevideo_engine/launcher/Main.qml index 154194b4..0b5f5ddd 100644 --- a/freevideo_engine/launcher/Main.qml +++ b/freevideo_engine/launcher/Main.qml @@ -1644,20 +1644,24 @@ ApplicationWindow { } FText { objectName: "downloadActionNotice"; visible: !!s.action_notice && s.action_notice !== s.busy_reason; text: s.action_notice || ""; color: theme.muted; Layout.fillWidth: true } FText { objectName: "downloadListError"; visible: text.length > 0; text: s.download_list.error; color: theme.danger; Layout.fillWidth: true; wrapMode: Text.WordWrap; font.pixelSize: theme.micro + 1 } - FText { objectName: "downloadListRawError"; visible: s.error.length > 0 && !s.download_list.inspection_error; text: s.error; color: theme.danger; Layout.fillWidth: true; maximumLineCount: 2; elide: Text.ElideRight; font.pixelSize: theme.micro } + FText { objectName: "downloadListRawError"; visible: s.error.length > 0 && !s.download_list.inspection_failed; text: s.error; color: theme.danger; Layout.fillWidth: true; maximumLineCount: 2; elide: Text.ElideRight; font.pixelSize: theme.micro } } } FText { objectName: "downloadBusyReason"; visible: !s.can_manage && (!s.action_notice || s.action_notice === s.busy_reason); text: s.busy_reason; color: theme.muted; Layout.fillWidth: true } - RowLayout { - objectName: "downloadListActions"; Layout.fillWidth: true; Layout.minimumWidth: 0; spacing: 8 - // The actions share what the Close button leaves, so a long label wraps instead of leaving the dialog. + GridLayout { + id: downloadListActions; objectName: "downloadListActions"; Layout.fillWidth: true; Layout.minimumWidth: 0 + columnSpacing: 8; rowSpacing: 8 + // Wide: one row, Close on the left, every label on one line. Narrow, or wherever the row would not + // hold every label on one line: one full-width column, the actions first and Close last. + readonly property real rowWidth: closeDownloadList.implicitWidth + (checkDownloadList.visible ? checkDownloadList.implicitWidth : 0) + + (useDownloadedFolder.visible ? useDownloadedFolder.implicitWidth : 0) + exportDownloadList.implicitWidth + (actionCount + 1) * columnSpacing + readonly property bool stacked: win.narrow || rowWidth > width readonly property int actionCount: (checkDownloadList.visible ? 1 : 0) + (downloads.scope === "image" ? 1 : 2) - readonly property real actionWidth: Math.max(80, Math.floor((width - closeDownloadList.implicitWidth - (actionCount + 1) * spacing) / actionCount)) - FButton { id: closeDownloadList; text: downloads.copy.close; flat: true; onClicked: modelInfoOpen = false } - Item { Layout.fillWidth: true } - FButton { id: checkDownloadList; objectName: "checkDownloadList"; Layout.maximumWidth: parent.actionWidth; visible: downloads.scope !== "image" && (!downloads.available || s.download_list.needs_check); text: downloads.copy.check; enabled: !s.busy; onClicked: backend.prepareDownloadList() } - FButton { objectName: "useDownloadedFolder"; visible: downloads.scope !== "image"; Layout.maximumWidth: parent.actionWidth; text: downloads.copy.use_folder; enabled: s.can_manage; onClicked: backend.useDownloadedFolder() } - FButton { objectName: "exportDownloadList"; Layout.maximumWidth: parent.actionWidth; text: downloads.copy.export; primary: true; enabled: downloads.available && (downloads.scope === "image" || !s.busy); onClicked: backend.exportDownloadList() } + FButton { id: closeDownloadList; objectName: "closeDownloadList"; Layout.row: downloadListActions.stacked ? 3 : 0; Layout.column: 0; Layout.fillWidth: downloadListActions.stacked; text: downloads.copy.close; flat: true; onClicked: modelInfoOpen = false } + Item { visible: !downloadListActions.stacked; Layout.row: 0; Layout.column: 1; Layout.fillWidth: true } + FButton { id: checkDownloadList; objectName: "checkDownloadList"; Layout.row: 0; Layout.column: downloadListActions.stacked ? 0 : 2; Layout.fillWidth: downloadListActions.stacked; visible: downloads.scope !== "image" && (!downloads.available || s.download_list.needs_check); text: downloads.copy.check; enabled: !s.busy; onClicked: backend.prepareDownloadList() } + FButton { id: useDownloadedFolder; objectName: "useDownloadedFolder"; visible: downloads.scope !== "image"; Layout.row: downloadListActions.stacked ? 1 : 0; Layout.column: downloadListActions.stacked ? 0 : 3; Layout.fillWidth: downloadListActions.stacked; text: downloads.copy.use_folder; enabled: s.can_manage; onClicked: backend.useDownloadedFolder() } + FButton { id: exportDownloadList; objectName: "exportDownloadList"; Layout.row: downloadListActions.stacked ? 2 : 0; Layout.column: downloadListActions.stacked ? 0 : 4; Layout.fillWidth: downloadListActions.stacked; text: downloads.copy.export; primary: true; enabled: downloads.available && (downloads.scope === "image" || !s.busy); onClicked: backend.exportDownloadList() } } } } diff --git a/freevideo_engine/launcher_session.py b/freevideo_engine/launcher_session.py index 66b25595..578b1be9 100644 --- a/freevideo_engine/launcher_session.py +++ b/freevideo_engine/launcher_session.py @@ -2424,6 +2424,8 @@ def snapshot(self): download_state = dict(checking=bool(self.controller.busy), error=self.download_error, needs_check=self.download_error_key in ('unchecked', 'changed'), folder=self.download_export, inspection_error=inspection_read_error, + # Any failed check: the list's own line asks for Check again, so the raw error stays in the details. + inspection_failed=row.get('status') == 'failed' and row.get('action') == 'inspect', notice=downloads['copy']['exported'].format(folder=self.download_export) if self.download_export else '') if inspection_read_error: download_state.update(error=self.t(*DOWNLOAD_COPY['error_read']), needs_check=True) From 15df89dcd95276837f10cd324f849d43b4579d66 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Thu, 5 Nov 2026 03:21:00 +0000 Subject: [PATCH 12/14] Leave the sound wording of prompts as written; drop the added natural-sounds sentence --- freevideo_engine/generate.py | 3 -- freevideo_engine/sound_prompt.py | 78 -------------------------------- 2 files changed, 81 deletions(-) delete mode 100644 freevideo_engine/sound_prompt.py diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.py index f4705690..b91e6f4d 100644 --- a/freevideo_engine/generate.py +++ b/freevideo_engine/generate.py @@ -843,9 +843,6 @@ def interrupted(signum, frame): from .continuation import first_frame_prompt prompt_text = first_frame_prompt(prompt_text) prompt_text = without_music(prompt_text) - # The model has no negative prompt: sound the user excluded is worded as the sounds the scene has. - from .sound_prompt import positive_sound - prompt_text = positive_sound(prompt_text) encoding = {'prompt': prompt_text, 'encoder': args.encoder, 'comfy_root': str(args.comfy_root or comfy_root()), 'model_paths': str(args.model_paths.resolve()) if args.model_paths else None, diff --git a/freevideo_engine/sound_prompt.py b/freevideo_engine/sound_prompt.py deleted file mode 100644 index 8fb8c319..00000000 --- a/freevideo_engine/sound_prompt.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Positive sound wording for the text encoder. - -The model has no negative prompt, so "no music" only draws attention to music. Sound requests that -exclude something are replaced by the sounds the scene actually has. A prompt that asks for music -is returned unchanged, and quoted or tagged speech is never edited. A prompt with spoken lines, quoted or described, gets no added -sentence either: it only lost the excluded sound words, because the sentence made the spoken lines less accurate. -""" -import re - -from .prompt_vlm.rules import QUOTED, QUOTES - -MUSIC = (r'(?:background\s+music|music|soundtrack|BGM|cinematic\s+(?:orchestral\s+)?score|orchestral\s+score|song|' - r'背景音乐|音乐|配乐|歌曲|歌声)') -PERFORMED = (r'(?:\b(?:sings?|singing|singers?|bands?|DJ|whistl(?:e|es|ed|ing)|choir)\b|' - r'\b(?:plays?|playing|played)\s+(?:an?\s+|the\s+)?(?:\w+\s+)?(?:guitar|piano|violin|cello|drums?|saxophone|trumpet|flute|keyboard|harp|accordion|ukulele|bass)\b|' - r'唱歌|歌手|乐队|演奏|弹(?:吉他|钢琴)|拉(?:小提琴|大提琴)|吹(?:口哨|笛子|萨克斯)|口哨|打鼓|合唱)') -ANY_MUSIC = r'(?:' + MUSIC + '|' + PERFORMED + ')' -NEGATION = (r'(?:\b(?:no|without|avoid|exclude)\b|\bdo\s+not\s+(?:add|include|use|play)\b|' - r'请?(?:不要|别|不用|不需要|无需|不加|没有|去掉|去除|禁止|无)(?:任何|有|放|加|使用|出现)?)') -SOUND_WORD = (r'(?:background\s+music|music|soundtrack|BGM|subtitles?|captions?|narration|voiceover|singing|audio|sound|' - r'背景音乐|音乐|配乐|歌曲|歌声|字幕|旁白|解说|声音|音效)') -# An exclusion is cut out only when it ends the clause; "no music at all" or "没有音乐的派对" would be left broken. -NEGATED = re.compile(r'(?:(?:[,，]\s*)|\b(?:with|and|but|plus)\s+)?' + NEGATION + r'\s*(?:any\s+)?' + SOUND_WORD + - r'(?:\s+(?:or|and)\s+(?:music|subtitles?|captions?|narration|voiceover)|[和或及、]\s*(?:音乐|背景音乐|字幕|旁白|解说|音效))*' - r'(?:\s+of\s+[^,.;，。]*)?(?=\s*(?:[,，;；.。!?！？]|$))', re.I) -SILENT = re.compile(r'\bsilent\b|\bmute\b|\bno\s+(?:sound|audio)\b(?!\s+of)|无声|静音', re.I) -TAGGED = re.compile(r'<(d|en|zh|ja|ko|cn)>.*?', re.S) -SOUNDSCAPE = re.compile(r'(?m)^[ \t]*overall_soundscape[ \t]*:') -CLAUSE_EN = 'The sounds in the scene are the natural sounds of what is happening and its surroundings.' -CLAUSE_ZH = '画面里的声音是正在发生的动作和周围环境自然发出的声音。' -SPEECH = re.compile(r'\b(?:says?|said|speaks?|speaking|talks?|talking|whispers?|shouts?|asks?|replies|tells?|announces?)\b|' - r'说|讲|喊|问|答|念|朗读|台词|对白', re.I) -CHINESE = re.compile(r'[一-鿿]') - - -def _protect(prompt): - """Prompt with quoted and tagged speech swapped for placeholders, and the saved pieces.""" - kept = [] - - def save(match): - kept.append(match.group()) - return '\x00%d\x00' % (len(kept) - 1) - - return QUOTED.sub(save, TAGGED.sub(save, prompt)), kept - - -def _restore(text, kept): - return re.sub('\x00(\\d+)\x00', lambda m: kept[int(m.group(1))], text) - - -def wants_music(prompt): - """True when music is requested. Negated mentions and the visual 'music video' style are not requests.""" - body, _ = _protect(prompt) - excluded = any(re.search(MUSIC, m.group(), re.I) for m in NEGATED.finditer(body)) - body = NEGATED.sub('', body) - body = re.sub(r'\bmusic\s+video\b|\bMV\b|音乐视频', '', body, flags=re.I) - # Music that was explicitly excluded is not requested back by someone who sings or plays in the scene. - return bool(re.search(MUSIC if excluded else ANY_MUSIC, body, re.I)) - - -def positive_sound(prompt): - if not prompt.strip() or SOUNDSCAPE.search(prompt) or wants_music(prompt): - return prompt - body, kept = _protect(prompt.strip()) - if SILENT.search(body): - return prompt - chinese = bool(CHINESE.search(body)) - body = re.sub(r'\bmusic\s+video\b', '\x01', body, flags=re.I) - body = NEGATED.sub('', body) - body = re.sub(r'\s+([,.;!?])', r'\1', body) - body = re.sub(r'[,，;；]\s*(?=[,，;；.。!?！？]|$)', '', body) - body = re.sub(r'([.。!?！？])(?:\s*[.。])+', r'\1', body).strip(' ,，;；').lstrip('.。 ').replace('\x01', 'music video') - clause = CLAUSE_ZH if chinese else CLAUSE_EN - if kept or SPEECH.search(body) or clause in _restore(body, kept): - return _restore(body, kept) - if body and _restore(body, kept)[-1] not in '.。!?！？' + ''.join(QUOTES.values()): - body += '。' if chinese else '.' - return (_restore(body, kept) + ' ' + clause).strip() From a961fe08999ae9657bc47b735ba654bcf8ffaf72 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Thu, 5 Nov 2026 03:58:13 +0000 Subject: [PATCH 13/14] Launcher: restore the previous setup check message and download list layout This change returns in a later version. --- freevideo_engine/failure_details.py | 8 -------- freevideo_engine/launcher/Main.qml | 24 ++++++++++-------------- freevideo_engine/launcher_session.py | 2 -- 3 files changed, 10 insertions(+), 24 deletions(-) diff --git a/freevideo_engine/failure_details.py b/freevideo_engine/failure_details.py index b361d86f..b4359135 100644 --- a/freevideo_engine/failure_details.py +++ b/freevideo_engine/failure_details.py @@ -864,11 +864,6 @@ def _launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_re # Validation messages are already actionable. Show the cause, not a stack # trace or a guessed hardware diagnosis, and leave long details expandable. lines = [line.strip() for line in text.splitlines() if line.strip()] - if lines and all(re.fullmatch(INSPECTION_SUMMARY, line) for line in lines): - # The check printed nothing but its own summary line (once per place that kept it); that line - # is for the report, not the card. The next step goes under it, as for an unexpected failure. - return dict(title='暂时未能完成' if zh else 'Couldn’t complete this step', - detail=CHECK_UNFINISHED[zh] + '\n' + UNKNOWN_NEXT_STEP[zh], action='', kind='unknown') causes = [line for line in lines if re.match(r'^(?:[\w.]+(?:Error|Exception):|ERROR[: ]|Failed step:|Setup failed:)', line)] detail = causes[-1] if causes else next((line for line in reversed(lines) if not line.startswith(('File ', 'Traceback', '^', 'All files retained.', 'Existing files are retained.', 'Rerun the same setup command'))), '') @@ -894,9 +889,6 @@ def _launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_re return dict(title=title, detail=detail, action='', kind='unknown') -# comfy_setup.inspection_failed: the first line of a failed check. -INSPECTION_SUMMARY = r'Setup inspection failed(?: \([^()\r\n]*(?:\([^()\r\n]*\)[^()\r\n]*)*\))?' -CHECK_UNFINISHED = ('The check did not finish.', '这次检查没有完成。') UNKNOWN_NEXT_STEP = ('Try again. If it fails again, export the report, then use "Report a problem" and attach it.', '请重试。仍然失败时，请先导出报告，再点“提交问题”并附上报告。') diff --git a/freevideo_engine/launcher/Main.qml b/freevideo_engine/launcher/Main.qml index 0b5f5ddd..154194b4 100644 --- a/freevideo_engine/launcher/Main.qml +++ b/freevideo_engine/launcher/Main.qml @@ -1644,24 +1644,20 @@ ApplicationWindow { } FText { objectName: "downloadActionNotice"; visible: !!s.action_notice && s.action_notice !== s.busy_reason; text: s.action_notice || ""; color: theme.muted; Layout.fillWidth: true } FText { objectName: "downloadListError"; visible: text.length > 0; text: s.download_list.error; color: theme.danger; Layout.fillWidth: true; wrapMode: Text.WordWrap; font.pixelSize: theme.micro + 1 } - FText { objectName: "downloadListRawError"; visible: s.error.length > 0 && !s.download_list.inspection_failed; text: s.error; color: theme.danger; Layout.fillWidth: true; maximumLineCount: 2; elide: Text.ElideRight; font.pixelSize: theme.micro } + FText { objectName: "downloadListRawError"; visible: s.error.length > 0 && !s.download_list.inspection_error; text: s.error; color: theme.danger; Layout.fillWidth: true; maximumLineCount: 2; elide: Text.ElideRight; font.pixelSize: theme.micro } } } FText { objectName: "downloadBusyReason"; visible: !s.can_manage && (!s.action_notice || s.action_notice === s.busy_reason); text: s.busy_reason; color: theme.muted; Layout.fillWidth: true } - GridLayout { - id: downloadListActions; objectName: "downloadListActions"; Layout.fillWidth: true; Layout.minimumWidth: 0 - columnSpacing: 8; rowSpacing: 8 - // Wide: one row, Close on the left, every label on one line. Narrow, or wherever the row would not - // hold every label on one line: one full-width column, the actions first and Close last. - readonly property real rowWidth: closeDownloadList.implicitWidth + (checkDownloadList.visible ? checkDownloadList.implicitWidth : 0) - + (useDownloadedFolder.visible ? useDownloadedFolder.implicitWidth : 0) + exportDownloadList.implicitWidth + (actionCount + 1) * columnSpacing - readonly property bool stacked: win.narrow || rowWidth > width + RowLayout { + objectName: "downloadListActions"; Layout.fillWidth: true; Layout.minimumWidth: 0; spacing: 8 + // The actions share what the Close button leaves, so a long label wraps instead of leaving the dialog. readonly property int actionCount: (checkDownloadList.visible ? 1 : 0) + (downloads.scope === "image" ? 1 : 2) - FButton { id: closeDownloadList; objectName: "closeDownloadList"; Layout.row: downloadListActions.stacked ? 3 : 0; Layout.column: 0; Layout.fillWidth: downloadListActions.stacked; text: downloads.copy.close; flat: true; onClicked: modelInfoOpen = false } - Item { visible: !downloadListActions.stacked; Layout.row: 0; Layout.column: 1; Layout.fillWidth: true } - FButton { id: checkDownloadList; objectName: "checkDownloadList"; Layout.row: 0; Layout.column: downloadListActions.stacked ? 0 : 2; Layout.fillWidth: downloadListActions.stacked; visible: downloads.scope !== "image" && (!downloads.available || s.download_list.needs_check); text: downloads.copy.check; enabled: !s.busy; onClicked: backend.prepareDownloadList() } - FButton { id: useDownloadedFolder; objectName: "useDownloadedFolder"; visible: downloads.scope !== "image"; Layout.row: downloadListActions.stacked ? 1 : 0; Layout.column: downloadListActions.stacked ? 0 : 3; Layout.fillWidth: downloadListActions.stacked; text: downloads.copy.use_folder; enabled: s.can_manage; onClicked: backend.useDownloadedFolder() } - FButton { id: exportDownloadList; objectName: "exportDownloadList"; Layout.row: downloadListActions.stacked ? 2 : 0; Layout.column: downloadListActions.stacked ? 0 : 4; Layout.fillWidth: downloadListActions.stacked; text: downloads.copy.export; primary: true; enabled: downloads.available && (downloads.scope === "image" || !s.busy); onClicked: backend.exportDownloadList() } + readonly property real actionWidth: Math.max(80, Math.floor((width - closeDownloadList.implicitWidth - (actionCount + 1) * spacing) / actionCount)) + FButton { id: closeDownloadList; text: downloads.copy.close; flat: true; onClicked: modelInfoOpen = false } + Item { Layout.fillWidth: true } + FButton { id: checkDownloadList; objectName: "checkDownloadList"; Layout.maximumWidth: parent.actionWidth; visible: downloads.scope !== "image" && (!downloads.available || s.download_list.needs_check); text: downloads.copy.check; enabled: !s.busy; onClicked: backend.prepareDownloadList() } + FButton { objectName: "useDownloadedFolder"; visible: downloads.scope !== "image"; Layout.maximumWidth: parent.actionWidth; text: downloads.copy.use_folder; enabled: s.can_manage; onClicked: backend.useDownloadedFolder() } + FButton { objectName: "exportDownloadList"; Layout.maximumWidth: parent.actionWidth; text: downloads.copy.export; primary: true; enabled: downloads.available && (downloads.scope === "image" || !s.busy); onClicked: backend.exportDownloadList() } } } } diff --git a/freevideo_engine/launcher_session.py b/freevideo_engine/launcher_session.py index 578b1be9..66b25595 100644 --- a/freevideo_engine/launcher_session.py +++ b/freevideo_engine/launcher_session.py @@ -2424,8 +2424,6 @@ def snapshot(self): download_state = dict(checking=bool(self.controller.busy), error=self.download_error, needs_check=self.download_error_key in ('unchecked', 'changed'), folder=self.download_export, inspection_error=inspection_read_error, - # Any failed check: the list's own line asks for Check again, so the raw error stays in the details. - inspection_failed=row.get('status') == 'failed' and row.get('action') == 'inspect', notice=downloads['copy']['exported'].format(folder=self.download_export) if self.download_export else '') if inspection_read_error: download_state.update(error=self.t(*DOWNLOAD_COPY['error_read']), needs_check=True) From 11a9655f80d4e2036a5f1c0b1a4c164c2f38cfa8 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Thu, 5 Nov 2026 05:45:17 +0000 Subject: [PATCH 14/14] Release 0.4.33 --- freevideo_engine/release_notes.json | 67 +++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/freevideo_engine/release_notes.json b/freevideo_engine/release_notes.json index 4dc6c4d1..6e465075 100644 --- a/freevideo_engine/release_notes.json +++ b/freevideo_engine/release_notes.json @@ -1,31 +1,64 @@ { "schema": 1, - "product_version": "0.4.32", + "product_version": "0.4.33", "en": { - "summary": "A picture can be generated for a span of imported sound, and the top bar has a permanent Tutorial entry.", + "summary": "Prompt enhancement works with less free graphics memory, and Setup or Repair now waits for a video that is still generating.", "changes": [ - "Sound-led video: drag a span over imported sound in the Edit view, describe the picture, and generate a video for that span while the original sound is kept.", - "Tutorial: the top bar has a permanent Tutorial entry. Three new feature examples cover making a first video, characters and the Edit view. These and some tutorial chapters have \"Take me there\", which highlights the matching control.", - "Audio descriptions: when a description asks to leave out music or other sounds, that request is removed before generation and, where needed, a sentence is added saying the sound comes from the action and the surroundings, so unwanted background music appears less often. Descriptions that ask for music or silence, and quoted lines, are unchanged.", - "Two GPUs: a machine with two GPUs now decides whether tasks can run together from the memory each task actually uses. On Linux, two 16 GB cards with 64 GB of RAM can run two tasks at once. Single-GPU machines are unaffected.", - "Editor: deleting an edit no longer keeps failing after a brief loss of contact with the background service.", - "Editor import: a video that is cut off or damaged part-way now keeps the part that can be read instead of failing the whole import.", - "Reference audio: generating with reference audio works again on Mac, and no longer fails on other systems when the model has to be loaded first." + "Prompts no longer receive the added sentence about natural sounds that 0.4.32 introduced.", + "Audio references: an mp4 file that only contains sound is accepted. An Agent that needs only the voice of a video can send it at a small picture size, which makes generation much faster.", + "Agent: choosing a take cancels the takes of that shot that are still queued or running.", + "Agent: when a voice reference is uploaded, the first six seconds are checked for speech, and the Agent is told when there is too little, such as in a clip that opens with music, so another clip can be chosen.", + "Setup or Repair that is stopped by a video still generating now waits for it and continues by itself, instead of showing a general failure.", + "Prompt enhancement: rewriting now works with less free graphics memory. When other programs hold the memory it needs, it waits and starts by itself once there is room, and lists those programs when their names can be read.", + "Audio processing in videos no longer differs depending on when FreeVideo was installed.", + "With several graphics cards, a new generation waits when its card is still briefly held by the previous request, instead of failing within a few seconds.", + "The Autopilot run bar stays fully visible in the top bar on narrow windows.", + "Settings: on narrow screens, the three Getting started buttons no longer break their labels across lines." ] }, "zh": { - "summary": "可以为导入声音的一段生成画面；顶栏有常驻的教程入口。", + "summary": "提示词增强在可用显存较少时也能改写；安装或修复遇到还在生成的视频时会等待并自动继续。", "changes": [ - "声音带动画面：在剪辑视图里，在导入的声音上拖出一段，写上画面描述，为这一段生成视频，原声保留。", - "教程：顶栏有常驻的“教程”入口。功能示例新增三张：做出第一条视频、角色、剪辑视图。这三张和部分教程章节有“带我去”，点后高亮对应的位置。", - "声音描述：写了不要音乐或某种声音时，生成前会去掉这类说法，必要时补一句说明声音来自动作和环境，减少无意带出的背景音乐。要求音乐或静音的描述、引号里的台词不改。", - "双显卡：有两张显卡的机器按每个任务的实际内存用量判断能否同时运行。在 Linux 上，两张 16 GB 显卡配 64 GB 内存可以同时运行两个任务。单卡机器不受影响。", - "剪辑：与后台的通信短暂中断后，删除剪辑不再连续失败。", - "剪辑导入：中途被截断或损坏的视频，现在会保留可读取的部分，不再整个导入失败。", - "参考音频：使用参考音频生成在 Mac 上恢复可用；其他系统需要先载入模型时也不再失败。" + "提示词不再被补写 0.4.32 加入的自然声音描述。", + "音频参考：只有声音的 mp4 文件也可以使用。Agent 只需要视频里的声音时，可用小画面传入，生成明显更快。", + "Agent：选定一版后，同一镜头还在排队或生成的其他版本自动撤下。", + "Agent：上传声音参考时检查前 6 秒的说话时长，说话太少（例如以音乐开头）时告知 Agent，便于换一段。", + "安装或修复被还在生成的视频拦下时，启动器现在会等视频结束后自动继续，不再显示通用的失败提示。", + "提示词增强：可用显存较少时也能改写；显存被其他程序占用时会等待，空出后自动改写，能读取到程序名时一并列出。", + "视频声音的处理结果不再因安装时间不同而有差异。", + "多张显卡时，新的生成遇到显卡还被上一个请求短暂占用会等待，不再几秒内失败。", + "AI 全自动运行时，窄窗口顶栏的运行条完整显示。", + "设置：窄屏上“新手指引”的三个按钮不再折行断字。" ] }, "history": [ + { + "product_version": "0.4.32", + "en": { + "summary": "A picture can be generated for a span of imported sound, and the top bar has a permanent Tutorial entry.", + "changes": [ + "Sound-led video: drag a span over imported sound in the Edit view, describe the picture, and generate a video for that span while the original sound is kept.", + "Tutorial: the top bar has a permanent Tutorial entry. Three new feature examples cover making a first video, characters and the Edit view. These and some tutorial chapters have \"Take me there\", which highlights the matching control.", + "Audio descriptions: when a description asks to leave out music or other sounds, that request is removed before generation and, where needed, a sentence is added saying the sound comes from the action and the surroundings, so unwanted background music appears less often. Descriptions that ask for music or silence, and quoted lines, are unchanged.", + "Two GPUs: a machine with two GPUs now decides whether tasks can run together from the memory each task actually uses. On Linux, two 16 GB cards with 64 GB of RAM can run two tasks at once. Single-GPU machines are unaffected.", + "Editor: deleting an edit no longer keeps failing after a brief loss of contact with the background service.", + "Editor import: a video that is cut off or damaged part-way now keeps the part that can be read instead of failing the whole import.", + "Reference audio: generating with reference audio works again on Mac, and no longer fails on other systems when the model has to be loaded first." + ] + }, + "zh": { + "summary": "可以为导入声音的一段生成画面；顶栏有常驻的教程入口。", + "changes": [ + "声音带动画面：在剪辑视图里，在导入的声音上拖出一段，写上画面描述，为这一段生成视频，原声保留。", + "教程：顶栏有常驻的“教程”入口。功能示例新增三张：做出第一条视频、角色、剪辑视图。这三张和部分教程章节有“带我去”，点后高亮对应的位置。", + "声音描述：写了不要音乐或某种声音时，生成前会去掉这类说法，必要时补一句说明声音来自动作和环境，减少无意带出的背景音乐。要求音乐或静音的描述、引号里的台词不改。", + "双显卡：有两张显卡的机器按每个任务的实际内存用量判断能否同时运行。在 Linux 上，两张 16 GB 显卡配 64 GB 内存可以同时运行两个任务。单卡机器不受影响。", + "剪辑：与后台的通信短暂中断后，删除剪辑不再连续失败。", + "剪辑导入：中途被截断或损坏的视频，现在会保留可读取的部分，不再整个导入失败。", + "参考音频：使用参考音频生成在 Mac 上恢复可用；其他系统需要先载入模型时也不再失败。" + ] + } + }, { "product_version": "0.4.31", "en": {