FreeVideo 0.4.33

freevideo_engine/agent/builtin_director.py

@@ -26,7 +26,7 @@
 import urllib.request
 from urllib.parse import urlsplit
 
-from . import drama, spec
+from . import drama, spec, take_qa
 
 
 @dataclass(frozen=True)
@@ -367,17 +367,27 @@ def judge(shot_view, sheets, *, ask):
     shot = shot_view
     if shot.get('chosen_by') == 'user' or shot.get('chosen') is not None:
         return Decision('keep', shot['id'], '保留已选定的版本。')
-    if shot.get('state') in ('waiting', 'shooting'):
-        return Decision('wait', shot['id'], '等待这一镜的各版完成。')
+    shooting = shot.get('state') in ('waiting', 'shooting')
     uncertain = Decision('ask_user', shot['id'], '拿不准，请挑一版。')
-    takes = _selectable(shot)
+    finished = _selectable(shot)
+    unchecked = any('qa' in t and t['qa'] is None for t in finished)
+    if shooting and (not finished or any(t.get('qa') is None for t in finished)) or unchecked:
+        return Decision('wait', shot['id'], '等待这一镜的各版完成并质检。')
+    # A take that failed the picture or sound check is never chosen; a shot still shooting is judged as takes arrive.
+    takes = [t for t in finished if (t.get('qa') or {}).get('ok', True)]
     if not takes:
-        return uncertain
+        if shooting:
+            return Decision('wait', shot['id'], '已完成的版本没有通过质检，等待其余版本。')
+        if not finished or _retaken(shot):
+            return uncertain
+        faults = '；'.join(sorted({p for t in finished for p in t['qa']['problems']}))
+        return Decision('reshoot', shot['id'], '各版都没有通过质检。',
+                        note=('上一次的问题：' + faults)[:300], takes=_take_count(shot))
     try:
         rows = sheets['takes']
         images = {row['take']: row['image'] for row in rows}
         numbers = [t['take'] for t in takes]
-        if len(images) != len(rows) or set(images) != set(numbers):
+        if len(images) != len(rows) or not set(numbers) <= set(images):
             return uncertain
         previous = sheets.get('previous')
         batch_size = 15 if previous is not None else 16
@@ -391,9 +401,11 @@ def judge(shot_view, sheets, *, ask):
                              ('ratings',))
             prompt = ('评价这一镜的每个版本：0–10 分，reason 用不超过30字的中文指出具体优点或问题'
                       '（多余的字、人物变形、和描述不符等）；note 给下一次生成能用的具体修正。'
-                      'captions 表示有叠加字幕/文字，场景内真实招牌不算。不评价声音。'
+                      'captions 表示有叠加字幕/文字，场景内真实招牌不算。声音由自动质检评价，这里不评价。'
                       '\n镜头描述：' + _dump({k: shot[k] for k in ('screenplay', 'characters', 'style', 'captions') if k in shot})
                       + '\n图片顺序对应版号：' + _dump(batch)
+                      + ''.join('\n版%d的%s' % (t['take'], take_qa.summary(t.get('qa'))) for t in takes
+                                if t['take'] in batch and t.get('qa'))
                       + ('；最后一张是上一镜选定版的末帧，仅供接续参考。' if previous is not None else '。'))
             reply = _ask(ask, prompt, [images[n] for n in batch] + ([previous] if previous is not None else []), schema)
             if reply is None or sorted(r['take'] for r in reply['ratings']) != sorted(batch):
@@ -405,6 +417,8 @@ def judge(shot_view, sheets, *, ask):
             if row['captions'] and not shot.get('captions', False):
                 row['score'] = min(row['score'], 3)
         best = max(ratings, key=lambda r: (r['score'], -r['take']))
+        if best['score'] < 7 and shooting:
+            return Decision('wait', shot['id'], '已完成的版本不够好，等待其余版本。')
         if best['score'] >= 7:
             return Decision('choose', shot['id'], best['reason'], take=best['take'])
         if _retaken(shot):

freevideo_engine/agent/comfy.py

@@ -60,6 +60,15 @@ def output_path(self, relative, create=False):
     def output_relative(self, path):
         return Path(path).resolve().relative_to(self.output_root).as_posix()
 
+    def voice_window(self, path, seconds):
+        """Speech in the first seconds of an uploaded sound, or None without the speech model."""
+        from ..comfy_bridge import installation
+        from ..image_gen import image_pack
+        from ..speech import MODEL_FILE
+        from ..voice_window import inspect
+        model = image_pack.base(installation()[1]['model_root']) / MODEL_FILE
+        return inspect(path, seconds, model, self.output_root / 'FreeVideo' / '.editor' / 'speech')
+
     def library_video(self, identity):
         from ..comfy_library import _video, _report
         try:

freevideo_engine/agent/jobs.py

@@ -1655,8 +1655,9 @@ def resolve():
                                                role=role))
                     for i, item in enumerate(spec['media'].get('references', []), 1):
                         asset = dict(file=self.media_file(row, item, 'reference-%02d' % i, target=target), role='reference')
-                        if 'early_steps' in item:
-                            asset['early_steps'] = item['early_steps']
+                        for key in ('early_steps', 'voice_only'):
+                            if key in item:
+                                asset[key] = item[key]
                         assets.append(asset)
                     return assets
                 assets = resolve()

freevideo_engine/agent/media.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 import uuid
 
-from ..comfy_assets import media_kind
+from ..comfy_assets import media_kind_of
 
 FPS = 24
 MAX_SAMPLES = 64
@@ -20,7 +20,7 @@
 
 def inspect(path):
     path = _file(path)
-    kind = media_kind(path.name)
+    kind = media_kind_of(path)
     info = dict(path=str(path), name=path.name, kind=kind, bytes=path.stat().st_size)
     if kind == 'image':
         from PIL import Image
@@ -726,7 +726,7 @@ def _file(path, kind=None):
     path = Path(path)
     if not path.is_file():
         raise ValueError('No such media file: %s' % path.name)
-    if kind and media_kind(path.name) != kind:
+    if kind and media_kind_of(path) != kind:
         raise ValueError('%s is not a %s file' % (path.name, kind))
     return path
 

freevideo_engine/agent/production.py

@@ -118,11 +118,12 @@ def plan(request, *, fields=True):
         problems.extend(dict(path='/screenplay' + p['path'], message=_rule(p)) for p in error.problems)
     if fields:
         _fields(play, problems)
-    count = request.get('takes_per_shot', 1)
-    count_ok = _count(count, '/takes_per_shot', problems)
     director = request.get('director', 'agent')
     if director not in ('agent', 'builtin'):
         problems.append(dict(path='/director', message='must be "agent" or "builtin"'))
+    # The built-in director judges takes as they arrive and withdraws the rest once one is chosen, so two cost little.
+    count = request.get('takes_per_shot', 2 if director == 'builtin' else 1)
+    count_ok = _count(count, '/takes_per_shot', problems)
     rows = play.get('shots', []) if isinstance(play, dict) else []
     total = 0
     if isinstance(rows, list):
@@ -530,6 +531,22 @@ def resume(state, *, by, now):
     _commit(state, draft, now)
 
 
+def _withdraw_others(row, chosen):
+    """Stop the shot's other unfinished takes and unsent manual batches once one is chosen.
+
+    Finished takes stay, so the choice can still change; automatic reconnects are not ours to drop.
+    """
+    for other in row['takes']:
+        if other['take'] == chosen or other['stale'] or other['cancel_requested']:
+            continue
+        if other['state'] == 'paused':
+            other['state'] = 'cancelled'
+            other['cancel_requested'] = True
+        elif other['state'] in ACTIVE:
+            other['cancel_requested'] = True
+    row['pending'] = [g for g in row['pending'] if g[0].get('automatic')]
+
+
 def choose(state, shot, take, *, by, reason=None, cascade=True, now):
     """Choose a current take, optionally reconnecting descendants automatically."""
     _editable(state, by)
@@ -550,6 +567,7 @@ def choose(state, shot, take, *, by, reason=None, cascade=True, now):
             _commit(state, draft, now)
         return
     row['chosen'], row['chosen_by'] = take, by
+    _withdraw_others(row, take)
     _refresh(draft)
     if by != 'user':
         affected = [dict(shot=old['id']) for old, child in zip(state['shots'], draft['shots'])

freevideo_engine/agent/productions.py

@@ -21,7 +21,7 @@
 import time
 import uuid
 
-from . import drama, production, schema, spec, errors, operation_receipts
+from . import drama, production, schema, spec, errors, operation_receipts, take_qa
 from .jobs import DONE, JobError, iso, parts_problems
 from .forecast import generation_estimate
 
@@ -173,6 +173,9 @@ def __init__(self, jobs):
         self._waiter_lock = threading.Lock()
         # gate(action, *, by, estimate_seconds, refs, question) raises when the user must approve first.
         self.gate = None
+        # transcribe(path) -> heard text, optional; without it the QA does not compare dialogue.
+        self.transcribe = None
+        self._inspecting = set()
         self.load()
         jobs.parent_allowed = self.allows
 
@@ -514,11 +517,17 @@ def fraction(row, status):
                 return 1.
             value = (self.jobs.progress(row) or {}).get('fraction') if row and status == 'running' else None
             return min(1., max(0., value)) if type(value) in (int, float) and math.isfinite(value) else 0.
+        waiting = set()  # built-in shots whose first live take is still going: the others are withdrawn once it is chosen
         for shot, take in self._takes(state):
             child = children.get(take['job'])
             status = child['status'] if child and take['state'] != 'paused' else take['state']
             if take['stale'] or status == 'cancelled' or (take['cancel_requested'] and status != 'succeeded'):
                 continue
+            if state['director'] == 'builtin':
+                if shot['id'] in waiting:
+                    continue
+                if status not in ('succeeded', 'failed', 'interrupted'):
+                    waiting.add(shot['id'])
             part = fraction(child, status)
             completed += status in ('succeeded', 'failed', 'interrupted')
             fractions.append(part)
@@ -649,6 +658,7 @@ def contextual(error, join=False):
             for take, saved in zip(shot['takes'], internal['takes']):
                 take['error'] = contextual(saved.get('error'))
                 take['task'] = tasks.get(take['job'])
+                take['qa'] = deepcopy(record.get('qa', {}).get(take['job']))
         if value['join'] is not None:
             ref = record['state']['current_join']
             join = next(j for j in record['state']['joins'] if j['ref'] == ref)
@@ -884,6 +894,50 @@ def _advance(self, identity):
             self._bump(draft)  # Wake since/wait readers when the elapsed reminder appears or clears.
         if draft != record:
             self._save(draft)
+        self._inspect_takes(identity)
+
+    def _inspect_takes(self, identity):
+        """Start the picture and sound check of each finished take that has none; it runs off the lock.
+
+        Only the built-in director reads the result, so films run by an agent are left alone.
+        """
+        record = self._record(identity)
+        if record['state']['director'] != 'builtin':
+            return
+        for shot in record['state']['shots']:
+            for take in shot['takes']:
+                job = take['job']
+                if (take['state'] != 'succeeded' or take['stale']
+                        or job in record.get('qa', {}) or (identity, job) in self._inspecting):
+                    continue
+                if not take['library_id']:
+                    # Nothing to open: pass it as unchecked, or the judge would wait on it forever.
+                    draft = deepcopy(self._record(identity))
+                    draft.setdefault('qa', {})[job] = dict(error='NoVideo', problems=[], notes=[], ok=True)
+                    self._bump(draft)
+                    self._save(draft)
+                    continue
+                self._inspecting.add((identity, job))
+                lines = [row['line'] for row in shot.get('lines') or [] if row.get('line')]
+                threading.Thread(target=self._inspect, daemon=True, args=(
+                    identity, job, take['library_id'], shot.get('seconds'), lines)).start()
+
+    def _inspect(self, identity, job, library, seconds, lines):
+        try:
+            with self.jobs.cpu:
+                result = take_qa.check(self.jobs.port.library_video(library), seconds=seconds,
+                                       lines=lines, transcribe=self.transcribe)
+        except Exception as error:  # noqa: BLE001 - a failed check must not block the film; the judge sees it as unchecked
+            logging.getLogger(__name__).warning('A take could not be checked (%s)', type(error).__name__)
+            result = dict(error=type(error).__name__, problems=[], notes=[], ok=True)
+        with self.changed:
+            self._inspecting.discard((identity, job))
+            if identity not in self.records:
+                return
+            draft = deepcopy(self._record(identity))
+            draft.setdefault('qa', {})[job] = result
+            self._bump(draft)
+            self._save(draft)
 
     def _execute(self, identity, action):
         """Carry out one action; True when it changed a child, a receipt or a tombstone."""

freevideo_engine/agent/routes.py

@@ -1889,6 +1889,13 @@ def digest():
             os.replace(target, out)
             info['name'] = identity + suffix
             info['original_name'] = name
+            if info['kind'] in ('audio', 'video') and info.get('has_audio', info['kind'] == 'audio'):
+                try:
+                    voice = await asyncio.to_thread(port.voice_window, out, info['seconds'])
+                except Exception:  # noqa: BLE001 - the check is advice; an upload never fails for it
+                    voice = None
+                if voice is not None:
+                    info['voice'] = voice
             return reply(dict(**assets.view(relative), **info), 201)
 
     @routes.get(PREFIX + '/media/{kind}/{file:.+}/file')

freevideo_engine/agent/schema.py

@@ -138,6 +138,9 @@ def media(image=False, *, local=False, clip=False, video=False, early=False):
         properties['early_steps'] = number(1, 32, integer=True, description=
             'A reference video only: it guides the first pass for this many steps and is dropped afterwards, '
             'including from the second pass. Omitted: the video guides every step.')
+        properties['voice_only'] = {'type': 'boolean', 'description':
+            'A reference video used only for its voice: the picture is encoded small and at a low frame rate, '
+            'the sound is unchanged. Not with early_steps.'}
     if clip:
         properties.pop('frame')
         properties.pop('edit')
@@ -166,6 +169,7 @@ def common():
 def generation_media(*, local=False, enhance=False):
     return obj({'first': nullable(media(True, local=local)), 'last': nullable(media(True, local=local)),
                 'references': nullable(array(media(enhance, local=local, early=not enhance), high=8 if enhance else 32))},
+               description='A first or last frame request does not take a voice reference.',
                allOf=[{'if': {'required': ['references'], 'properties': {'references': {'type': 'array', 'minItems': 1}}},
                        'then': {'properties': {'first': {'type': 'null'}, 'last': {'type': 'null'}}}}])