FreeVideo 0.4.35

freevideo_engine/agent/builtin_director.py

@@ -17,7 +17,7 @@
 """
 import base64
 from copy import deepcopy
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 import hashlib
 import ipaddress
 import json
@@ -428,6 +428,69 @@ def judge(shot_view, sheets, *, ask):
         return uncertain
 
 
+CUT_AGREE = 0.3  # seconds by which the same cut may differ between takes
+
+
+GOOD = 7  # a part is good enough to keep when it scores this or more
+
+
+def rate_segments(shot_view, segments, sheets, *, ask):
+    """{take: [score of every part]} for takes that share their cuts, or None when the model cannot say."""
+    images = {row['take']: row['image'] for row in sheets['takes']}
+    if not segments or any(s['take'] not in images for s in segments):
+        return None
+    count = len(segments[0]['cuts']) + 1
+    numbers = [s['take'] for s in segments]
+    rating = _object(dict(take=dict(type='integer', enum=numbers),
+                          scores=dict(type='array', minItems=count, maxItems=count,
+                                      items=dict(type='number', minimum=0, maximum=10))), ('take', 'scores'))
+    schema = _object(dict(ratings=dict(type='array', minItems=len(numbers), maxItems=len(numbers), items=rating)),
+                     ('ratings',))
+    parts = ''.join('\n版%d：%s' % (s['take'], '、'.join('第%d段 %.2f–%.2f 秒' % (i + 1, a, b) for i, (a, b) in enumerate(
+        zip([0.0] + s['cuts'], s['cuts'] + [s['seconds']])))) for s in segments)
+    prompt = ('同一镜头有几个版本，每版在画面硬切处分成相同的几段。按图上的时间标注，给每版的每一段打 0–10 分'
+              '（人物变形、多余的字、和描述不符都扣分），scores 按段的顺序给出。'
+              '\n镜头描述：' + _dump({k: shot_view[k] for k in ('screenplay', 'characters', 'style') if k in shot_view})
+              + '\n图片顺序对应版号：' + _dump(numbers) + '\n各版的分段：' + parts)
+    reply = _ask(ask, prompt, [images[n] for n in numbers], schema)
+    if reply is None or sorted(r['take'] for r in reply['ratings']) != sorted(numbers):
+        return None
+    return {r['take']: r['scores'] for r in reply['ratings']}
+
+
+def first_take_is_good(tools, production_id, shot, sheets, take, *, ask):
+    """Whether a shot still being filmed may stop at this take: it has no numbered cuts, or every part of it scores
+    GOOD. Otherwise the other takes are waited for so each cut can take its best part."""
+    try:
+        own = [s for s in tools.segments(production_id, shot['id']) if s['take'] == take]
+        if not own or not own[0]['cuts']:
+            return True
+        scores = rate_segments(shot, own, sheets, ask=ask)
+    except Exception:  # noqa: BLE001 - without the cut scores the whole-take score decides, as before
+        return True
+    return scores is None or all(x >= GOOD for x in scores.get(take, ()))
+
+
+def stitch_shot(shot_view, segments, sheets, *, ask):
+    """Pieces taking the best part of every cut from different takes, or None to keep one whole take.
+
+    Needs two or more takes that passed the check, every one with the same two or more cuts. The last piece
+    comes from the take that is chosen, so the next shot still continues from the chosen take's last frame."""
+    from . import cut_pick
+    if len(segments) < 2 or any(len(s['cuts']) != len(segments[0]['cuts']) for s in segments) or not segments[0]['cuts']:
+        return None
+    if any(abs(a - b) > CUT_AGREE for s in segments for a, b in zip(s['cuts'], segments[0]['cuts'])):
+        return None  # The same cut sits at different times: these are not the same parts.
+    scores = rate_segments(shot_view, segments, sheets, ask=ask)
+    if scores is None:
+        return None
+    candidates = [dict(s, scores=scores[s['take']]) for s in segments]
+    result = cut_pick.stitch(candidates)
+    if result['fallback'] or len(result['pieces']) < 2:
+        return None
+    return result
+
+
 def revise(view, sentence, *, ask):
     """Translate one request into one checked reshoot, or ask for clarification."""
     unclear = Problem('unclear', '请说明要改哪一镜，以及画面或时长要怎样改？')
@@ -480,6 +543,19 @@ def _ref(identity, action, snapshot):
     return identity + ':builtin:' + action + ':' + digest
 
 
+def _with_stitch(tools, production_id, shot, sheets, decision, ask):
+    """Take the best part of every cut when the takes allow it; the choice becomes the take that ends the film part.
+    Returns the decision and the parts, or None for the parts when one whole take stands."""
+    try:
+        pieces = stitch_shot(shot, tools.segments(production_id, shot['id']), sheets, ask=ask)
+    except Exception:  # noqa: BLE001 - stitching is an extra; one whole take is always the fallback
+        return decision, None
+    if pieces is None:
+        return decision, None
+    parts = [dict(take=p['take'], start=p['start'], end=p['end']) for p in pieces['pieces']]
+    return replace(decision, take=parts[-1]['take'], reason=(decision.reason + '；各切镜取最好的一版拼接')[:300]), parts
+
+
 def run(tools, production_id, *, ask, stop, wait=25):
     """Yield {shot, action, reason}; stop never cancels generation jobs.
 
@@ -506,6 +582,8 @@ def run(tools, production_id, *, ask, stop, wait=25):
                                for s in view['shots']):
                         continue
                     snapshot = [(s['id'], s['chosen']) for s in view['shots']]
+                    chosen = dict(snapshot)
+                    # The parts of a shot are kept with its chosen take in the production.
                     tools.finish(production_id, by='builtin', ref=_ref(production_id, 'finish', snapshot))
                     yield dict(shot=None, action='finish', reason='每镜已选定，开始接成影片。')
                     return
@@ -534,7 +612,12 @@ def run(tools, production_id, *, ask, stop, wait=25):
                 elif decision.action in ('choose', 'reshoot'):
                     ref = _ref(production_id, decision.action, shot)
                     if decision.action == 'choose':
-                        tools.choose(production_id, shot['id'], decision.take, by='builtin', reason=decision.reason, ref=ref)
+                        if shot.get('state') in ('waiting', 'shooting') and not first_take_is_good(
+                                tools, production_id, shot, sheets, decision.take, ask=ask):
+                            continue  # the first take is not good in every cut: wait for the others
+                        decision, parts = _with_stitch(tools, production_id, shot, sheets, decision, ask)
+                        tools.choose(production_id, shot['id'], decision.take, by='builtin', reason=decision.reason, ref=ref,
+                                     **({'pieces': parts} if parts else {}))
                     else:
                         tools.reshoot(production_id, shot['id'], by='builtin', note=decision.note,
                                       takes=decision.takes, reason=decision.reason, ref=ref)

freevideo_engine/agent/cut_pick.py

@@ -0,0 +1,218 @@
+"""Cut detection, per-segment checks and stitching of same-shot takes.
+
+Pure CPU helpers: `detect_cuts` finds the picture cuts of a take, `segment_qa` applies the take checks to one
+segment, `seam` measures how a join between two takes sounds and `stitch` picks the best take for every
+segment. Thresholds are those of take_qa.
+"""
+from functools import lru_cache
+from pathlib import Path
+
+import numpy as np
+
+from . import take_qa
+from .take_qa import _db, _runs
+
+CUT_DIFF = 25.0          # 0-255 mean change between consecutive frames of a hard cut
+CUT_SPIKE = 3.0          # a cut stands out this much from the frame changes around it
+MATCH_DIFF = 12.0        # weakest change accepted as an expected cut
+SEAM_LEVEL_DB = 20.0     # level step at which the sound penalty is full
+SEAM_CLICK = 0.5         # sample jump at which the click penalty is full
+SEAM_SPEECH = 0.4        # penalty added for a join inside speech
+SEAM_QUIET = 0.25        # weight of level and click at a join in silence
+
+
+@lru_cache(maxsize=4)
+def _decoded(path, stamp):
+    return take_qa._pictures(path)
+
+
+@lru_cache(maxsize=4)
+def _heard(path, stamp):
+    return take_qa._samples(path)
+
+
+def _stamp(path):
+    stat = Path(path).stat()
+    return stat.st_mtime_ns, stat.st_size
+
+
+def _pictures(path):
+    """take_qa's frames, decoded once while the same file is checked part by part."""
+    return _decoded(str(path), _stamp(path))
+
+
+def _samples(path):
+    return _heard(str(path), _stamp(path))
+
+
+def _changes(small):
+    return [0.0] + [float(np.abs(small[i] - small[i - 1]).mean()) for i in range(1, len(small))]
+
+
+def _spike(diffs, index, lo, hi, floor):
+    """Whether frame `index` is a cut: a large change that stands out from its neighbours in lo..hi."""
+    around = [diffs[i] for i in range(lo, hi) if i != index]
+    return diffs[index] >= floor and diffs[index] >= CUT_SPIKE * (float(np.median(around)) if around else 0.0)
+
+
+def cuts_in(small, times):
+    """Seconds of the hard picture cuts in decoded frames, ascending, excluding 0 and the end."""
+    diffs = _changes(small)
+    return [round(times[i], 3) for i in range(1, len(times))
+            if (diffs[i] >= CUT_DIFF and diffs[i] >= diffs[i - 1] and diffs[i] > (diffs[i + 1] if i + 1 < len(diffs) else -1)
+                and _spike(diffs, i, max(1, i - 3), min(len(diffs), i + 4), CUT_DIFF))]
+
+
+def detect_cuts(path, expected=None, *, tolerance=0.6):
+    """Seconds of the hard picture cuts, ascending, excluding 0 and the end. With `expected`, each expected cut
+    takes the strongest frame change within `tolerance` s; one with no such change is None, not guessed."""
+    _, small, times = _pictures(path)
+    if not expected:
+        return cuts_in(small, times)
+    diffs = _changes(small)
+    found = []
+    for wanted in expected:
+        near = [i for i in range(1, len(times)) if abs(times[i] - wanted) <= tolerance]
+        best = max(near, key=lambda i: (diffs[i], -abs(times[i] - wanted)), default=None)
+        if best is not None and _spike(diffs, best, max(1, min(near) - 1), min(len(diffs), max(near) + 2), MATCH_DIFF):
+            found.append(times[best])
+        else:
+            found.append(None)
+    # one frame cannot be two cuts: the cut nearer its expected time keeps it, the other is not found
+    for index, at in enumerate(found):
+        for other in range(index):
+            if at is not None and found[other] == at:
+                loser = index if abs(at - expected[index]) >= abs(at - expected[other]) else other
+                found[loser] = None
+                if loser == index:
+                    break
+    return found
+
+
+def segment_qa(path, start, end):
+    """Black picture, frozen picture and a long silence in the middle of speech, inside [start, end)."""
+    problems, notes = [], []
+    means, small, times = _pictures(path)
+    keep = [i for i, t in enumerate(times) if start <= t < end]
+    if keep:
+        means, small, times = [means[i] for i in keep], [small[i] for i in keep], [times[i] for i in keep]
+        black = _runs([m < take_qa.BLACK_LUMA and float(g.std()) < take_qa.BLACK_SPREAD for m, g in zip(means, small)],
+                      times, take_qa.BLACK_SECONDS)
+        flags = [False] * len(small)
+        for index in range(1, len(small)):
+            if float(np.abs(small[index] - small[index - 1]).mean()) <= take_qa.FREEZE_DIFF:
+                flags[index] = flags[index - 1] = True
+        frozen = _runs(flags, times, take_qa.FREEZE_SECONDS)
+        if black:
+            problems.append('black picture at %s' % ', '.join('%.1f-%.1f s' % s for s in black))
+        if frozen:
+            problems.append('frozen picture at %s' % ', '.join('%.1f-%.1f s' % s for s in frozen))
+    samples = _samples(path)[int(start * take_qa.SAMPLE_RATE):int(end * take_qa.SAMPLE_RATE)]
+    step = int(take_qa.SAMPLE_RATE * take_qa.WINDOW)
+    count = samples.size // step
+    if count:
+        levels = [_db(np.sqrt(np.mean(samples[i * step:(i + 1) * step] ** 2))) for i in range(count)]
+        moments = [start + i * take_qa.WINDOW for i in range(count)]
+        if any(level > take_qa.SPEECH_DB for level in levels):
+            gaps = [(a, b) for a, b in _runs([level < take_qa.GAP_DB for level in levels], moments, take_qa.GAP_SECONDS)
+                    if a > start + take_qa.GAP_EDGE and b < moments[-1] - take_qa.GAP_EDGE]
+            if gaps:
+                problems.append('silent gap at %s' % ', '.join('%.1f-%.1f s' % g for g in gaps))
+    return dict(problems=problems, notes=notes, ok=not problems)
+
+
+def _side(samples, at, span, before):
+    rate = take_qa.SAMPLE_RATE
+    n = max(1, int(span * rate))
+    cut = min(samples.size, max(0, int(round(at * rate))))
+    return samples[max(0, cut - n):cut] if before else samples[cut:cut + n]
+
+
+def _seam(a, a_end, b, b_start, span):
+    left, right = _side(a, a_end, span, True), _side(b, b_start, span, False)
+    if not left.size or not right.size:
+        return dict(level_db=0.0, click=0.0, in_speech=False, penalty=0.0)
+    levels = [_db(np.sqrt(np.mean(x ** 2))) for x in (left, right)]
+    level_db = abs(levels[0] - levels[1])
+    click = float(abs(right[0] - left[-1]))
+    in_speech = any(level > take_qa.SPEECH_DB for level in levels)
+    base = 0.6 * min(1.0, level_db / SEAM_LEVEL_DB) + 0.4 * min(1.0, click / SEAM_CLICK)
+    penalty = base + SEAM_SPEECH if in_speech else base * SEAM_QUIET
+    return dict(level_db=round(level_db, 2), click=round(click, 4), in_speech=in_speech,
+                penalty=round(min(1.0, penalty), 4))
+
+
+def seam(a_path, a_end, b_path, b_start, *, span=0.1):
+    """How a join sounds: level step over `span` s each side, sample jump, whether it falls in speech, penalty 0..1."""
+    return _seam(_samples(a_path), a_end, _samples(b_path), b_start, span)
+
+
+def _whole(candidates):
+    best = min(candidates, key=lambda c: (-sum(c.get('scores') or [0]), c['take']))
+    return dict(pieces=[dict(take=best['take'], start=0.0, end=best['seconds'])],
+                score=round(float(sum(best.get('scores') or [0])), 6), fallback=True)
+
+
+def stitch(candidates, *, seam_weight=0.5):
+    """Choose a take for every segment, maximizing segment scores minus seam_weight * 10 * seam penalties.
+    A join between different takes inside speech is not allowed. Falls back to the best whole take when cut counts
+    differ, a segment has no usable candidate or no allowed sequence of takes exists."""
+    if not candidates:
+        return dict(pieces=[], score=0.0, fallback=True)
+    candidates = sorted(candidates, key=lambda c: c['take'])
+    count = len(candidates[0]['cuts']) + 1
+    if any(len(c['cuts']) + 1 != count or len(c['scores']) != count or len(c['qa']) != count for c in candidates):
+        return _whole(candidates)
+    bounds = {c['take']: [0.0] + [float(t) for t in c['cuts']] + [float(c['seconds'])] for c in candidates}
+    if any(b <= a for row in bounds.values() for a, b in zip(row, row[1:])):
+        return _whole(candidates)  # a cut at or before the previous one leaves an empty piece
+    usable = [[c for c in candidates if c['qa'][k]] for k in range(count)]
+    if not all(usable):
+        return _whole(candidates)
+    audio, joins = {}, {}
+
+    def penalty(a, b, k):
+        """The seam penalty, or None when the join is not allowed: a join between takes inside speech, or unreadable."""
+        key = (a['take'], b['take'], k)
+        if key not in joins:
+            try:
+                for c in (a, b):
+                    if c['take'] not in audio:
+                        audio[c['take']] = _samples(c['path'])
+                row = _seam(audio[a['take']], bounds[a['take']][k + 1], audio[b['take']], bounds[b['take']][k + 1], 0.1)
+                joins[key] = None if row['in_speech'] else row['penalty']
+            except Exception:
+                joins[key] = None
+        return joins[key]
+
+    # state: candidate take -> (score, takes so far); ties keep the smaller take sequence
+    state = {c['take']: (float(c['scores'][0]), [c['take']]) for c in usable[0]}
+    for k in range(1, count):
+        step = {}
+        for c in usable[k]:
+            options = []
+            for p in usable[k - 1]:
+                if p['take'] not in state:
+                    continue
+                score, path = state[p['take']]
+                cost = 0.0
+                if p['take'] != c['take']:
+                    seam_penalty = penalty(p, c, k - 1)
+                    if seam_penalty is None:
+                        continue
+                    cost = seam_weight * 10 * seam_penalty
+                options.append((round(score + float(c['scores'][k]) - cost, 9), path + [c['take']]))
+            if options:
+                step[c['take']] = min(options, key=lambda o: (-o[0], o[1]))
+        if not step:
+            return _whole(candidates)
+        state = step
+    score, takes = min(state.values(), key=lambda o: (-o[0], o[1]))
+    pieces = []
+    for k, take in enumerate(takes):
+        start, end = bounds[take][k], bounds[take][k + 1]
+        if pieces and pieces[-1]['take'] == take:
+            pieces[-1]['end'] = end
+        else:
+            pieces.append(dict(take=take, start=start, end=end))
+    return dict(pieces=pieces, score=round(score, 6), fallback=False)

freevideo_engine/agent/guide.py

@@ -197,6 +197,11 @@
         'job (30 s or more, or beyond the device limit, continues and joins automatically); to polish an idea first, run enhance, then generate with "prompt": {"from_job": "<enhance id>"} rather than retyping the text. '
         'A film is cut into shots: see the rule "Cut by default" before making one long continued video.',
         'Use drama plan → start → get → sheet → choose/reshoot → finish; inspect todo after every change.',
+        'A shot whose prompt has numbered cuts and several takes: do not stop at the first take unless every cut of it is good '
+        '(scores 7 or more of 10); wait for the other takes, read segments, and choose with pieces '
+        '[{take, start, end}] to take the best part of every cut from different takes. Cut only where the takes cut, '
+        'never inside speech (in_speech), and skip a part with parts_ok false. The last part comes from the chosen take. Pass the cut times back exactly as returned, unrounded. '
+        'finish accepts pieces for any shot too. One whole take stays the fallback.',
         'At start tell the user the estimated total time; on over_budget decide whether to ask them; '
         'without an estimate report only the current work and elapsed time.',
         'Use get with since=revision and wait=25. changed=false is a compact timeout; retain the last full board.',
@@ -353,6 +358,7 @@
     ('GET', '/drama/{id}'): '读取待办事项 todo 和版本 revision；用 since 和 wait=25 等待变化。'
         'paused 有值时停止操作，持续等待用户继续',
     ('GET', '/drama/{id}/shots/{shot}/sheet'): '各条生成完成后，查看带时间标记的画面和前一镜所选片段的末帧',
+    ('GET', '/drama/{id}/shots/{shot}/segments'): '镜头有几版且提示词写了同样的编号切镜时，读取每版的切点、各段检查和各版在每个切点处的衔接，再用 pieces 选出每个切镜最好的一段',
     ('POST', '/drama/{id}/shots/{shot}/choose'): '查看画面后选择一条；选定后才能拍摄承接它的下一镜',
     ('POST', '/drama/{id}/shots/{shot}/reshoot'): '按修改要求重拍，再查看画面并重新选片',
     ('POST', '/drama/{id}/finish'): '每一镜都选定后提交拼接，继续查询影片直到成片就绪',