From 1496a1b5019fda58e07fdbffbae3c1b4ca379d17 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Thu, 5 Nov 2026 06:14:00 +0000 Subject: [PATCH 01/31] Voice change: dubbed lines keep the original pace and loudness: the tempo change closes a line... Voice change: dubbed lines keep the original pace and loudness: the tempo change closes a line first and each new line comes back as loud as the one it replaces --- freevideo_engine/agent/voice.py | 11 +++++++++++ freevideo_engine/dubbing/align.py | 10 ++++++++-- freevideo_engine/dubbing/service.py | 5 +++++ freevideo_engine/dubbing/worker.py | 4 +++- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/freevideo_engine/agent/voice.py b/freevideo_engine/agent/voice.py index ffd31741..fc3602db 100644 --- a/freevideo_engine/agent/voice.py +++ b/freevideo_engine/agent/voice.py @@ -471,6 +471,14 @@ def _measure(audio, first, last): return (low + span[0]) / RATE, (low + span[1]) / RATE +def _level(audio, first, last): + """RMS of the sound in [first, last) of a mono 44.1 kHz file: the loudness the new voice is to come back with.""" + import numpy as np + low, high = max(0, int(first * RATE)), min(len(audio), int(last * RATE)) + part = np.asarray(audio[low:high], dtype=np.float64) + return round(float(np.sqrt(np.mean(np.square(part)))), 6) if len(part) else None + + def _windows(every, todo, spans, base, limit): """Where each changed line's new voice goes: the original line plus a little quiet each side, and how far it may reach.""" windows, failures = {}, {} @@ -571,6 +579,9 @@ def item(line, length, offset): window = windows[line['id']] made = dict(key=line['id'], text=line['text'].strip(), seconds=round(length, 3), onset=round(window['onset'] - window['first'], 3), offset=round(offset - window['first'], 3)) + level = _level(mono, windows[line['id']]['onset'], windows[line['id']]['offset']) + if level: + made['level'] = level if own and line['end'] - line['start'] >= 1. and line['original_text'].strip(): made['reference'] = dict(file=str(original_wav), text=line['original_text'].strip(), start=base + line['start'], end=base + line['end']) diff --git a/freevideo_engine/dubbing/align.py b/freevideo_engine/dubbing/align.py index 5c3ec0d2..355b04c8 100644 --- a/freevideo_engine/dubbing/align.py +++ b/freevideo_engine/dubbing/align.py @@ -180,14 +180,20 @@ def run(speed): tried.append((speed, voice, rate, seconds(voice, rate), edges)) return tried[-1] + # A line that is too long is first closed by the tempo change, which keeps the voice natural; the model + # speed only takes what the tempo change cannot (a faster model speed costs naturalness quickly). + # A line that is too short is slowed by the tempo change in the same way, inside the same range. + def aim(length): + return min(max(length, wanted / (max_stretch * .96)), wanted * max_stretch * .96) # inside this range no model speed is needed + def off(entry): - return abs(entry[3] - wanted) / wanted + return abs(entry[3] - aim(entry[3])) / wanted speed = 1.0 entry = run(speed) while off(entry) > tolerance and len(tried) < 3: # the length scales with 1/speed, so the next speed is the measured ratio - speed = min(max(speed * entry[3] / wanted, low), high) + speed = min(max(speed * entry[3] / aim(entry[3]), low), high) if speed == tried[-1][0]: break entry = run(speed) diff --git a/freevideo_engine/dubbing/service.py b/freevideo_engine/dubbing/service.py index a0ae7b23..0c36c65f 100644 --- a/freevideo_engine/dubbing/service.py +++ b/freevideo_engine/dubbing/service.py @@ -90,6 +90,11 @@ def request(value): raise ValueError('invalid_request') if line.get('reference') is not None: row['reference'] = reference_of(line['reference'], required=True) + level = line.get('level') + if level is not None: + if isinstance(level, bool) or not isinstance(level, (int, float)) or not 0 < level <= 1: + raise ValueError('invalid_request') + row['level'] = float(level) instruction = line.get('instruction') if instruction not in (None, ''): if not isinstance(instruction, str) or len(instruction) > 200: diff --git a/freevideo_engine/dubbing/worker.py b/freevideo_engine/dubbing/worker.py index 8bca3594..7ce212ec 100644 --- a/freevideo_engine/dubbing/worker.py +++ b/freevideo_engine/dubbing/worker.py @@ -181,11 +181,13 @@ def prepare(source, key): used = [] match = {} # per line with its own reference: the voiced RMS the line is to come back with for index, line in enumerate(lines): + if line.get('level'): + match[index] = float(line['level']) own = line.get('reference') if own: try: used.append(('line',) + prepare(own, 'line%d' % index) + (cut['reference-line%d.wav' % index], None)) - match[index] = levels.get('line%d' % index) + match.setdefault(index, levels.get('line%d' % index)) continue except ValueError as error: if str(error) != 'reference_short' or job_ref is None: From 08e6456d80a2a5231747501f2c4a865b373d64fa Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Thu, 5 Nov 2026 06:14:34 +0000 Subject: [PATCH 02/31] Drama: numbered cuts in one shot, six reference sections for picture shots, length from the... Drama: numbered cuts in one shot, six reference sections for picture shots, length from the capability table, speaker numbers by order in the video, continuity table in every shot, defined and capitalised names, blank notes --- freevideo_engine/agent/drama.py | 279 ++++++++++++++++++++++---- freevideo_engine/agent/guide.py | 2 +- freevideo_engine/agent/production.py | 6 +- freevideo_engine/agent/productions.py | 42 +++- freevideo_engine/agent/schema.py | 2 +- freevideo_engine/agent/voice.py | 11 - freevideo_engine/chain_plan.py | 8 + freevideo_engine/dubbing/align.py | 10 +- freevideo_engine/dubbing/service.py | 5 - freevideo_engine/dubbing/worker.py | 4 +- 10 files changed, 288 insertions(+), 81 deletions(-) diff --git a/freevideo_engine/agent/drama.py b/freevideo_engine/agent/drama.py index ac0b32b9..e1743683 100644 --- a/freevideo_engine/agent/drama.py +++ b/freevideo_engine/agent/drama.py @@ -20,7 +20,7 @@ import time from . import spec as specs -from ..chain_plan import CONTINUE_SECONDS +from ..chain_plan import CONTINUE_SECONDS, BLOCK_SECONDS EXAMPLE = { 'title': '末班车', 'name': 'the-last-train', 'aspect': '9:16', 'quality': 'light', @@ -67,8 +67,10 @@ def __init__(self, problems): MAX_REFERENCES = 32 -def check(play): - """Validate every independent field, mutating only missing shot ids.""" +def check(play, single_seconds=BLOCK_SECONDS): + """Validate every independent field, mutating only missing shot ids. + + single_seconds: the longest shot this machine generates at once, which bounds a shot that carries pictures.""" problems = [] def problem(path, message): @@ -109,6 +111,12 @@ def validate(function, *args, **kwargs): validate(specs._text, play[field], '/' + field, 20000, empty=True) if 'name' in play and (not isinstance(play['name'], str) or not specs.SLUG.fullmatch(play['name'])): problem('/name', 'must be a short project name') + if 'continuity' in play: + if not isinstance(play['continuity'], dict): + problem('/continuity', 'must be an object of text by character name, and optionally "set"') + else: + for key, text in play['continuity'].items(): + validate(specs._text, text, '/continuity/' + str(key).replace('~', '~0').replace('/', '~1'), 20000, empty=True) if 'captions' in play: validate(specs._bool, play['captions'], '/captions') if 'quality' in play and (not isinstance(play['quality'], str) or play['quality'] not in specs.QUALITY): @@ -136,8 +144,8 @@ def validate(function, *args, **kwargs): problem(path + '/id', '/shots/%d/id must be a unique short name' % i) if isinstance(shot['id'], str): names.add(shot['id']) - if not any(isinstance(shot.get(key), str) and shot[key].strip() for key in ('action', 'idea', 'prompt')): - problem(path, '/shots/%d needs action, idea or prompt' % i) + if not any(isinstance(shot.get(key), str) and shot[key].strip() for key in ('action', 'idea', 'prompt')) and not shot.get('cuts'): + problem(path, '/shots/%d needs action, idea, prompt or cuts' % i) if shot.get('continue') and i == 0: problem(path + '/continue', '/shots/0 cannot continue a previous shot') if 'continue' in shot: @@ -159,12 +167,13 @@ def validate(function, *args, **kwargs): if ((shot.get('continue') or long) and play.get('quality') in specs.QUALITY and not specs.QUALITY[play['quality']][0] and all(p['path'] != '/quality' for p in problems)): problem('/quality', '"light" or "medium" (two-pass) when a shot continues another or is longer than 15 s') - if long and (shot.get('first') is not None or shot.get('references') and not shot.get('continue')): - problem(path + '/seconds', 'at most 15 with a first frame or references') - elif long and not shot.get('continue') and _carded(play, shot): - # Cast cards travel as references, which one generation of at most 15 s carries. - problem(path + '/seconds', 'at most 15 when a character in the shot has a cast card ' - '(a following shot with continue=true carries the same people further)') + over = (isinstance(seconds, (int, float)) and not isinstance(seconds, bool) and seconds > single_seconds) + if over and (shot.get('first') is not None or shot.get('references') and not shot.get('continue')): + problem(path + '/seconds', 'at most %d with a first frame or references on this machine' % single_seconds) + elif over and not shot.get('continue') and _carded(play, shot): + # Cast cards travel as references, which one generation (at most single_seconds) carries. + problem(path + '/seconds', 'at most %d on this machine when a character in the shot has a cast card ' + '(a following shot with continue=true carries the same people further)' % single_seconds) if shot.get('first') is not None: validate(specs.media_item, shot['first'], path + '/first', image=True) if 'references' in shot: @@ -185,6 +194,8 @@ def validate(function, *args, **kwargs): for j, key in enumerate(shot['cast']): if not isinstance(key, str) or key not in characters: problem(path + '/cast/%d' % j, 'must name a character') + if 'cuts' in shot: + _check_cuts(shot, path, characters, seconds, problem, validate) lines = shot.get('lines', []) if not isinstance(lines, list): problem(path + '/lines', 'must be a list') @@ -205,12 +216,62 @@ def validate(function, *args, **kwargs): return play +def _check_cuts(shot, path, characters, seconds, problem, validate): + """Numbered cuts inside one generation: each starts at "at" seconds (the first at 0), strictly later than the one before.""" + cuts = shot['cuts'] + if not isinstance(cuts, list) or not 1 <= len(cuts) <= 8: + problem(path + '/cuts', 'must be a list of 1 to 8 cuts') + return + last = None + for j, cut in enumerate(cuts): + here = '%s/cuts/%d' % (path, j) + if not isinstance(cut, dict): + problem(here, 'must be an object') + continue + for key in cut: + if key not in CUT_FIELDS: + problem(here + '/' + key, 'is not a cut field (%s)' % ', '.join(CUT_FIELDS)) + at = cut.get('at', 0 if j == 0 else None) + if isinstance(at, bool) or not isinstance(at, (int, float)) or at < 0 or (j == 0 and at != 0): + problem(here + '/at', 'must be the second the cut starts (0 for the first cut)') + elif last is not None and at <= last: + problem(here + '/at', 'must be later than the cut before') + elif isinstance(seconds, (int, float)) and not isinstance(seconds, bool) and at >= seconds: + problem(here + '/at', 'must be before the end of the shot (%s s)' % seconds) + else: + last = at + for field in ('framing', 'action', 'camera'): + if field in cut: + validate(specs._text, cut[field], here + '/' + field, 20000, empty=True) + if not any(isinstance(cut.get(k), str) and cut[k].strip() for k in ('action', 'framing')): + problem(here, 'needs an action or framing') + for k, line in enumerate(cut.get('lines', []) if isinstance(cut.get('lines', []), list) else [None]): + if not isinstance(line, dict) or line.get('who') not in characters: + problem('%s/lines/%d' % (here, k), 'must be a line from a named character') + else: + validate(specs._text, line.get('line'), '%s/lines/%d/line' % (here, k), 20000) + + +CUT_FIELDS = ('at', 'framing', 'action', 'camera', 'lines') + + +def _stamp(seconds): + whole = int(seconds) + return '%02d:%02d.%03d' % (whole // 60, whole % 60, round((seconds - whole) * 1000)) + + +def _mentions(text, word): + """Whole-word, case-insensitive match; a blank word never matches.""" + word = (word or '').strip() + return bool(word) and re.search(r'(?[%s] %s' % ((person.get('name') or who).capitalize(), voice, - speakers[who], line.get('how') or 'says', language, line['line'])) + return '%s%s (S%d) %s: [%s] %s' % (_cap(person.get('name') or who), voice, + numbers[who], line.get('how') or 'says', language, line['line']) + + def looks(): + return ['%s is %s.' % (_cap(characters[key].get('name') or key), characters[key]['look'].rstrip('.')) + for key in people if characters[key].get('look')] + + def continuity(): + """The film's table of clothes, props and places, the rows of the people in this shot (and 'set'), in every shot.""" + table = play.get('continuity') + if not isinstance(table, dict): + return [] + rows = ['%s: %s.' % (_cap(characters[key].get('name') or key), table[key].strip().rstrip('.')) + for key in people if isinstance(table.get(key), str) and table[key].strip()] + if isinstance(table.get('set'), str) and table['set'].strip(): + rows.append(table['set'].strip().rstrip('.') + '.') + return ['Continuity - ' + ' '.join(rows)] if rows else [] + + cuts = shot.get('cuts') + if cuts: + # Numbered cuts inside one generation: [Shot 1] has no time, the others say when they start. + seconds = shot.get('seconds') + parts = [] + for number, cut in enumerate(cuts, 1): + row = [('[Shot 1] ' if number == 1 else '[Shot %d] At %s, ' % (number, _stamp(cut['at']))).rstrip()] + if number == 1: + row += [lead + '.' if lead else ''] + if shot.get('continue'): + row.append('Continuing from the opening frame, the scene keeps its exact layout, lighting and characters.') + row += looks() + continuity() + elif cut.get('framing'): + row.append(cut['framing'].strip().rstrip('.') + '.') + if number == 1 and cut.get('framing'): + row.append(cut['framing'].strip().rstrip('.') + '.') + if cut.get('action'): + row.append(named(play, people, cut['action'])) + if cut.get('camera'): + row.append(cut['camera']) + row += [say(line) for line in cut.get('lines', [])] + if number == len(cuts) and isinstance(seconds, (int, float)) and not isinstance(seconds, bool): + row.append('The shot ends at %s.' % _stamp(seconds)) + parts.append(' '.join(x for x in row if x)) + body = '\n'.join(parts) + else: + parts = ['[Shot 1] ' + (lead + '.' if lead else '')] + if shot.get('continue'): + parts.append('Continuing from the opening frame, the scene keeps its exact layout, lighting and characters.') + parts += looks() + continuity() + parts.append(named(play, people, shot.get('action', ''))) + if shot.get('camera'): + parts.append(shot['camera']) + parts += [say(line) for line in shot.get('lines', [])] + body = ' '.join(p for p in parts if p) if not play.get('captions'): # Quoted words are text the shot shows (a sign, a menu); then only captions are ruled out. scene = ' '.join(shot.get(k) or '' for k in ('framing', 'action', 'camera')) + ' ' + (play.get('style') or '') - parts.append('No subtitles or captions.' if SHOWN_TEXT.search(scene) else 'No subtitles, no on-screen text.') + scene += ' ' + ' '.join(c.get(k) or '' for c in (cuts or []) for k in ('framing', 'action', 'camera')) + body += ' ' + ('No subtitles or captions.' if SHOWN_TEXT.search(scene) else 'No subtitles, no on-screen text.') + if PAPER.search(scene) and not SHOWN_TEXT.search(scene): + # A model writes the spoken line on any note it is asked to show close; a blank or unreadable face avoids it. + body += ' Any note, paper or sign in frame has a blank face with only faint folds, seen from the side or out of focus.' sound = shot.get('sound') or 'Natural ambience of the scene.' # Only a shot's own music is written; a score for the whole film is imported and laid on the sound track. music = shot.get('music') or 'N/A' - return ('integrated_multimodal_description: ' + ' '.join(p for p in parts if p) + + return ('integrated_multimodal_description: ' + body + '\n\noverall_soundscape: ' + sound + '\n\nnon_diegetic_music: ' + music) @@ -325,7 +444,7 @@ def cast_references(play, shot, cards, *, continues=False): function serves plan previews and the moment a take is submitted. pictures_each is the most pictures one person carries in this shot. cast_reference: None (no character in the shot has a card), 'own' (the shot - names its own first frame or references; an empty list means no pictures), 'missing' (a card was deleted), + names its own first frame, which carries only the written descriptions, or an empty list of references), 'missing' (a card was deleted), 'partial' (not everyone fit) or 'cards'. 'omitted' lists people left out entirely; 'voice_omitted' lists people whose pictures are carried but whose voice did not fit. A shot that continues the previous one carries no voice @@ -334,13 +453,31 @@ def cast_references(play, shot, cards, *, continues=False): characters = play.get('characters', {}) keys = [k for k in cast(play, shot) if isinstance(characters[k].get('card'), str)] result = dict(cast_reference=None, people=[], omitted=[], voice_omitted=[], missing=[], references=[], - prefix=None, subjects=[], cards={}, pictures_each=None, clip_seconds=[]) + prefix=None, subjects=[], cards={}, pictures_each=None, clip_seconds=[], own_pictures=0) if not keys: return result - # A references list, even an empty one, is the shot's own choice: cards are added only when it is left out. - if shot.get('first') is not None or isinstance(shot.get('references'), list): + own = shot.get('references') if isinstance(shot.get('references'), list) else None + if own is not None and not own: + # An empty list is the shot's own choice of no pictures: no cards are added. result['cast_reference'] = 'own' return result + if shot.get('first') is not None and own is None: + # A first frame cannot travel with reference pictures; the people still get their written descriptions. + subjects = [] + for key in keys: + card = cards.get(characters[key]['card']) + if card is None: + result['missing'].append(key) + continue + subjects.append(dict(name=characters[key].get('name') or key, description=card.get('description'), + pictures=None, audio=None)) + subjects = [s for s in subjects if (s['description'] or '').strip()] + if subjects: + result['prefix'] = subject_definitions(subjects) + result['subjects'] = subjects + result['cast_reference'] = 'missing' if result['missing'] else 'own' + return result + offset = len(own or []) # the shot's own pictures come first; the cards follow them spoken = {} for line in shot.get('lines', []): spoken[line['who']] = spoken.get(line['who'], 0) + 1 @@ -356,13 +493,13 @@ def cast_references(play, shot, cards, *, continues=False): continue images = person_pictures(card, alone=alone, continues=continues) if continues: - if images and len(chosen) < specs.CONTINUED_PICTURES: + if images and offset + len(chosen) < specs.CONTINUED_PICTURES: chosen.append((key, card, images, False)) else: result['omitted'].append(key) continue voice = card.get('voice') is not None and voices < MAX_VOICES - if pictures + len(images) + voices + voice > MAX_REFERENCES: + if offset + pictures + len(images) + voices + voice > MAX_REFERENCES: result['omitted'].append(key) continue if card.get('voice') is not None and not voice: @@ -370,7 +507,7 @@ def cast_references(play, shot, cards, *, continues=False): pictures, voices = pictures + len(images), voices + voice chosen.append((key, card, images, voice)) chosen.sort(key=lambda row: keys.index(row[0])) - subjects, clips, picture = [], [], 0 + subjects, clips, picture = [], [], offset for key, card, images, voice in chosen: result['cards'][card['id']] = card['revision'] result['references'].extend(dict(item) for item in images) @@ -386,7 +523,15 @@ def cast_references(play, shot, cards, *, continues=False): subjects.append(subject) result['people'].append(dict(character=key, card=card['id'], revision=card['revision'], pictures=len(images), voice=voice)) + # Someone left out of the pictures still gets a written definition, numbered after the pictured people, + # so every carded person in the shot is a Subject and keeps one number in the story and in the voice lines. + for key in result['omitted']: + card = cards.get(characters[key]['card']) + if card is not None and (card.get('description') or '').strip(): + subjects.append(dict(name=characters[key].get('name') or key, description=card.get('description'), + pictures=None, audio=None)) result['references'].extend(clips) + result['own_pictures'] = offset result['pictures_each'] = max((len(images) for _, _, images, _ in chosen), default=0) or None if subjects: result['prefix'] = subject_definitions(subjects) @@ -400,6 +545,56 @@ def cast_references(play, shot, cards, *, continues=False): DESCRIPTION_CUTS = (500, 250, 120, 0) +def ref2va(text, resolved): + """A prompt written for pictures: tags in the story, then summary, retention_analysis and detailed_description. + + Reference generation takes six sections in a fixed order (subject_definitions comes from the cards, see with_cast); + the first-frame form keeps integrated_multimodal_description. Names become everywhere in the story.""" + head = 'integrated_multimodal_description: ' + subjects = resolved.get('subjects') or [] + if not resolved.get('references') or not subjects or not text.startswith(head) or '\n\noverall_soundscape:' not in text: + return text + body, rest = text[len(head):].split('\n\noverall_soundscape:', 1) + tags = {} + for number, subject in enumerate(subjects, 1): + name = (subject.get('name') or '').strip() + if name: + tags[name] = number + for name, number in sorted(tags.items(), key=lambda kv: -len(kv[0])): + body = re.sub(r'(?' % number, body, flags=re.I) + marks = [(m.start(), int(m.group(1))) for m in re.finditer(r'\[Shot (\d+)\]', body)] + total = len(marks) + + def appears(number): + tag = '' % number + shots = [n for i, (at, n) in enumerate(marks) + if tag in body[at:marks[i + 1][0] if i + 1 < total else len(body)]] + if total and len(shots) == total: + return 'all shots' + return ', '.join('[Shot %d]' % n for n in shots) if shots else 'the story' + + voiced = any(s.get('audio') for s in subjects) + first = re.sub(r'\[Shot \d+\](?: At [\d:.]+,)?\s*', '', body.split('\n')[0]).strip() + if len(first) > 240: + cut = first[:240] + first = (cut[:cut.rfind('. ') + 1] if '. ' in cut else cut.rstrip()).strip() + summary = ('[reference generation%s] ' % (' + audio reference' if voiced else '')) + first + lines = [] + for number, subject in enumerate(subjects, 1): + if not subject.get('pictures'): + lines.append(' (appears in %s): described - identity, hair, clothing and accessories follow ' + 'the written description.' % (number, appears(number))) + continue + lines.append(' (appears in %s): fully_preserved - identity, hair, clothing and accessories are kept ' + 'exactly as in the reference pictures.' % (number, appears(number))) + for number, subject in enumerate(subjects, 1): + if subject.get('audio'): + lines.append("