diff --git a/MANIFEST.in b/MANIFEST.in
index f4dd048c..e6ec5836 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -74,5 +74,25 @@ include web/assets/tour/c12-s1.mp4
include web/assets/tour/c12-s1.webp
include web/assets/tour/c13-s1.mp4
include web/assets/tour/c13-s1.webp
+include web/assets/tour/cb-ambience.mp4
+include web/assets/tour/cb-ambience.webp
+include web/assets/tour/cb-dialogue.mp4
+include web/assets/tour/cb-dialogue.webp
+include web/assets/tour/cb-music.mp4
+include web/assets/tour/cb-music.webp
+include web/assets/tour/cb-source.mp4
+include web/assets/tour/cb-source.webp
+include web/assets/tour/ca-music.mp4
+include web/assets/tour/ca-music.webp
+include web/assets/tour/ca-music-wave.webp
+include web/assets/tour/ca-dub.mp4
+include web/assets/tour/ca-dub.webp
+include web/assets/tour/ca-dub-wave.webp
+include web/assets/tour/ca-record.mp4
+include web/assets/tour/ca-record.webp
+include web/assets/tour/ca-record-wave.webp
+include web/assets/tour/cc-speech.mp4
+include web/assets/tour/cc-speech.webp
+include web/assets/tour/cc-speech-wave.webp
include web/assets/tour/c10-source.webp
include web/assets/tour/fox-sheet.webp
diff --git a/freevideo_engine/agent/drama.py b/freevideo_engine/agent/drama.py
index ac0b32b9..56243c7a 100644
--- a/freevideo_engine/agent/drama.py
+++ b/freevideo_engine/agent/drama.py
@@ -20,7 +20,7 @@
import time
from . import spec as specs
-from ..chain_plan import CONTINUE_SECONDS
+from ..chain_plan import CONTINUE_SECONDS, BLOCK_SECONDS
EXAMPLE = {
'title': '末班车', 'name': 'the-last-train', 'aspect': '9:16', 'quality': 'light',
@@ -67,8 +67,10 @@ def __init__(self, problems):
MAX_REFERENCES = 32
-def check(play):
- """Validate every independent field, mutating only missing shot ids."""
+def check(play, single_seconds=BLOCK_SECONDS):
+ """Validate every independent field, mutating only missing shot ids.
+
+ single_seconds: the longest shot this machine generates at once, which bounds a shot that carries pictures."""
problems = []
def problem(path, message):
@@ -109,6 +111,12 @@ def validate(function, *args, **kwargs):
validate(specs._text, play[field], '/' + field, 20000, empty=True)
if 'name' in play and (not isinstance(play['name'], str) or not specs.SLUG.fullmatch(play['name'])):
problem('/name', 'must be a short project name')
+ if 'continuity' in play:
+ if not isinstance(play['continuity'], dict):
+ problem('/continuity', 'must be an object of text by character name, and optionally "set"')
+ else:
+ for key, text in play['continuity'].items():
+ validate(specs._text, text, '/continuity/' + str(key).replace('~', '~0').replace('/', '~1'), 20000, empty=True)
if 'captions' in play:
validate(specs._bool, play['captions'], '/captions')
if 'quality' in play and (not isinstance(play['quality'], str) or play['quality'] not in specs.QUALITY):
@@ -136,8 +144,8 @@ def validate(function, *args, **kwargs):
problem(path + '/id', '/shots/%d/id must be a unique short name' % i)
if isinstance(shot['id'], str):
names.add(shot['id'])
- if not any(isinstance(shot.get(key), str) and shot[key].strip() for key in ('action', 'idea', 'prompt')):
- problem(path, '/shots/%d needs action, idea or prompt' % i)
+ if not any(isinstance(shot.get(key), str) and shot[key].strip() for key in ('action', 'idea', 'prompt')) and not shot.get('cuts'):
+ problem(path, '/shots/%d needs action, idea, prompt or cuts' % i)
if shot.get('continue') and i == 0:
problem(path + '/continue', '/shots/0 cannot continue a previous shot')
if 'continue' in shot:
@@ -145,6 +153,12 @@ def validate(function, *args, **kwargs):
for field in ('action', 'idea', 'prompt', 'camera', 'framing', 'sound', 'music'):
if field in shot:
validate(specs._text, shot[field], path + '/' + field, 20000, empty=True)
+ if 'continuity' in shot:
+ if not isinstance(shot['continuity'], dict):
+ problem(path + '/continuity', 'must be an object of text by character name, and optionally "set"')
+ else:
+ for key, text in shot['continuity'].items():
+ validate(specs._text, text, path + '/continuity/' + str(key).replace('~', '~0').replace('/', '~1'), 20000, empty=True)
for field, low, high in (('takes', 1, 8), ('seed', 0, specs.MAX_SEED)):
if field in shot:
validate(specs._int, shot[field], path + '/' + field, low, high)
@@ -159,12 +173,13 @@ def validate(function, *args, **kwargs):
if ((shot.get('continue') or long) and play.get('quality') in specs.QUALITY
and not specs.QUALITY[play['quality']][0] and all(p['path'] != '/quality' for p in problems)):
problem('/quality', '"light" or "medium" (two-pass) when a shot continues another or is longer than 15 s')
- if long and (shot.get('first') is not None or shot.get('references') and not shot.get('continue')):
- problem(path + '/seconds', 'at most 15 with a first frame or references')
- elif long and not shot.get('continue') and _carded(play, shot):
- # Cast cards travel as references, which one generation of at most 15 s carries.
- problem(path + '/seconds', 'at most 15 when a character in the shot has a cast card '
- '(a following shot with continue=true carries the same people further)')
+ over = (isinstance(seconds, (int, float)) and not isinstance(seconds, bool) and seconds > single_seconds)
+ if over and (shot.get('first') is not None or shot.get('references') and not shot.get('continue')):
+ problem(path + '/seconds', 'at most %d with a first frame or references on this machine' % single_seconds)
+ elif over and not shot.get('continue') and _carded(play, shot):
+ # Cast cards travel as references, which one generation (at most single_seconds) carries.
+ problem(path + '/seconds', 'at most %d on this machine when a character in the shot has a cast card '
+ '(a following shot with continue=true carries the same people further)' % single_seconds)
if shot.get('first') is not None:
validate(specs.media_item, shot['first'], path + '/first', image=True)
if 'references' in shot:
@@ -185,6 +200,8 @@ def validate(function, *args, **kwargs):
for j, key in enumerate(shot['cast']):
if not isinstance(key, str) or key not in characters:
problem(path + '/cast/%d' % j, 'must name a character')
+ if 'cuts' in shot:
+ _check_cuts(shot, path, characters, seconds, problem, validate)
lines = shot.get('lines', [])
if not isinstance(lines, list):
problem(path + '/lines', 'must be a list')
@@ -205,12 +222,62 @@ def validate(function, *args, **kwargs):
return play
+def _check_cuts(shot, path, characters, seconds, problem, validate):
+ """Numbered cuts inside one generation: each starts at "at" seconds (the first at 0), strictly later than the one before."""
+ cuts = shot['cuts']
+ if not isinstance(cuts, list) or not 1 <= len(cuts) <= 8:
+ problem(path + '/cuts', 'must be a list of 1 to 8 cuts')
+ return
+ last = None
+ for j, cut in enumerate(cuts):
+ here = '%s/cuts/%d' % (path, j)
+ if not isinstance(cut, dict):
+ problem(here, 'must be an object')
+ continue
+ for key in cut:
+ if key not in CUT_FIELDS:
+ problem(here + '/' + key, 'is not a cut field (%s)' % ', '.join(CUT_FIELDS))
+ at = cut.get('at', 0 if j == 0 else None)
+ if isinstance(at, bool) or not isinstance(at, (int, float)) or at < 0 or (j == 0 and at != 0):
+ problem(here + '/at', 'must be the second the cut starts (0 for the first cut)')
+ elif last is not None and at <= last:
+ problem(here + '/at', 'must be later than the cut before')
+ elif isinstance(seconds, (int, float)) and not isinstance(seconds, bool) and at >= seconds:
+ problem(here + '/at', 'must be before the end of the shot (%s s)' % seconds)
+ else:
+ last = at
+ for field in ('framing', 'action', 'camera'):
+ if field in cut:
+ validate(specs._text, cut[field], here + '/' + field, 20000, empty=True)
+ if not any(isinstance(cut.get(k), str) and cut[k].strip() for k in ('action', 'framing')):
+ problem(here, 'needs an action or framing')
+ for k, line in enumerate(cut.get('lines', []) if isinstance(cut.get('lines', []), list) else [None]):
+ if not isinstance(line, dict) or line.get('who') not in characters:
+ problem('%s/lines/%d' % (here, k), 'must be a line from a named character')
+ else:
+ validate(specs._text, line.get('line'), '%s/lines/%d/line' % (here, k), 20000)
+
+
+CUT_FIELDS = ('at', 'framing', 'action', 'camera', 'lines')
+
+
+def _stamp(seconds):
+ whole = int(seconds)
+ return '%02d:%02d.%03d' % (whole // 60, whole % 60, round((seconds - whole) * 1000))
+
+
+def _mentions(text, word):
+ """Whole-word, case-insensitive match; a blank word never matches."""
+ word = (word or '').strip()
+ return bool(word) and re.search(r'(? BLOCK_SECONDS
+ and (shot.get('first') is not None
+ or not shot.get('continue') and (shot.get('references') or _carded(play, shot)))):
+ return True
+ return False
+
+
SHOWN_TEXT = re.compile(r'["“「][^"”」]+["”」]')
+def _cap(name):
+ """A name as written, with its first letter upper-case: 'Old Ren' stays 'Old Ren'."""
+ return name[:1].upper() + name[1:]
+
+
+PAPER = re.compile(r'\b(note|letter|paper|card|sign|screen|book|poster|message|menu|map)\b', re.I)
+
+
def prompt(play, shot):
"""The H3 prompt of one shot, from its structured fields."""
if shot.get('prompt'):
@@ -232,34 +318,86 @@ def prompt(play, shot):
characters = play.get('characters', {})
people = cast(play, shot)
lead = ', '.join(x for x in (play.get('style'), shot.get('framing')) if x)
- parts = ['[Shot 1] ' + (lead + '.' if lead else '')]
- if shot.get('continue'):
- parts.append('Continuing from the opening frame, the scene keeps its exact layout, lighting and characters.')
- for key in people:
- person = characters[key]
- if person.get('look'):
- parts.append('%s is %s.' % ((person.get('name') or key).capitalize(), person['look'].rstrip('.')))
- parts.append(named(play, people, shot.get('action', '')))
- if shot.get('camera'):
- parts.append(shot['camera'])
- speakers = {}
- for line in shot.get('lines', []):
+ # A speaker's number is its place in the order of vocal events of this video (the official rule): the first
+ # voice heard is (S1), the next new one (S2); the number is reused at every later line of the same shot.
+ heard = [line['who'] for cut in (shot.get('cuts') or [shot]) for line in cut.get('lines', [])]
+ numbers = {key: number for number, key in enumerate(dict.fromkeys(heard), 1)}
+ spoken = set()
+
+ def say(line):
who = line['who']
person = characters[who]
- first = who not in speakers
- speakers.setdefault(who, len(speakers) + 1)
+ first = who not in spoken
+ spoken.add(who)
voice = (' with a %s voice' % person['voice']) if first and person.get('voice') else ''
language = line.get('language') or person.get('language') or play.get('language') or guess(line['line'])
- parts.append('%s%s (S%d) %s: [%s] %s ' % ((person.get('name') or who).capitalize(), voice,
- speakers[who], line.get('how') or 'says', language, line['line']))
+ return '%s%s (S%d) %s: [%s] %s ' % (_cap(person.get('name') or who), voice,
+ numbers[who], line.get('how') or 'says', language, line['line'])
+
+ def looks():
+ return ['%s is %s.' % (_cap(characters[key].get('name') or key), characters[key]['look'].rstrip('.'))
+ for key in people if characters[key].get('look')]
+
+ def continuity():
+ """The film's table of clothes, props and places, the rows of the people in this shot (and 'set'), in every shot.
+ A row written in the shot itself replaces the film's row of that person (or 'set') for this shot only."""
+ table = {**(play.get('continuity') if isinstance(play.get('continuity'), dict) else {}),
+ **(shot.get('continuity') if isinstance(shot.get('continuity'), dict) else {})}
+ if not table:
+ return []
+ rows = ['%s: %s.' % (_cap(characters[key].get('name') or key), table[key].strip().rstrip('.'))
+ for key in people if isinstance(table.get(key), str) and table[key].strip()]
+ if isinstance(table.get('set'), str) and table['set'].strip():
+ rows.append(table['set'].strip().rstrip('.') + '.')
+ return ['Continuity - ' + ' '.join(rows)] if rows else []
+
+ cuts = shot.get('cuts')
+ if cuts:
+ # Numbered cuts inside one generation: [Shot 1] has no time, the others say when they start.
+ seconds = shot.get('seconds')
+ parts = []
+ for number, cut in enumerate(cuts, 1):
+ row = [('[Shot 1] ' if number == 1 else '[Shot %d] At %s, ' % (number, _stamp(cut['at']))).rstrip()]
+ if number == 1:
+ row += [lead + '.' if lead else '']
+ if shot.get('continue'):
+ row.append('Continuing from the opening frame, the scene keeps its exact layout, lighting and characters.')
+ row += looks() + continuity()
+ elif cut.get('framing'):
+ row.append(cut['framing'].strip().rstrip('.') + '.')
+ if number == 1 and cut.get('framing'):
+ row.append(cut['framing'].strip().rstrip('.') + '.')
+ if cut.get('action'):
+ row.append(named(play, people, cut['action']))
+ if cut.get('camera'):
+ row.append(cut['camera'])
+ row += [say(line) for line in cut.get('lines', [])]
+ if number == len(cuts) and isinstance(seconds, (int, float)) and not isinstance(seconds, bool):
+ row.append('The shot ends at %s.' % _stamp(seconds))
+ parts.append(' '.join(x for x in row if x))
+ body = '\n'.join(parts)
+ else:
+ parts = ['[Shot 1] ' + (lead + '.' if lead else '')]
+ if shot.get('continue'):
+ parts.append('Continuing from the opening frame, the scene keeps its exact layout, lighting and characters.')
+ parts += looks() + continuity()
+ parts.append(named(play, people, shot.get('action', '')))
+ if shot.get('camera'):
+ parts.append(shot['camera'])
+ parts += [say(line) for line in shot.get('lines', [])]
+ body = ' '.join(p for p in parts if p)
if not play.get('captions'):
# Quoted words are text the shot shows (a sign, a menu); then only captions are ruled out.
scene = ' '.join(shot.get(k) or '' for k in ('framing', 'action', 'camera')) + ' ' + (play.get('style') or '')
- parts.append('No subtitles or captions.' if SHOWN_TEXT.search(scene) else 'No subtitles, no on-screen text.')
+ scene += ' ' + ' '.join(c.get(k) or '' for c in (cuts or []) for k in ('framing', 'action', 'camera'))
+ body += ' ' + ('No subtitles or captions.' if SHOWN_TEXT.search(scene) else 'No subtitles, no on-screen text.')
+ if PAPER.search(scene) and not SHOWN_TEXT.search(scene):
+ # A model writes the spoken line on any note it is asked to show close; a blank or unreadable face avoids it.
+ body += ' Any note, paper or sign in frame has a blank face with only faint folds, seen from the side or out of focus.'
sound = shot.get('sound') or 'Natural ambience of the scene.'
# Only a shot's own music is written; a score for the whole film is imported and laid on the sound track.
music = shot.get('music') or 'N/A'
- return ('integrated_multimodal_description: ' + ' '.join(p for p in parts if p) +
+ return ('integrated_multimodal_description: ' + body +
'\n\noverall_soundscape: ' + sound + '\n\nnon_diegetic_music: ' + music)
@@ -325,7 +463,7 @@ def cast_references(play, shot, cards, *, continues=False):
function serves plan previews and the moment a take is submitted.
pictures_each is the most pictures one person carries in this shot.
cast_reference: None (no character in the shot has a card), 'own' (the shot
- names its own first frame or references; an empty list means no pictures), 'missing' (a card was deleted),
+ names its own first frame, which carries only the written descriptions, or an empty list of references), 'missing' (a card was deleted),
'partial' (not everyone fit) or 'cards'. 'omitted' lists people left out
entirely; 'voice_omitted' lists people whose pictures are carried but whose
voice did not fit. A shot that continues the previous one carries no voice
@@ -334,13 +472,31 @@ def cast_references(play, shot, cards, *, continues=False):
characters = play.get('characters', {})
keys = [k for k in cast(play, shot) if isinstance(characters[k].get('card'), str)]
result = dict(cast_reference=None, people=[], omitted=[], voice_omitted=[], missing=[], references=[],
- prefix=None, subjects=[], cards={}, pictures_each=None, clip_seconds=[])
+ prefix=None, subjects=[], cards={}, pictures_each=None, clip_seconds=[], own_pictures=0)
if not keys:
return result
- # A references list, even an empty one, is the shot's own choice: cards are added only when it is left out.
- if shot.get('first') is not None or isinstance(shot.get('references'), list):
+ own = shot.get('references') if isinstance(shot.get('references'), list) else None
+ if own is not None and not own:
+ # An empty list is the shot's own choice of no pictures: no cards are added.
result['cast_reference'] = 'own'
return result
+ if shot.get('first') is not None and own is None:
+ # A first frame cannot travel with reference pictures; the people still get their written descriptions.
+ subjects = []
+ for key in keys:
+ card = cards.get(characters[key]['card'])
+ if card is None:
+ result['missing'].append(key)
+ continue
+ subjects.append(dict(name=characters[key].get('name') or key, description=card.get('description'),
+ pictures=None, audio=None))
+ subjects = [s for s in subjects if (s['description'] or '').strip()]
+ if subjects:
+ result['prefix'] = subject_definitions(subjects)
+ result['subjects'] = subjects
+ result['cast_reference'] = 'missing' if result['missing'] else 'own'
+ return result
+ offset = len(own or []) # the shot's own pictures come first; the cards follow them
spoken = {}
for line in shot.get('lines', []):
spoken[line['who']] = spoken.get(line['who'], 0) + 1
@@ -356,13 +512,13 @@ def cast_references(play, shot, cards, *, continues=False):
continue
images = person_pictures(card, alone=alone, continues=continues)
if continues:
- if images and len(chosen) < specs.CONTINUED_PICTURES:
+ if images and offset + len(chosen) < specs.CONTINUED_PICTURES:
chosen.append((key, card, images, False))
else:
result['omitted'].append(key)
continue
voice = card.get('voice') is not None and voices < MAX_VOICES
- if pictures + len(images) + voices + voice > MAX_REFERENCES:
+ if offset + pictures + len(images) + voices + voice > MAX_REFERENCES:
result['omitted'].append(key)
continue
if card.get('voice') is not None and not voice:
@@ -370,7 +526,7 @@ def cast_references(play, shot, cards, *, continues=False):
pictures, voices = pictures + len(images), voices + voice
chosen.append((key, card, images, voice))
chosen.sort(key=lambda row: keys.index(row[0]))
- subjects, clips, picture = [], [], 0
+ subjects, clips, picture = [], [], offset
for key, card, images, voice in chosen:
result['cards'][card['id']] = card['revision']
result['references'].extend(dict(item) for item in images)
@@ -386,7 +542,15 @@ def cast_references(play, shot, cards, *, continues=False):
subjects.append(subject)
result['people'].append(dict(character=key, card=card['id'], revision=card['revision'],
pictures=len(images), voice=voice))
+ # Someone left out of the pictures still gets a written definition, numbered after the pictured people,
+ # so every carded person in the shot is a Subject and keeps one number in the story and in the voice lines.
+ for key in result['omitted']:
+ card = cards.get(characters[key]['card'])
+ if card is not None and (card.get('description') or '').strip():
+ subjects.append(dict(name=characters[key].get('name') or key, description=card.get('description'),
+ pictures=None, audio=None))
result['references'].extend(clips)
+ result['own_pictures'] = offset
result['pictures_each'] = max((len(images) for _, _, images, _ in chosen), default=0) or None
if subjects:
result['prefix'] = subject_definitions(subjects)
@@ -400,6 +564,56 @@ def cast_references(play, shot, cards, *, continues=False):
DESCRIPTION_CUTS = (500, 250, 120, 0)
+def ref2va(text, resolved):
+ """A prompt written for pictures: tags in the story, then summary, retention_analysis and detailed_description.
+
+ Reference generation takes six sections in a fixed order (subject_definitions comes from the cards, see with_cast);
+ the first-frame form keeps integrated_multimodal_description. Names become everywhere in the story."""
+ head = 'integrated_multimodal_description: '
+ subjects = resolved.get('subjects') or []
+ if not resolved.get('references') or not subjects or not text.startswith(head) or '\n\noverall_soundscape:' not in text:
+ return text
+ body, rest = text[len(head):].split('\n\noverall_soundscape:', 1)
+ tags = {}
+ for number, subject in enumerate(subjects, 1):
+ name = (subject.get('name') or '').strip()
+ if name:
+ tags[name] = number
+ for name, number in sorted(tags.items(), key=lambda kv: -len(kv[0])):
+ body = re.sub(r'(?' % number, body, flags=re.I)
+ marks = [(m.start(), int(m.group(1))) for m in re.finditer(r'\[Shot (\d+)\]', body)]
+ total = len(marks)
+
+ def appears(number):
+ tag = '' % number
+ shots = [n for i, (at, n) in enumerate(marks)
+ if tag in body[at:marks[i + 1][0] if i + 1 < total else len(body)]]
+ if total and len(shots) == total:
+ return 'all shots'
+ return ', '.join('[Shot %d]' % n for n in shots) if shots else 'the story'
+
+ voiced = any(s.get('audio') for s in subjects)
+ first = re.sub(r'\[Shot \d+\](?: At [\d:.]+,)?\s*', '', body.split('\n')[0]).strip()
+ if len(first) > 240:
+ cut = first[:240]
+ first = (cut[:cut.rfind('. ') + 1] if '. ' in cut else cut.rstrip()).strip()
+ summary = ('[reference generation%s] ' % (' + audio reference' if voiced else '')) + first
+ lines = []
+ for number, subject in enumerate(subjects, 1):
+ if not subject.get('pictures'):
+ lines.append(' (appears in %s): described - identity, hair, clothing and accessories follow '
+ 'the written description.' % (number, appears(number)))
+ continue
+ lines.append(' (appears in %s): fully_preserved - identity, hair, clothing and accessories are kept '
+ 'exactly as in the reference pictures.' % (number, appears(number)))
+ for number, subject in enumerate(subjects, 1):
+ if subject.get('audio'):
+ lines.append(": reference - the target references the voice timbre of for 's lines."
+ % (subject['audio'], subject['audio'], number))
+ return ('summary: ' + summary + '\n\nretention_analysis:\n' + '\n'.join(lines) +
+ '\n\ndetailed_description:\n' + body + '\n\noverall_soundscape:' + rest)
+
+
def with_cast(text, resolved, *, limit=PROMPT_LIMIT):
"""The submitted prompt: the subject definitions, then the shot's own prompt.
@@ -407,6 +621,7 @@ def with_cast(text, resolved, *, limit=PROMPT_LIMIT):
shortened (the pictures still show each person). Plan previews and submissions both call this."""
if not resolved.get('prefix'):
return text
+ text = ref2va(text, resolved)
full = resolved['prefix'] + '\n\n' + text
if len(full) <= limit or not resolved.get('subjects'):
return full
@@ -424,8 +639,9 @@ def named(play, people, text):
for key in people:
name = play.get('characters', {})[key].get('name')
if name and key.lower() != name.lower():
- text = text.replace(key.capitalize(), name.capitalize() if text.startswith(key.capitalize())
- else name).replace(key, name)
+ text = re.sub(r'(?[Language] ..., verbatim: The woman (S1) says: [Chinese] 我们走吧。 ',
'Voiceover: "says in an off-screen voiceover: ... while his lips remain completely closed."',
'Visible text goes in double quotes: a neon sign reading "营业中". Never put spoken words in quotes: '
@@ -74,6 +74,7 @@
"User message attachments are reference images by default. Use each name directly in media.references, media.first or media.last according to the user's words; if the intended role is unclear, ask one question. Text can be empty when a message has attachments.",
'Tags are numbered per kind in reference order: , , , . '
'For keyframes the first frame is and the last frame the next picture.',
+ 'A reference video marked voice_only is an audio reference: its sound is and it has no tag.',
'Name subjects , in subject_definitions and reuse those labels in every section.',
'Clips are cut to the generated length (at most 15 s); the job result lists any cut.',
],
@@ -122,7 +123,8 @@
],
'agent_jobs': [
'GET /capabilities reads the studio cache with the same size/task/material query parameters. '
- 'status=computing includes retry_after_seconds; unknown plan estimates have seconds=null.',
+ 'status=computing includes retry_after_seconds; unknown plan estimates have seconds=null. '
+ 'A request answered with capabilities_computing waits retry_after_seconds and repeats unchanged.',
'GET /jobs/{id}?wait=25 waits at most 25 seconds and returns poll_after_ms. Like POST /jobs, the answer '
'has the job under job (its fields are also at the top level). Poll the same ID until job.status is '
'succeeded, failed, cancelled or interrupted; disconnecting a wait or SSE does not cancel generation.',
@@ -145,7 +147,8 @@
'POST /projects/{id}/generate reserves pending parts for clip_ids or a start/frames range and fills them as the returned job finishes. POST /projects/{id}/export renders the revision you read; wait for its job.',
'Music: POST /projects/{id}/music with the revision you read and a caption; wait for the job, then read the project: result.clip_ids are the new clips on a2. Run it after the cut is joined; a second run needs the earlier score removed from a2.',
'Split sound: POST /projects/{id}/stems with the revision you read and the clip_id of a sound clip; wait for the job. The clip\'s sound is split into dialogue, music and ambience on tracks a4, a5 and a6 over the clip\'s range, and the clip itself goes silent. result.parts lists each track with its name, clip_id, start, end and frames. Offered only where features lists projects.stems; a clip is split once, and deleting the three parts gives its sound back. To save the three tracks and the original mix as four wav files, POST /projects/{id}/stems/export with the revision and the split clip\'s clip_id; the answer lists folder and files at once.',
- 'Change a voice: POST /projects/{id}/voice/lines with the revision you read and a selection (clip_id, and start and end in seconds if only part) to write down the lines; read result.lines. Change the text of the lines to redo and POST /projects/{id}/voice/replace with the same selection and all lines (original_text unchanged); only the changed lines are spoken again, in the voice of the original, and the rest keep their sound. result.lines gives where each new line starts and ends; a line that cannot fit before the next one keeps its original voice with replaced false and warning too_long. POST /projects/{id}/voice/estimate with the lines answers at once with the seconds the changed lines take. Offered only where features lists projects.voice.',
+ 'Finish the sound: after the cut is joined, POST /projects/{id}/finish_sound with the revision you read; wait for the job. In one job it splits each picture clip\'s sound where separation is offered and removes the separated music where it is a sustained score (shots, explosions and crowds filed there stay, at the ambience level), lays one score on a2 for the whole cut (music.caption composes it, music.source lays a file; music already on a2 stays), evens the dialogue, sets the music 16 LU under it, brings the cut to -18 LUFS with a -1 dBTP peak ceiling and fills digital silence with room tone. result gives loudness_lufs, true_peak_db, music_removed (per block: removed or kept, with the levels) and the steps in skipped; skipped.music is too_long for a cut over 600 s, and the rest still runs; then tell the user: The cut is over 10 minutes, so no music was composed. The other steps are done. To add music, run this again with a music file. Run again, only the clips split this time are judged and levelled.',
+ 'Change a voice: POST /projects/{id}/voice/lines with the revision you read and a selection (clip_id, and start and end in seconds if only part) to write down the lines; read result.lines. Change the text of the lines to redo and POST /projects/{id}/voice/replace with the same selection and all lines (original_text unchanged); only the changed lines are spoken again, in the voice of the original, and the rest keep their sound. result.lines gives where each new line starts and ends; a line that cannot fit before the next one keeps its original voice with replaced false and warning too_long. POST /projects/{id}/voice/estimate with the lines answers at once with the seconds the changed lines take. Offered only where features lists projects.voice. Where features lists projects.voice.install instead, the dubbing model is not installed yet: the person downloads it (about 4.9 GB) from the voice change bar in the editor, and no agent route does it.',
'Subtitles: POST /projects/{id}/listen with mode transcribe (writes down what is said) or align (times the lines you give) and the revision you read; wait for the job and read result.lines, each with start and end frames. Then add them with a subtitles operation of cues {start, frames, text}; replace true swaps the whole track. With no speech model the job fails with no_speech_model: write the lines yourself.',
'The user sees one line under each picture clip: its note, or else the first line of its description. A description is generation text, often long and in English, so after you add, generate or change a clip, write a note (operation note) in the user\'s language saying what is in the picture: the subject first, then the setting, for example "Train through morning mist" (in Chinese "小火车穿过晨雾"). A short clip shows only the first few characters, about four in Chinese, and those must already make sense on their own. Do not write where the clip sits (opening, middle, ending, part 2): the order of the clips shows that. Do not write how far along it is: the clip already shows its progress. Keep it true when you change the clip again.',
'Reuse ref or Idempotency-Key only for the identical request. Source video/audio/thumbnail URLs require the same Agent token.',
@@ -381,6 +384,8 @@
'剪辑拼好后先读取 revision，并删去该轨道上原有的配乐；首次使用会下载音乐模型（约 20 GB）',
('POST', '/projects/{id}/stems'): '把一个声音片段的声音分成人声、音乐和环境声，放在其下方的 a4、a5、a6 三条轨道上，原片段静音。'
'先读取 revision 与片段 ID，等待返回的任务完成，结果 parts 给出各轨道的名称、起止帧；仅在 features 含 projects.stems 时可用',
+ ('POST', '/projects/{id}/finish_sound'): '剪辑拼好后，用一个任务完成整条剪辑的声音：分离人声、音乐与环境声，在 a2 铺一段贯穿全片的配乐，'
+ '统一人声响度，整片调至 -18 LUFS，并以环境底噪填补无声处。先读取 revision，等待返回的任务完成',
('POST', '/projects/{id}/voice/lines'): '把选中声音片段里说的话逐句写下来，每句带起止秒数；先读取 revision 与片段 ID，等待返回的任务完成，结果 lines 给出各句；仅在 features 含 projects.voice 时可用',
('POST', '/projects/{id}/voice/replace'): '把改过文字的句子用原说话人的声音重新说一遍，放回原句的位置，没改的句子保持原声；提交 voice/lines 给出的全部句子，只改 text；结果 lines 给出每句新声的起止，放不进下一句之前的句子保留原声',
('POST', '/projects/{id}/voice/shorten'): '按目标秒数改短一句的文字；只返回文字，不保存也不配音',
@@ -626,7 +631,7 @@ def connection_text(value):
'continue the previous one carry the sheet and the face for a single person, else one picture each (the face, '
'else the front) for at most 3 people, and keep the voices of the previous shot; POST /drama/plan shows each shot\'s cast. '
'A shot\'s own references replace the card pictures; references=[] sends none. '
- 'A shot with a carded character is at most 15 s; continue it in the next shot to go longer.',
+ 'A shot with a carded character is at most one generation long on this machine (freevideo_capabilities reports it; 15 s when unknown); continue it in the next shot to go longer.',
'POST /cast/{id}/generate makes reference pictures from the description as a normal job (face=true adds a close-up). '
'It defaults to the installed image model, otherwise video frames (labels guessed, label_guess). '
'When generation ends the card gets the pictures and a new revision; a failure '
diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py
index cff93a7a..2224c097 100644
--- a/freevideo_engine/agent/jobs.py
+++ b/freevideo_engine/agent/jobs.py
@@ -32,7 +32,7 @@
ACTIVE = ('waiting', 'queued', 'running', 'cancelling')
CAST_AGAIN = 'POST /freevideo/agent/v1/cast/%s/generate to make the reference pictures again.'
DONE = ('succeeded', 'failed', 'cancelled', 'interrupted')
-GPU_KINDS = ('vlm', 'enhance', 'image', 'music', 'stems', 'voice')
+GPU_KINDS = ('vlm', 'enhance', 'image', 'music', 'stems', 'finish_sound', 'voice')
IMAGE_ERRORS = errors.IMAGE_ERRORS
IMAGE_WAIT_ERRORS = ('busy', 'busy_video')
IMAGE_WAIT_NOTICES = {
@@ -1488,6 +1488,9 @@ def run():
elif row['kind'] == 'stems':
from .stems import run_stems
result = run_stems(self, row)
+ elif row['kind'] == 'finish_sound':
+ from .sound import run_finish_sound
+ result = run_finish_sound(self, row)
elif row['kind'] == 'voice_lines':
from .voice import run_voice_lines
result = run_voice_lines(self, row)
diff --git a/freevideo_engine/agent/music.py b/freevideo_engine/agent/music.py
index 8a377c9d..388a5712 100644
--- a/freevideo_engine/agent/music.py
+++ b/freevideo_engine/agent/music.py
@@ -49,10 +49,10 @@ def phase(name, fraction=None):
generate_seconds=numbers.get('generate_s'), load_seconds=numbers.get('load_s'))
-def _compose(jobs, row, service, out, frames, seed, quiet, phase, poll):
+def _compose(jobs, row, service, out, frames, seed, quiet, phase, poll, *, caption=None):
for attempt in range(PREEMPTIONS + 1):
try:
- return service.compose(out, row['spec']['caption'], frames, seed=seed, quiet=quiet, fps=comfy_editor.FPS,
+ return service.compose(out, row['spec']['caption'] if caption is None else caption, frames, seed=seed, quiet=quiet, fps=comfy_editor.FPS,
server=getattr(jobs.port, 'server', None), cancelled=lambda: bool(row.get('_cancel')),
phase=phase)
except service.Preempted:
diff --git a/freevideo_engine/agent/production.py b/freevideo_engine/agent/production.py
index ce05deba..5229df82 100644
--- a/freevideo_engine/agent/production.py
+++ b/freevideo_engine/agent/production.py
@@ -73,7 +73,7 @@ def _compiled(play, shot):
# What a screenplay shot and a line may say; anything else would be ignored without a word.
SHOT_FIELDS = ('id', 'seconds', 'action', 'idea', 'prompt', 'camera', 'framing', 'sound', 'music', 'takes', 'seed',
- 'continue', 'first', 'references', 'cast', 'lines')
+ 'continue', 'first', 'references', 'cast', 'lines', 'cuts', 'continuity')
LINE_FIELDS = ('who', 'line', 'how', 'language')
@@ -101,7 +101,7 @@ def _fields(play, problems):
problems.append(dict(path=prefix + name, message='is not a field here; use ' + ', '.join(known)))
-def plan(request, *, fields=True):
+def plan(request, *, fields=True, single_seconds=drama.BLOCK_SECONDS):
"""Return {'request': normalized copy, 'preview': {'shots': [...], 'total': N}}.
fields=False rechecks a saved screenplay, which may predate the unknown-field rule."""
@@ -112,7 +112,7 @@ def plan(request, *, fields=True):
play = deepcopy(request.get('screenplay'))
play_ok = True
try:
- drama.check(play)
+ drama.check(play, single_seconds)
except drama.Screenplay as error:
play_ok = False
problems.extend(dict(path='/screenplay' + p['path'], message=_rule(p)) for p in error.problems)
@@ -403,9 +403,9 @@ def _commit(state, draft, now):
state.update(draft)
-def new(identity, request, *, now):
+def new(identity, request, *, now, single_seconds=drama.BLOCK_SECONDS):
"""Create a JSON receipt. Shot indices start at 1; generations start at 0."""
- request = plan(request)['request']
+ request = plan(request, single_seconds=single_seconds)['request']
if not isinstance(identity, str) or not spec.SLUG.fullmatch(identity):
_invalid([dict(path='/id', message='Production id must be a unique short name')])
state = dict(version=1, id=identity, title=request['screenplay'].get('title'), revision=1,
@@ -583,7 +583,7 @@ def choose(state, shot, take, *, by, reason=None, cascade=True, now):
_commit(state, draft, now)
-def reshoot(state, shot, *, by, note=None, changes=None, takes=1, reason=None, now):
+def reshoot(state, shot, *, by, note=None, changes=None, takes=1, reason=None, now, single_seconds=drama.BLOCK_SECONDS):
"""Reserve a fresh generation; keep all previous non-stale takes selectable."""
_editable(state, by)
draft = deepcopy(state)
@@ -604,7 +604,7 @@ def reshoot(state, shot, *, by, note=None, changes=None, takes=1, reason=None, n
source.update(deepcopy(changes or {}))
# Recheck the complete effective screenplay, transactionally.
plan(dict(screenplay=draft['screenplay'], takes_per_shot=draft['takes_per_shot'], director=draft['director']),
- fields=False)
+ fields=False, single_seconds=single_seconds)
_schedule(draft, row, takes, by, note)
draft['cancelled'], draft['current_join'] = False, None
_decision(draft, now, by, 'reshoot', shot, reason=reason)
diff --git a/freevideo_engine/agent/productions.py b/freevideo_engine/agent/productions.py
index 3c9f7d25..0b9ca030 100644
--- a/freevideo_engine/agent/productions.py
+++ b/freevideo_engine/agent/productions.py
@@ -24,6 +24,8 @@
from . import drama, production, schema, spec, errors, operation_receipts, take_qa
from .jobs import DONE, JobError, iso, parts_problems
from .forecast import generation_estimate
+from .length import WAIT_SECONDS, capability_table, single_frame_limit
+from .. import chain_plan
SHEET_LEAST, SHEET_MOST = 6, 24 # frames per take when the caller gives no count: one per second
@@ -31,6 +33,7 @@
MAX_CHILDREN = 60 # Includes assembly, rejected submissions and resumed attempts.
MAX_WAITERS = 32 # Long GET waits served at once; more callers receive the current state.
MAX_WAIT = 25
+READY_SECONDS = 20. # A long shot that needs the first table after a start waits this long more, then is asked to retry.
REQUEST_ID = re.compile(r'r-[0-9a-f]{32}')
_NAME = re.compile(r'[A-Za-z0-9_-]{1,64}')
_POINTER = re.compile(r'(?:/(?:[A-Za-z0-9_-]{1,64}|\?))*|/')
@@ -173,8 +176,8 @@ def __init__(self, jobs):
self._waiter_lock = threading.Lock()
# gate(action, *, by, estimate_seconds, refs, question) raises when the user must approve first.
self.gate = None
- # transcribe(path) -> heard text, optional; without it the QA does not compare dialogue.
- self.transcribe = None
+ # transcribe(path) -> heard text; a machine without a speech model gets a note instead of a comparison.
+ self.transcribe = take_qa.recognise
self._inspecting = set()
self.load()
jobs.parent_allowed = self.allows
@@ -206,7 +209,7 @@ def load(self):
raise ValueError('Invalid submission marker')
production.view(state)
try:
- checked = production.plan(request, fields=False)
+ checked = production.plan(request, fields=False, single_seconds=math.inf) # saved: the limit held when it was accepted
if [s['id'] for s in state['shots']] != [s['id'] for s in checked['preview']['shots']]:
raise JobError('internal_error', 'Saved shots differ from the current plan')
production.advance(deepcopy(state), deepcopy(record['children']), now=state['updated_at'])
@@ -366,9 +369,46 @@ def _material(self, state, *, local=True):
if problems:
_domain(production._invalid, problems)
+ def _single_seconds(self, request):
+ """The longest shot this machine generates at once for the film's size and quality; unknown keeps 15 s."""
+ play = request.get('screenplay') if isinstance(request, dict) else None
+ try:
+ probe = dict(prompt='x', aspect=play.get('aspect', '9:16'), quality=play.get('quality', 'light'), seconds=15)
+ if play.get('megapixels'):
+ probe['megapixels'] = play['megapixels']
+ probe = spec.normalize(probe, port=self.jobs.port)
+ # Pictures and cards make the shot a reference generation, which has its own limit.
+ table = capability_table(self.jobs.port, probe, wait=WAIT_SECONDS, task='ref2va')
+ waiting = table.get('status') == 'computing' and drama.needs_machine_limit(play)
+ if waiting:
+ table = capability_table(self.jobs.port, probe, wait=READY_SECONDS, task='ref2va')
+ waiting = table.get('status') == 'computing'
+ if not waiting:
+ return chain_plan.single_seconds(single_frame_limit(probe, table=table))
+ except Exception: # A film that does not normalize is reported by its own checks.
+ return chain_plan.BLOCK_SECONDS
+ # A limit not worked out yet is not a low limit: the same request can succeed in a moment.
+ raise JobError('capabilities_computing', 'Capabilities are being worked out',
+ retry_after_seconds=table.get('retry_after_seconds'))
+
+ def _reshoot_seconds(self, identity, shot, changes):
+ """The limit for a shot that changes; a reshoot with no changes keeps what was accepted. Call without the lock."""
+ if not changes or not isinstance(changes, dict):
+ return math.inf
+ with self.changed:
+ state = deepcopy(self._record(identity)['state'])
+ try:
+ state['screenplay']['shots'][production._shot(state, shot)['index'] - 1].update(changes)
+ except production.ProductionError: # production.reshoot reports it.
+ return math.inf
+ return self._single_seconds(dict(screenplay=state['screenplay']))
+
def plan(self, request, *, local=True):
- result = _domain(production.plan, {k: v for k, v in request.items() if k != 'ref'})
- state = _domain(production.new, 'dplanpreview', result['request'], now=iso(self.jobs.clock()))
+ single_seconds = self._single_seconds(request)
+ result = _domain(production.plan, {k: v for k, v in request.items() if k != 'ref'},
+ single_seconds=single_seconds)
+ state = _domain(production.new, 'dplanpreview', result['request'], now=iso(self.jobs.clock()),
+ single_seconds=single_seconds)
play = result['request']['screenplay']
with self.changed:
cards = self._cards(play)
@@ -382,7 +422,7 @@ def plan(self, request, *, local=True):
row.update(cast_reference=resolved['cast_reference'], cast=resolved['people'],
cast_omitted=resolved['omitted'], cast_voice_omitted=resolved['voice_omitted'],
cast_missing=resolved['missing'], cast_pictures_each=pictures_each(resolved),
- references=len(resolved['references']))
+ references=len(resolved['references']) + resolved['own_pictures'])
try:
spec._text(row['prompt'], '/screenplay/shots/%d/prompt' % i, 20000)
except spec.SpecError as error:
@@ -411,7 +451,7 @@ def _take_estimate(self, state, shot, take, item=None):
resolved = self._carry(state['screenplay'], state['screenplay']['shots'][shot['index'] - 1],
self._current(state['screenplay']), item, continues=shot['continues'])
if resolved['references']:
- item['media'] = dict(references=deepcopy(resolved['references']))
+ item['media'] = dict(references=self._own(item) + deepcopy(resolved['references']))
estimate, revision = generation_estimate(self.jobs.port, spec.normalize(item, port=self.jobs.port))
return dict(estimate, revision=revision)
except Exception: # Timing is optional; a forecast failure must not reject a film or job.
@@ -449,7 +489,8 @@ def _join_estimate(self, state, previous=None):
shapes.append([normalized[k] for k in ('width', 'height', 'frames')])
current = next((j for j in state['joins'] if j['ref'] == state['current_join']), None)
transition = current['transition'] if current else state['screenplay'].get('transition', 'cut')
- signature = dict(shapes=shapes, transition=transition)
+ seconds = state['screenplay'].get('transition_seconds', .5)
+ signature = dict(shapes=shapes, transition=transition, transition_seconds=seconds)
if previous and previous.get('signature') == signature and previous.get('ref') == state['current_join']:
return previous
samples = []
@@ -460,7 +501,7 @@ def _join_estimate(self, state, previous=None):
if saved.get('transition', 'cut') != transition or saved.get('width') or saved.get('height'):
continue
if saved.get('audio', True) is not True or (
- transition == 'crossfade' and saved.get('transition_seconds', .5) != .5):
+ transition == 'crossfade' and saved.get('transition_seconds', .5) != seconds):
continue
if any(set(clip) != {'from_job'} for clip in saved['clips']):
continue
@@ -592,12 +633,13 @@ def start(self, request, *, by='agent', ref=None, local=True, client=None):
if ref and alias and ref != alias:
raise JobError('idempotency_conflict', 'Conflicting keys')
ref = ref or alias
+ single_seconds = self._single_seconds(request) # May wait, so not under the lock.
with self.changed:
digest, replay = self._receipt('start', dict(request=request, by=by), ref, client)
if replay is not None:
return replay
identity = 'd' + uuid.uuid4().hex[:12]
- state = _domain(production.new, identity, request, now=iso(self.jobs.clock()))
+ state = _domain(production.new, identity, request, now=iso(self.jobs.clock()), single_seconds=single_seconds)
# new() only takes director, not the initiating actor. Attribution
# belongs to this boundary, including the future in-process UI.
for decision in state['decisions']:
@@ -746,7 +788,7 @@ def list(self, *, status='all', limit=50):
paused=view['paused'], stalled=view['stalled'], session=record.get('session'), error_zh=view['error_zh'], error_en=view['error_en']))
return result
- def _mutate(self, action, identity, *, by, ref=None, client=None, **values):
+ def _mutate(self, action, identity, *, by, ref=None, client=None, limit=None, **values):
with self.changed:
record = self._record(identity)
if action in ('choose', 'reshoot', 'finish'):
@@ -758,7 +800,7 @@ def _mutate(self, action, identity, *, by, ref=None, client=None, **values):
return replay
draft = deepcopy(record)
state = draft['state']
- _domain(getattr(production, action), state, by=by, now=iso(self.jobs.clock()), **values)
+ _domain(getattr(production, action), state, by=by, now=iso(self.jobs.clock()), **values, **(limit or {}))
# The next advance is part of this operation: if it cannot run, nothing changes.
self._preflight(record, draft)
self._prepare_estimates(draft)
@@ -786,7 +828,8 @@ def choose(self, identity, shot, take, *, by='agent', ref=None, **kwargs):
return self._mutate('choose', identity, shot=shot, take=take, by=by, ref=ref, **kwargs)
def reshoot(self, identity, shot, *, by='agent', ref=None, **kwargs):
- return self._mutate('reshoot', identity, shot=shot, by=by, ref=ref, **kwargs)
+ seconds = self._reshoot_seconds(identity, shot, kwargs.get('changes'))
+ return self._mutate('reshoot', identity, shot=shot, by=by, ref=ref, limit=dict(single_seconds=seconds), **kwargs)
def finish(self, identity, *, by='agent', ref=None, **kwargs):
return self._mutate('finish', identity, by=by, ref=ref, **kwargs)
@@ -970,7 +1013,8 @@ def _execute(self, identity, action):
raise JobError('production_limit', 'Too many child jobs')
if 'join' in action:
item = dict(kind='assemble', clips=[children[r]['id'] for r in item['takes']],
- transition=item['transition'])
+ transition=item['transition'],
+ transition_seconds=record['state']['screenplay'].get('transition_seconds', .5))
else:
take = next(t for s in record['state']['shots'] for t in s['takes'] if t['job'] == ref)
if take['generation'] > MAX_GENERATION:
@@ -1025,12 +1069,18 @@ def _cast(self, state, take, item):
if resolved['cast_reference'] is None:
return resolved
if resolved['references']:
- item['media'] = dict(references=deepcopy(resolved['references']))
+ item['media'] = dict(references=self._own(item) + deepcopy(resolved['references']))
+ if resolved['prefix']:
item['prompt'] = drama.with_cast(item['prompt'], resolved)
item['metadata'] = dict(item.get('metadata') or {}, cast=dict(resolved['cards']),
cast_reference=resolved['cast_reference'])
return resolved
+ @staticmethod
+ def _own(item):
+ """The pictures the shot itself names; the cast cards are added after them."""
+ return deepcopy(((item or {}).get('media') or {}).get('references') or [])
+
def _carry(self, play, shot, cards, item, *, continues):
"""The cast a shot carries: each person's sheet and face close-up on every device (one picture a person when
a continuing shot has several people). Plan previews and submissions both come here, so they agree."""
diff --git a/freevideo_engine/agent/projects.py b/freevideo_engine/agent/projects.py
index 803cd886..d12a4842 100644
--- a/freevideo_engine/agent/projects.py
+++ b/freevideo_engine/agent/projects.py
@@ -419,6 +419,24 @@ def music(self, identity, value, *, jobs, ref=None, client=None):
self._remember(ref, digest, result, client)
return result
+ def finish_sound(self, identity, value, *, jobs, ref=None, client=None):
+ """Queue one job that finishes the sound of the cut the caller read; a stale revision starts nothing."""
+ schema.validate(schema.project_schema('finish_sound'), value)
+ with self.listen_lock:
+ digest, replay = self._receipt('projects:finish_sound', dict(id=identity, body=value), ref, client)
+ if replay is not None:
+ return replay
+ stored = self._load(identity)['project']
+ if stored['revision'] != value['revision']:
+ raise ProjectError('project_changed', project=stored)
+ if not comfy_editor.duration(stored):
+ raise _invalid('/revision', 'a revision whose timeline has clips', value['revision'])
+ job = jobs.submit(dict(kind='finish_sound', editor_project=identity,
+ **{k: v for k, v in value.items() if k != 'ref'}), client=client)[0]
+ result = dict(job=jobs.view(job), jobs=[jobs.view(job)])
+ self._remember(ref, digest, result, client)
+ return result
+
def stems(self, identity, value, *, jobs, ref=None, client=None):
"""Queue a sound-splitting job for one sound clip of the revision the caller read."""
schema.validate(schema.project_schema('stems'), value)
diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py
index 1df729cd..9810b2f7 100644
--- a/freevideo_engine/agent/routes.py
+++ b/freevideo_engine/agent/routes.py
@@ -172,7 +172,7 @@ def submit_batch(jobs, body, client):
return jobs.submit_request(body, client=client)[0]
-JOB_OPERATIONS = ('generate', 'export', 'listen', 'music', 'stems', 'voice_lines', 'voice_replace')
+JOB_OPERATIONS = ('generate', 'export', 'listen', 'music', 'stems', 'finish_sound', 'voice_lines', 'voice_replace')
def stems_supported(machine):
@@ -181,18 +181,25 @@ def stems_supported(machine):
return supported(machine.get('vram_gib'), uuid=machine.get('gpu_uuid')) and machine.get('device_backend', 'cuda') == 'cuda'
-def voice_ready(port):
- """Whether the dubbing model that changes a voice is on this machine."""
+def voice_state(port):
+ """(model ready, model can be downloaded here): the dubbing model is on this machine, or this machine could install it."""
service = getattr(port.server, '_freevideo_dubbing', None)
if service is None:
- return False
+ return False, False
try:
- return bool(service.info(wait=False).get('ready'))
+ info = service.info(wait=False)
except (OSError, ValueError):
- return False
+ return False, False
+ ready = bool(info.get('ready'))
+ return ready, not ready and bool(info.get('supported')) and bool(info.get('published', True))
+
+
+def voice_ready(port):
+ """Whether the dubbing model that changes a voice is on this machine."""
+ return voice_state(port)[0]
-def features(continuation, vlm, stems=False, voice=True):
+def features(continuation, vlm, stems=False, voice=True, voice_install=False):
"""What an agent can use here (ready says whether the engine can run now). A missing name is not offered."""
from .editor_runtime import available
rules = available()
@@ -203,8 +210,11 @@ def features(continuation, vlm, stems=False, voice=True):
'queue.shared': True, 'queue.busy': True, 'chain.auto': bool(continuation),
'enhance': bool(vlm.get('ready')), 'drama.film': True, 'images.edit': rules,
'images.layers': rules, 'images.heal': rules, 'images.render': rules, 'projects.read': True,
- **{'projects.' + name: rules for name in ('insert', 'split', 'move', 'trim', 'remove', 'export', 'generate', 'listen', 'music')},
- 'projects.stems': bool(rules and stems), 'projects.voice': bool(rules and stems and voice)}
+ **{'projects.' + name: rules for name in ('insert', 'split', 'move', 'trim', 'remove', 'export', 'generate', 'listen', 'music',
+ 'finish_sound')},
+ 'projects.stems': bool(rules and stems), 'projects.voice': bool(rules and stems and voice),
+ # The machine could change a voice but the dubbing model is not there yet: the page offers its download.
+ 'projects.voice.install': bool(rules and stems and voice_install and not voice)}
def whitebox_support(jobs):
@@ -233,6 +243,7 @@ def capabilities(port, jobs):
vlm = {k: v for k, v in service.info(wait=False).items() if k in ('ready', 'model', 'license', 'bytes', 'busy')}
except (OSError, ValueError):
vlm = dict(ready=False)
+ voice = voice_state(port)
running, pending = port.snapshot()
from ..chain_plan import BLOCK_SECONDS
# One block is the limit when the backend cannot continue a video; plan and submit refuse the same requests.
@@ -242,7 +253,7 @@ def capabilities(port, jobs):
return dict(
api_version=API_VERSION, freevideo_version=release or __version__, build=__version__, ready=ready, problem=problem,
continuation=continuation,
- features=features(continuation, vlm, stems=stems_supported(machine), voice=voice_ready(port)),
+ features=features(continuation, vlm, stems=stems_supported(machine), voice=voice[0], voice_install=voice[1]),
whitebox=whitebox_support(jobs),
device=dict(backend=machine.get('device_backend', 'cuda') if machine else None,
vram_limit_gib=machine.get('vram_gib'), ram_limit_gib=machine.get('ram_gib')),
@@ -255,6 +266,7 @@ def capabilities(port, jobs):
'listen': 'Listen to the sound of an editing project for subtitle lines and times',
'music': 'Compose music for an editing project and lay it on its second sound track',
'stems': 'Split the sound of a clip into dialogue, music and ambience on sub-tracks',
+ 'finish_sound': 'Finish the sound of a joined cut: one score, even loudness, room tone',
'voice_lines': 'Write down the spoken lines of a sound clip with their start and end',
'voice': 'Say edited lines again in the original voice and put them back where the originals were',
'image_edit': 'Render a saved revision of an image session to a picture'},
diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py
index 1adc468e..e272e1b5 100644
--- a/freevideo_engine/agent/schema.py
+++ b/freevideo_engine/agent/schema.py
@@ -139,8 +139,8 @@ def media(image=False, *, local=False, clip=False, video=False, early=False):
'A reference video only: it guides the first pass for this many steps and is dropped afterwards, '
'including from the second pass. Omitted: the video guides every step.')
properties['voice_only'] = {'type': 'boolean', 'description':
- 'A reference video used only for its voice: the picture is encoded small and at a low frame rate, '
- 'the sound is unchanged. Not with early_steps.'}
+ 'A reference video used only for its voice: only its sound track is read, as an audio reference, and no '
+ 'picture is encoded. A clip without sound is refused. Not with early_steps.'}
if clip:
properties.pop('frame')
properties.pop('edit')
@@ -234,7 +234,7 @@ def generate(*, local=False):
def assemble(*, local=False):
return obj(dict(kind={'const': 'assemble', 'type': 'string'},
clips=array({'anyOf': [JOB, SAVED, media(local=local, clip=True)]}, 1, 200),
- transition=enum(('cut', 'crossfade'), default='cut'), transition_seconds=number(.04, 5, default=.5),
+ transition=enum(('cut', 'crossfade'), default='cut'), transition_seconds=number(1 / 24, 3, default=.5),
width=nullable(number(64, 4096, integer=True, multipleOf=2)), height=nullable(number(64, 4096, integer=True, multipleOf=2)),
audio={'type': 'boolean', 'default': True}, name=SLUG, **common()), ('kind', 'clips'),
**{'x-job-kind': 'assemble'})
@@ -355,6 +355,22 @@ def stems(*, local=False):
**{'x-job-kind': 'stems'})
+SOUND_FIELDS = dict(
+ music={'oneOf': [obj(dict(caption=MUSIC_FIELDS['caption'], seed=MUSIC_FIELDS['seed']), ('caption',)),
+ obj(dict(source=EDITOR_SOURCE), ('source',))],
+ 'description': 'For an empty a2: a caption to compose, or a source to lay. Omit for no music.'},
+ keep_dialogue_volume={'type': 'boolean', 'default': False})
+
+
+def finish_sound(*, local=False):
+ return obj(dict(kind={'const': 'finish_sound', 'type': 'string'},
+ editor_project=string(34, pattern=r'^e-[0-9a-f]{32}$'),
+ revision=number(1, 2**53 - 1, integer=True),
+ **SOUND_FIELDS, **common()), ('kind', 'editor_project', 'revision'),
+ description='Finish the sound of a joined cut: split, one score on a2, even loudness at -18 LUFS, room tone.',
+ **{'x-job-kind': 'finish_sound'})
+
+
VOICE_SELECTION = obj(dict(clip_id=string(64), start=number(0, 36000, description='Seconds from the clip start; default 0.'),
end=number(0, 36000, description='Seconds from the clip start; default the clip end.')), ('clip_id',),
description='The sound clip, or part of it, whose voice to change: the original sound, or its dialogue sub-track.')
@@ -366,8 +382,11 @@ def stems(*, local=False):
('id', 'start', 'end', 'text', 'original_text'))
VOICE_REFERENCE = obj(dict(clip_id=string(64), start=number(0, 36000), end=number(0, 36000), text=string(1000)),
('clip_id', 'start', 'text'), description='A stretch of a sound clip whose voice every line takes.')
+VOICE_SCRIPT = array(obj(dict(text=string(2000), speaker=string(64, description='Who says it; the same name is the same number in the result.')),
+ ('text',)), 1, 400,
+ description='What the selection is known to say, in order. A recognised line that is one of these says it as written, and the lines are numbered by who says them.')
VOICE_LINES_FIELDS = dict(
- selection=VOICE_SELECTION, language=VOICE_LANGUAGE,
+ selection=VOICE_SELECTION, language=VOICE_LANGUAGE, script=VOICE_SCRIPT,
separate_first={'type': 'boolean', 'default': False,
'description': 'Separate the sound first and recognise on the dialogue alone.'})
VOICE_FIELDS = dict(
@@ -417,12 +436,13 @@ def image_edit(*, local=False):
KINDS = dict(generate=generate, assemble=assemble, vlm=vlm, enhance=enhance, image=image,
- export=export, listen=listen, music=music, stems=stems, voice_lines=voice_lines, voice=voice, image_edit=image_edit)
+ export=export, listen=listen, music=music, stems=stems, finish_sound=finish_sound, voice_lines=voice_lines, voice=voice,
+ image_edit=image_edit)
def job_schema(*, local=False, refs=False, batch=False):
# Music scores a joined cut, so it is not a storyboard row.
- value = {'oneOf': [f(local=local) for kind, f in KINDS.items() if not (batch and kind in ('music', 'stems', 'voice_lines', 'voice'))], 'x-discriminator': 'kind',
+ value = {'oneOf': [f(local=local) for kind, f in KINDS.items() if not (batch and kind in ('music', 'stems', 'finish_sound', 'voice_lines', 'voice'))], 'x-discriminator': 'kind',
'description': 'x-discriminator selects field diagnostics by kind; all accepted branches use standard oneOf.'}
return batch_references(value) if refs else value
@@ -539,7 +559,7 @@ def route(summary, body=None, query=None, **kw):
if method == 'GET':
extra['query'] = shape
else:
- extra.update(body=shape, idempotent=True, **({'status': 201} if name in ('create', 'generate', 'export', 'listen', 'music', 'stems', 'voice_lines', 'voice_replace') else {}))
+ extra.update(body=shape, idempotent=True, **({'status': 201} if name in ('create', 'generate', 'export', 'listen', 'music', 'stems', 'finish_sound', 'voice_lines', 'voice_replace') else {}))
routes[method, suffix] = route(summary, **extra)
for name, (method, suffix, summary) in WHITEBOX_OPERATIONS.items():
extra = dict(path=obj({'id': WHITEBOX}, ('id',))) if '{id}' in suffix else {}
@@ -616,9 +636,11 @@ def route(summary, body=None, query=None, **kw):
auth=False, page_or_token=True)
routes['PUT', '/confirmations'] = route('Only the page user may change which operations need approval.',
body=confirm_settings(), auth=False, page_or_token=True)
- # The editor page separates sound itself: these four also accept its page session, which never holds the token.
+ # The editor page separates sound and changes voices itself: these also accept its page session, which never holds the token.
for key in (('POST', '/projects/{id}/stems'), ('POST', '/projects/{id}/stems/export'),
- ('GET', '/jobs/{id}'), ('POST', '/jobs/{id}/cancel')):
+ ('GET', '/jobs/{id}'), ('POST', '/jobs/{id}/cancel'), ('GET', ''),
+ ('POST', '/projects/{id}/voice/lines'), ('POST', '/projects/{id}/voice/replace'),
+ ('POST', '/projects/{id}/voice/shorten'), ('POST', '/projects/{id}/voice/estimate')):
routes[key]['page_ok'] = True
return routes
@@ -935,11 +957,12 @@ def image_schema(name):
'voice_shorten': ('POST', '/projects/{id}/voice/shorten', 'Shorten one line to fit a number of seconds. The answer is a text; nothing is saved or spoken.'),
'voice_estimate': ('POST', '/projects/{id}/voice/estimate', 'Estimate the seconds the changed lines take to make. The answer comes at once.'),
'music': ('POST', '/projects/{id}/music', 'Compose music as long as the joined cut and lay it on sound track a2 at frame 0, under the video sound. First get the revision; a2 must be empty. Wait for the returned job; the first use downloads the model (20 GB).'),
+ 'finish_sound': ('POST', '/projects/{id}/finish_sound', 'Finish the sound of a joined cut in one job: split each clip\'s sound where offered, one score on a2, even dialogue, -18 LUFS, room tone. First get the revision; wait for the job.'),
}
PROJECT_TOOLS = dict(list='freevideo_projects', get='freevideo_project_read', create='freevideo_project_create',
write='freevideo_project_write', operations='freevideo_editor_operations',
generate='freevideo_project_generate', export='freevideo_export',
- listen='freevideo_project_listen', music='freevideo_project_music',
+ listen='freevideo_project_listen', music='freevideo_project_music', finish_sound='freevideo_project_finish_sound',
stems='freevideo_project_stems', stems_export='freevideo_project_stems_export',
voice_lines='freevideo_project_voice_lines', voice_replace='freevideo_project_voice_replace',
voice_shorten='freevideo_project_voice_shorten', voice_estimate='freevideo_project_voice_estimate')
@@ -1063,6 +1086,8 @@ def project_schema(name):
return obj(dict(revision=revision, **LISTEN_FIELDS, ref=REF), ('revision', 'mode'))
if name == 'music':
return obj(dict(revision=revision, **MUSIC_FIELDS, ref=REF), ('revision', 'caption'))
+ if name == 'finish_sound':
+ return obj(dict(revision=revision, **SOUND_FIELDS, ref=REF), ('revision',))
if name in ('stems', 'stems_export'):
return obj(dict(revision=revision, **STEMS_FIELDS, **(STEMS_LAY if name == 'stems' else {}), ref=REF), ('revision', 'clip_id'))
if name == 'voice_lines':
diff --git a/freevideo_engine/agent/skill/freevideo/SKILL.txt b/freevideo_engine/agent/skill/freevideo/SKILL.txt
index bf1211f1..4b747e6a 100644
--- a/freevideo_engine/agent/skill/freevideo/SKILL.txt
+++ b/freevideo_engine/agent/skill/freevideo/SKILL.txt
@@ -16,17 +16,32 @@ FreeVideo runs on the user's own computer. Everything below goes through its MCP
## Pick the right task
-- One video, up to 60 seconds: `freevideo_generate`. Longer than the device can do in one run is cut into parts and joined automatically.
+- One video, up to 60 seconds: `freevideo_generate`. Longer than the device can do in one run is cut into parts and joined automatically. The device limit is a hard cap, not a target: aim for 15 to 20 seconds per generation.
- Start from or end on a picture: pass it as first or last frame. Make the picture first with `freevideo_generate_image`.
-- A film of several shots to choose between: the `freevideo_drama_*` tools. See `references/film.md`.
+- A story, scene or short drama: follow "Making a short drama" below. Several shots to choose between: the `freevideo_drama_*` tools; see `references/film.md`.
- Existing footage on a timeline: the editing project tools. See `references/edit.md`.
- A picture to change: `freevideo_edit_image` for a quick change, the `freevideo_image_*` tools for layered edits. See `references/image.md`.
-- Several independent jobs: `freevideo_submit_batch`.
+- Several independent jobs, or storyboard shots that do not depend on each other: put all of them in a single `freevideo_submit_batch` call, one job per shot with its own `ref`, then join the results in order. A shot that starts from the previous shot's last frame waits until that shot is done. Shots cut inside one generation with `[Shot N]` markers are one job.
+- What this computer can make: read the limits from `freevideo_capabilities` and report them as given (maximum width and height, shortest and longest seconds, frame rate). Do not answer from memory or round them.
+
+## Making a short drama
+
+Step one is the story, before any generation: a one-page outline with the lead's goal, the opponent's moves, the escalation, the low point, the reversal and the ending, each beat causing the next. The user confirms it; only then write dialogue and generate. Famous scenes may decorate a story, never replace it.
+
+One generation or several shots (the recommended length decides, not the device cap; `freevideo_capabilities` reports the cap as given):
+1. One `freevideo_generate` with `[Shot N]` cuts inside it (four to six cuts) when the whole piece is about 20 seconds or less, stays in one place, has at most three speakers and needs no shot to be chosen or reshot separately.
+2. Otherwise the `freevideo_drama_*` tools, even when this computer can make 60 seconds in one run. Each shot is one generation of 15 to 20 seconds, cut inside with `[Shot N]`; a shot with a cast card or reference pictures is at most 15 seconds, and longer content goes into the next shot with continue=true; a plan that is refused for length is shortened to that limit, never forced; never write 3 to 6 second shots. When the computer's limit is below 15 seconds, use that limit. A continuous action across two beats (a kick and its catch, a handover) stays in the same shot. Move from shot to shot through the story (someone arrives, looks, answers), not by forcing the last frame onto the next shot. Independent shots go in one batch; a shot that continues the previous one waits for it.
+3. When unsure, choose the drama tools.
+
+The short version, for every model (a small model can rely on point 7 alone): (1) story outline first; (2) about 20 seconds or less in one place with up to three people is one generation, longer is the drama tools; (3) cuts of 2 to 4 seconds in `[Shot N]`; (4) six sections in order: `subject_definitions:`, `summary:`, `retention_analysis:`, `detailed_description:`, `overall_soundscape:`, `non_diegetic_music:`; (5) people tagged `` with speakers numbered (S1), (S2) by first spoken line in that video; (6) say what is there and keep negation words out; (7) with the `freevideo_drama_*` tools, fill the structure fields (characters, and per shot the cuts with `at`, `framing`, `action`, `camera`, `lines`) and leave `prompt` empty: the tool writes the six sections and the (S#) numbers itself; (8) the full example and the rest are in `references/film-script.md`.
+
+Before each generation: one card per person (`freevideo_cast_list` first), clothes fixed for the whole film, a seating or standing plan for every beat, only what is seen and heard in the prompt (say what is there, never what is not), dialogue word for word, speaker numbers by order of first line in the video. Rules and examples for each of these, and the five faults to check before generating (stray lettering, small faces in wide shots, text under 14 pixels, third-party marks, voice references), are in `references/film-script.md`; the rest of the checks are in `references/film.md`. Generate two seeds of the first shot, look at them at full size, and only then continue. Whoever makes a shot looks at every take before choosing it.
## Running jobs
- A job takes minutes. Call `freevideo_wait` with the job id; it returns after at most 25 seconds. If the job is still active, call it again with the same id. Never submit the same job twice.
-- A failed or interrupted generation is continued with `freevideo_resume`; only unfinished parts run.
+- A failed or interrupted generation is continued with `freevideo_resume`; only unfinished parts run. When a generation fails with `retryable` true, resume it once yourself; if it fails again, tell the user what happened.
+- An `invalid_request` about size or length lists what is allowed. Resubmit once with the legal value nearest to what the user asked for (on a tie, the smaller one) and change nothing else; then tell the user which size was used.
- `freevideo_cancel` only for a job you started and the user wants stopped.
- Look at a result before reporting it: `freevideo_look` (a frame or a contact sheet).
- Report what you made with the saved-video id (`freevideo_library`), not a local path.
diff --git a/freevideo_engine/agent/skill/freevideo/references/edit.txt b/freevideo_engine/agent/skill/freevideo/references/edit.txt
index 18dfcf41..2c0f5852 100644
--- a/freevideo_engine/agent/skill/freevideo/references/edit.txt
+++ b/freevideo_engine/agent/skill/freevideo/references/edit.txt
@@ -5,7 +5,9 @@ Projects are the timelines shared with the FreeVideo editor page.
1. `freevideo_projects` lists them; `freevideo_project_create` starts an empty one.
2. Footage from the user's disk: `freevideo_import_media`, then poll `freevideo_imports` until `ready` (`failed` is final). Insert the returned `import:` source.
3. Always `freevideo_project_read` first. Edits carry the revision and clip ids you just read. On a `project_changed` error, read again and recompute; do not retry the old edit.
+ To trim a clip to its first N seconds, cut its end edge at frame (N seconds × the project frame rate); clips linked to it (its sound) go in the same operation.
4. `freevideo_editor_operations` applies 1 to 64 edits atomically. Prefer it to `freevideo_project_write`, which replaces the whole timeline.
5. `freevideo_project_generate` makes new picture and sound for selected clips or a frame range, as the Generate button does.
6. Subtitles: `freevideo_project_listen` with mode `align` times lines you give, `transcribe` writes down what is said.
-7. `freevideo_export` renders one saved revision to MP4 in the saved videos. Wait for the returned job.
+7. `freevideo_project_finish_sound` finishes the sound of a joined cut in one job: one score on a2, even dialogue, -18 LUFS, room tone. Wait for the returned job.
+8. `freevideo_export` renders one saved revision to MP4 in the saved videos. Wait for the returned job.
diff --git a/freevideo_engine/agent/skill/freevideo/references/film-script.txt b/freevideo_engine/agent/skill/freevideo/references/film-script.txt
new file mode 100644
index 00000000..26a1930c
--- /dev/null
+++ b/freevideo_engine/agent/skill/freevideo/references/film-script.txt
@@ -0,0 +1,92 @@
+# Writing a short drama: script, blocking, continuity, casting, prompt
+
+First version. Each part gives the few rules that matter most and one example. Every example is invented for this file.
+
+## Script
+
+- One page before any prompt: who wants what, who stands in the way, how it gets worse, the low point, the reversal, the ending. Each beat is caused by the one before it; "and then" is not a cause.
+- One beat is one thing the audience learns or one choice a person makes. A beat that does neither is cut.
+- Dialogue is short and carries the conflict: a person asks, refuses, lies or gives in. No line explains what the picture already shows.
+- Amount of speech (estimate, to be measured): about 2 spoken words per second in English and about 4 characters per second in Chinese, counting pauses. A 20 second generation holds about 30 words, three to five lines. More than that gets rushed or cut off.
+- The last beat of a generation ends on a look, a door or a held line, so the next shot has something to start from.
+
+Example, a 20 second piece in one generation: a girl wants the last seat on the night bus. Cut 1: she runs, the doors are closing. Cut 2: a man in the seat sees her and pulls his bag onto it. Cut 3: she says "Please." Cut 4: he says nothing, then moves the bag. Cut 5: she sits; the bus pulls away. Two spoken lines in all, about four words.
+
+## Blocking
+
+- Write a plan for every beat: who sits or stands where, who faces whom, where the camera is. Left and right are the camera's left and right.
+- Two people who talk keep the same sides in every cut; the camera stays on one side of the line between them.
+- Name the camera for each cut: wide, over the shoulder, close-up, and whether it moves.
+
+Example: Beat 2, the kitchen. Mia stands at the sink, left of frame, back to the window. Dad sits at the table, right of frame, facing her. Camera behind Dad's shoulder, locked off. Cut to a close-up of Mia from his side, still on the left.
+
+## Continuity across shots
+
+- Clothes, hair, props and time of day that stay the same are written once, in the film's continuity table; the prompt carries that row in every shot. A state that changes during the film (a scar that appears, a lantern that goes out, a bandage) is written in the shot where it holds, as that shot's own continuity row for the person: it replaces the table's row for that shot only. Do not put both states in the table, and keep a changing state out of the person's look in the cast as well; every shot would show both.
+- A new shot starts from what happened, not from the previous last frame: someone arrives, looks, answers. Say where everyone stands at the start.
+- A continuous action across two beats stays in one shot.
+- If the people in one scene look alike, give each a distinct colour of clothing so they are not swapped.
+
+Example: Mia wears a green raincoat in every shot and Dad a grey cardigan. Shot 2 opens with Dad still seated and Mia in the doorway, the mug she picked up in shot 1 in her hand.
+
+## Casting
+
+- One card per person: a clear front picture, and a voice sample of that person only. `freevideo_cast_list` first; reuse a card that matches.
+- Describe each person in the cast and in the prompt the same way, by what is visible: age, build, hair, clothes, one distinguishing item.
+- At most three people in one generation. Speaker numbers (S1), (S2) follow the order in which people first speak in that one video; the drama tools assign them.
+
+Example: (S1) Mia: girl of about ten, black bob, green raincoat, yellow school bag. (S2) Dad: tall man in his forties, short beard, grey cardigan.
+
+## Writing the prompt
+
+Rules, in this order:
+1. Cuts of 2 to 4 seconds each. A 15 to 20 second generation holds four to six cuts. The speaker starts within half a second of the cut. A walk, a door or a turn takes 2 seconds at most.
+2. Six sections in this order, a blank line between them, each starting with its name and a colon: `subject_definitions:`, `summary:`, `retention_analysis:`, `detailed_description:`, `overall_soundscape:`, `non_diegetic_music:`. Write these names exactly. Add no other headings such as [Style] or [Scene].
+3. `subject_definitions:` has one line per reference picture and per voice, tagged ``, ``, `` in the order the references are attached. Say what each looks like and keep that look in every cut. A `` is any reusable thing: a person, a place, a costume or prop, a style, a motion; one subject may draw on several pictures, for example the face from one and the outfit from another. Cite each voice where it is used: ` is the voice-timbre reference for (S1)`. The voice supplies timbre and delivery only; the lines are written fresh for the new video.
+4. `retention_analysis:` has one line per tag: ` (appears in [Shot 1], [Shot 2]): fully_preserved - ...` and `: reference - ...`.
+5. `summary:` opens with the task type in brackets, `[reference generation]`, or `[reference generation + audio reference]` when a voice reference is attached; one or two sentences of visual style (live-action, 2D animation, 3D CG, claymation, watercolor, vintage film) come before `[Shot 1]`. `detailed_description:` holds the cuts. `[Shot 1]` has no time; later cuts start `[Shot 2] At 00:03.500, ...` with times that only increase and the last cut ends at the length of the generation.
+6. Number speakers once, in the order of their first spoken line in this video, and reuse that number everywhere in it; the number is independent of the `` number and `retention_analysis` carries no (S#). Write it in brackets after the tag: ` (S1) says: [English] It is still warm. `. Write (S1), (S2); never "Speaker 1". Quote the line exactly.
+7. Cite each voice reference in the cut where it is active, and say whether it gives timbre or delivery. Every cut states: the shot size and the camera side and height; who is left, right and centre; where the camera is fixed or how it moves; where each person stands or sits and who they face; what the hands do; each face's expression; the clothes again; light and time of day; who is the only speaker; where each reference picture appears in the cut.
+8. Use concrete details in place of mood words: materials, light direction, gesture, texture, distance, sound source. A line that runs across a cut is marked `` on both sides and says the voice continues; a line the video ends in the middle of is marked ``. Say what is there. Write the positive version of every wish: "her face stays still, lips pressed together" in place of a negation; "plain blank walls" in place of "no text"; "everyone else listens with lips closed" in place of "nobody speaks". The words no, not, never, without, nobody and does not stay out of the prompt.
+9. Generate two seeds of the first shot, look at them at full size, and continue only when they are right.
+
+Which form the tools write: a shot with cast pictures or references gets the six sections; a shot from a first frame or without pictures gets the short form `integrated_multimodal_description:`, `overall_soundscape:`, `non_diegetic_music:`. Write the six sections only when you give `prompt` yourself for a picture shot. The alignment of a first or last frame to its second is added by the engine's prompt rewrite; do not write it.
+
+Minimal complete example (a 6 second generation with two cuts; a real one has four to six):
+
+subject_definitions:
+ is the first reference picture: a girl of about ten with a black bob, a green raincoat and a yellow school bag, round face, brown eyes. She looks like this in every cut.
+ is the second reference picture: a tall man in his forties with a short dark beard and a grey cardigan. He looks like this in every cut.
+ is the voice-timbre reference for (S1), a clear young girl's voice.
+ is the voice-timbre reference for (S2), a low calm man's voice.
+
+summary:
+[reference generation + audio reference] Live-action, warm natural light, shallow depth of field. In a warm kitchen at dusk, a girl holds out a mug to her father, who takes it and thanks her.
+
+retention_analysis:
+ (appears in [Shot 1], [Shot 2]): fully_preserved - face, black bob, green raincoat, yellow school bag.
+ (appears in [Shot 1], [Shot 2]): fully_preserved - face, beard, grey cardigan.
+: reference - the target uses the voice timbre of this reference for .
+: reference - the target uses the voice timbre of this reference for .
+
+detailed_description:
+[Shot 1] Medium shot over 's right shoulder, camera at eye level, locked-off tripod camera, the framing stays fixed for the whole shot. (S1) stands left of frame in her green raincoat with her yellow school bag on her back, facing him, and holds a white mug out with both hands. sits at the table on the right of frame, facing her, his grey cardigan in the soft orange light of the window. says warmly, her eyes on him: [English] It is still warm. Exactly as her voice stops, her lips close and stay closed. is the only speaker in this shot; listens with lips closed.
+[Shot 2] At 00:03.000, Close-up of from the left side, locked-off tripod camera. His bearded face fills the right of frame, his grey cardigan at the bottom edge; his right hand takes the white mug from the left edge of frame. His eyes soften and the corners of his mouth lift. (S2) says gently: [English] Thank you, Mia. Exactly as his voice stops, his lips close and stay closed. is the only speaker in this shot; listens with lips closed.
+
+overall_soundscape:
+Quiet kitchen at dusk, a soft clink of the mug, two voices close to the microphone.
+
+non_diegetic_music:
+N/A
+
+## Check before generating
+
+Five faults that came back in real test generations. Check the prompt and the references for each one.
+
+1. Stray lettering. Props and clothes grow unreadable made-up letters: the edge of a laptop, embroidery on a chest, a shop sign, a cup. Write every such surface as plain and unmarked: "a plain matte grey mug", "a smooth navy jacket", "a blank painted shop sign". Letters on screen only when a real picture of that screen is a reference.
+2. Faces in wide shots. A face that is 20 to 30 pixels tall comes out smooth and plastic. A person who matters is in a medium shot or a close-up; the wide shot shows people who do not need to be recognised.
+3. Reading size. Text the audience must read is at least 14 pixels tall in the finished video. Take interface screenshots at 175% to 200% zoom and attach that picture; an 8 pixel line is unreadable and the shot is sent back.
+4. Third-party marks. Write props as neutral and unbranded: "a neutral silver laptop with a smooth lid", "a plain white takeaway box". Brand logos and trademarks stay out of the prompt and out of the references unless the story needs that exact prop.
+5. Voice reference. The first 6 seconds of the voice file carry the timbre: at least 4 of them are the person speaking, and the file does not open with music. When the person's first frame has the mouth hidden (a closed visor or a mask), attach a voice reference whose picture shows the mouth. A video that is attached only for its voice carries `voice_only: true`: its sound track is read as an audio reference and its picture is ignored, so it does not serve as the mouth picture.
+
+Example, fixing fault 1 and 4 together. Before: "Lena opens her silver laptop and drinks from a coffee cup." After: "Lena opens a neutral silver laptop with a smooth lid and lifts a plain matte grey mug."
diff --git a/freevideo_engine/agent/skill/freevideo/references/film.txt b/freevideo_engine/agent/skill/freevideo/references/film.txt
index e2a2f758..0fe5c772 100644
--- a/freevideo_engine/agent/skill/freevideo/references/film.txt
+++ b/freevideo_engine/agent/skill/freevideo/references/film.txt
@@ -1,6 +1,6 @@
# Films of several shots
-Use this only when the film needs separately chosen shots. A single video of up to 60 seconds is one `freevideo_generate` job.
+Use this when the piece runs longer than about 20 seconds, changes place or costume, or needs shots chosen or reshot separately; otherwise one `freevideo_generate` with `[Shot N]` cuts is enough (see "Making a short drama" in SKILL.md). Each shot is one generation of 15 to 20 seconds; the device limit is only a cap, and a shot with a cast card cannot be longer than one generation on this computer.
1. Reuse or make cast cards: `freevideo_cast_list` first (reuse a matching card), then `freevideo_cast_create`; `freevideo_cast_generate` makes reference pictures when the user gave none.
2. `freevideo_drama_plan` previews the shots without starting.
diff --git a/freevideo_engine/agent/sound.py b/freevideo_engine/agent/sound.py
new file mode 100644
index 00000000..4c5d1976
--- /dev/null
+++ b/freevideo_engine/agent/sound.py
@@ -0,0 +1,745 @@
+"""Finishing the sound of a whole cut in one job, after the cut is made.
+
+Five steps, each an ordinary edit that stays visible and changeable on the timeline:
+1. the sound of each picture's clip is split into dialogue, music and ambience (a4-a6), once per source file;
+2. the separated music (a5) of each block goes where it is a sustained score (score_like); separation also puts shots,
+ explosions and crowds there, so a block whose music part is not a score keeps it;
+3. one piece of music lies on a2 from the first frame to the last, at one volume and lowered during speech
+ (a piece already on a2 stays, only its volume changes);
+4. one loudness: the dialogue of each block moves towards the common dialogue level, the ambience follows it down
+ but never up, the music sits MUSIC_UNDER_LU under the dialogue, every volume then moves by the same amount to
+ TARGET_LUFS, and the export holds the true peak under PEAK_DB (the project's master_peak_db);
+5. digital silence in the dialogue and ambience is filled with the block's own room tone (a new ambience file).
+
+Levels are measured on the export's own audio-only render, so speed, fades, crossfades and ducking count as the
+user will hear them. Without sound separation (Mac, AMD, small cards) steps 1, 2 and 5 are skipped."""
+import copy
+import json
+import math
+import secrets
+import shutil
+from pathlib import Path
+
+from .. import comfy_editor
+from .jobs import JobError
+
+FOLDER = 'freevideo-sound'
+RATE = 48000
+SAMPLES_PER_FRAME = RATE // comfy_editor.FPS
+TARGET_LUFS = -18.
+PEAK_DB = -1.
+SWING_DB = 4. # the most one block's dialogue moves towards the common level
+LIMIT_TAKES_DB = 5. # the most the export's limiter is left to take off a block's loudest peak
+MUSIC_UNDER_LU = 16.
+SPEECH_LUFS = -45. # a block whose dialogue is quieter has no speech; it keeps its level
+MAX_VOLUME = 2.
+HOLE_DB, HOLE_SECONDS = -70., .04
+CUT_GAP_SECONDS, CUT_SPAN_SECONDS = .01, .4
+LEVEL_ROUNDS = 3
+# score_like: a separated music part is a sustained score when it is this loud (RMS), sounds (0.5 s windows over that
+# RMS) more than this share of the time, and more than this share of its sounding frames are tonal: at least
+# TONAL_LINE of the 60 Hz-5 kHz energy on spectral lines 10 dB over their neighbourhood.
+SCORE_RMS, SCORE_SOUNDING, SCORE_TONAL, TONAL_LINE = .005, .5, .5, .3
+FADE_FRAMES = 72 # a given piece longer than the cut fades out over its last 3 s
+FADE_IN_FRAMES = 48 # and, when it starts inside the piece, fades in over 2 s
+
+
+def _cancelled(row):
+ if row.get('_cancel'):
+ raise JobError('cancelled', 'Cancelled')
+
+
+def _db(value):
+ return 20 * math.log10(value) if value > 0 else -math.inf
+
+
+def _round(value, digits=2):
+ return round(value, digits) if value is not None and math.isfinite(value) else None
+
+
+def _audio_tracks(project):
+ return {track['id']: track for track in project['tracks'] if track['kind'] == 'audio'}
+
+
+def blocks(project):
+ """The picture's clips in order, each with its linked sound clip (None when it has none) and that clip's parts."""
+ tracks = _audio_tracks(project)
+ clips = {clip['id']: clip for track in tracks.values() for clip in track['clips']}
+ parts = {}
+ for track in tracks.values():
+ if track.get('stem'):
+ for clip in track['clips']:
+ if clip.get('stem_of'):
+ parts.setdefault(clip['stem_of'], {})[track['stem']] = clip
+ video = next(track for track in project['tracks'] if track['id'] == 'v1')
+ result = []
+ for clip in sorted(video['clips'], key=lambda clip: clip['start']):
+ if not clip.get('source'):
+ continue
+ sound = clips.get(clip.get('link'))
+ sound = sound if sound is not None and sound.get('source') else None
+ result.append(dict(video=clip, sound=sound, parts=parts.get(sound['id'], {}) if sound else {}))
+ return result
+
+
+def cuts(found):
+ """Timeline frames where one picture clip meets the next."""
+ return [b['video']['start'] for a, b in zip(found, found[1:]) if a['video']['start'] + a['video']['frames'] == b['video']['start']]
+
+
+def level_gains(dialogue, peaks, *, keep_dialogue=False):
+ """Per-block dialogue and ambience gains (dB) towards the median dialogue level of the blocks with speech.
+
+ dialogue: integrated dialogue loudness per block (-inf for none); peaks: true peak of dialogue+ambience per block."""
+ speech = [value for value in dialogue if value > SPEECH_LUFS]
+ if not speech:
+ return None, [dict(dialogue_db=0., ambience_db=0.) for _ in dialogue]
+ speech.sort()
+ middle = len(speech) // 2
+ target = speech[middle] if len(speech) % 2 else (speech[middle - 1] + speech[middle]) / 2
+ gains = []
+ for loudness, peak in zip(dialogue, peaks):
+ gain = 0.
+ if loudness > SPEECH_LUFS and not keep_dialogue:
+ gain = max(-SWING_DB, min(SWING_DB, target - loudness))
+ if math.isfinite(peak):
+ gain = min(gain, PEAK_DB - peak + LIMIT_TAKES_DB)
+ # Lifting a quiet voice must not lift the crash or the crowd next to it.
+ gains.append(dict(dialogue_db=gain, ambience_db=min(gain, 0.)))
+ return target, gains
+
+
+def cut_steps(mix, music, cut_frames):
+ """At each cut: the music's level change (RMS 0.4 s each side, 10 ms clear of the cut) and the mix's short-term
+ loudness change (the 3 s window ending at the cut against the one ending 0.6 s later)."""
+ import numpy as np
+ from ..audio_loudness import short_term
+ curve = short_term(mix, RATE) if mix.shape[1] >= 3 * RATE else np.zeros(0)
+ gap, span = round(CUT_GAP_SECONDS * RATE), round(CUT_SPAN_SECONDS * RATE)
+
+ def rms_db(x):
+ return max(-70., _db(float(np.sqrt(np.mean(np.square(x))))) if x.size else -70.)
+
+ rows = []
+ for frame in cut_frames:
+ at = frame * SAMPLES_PER_FRAME
+ row = dict(frame=frame)
+ if music is not None:
+ row['music_step_db'] = round(rms_db(music[:, at + gap:at + gap + span]) - rms_db(music[:, max(0, at - gap - span):at - gap]), 2)
+ ending = round(frame / comfy_editor.FPS * 10) - 30
+ if curve.size and 0 <= ending and ending + 6 < curve.size:
+ row['mix_step_lu'] = round(float(max(curve[ending + 6], -70.) - max(curve[ending], -70.)), 2)
+ rows.append(row)
+ return rows
+
+
+def summary(rows, key):
+ import numpy as np
+ values = np.abs([row[key] for row in rows if key in row])
+ return dict(median=round(float(np.median(values)), 2), max=round(float(values.max()), 2)) if values.size else None
+
+
+def holes(audio, start, end):
+ """[first, last) sample runs in [start, end) where the audio stays under HOLE_DB for HOLE_SECONDS or longer."""
+ import numpy as np
+ hop = RATE // 100
+ count = (end - start) // hop
+ if count <= 0:
+ return []
+ frames = audio[:, start:start + count * hop].reshape(audio.shape[0], count, hop)
+ quiet = np.sqrt(np.mean(np.square(frames), axis=(0, 2))) < 10 ** (HOLE_DB / 20)
+ runs, index = [], 0
+ while index < count:
+ if quiet[index]:
+ stop = index
+ while stop < count and quiet[stop]:
+ stop += 1
+ if (stop - index) * hop >= HOLE_SECONDS * RATE:
+ runs.append((start + index * hop, start + stop * hop))
+ index = stop
+ else:
+ index += 1
+ return runs
+
+
+def _window_db(x, size, hop):
+ import numpy as np
+ count = max(0, (x.shape[1] - size) // hop + 1)
+ if not count:
+ return np.zeros(0)
+ total = np.concatenate([[0.], np.cumsum(np.mean(np.square(x.astype(np.float64)), axis=0))])
+ starts = np.arange(count) * hop
+ return 10 * np.log10(np.maximum((total[starts + size] - total[starts]) / size, 1e-20))
+
+
+def room_tone(ambience, dialogue, rate, start, end, quiet):
+ """A steady stretch of ambience with no speech in [start, end), at the block's usual background level
+ (25th percentile of the candidates), longest first: 1 s, 0.5 s, 0.25 s. None when the block has none.
+
+ quiet: a boolean per 10 ms of the range that is digital silence, which a candidate must not touch."""
+ import numpy as np
+ hop, hop_10 = round(.05 * rate), rate // 100
+ for seconds in (1., .5, .25):
+ size = round(seconds * rate)
+ if end - start < size:
+ continue
+ level = _window_db(ambience[:, start:end], size, hop)
+ speech = _window_db(dialogue[:, start:end], size, hop)
+ found = []
+ for index in np.nonzero((speech < -55) & (level > -68) & (level < -35))[0]:
+ at = start + index * hop
+ if np.any(quiet[(at - start) // hop_10:(at - start + size) // hop_10]):
+ continue
+ if np.std(_window_db(ambience[:, at:at + size], hop, hop)) <= 3.: # steady: room tone, not an effect hit
+ found.append((at, level[index]))
+ if found:
+ usual = np.percentile([level for _, level in found], 25)
+ at, _ = min(found, key=lambda item: abs(item[1] - usual))
+ return ambience[:, at:at + size].copy()
+ return None
+
+
+def tile(tone, count, rate):
+ """`count` samples of `tone` laid end to end, every other copy reversed, joined and faded with 15 ms
+ equal-power ramps."""
+ import numpy as np
+ fade = min(round(.015 * rate), tone.shape[1] // 2)
+ rise = np.sin(np.linspace(0, math.pi / 2, fade))[None, :]
+ fall = np.cos(np.linspace(0, math.pi / 2, fade))[None, :]
+ out = np.zeros((tone.shape[0], count + tone.shape[1] + fade))
+ position, flip = 0, False
+ while position < count + fade:
+ piece = (tone[:, ::-1] if flip else tone).astype(np.float64)
+ if position:
+ piece = piece.copy()
+ piece[:, :fade] *= rise
+ out[:, position:position + fade] *= fall
+ out[:, position + fade:position + piece.shape[1]] = 0
+ out[:, position:position + piece.shape[1]] += piece
+ position += piece.shape[1] - fade
+ flip = not flip
+ out = out[:, :count]
+ edge = min(fade, count // 2)
+ if edge:
+ out[:, :edge] *= rise[:, :edge]
+ out[:, count - edge:] *= fall[:, fade - edge:]
+ return out
+
+
+class _Renders:
+ """Audio-only renders of the project through the export, as float [2, samples] at 48 kHz."""
+
+ def __init__(self, jobs, row, folder):
+ self.jobs, self.row, self.folder, self.count = jobs, row, folder, 0
+
+ def __call__(self, project, *, solo=None, muted=(), master=False):
+ import numpy as np
+ from .. import editor_export
+ value = copy.deepcopy(project)
+ if not master:
+ value.pop('master_peak_db', None)
+ for track in value['tracks']:
+ if track['kind'] != 'audio':
+ continue
+ if solo is not None:
+ track['solo'] = track['id'] in solo
+ if track['id'] in muted:
+ track['muted'] = True
+ self.count += 1
+ target = self.folder / ('render-%d.wav' % self.count)
+ try:
+ editor_export.render(value, editor_export.resolve_source, target,
+ options=dict(only='sound', sound_format='wav'),
+ cancelled=lambda: bool(self.row.get('_cancel')))
+ except editor_export.ExportCancelled:
+ raise JobError('cancelled', 'Cancelled') from None
+ except editor_export.ExportError as error:
+ raise JobError('sound_failed', 'The sound of the cut cannot be rendered (%s)' % error, kind='sound') from None
+ try:
+ import av
+ with av.open(str(target)) as container:
+ chunks = [frame.to_ndarray() for frame in container.decode(audio=0)]
+ finally:
+ target.unlink(missing_ok=True)
+ samples = np.concatenate(chunks, axis=1) if chunks else np.zeros((1, 0), np.int16)
+ samples = samples.reshape(-1, 2).T if samples.shape[0] == 1 else samples
+ return samples.astype(np.float64) / 32768
+
+
+def run_finish_sound(jobs, row, *, poll=.5):
+ spec, port = row['spec'], jobs.port
+ identity = spec['editor_project']
+ try:
+ project = jobs.projects.get(identity)['project']
+ except FileNotFoundError as error:
+ raise JobError('not_found', 'The project does not exist') from error
+ if not any(found['sound'] for found in blocks(project)):
+ raise JobError('invalid_request', 'The cut has no picture clips with sound')
+ folder = '%s/%s-%s' % (FOLDER, row['id'], secrets.token_hex(4))
+ work = port.output_path(folder + '/work/.keep', create=True).parent
+ made = []
+
+ def phase(name, fraction=None):
+ row['_phase'] = name
+ row['_detail'] = json.dumps(dict(fraction=round(fraction, 3))) if fraction is not None else None
+
+ result = dict(editor_project=identity, skipped={})
+ try:
+ render = _Renders(jobs, row, work)
+ # Run again on a finished cut (it carries the export ceiling), only the blocks split this time are judged and
+ # levelled; the rest keep what the first run gave them, so gains do not add up.
+ finished = jobs.projects.get(identity)['project'].get('master_peak_db') is not None
+ separated = _split(jobs, row, identity, phase, poll, result, made)
+ fresh = set(result.get('split', {}).get('clips', ())) if finished else None
+ _cancelled(row)
+ if separated:
+ scores = _separated_music_levels(jobs.projects.get(identity)['project'], render, result, fresh)
+ _cancelled(row)
+ _set_volumes(jobs, identity, {identity_: 0. for identity_ in scores})
+ _cancelled(row)
+ _music(jobs, row, spec, identity, phase, poll, result, made)
+ _cancelled(row)
+ phase('balancing loudness')
+ _levels(jobs, row, identity, render, result, keep_dialogue=bool(spec.get('keep_dialogue_volume')), fresh=fresh)
+ _cancelled(row)
+ if separated:
+ phase('filling silence')
+ _fill(jobs, row, identity, render, port, folder, result, made)
+ else:
+ result['skipped'].setdefault('room_tone', 'stems_unavailable')
+ _cancelled(row)
+ phase('measuring loudness')
+ project = jobs.projects.get(identity)['project']
+ _measure(render, project, result)
+ except BaseException:
+ # Parts already placed stay with the project (they are ordinary edits); only unplaced files go.
+ try:
+ used = {clip['source']['id'] for track in jobs.projects.get(identity)['project']['tracks']
+ for clip in track['clips'] if clip.get('source') and clip['source']['kind'] == 'generated'}
+ except Exception: # noqa: BLE001 - unreadable: keep every file rather than break the project
+ used = None
+ for relative in made:
+ if used is None or any(name == relative or name.startswith(relative + '/') for name in used):
+ continue
+ path = port.output_path(relative)
+ if path.is_dir():
+ shutil.rmtree(path, ignore_errors=True)
+ else:
+ path.unlink(missing_ok=True)
+ raise
+ finally:
+ shutil.rmtree(work, ignore_errors=True)
+ result['revision'] = jobs.projects.get(identity)['project']['revision']
+ return result
+
+
+def _split(jobs, row, identity, phase, poll, result, made):
+ """Step 1; False when separation is not offered here, so nothing separated exists to work with."""
+ from ..comfy_bridge import installation
+ from ..editor_export import resolve_source
+ from ..stems import service
+ from . import stems
+ from .routes import stems_supported
+ try:
+ machine = installation()[1]
+ except (OSError, ValueError):
+ machine = {}
+ project = jobs.projects.get(identity)['project']
+ found = [block for block in blocks(project) if block['sound']]
+ if not stems_supported(machine):
+ result['skipped'].update(split='stems_unavailable', music_removed='stems_unavailable')
+ return any(block['parts'] for block in found)
+ waiting = [block['sound'] for block in found if not block['parts'] and block['sound']['volume'] > 0]
+ by_source = {}
+ for clip in waiting:
+ by_source.setdefault(_key(clip), []).append(clip)
+ laid = []
+ for index, (key, clips) in enumerate(by_source.items()):
+ _cancelled(row)
+ try:
+ source = resolve_source(key)['audio']
+ except (OSError, ValueError, KeyError) as error:
+ raise JobError('stems_failed', 'The clip media is not readable', kind='stems') from error
+ if source is None:
+ continue
+ name = '%s/%s-%d-%s' % (stems.FOLDER, row['id'], index, secrets.token_hex(4))
+ out = jobs.port.output_path(name + '/dialogue.mp4', create=True).parent
+ made.append(name)
+ stems._split(jobs, row, service, source, out, lambda step, fraction=None: phase(
+ 'separating %d/%d' % (index + 1, len(by_source)) if step not in ('waiting', 'downloading') else step, fraction),
+ poll)
+ sources = {part: 'generated:%s/%s.mp4' % (name, part) for part in stems.NAMES}
+ for clip in clips:
+ phase('placing')
+ jobs.projects.lay_stems(identity, clip['id'], sources)
+ laid.append(clip['id'])
+ result['split'] = dict(clips=laid, already=[block['sound']['id'] for block in found if block['parts']])
+ return True
+
+
+def score_like(samples, rate):
+ """Whether a separated music part ([2, n] floats) is a sustained score, with the numbers behind it. Separation puts
+ some effects with the music; shots, explosions and crowds are broadband, so they fail the tonal test, and short
+ stings fail the sounding one."""
+ import numpy as np
+ from scipy.ndimage import median_filter
+ mono = np.asarray(samples, np.float64).mean(axis=0)
+ rms = float(np.sqrt(np.mean(mono ** 2))) if mono.size else 0.
+ half = rate // 2
+ windows = mono[:mono.size // half * half].reshape(-1, half)
+ sounding = float(np.mean(np.sqrt(np.mean(windows ** 2, axis=1)) > SCORE_RMS)) if windows.size else 0.
+ size, hop = 4096, 2048
+ tonal = 0.
+ if mono.size >= size:
+ window = np.hanning(size)
+ frames = np.lib.stride_tricks.sliding_window_view(mono, size)[::hop] * window
+ power = np.abs(np.fft.rfft(frames, axis=1)) ** 2 + 1e-14
+ frequency = np.fft.rfftfreq(size, 1 / rate)
+ band = power[:, (frequency > 60) & (frequency < 5000)]
+ lines = (band * (band > 10 * median_filter(band, size=(1, 15)))).sum(axis=1) / band.sum(axis=1)
+ loud = np.sqrt(np.mean(frames ** 2, axis=1) / np.mean(window ** 2)) > SCORE_RMS
+ tonal = float(np.mean(lines[loud] >= TONAL_LINE)) if loud.any() else 0.
+ found = rms >= SCORE_RMS and sounding > SCORE_SOUNDING and tonal > SCORE_TONAL
+ return found, dict(music_rms=round(rms, 4), sounding=round(sounding, 2), tonal=round(tonal, 2))
+
+
+def _separated_music_levels(project, render, result, fresh=None):
+ """Before step 2: for each split block, whether its music part is a score (it goes) or effects (it stays), and the
+ block's level with and without that part and of the part alone. Levels are the K-weighted mean over the block,
+ ungated, so taking a part away can only lower them; the loudest values are the highest 400 ms momentary loudness.
+ Returns the ids of the music parts to silence. fresh: when given, only blocks whose sound clip is in it."""
+ import numpy as np
+ from ..audio_loudness import _block_powers, _lufs
+ tracks = _audio_tracks(project)
+ music = tuple(identity for identity, track in tracks.items() if track.get('stem') == 'music')
+ rest = tuple(identity for identity, track in tracks.items() if track.get('stem') != 'music' and identity != 'a2')
+ found = [block for block in blocks(project) if block['sound'] and block['parts'].get('music')
+ and block['parts']['music']['volume'] > 0 and (fresh is None or block['sound']['id'] in fresh)]
+ if not music or not found:
+ return []
+ before, after, alone = render(project, solo=rest + music), render(project, solo=rest), render(project, solo=music)
+
+ def level(samples):
+ powers = _block_powers(samples, RATE, samples.shape[1] / RATE) if samples.shape[1] else np.zeros(0)
+ return float(_lufs(powers[0])) if powers.size and powers[0] > 0 else -math.inf
+
+ def loudest(samples):
+ powers = _block_powers(samples, RATE, .4)
+ return float(_lufs(powers.max())) if powers.size and powers.max() > 0 else -math.inf
+
+ rows, scores = [], []
+ for block in found:
+ a, b = _span(block)
+ score, numbers = score_like(alone[:, a:b], RATE)
+ if score:
+ scores.append(block['parts']['music']['id'])
+ with_music, without = level(before[:, a:b]), level(after[:, a:b])
+ rows.append(dict(clip_id=block['video']['id'], removed=score, **numbers, with_music_db=_round(with_music),
+ without_music_db=_round(without),
+ drop_db=_round(with_music - without) if math.isfinite(with_music + without) else None,
+ music_db=_round(level(alone[:, a:b])), music_loudest_lufs=_round(loudest(alone[:, a:b])),
+ without_music_loudest_lufs=_round(loudest(after[:, a:b]))))
+ result['music_removed'] = dict(blocks=rows, removed=len(scores), kept=len(rows) - len(scores))
+ return scores
+
+
+def _music(jobs, row, spec, identity, phase, poll, result, made):
+ """Step 3: keep the music on a2, or lay the given piece, or compose one as long as the cut."""
+ from ..music import service
+ from . import music
+ project = jobs.projects.get(identity)['project']
+ track = _audio_tracks(project).get('a2')
+ if track is not None and any(clip.get('source') for clip in track['clips']):
+ result['music'] = dict(kept=[clip['id'] for clip in track['clips'] if clip.get('source')])
+ return
+ asked = spec.get('music') or {}
+ frames = comfy_editor.duration(project)
+ if not asked:
+ result['skipped']['music'] = 'not_asked'
+ return
+ if 'source' in asked:
+ source = asked['source']
+ else:
+ if frames / comfy_editor.FPS > service.MAX_SECONDS:
+ result['skipped']['music'] = 'too_long'
+ return
+ relative = '%s/%s-%s.mp4' % (music.FOLDER, row['id'], secrets.token_hex(4))
+ out = jobs.port.output_path(relative, create=True)
+ made.append(relative)
+ seed = asked.get('seed', secrets.randbelow(2**32))
+ music._compose(jobs, row, service, out, frames, seed, [], phase, poll, caption=asked['caption'])
+ source = 'generated:' + relative
+ result['music_seed'] = seed
+ phase('placing')
+ try:
+ saved, clip_ids = jobs.projects.lay_music(identity, source)
+ except (ValueError, KeyError) as error:
+ raise JobError('invalid_request', 'The music source is not a sound or video file of this installation') from error
+
+ laid = [clip for track in saved['tracks'] for clip in track['clips'] if clip['id'] in clip_ids]
+ skip = 0
+ if len(laid) == 1 and laid[0]['frames'] > frames:
+ # A given piece longer than the cut: start it where its own swells and breaks fall clear of the cuts.
+ try:
+ power = _music_power(source)
+ except Exception: # noqa: BLE001 - unreadable here: start at its beginning, as placed
+ power = None
+ if power is not None:
+ seconds = music_offset(power, [frame / comfy_editor.FPS for frame in cuts(blocks(project))],
+ frames / comfy_editor.FPS)
+ skip = min(round(seconds * comfy_editor.FPS), laid[0]['frames'] - frames)
+
+ def lowered(project):
+ # As on the page: placed music is lowered during speech. A longer piece ends with the cut, faded in where it
+ # starts inside the piece and faded out where it stops inside it, so the cut does not grow.
+ value = copy.deepcopy(project)
+ for track in value['tracks']:
+ for clip in track['clips']:
+ if clip['id'] in clip_ids:
+ clip['duck'] = True
+ if clip['start'] + clip['frames'] > frames:
+ clip['in'] += skip
+ clip['frames'] = max(1, frames - clip['start'])
+ if skip:
+ clip['fade_in'] = min(FADE_IN_FRAMES, clip['frames'] // 2)
+ if clip['in'] + clip['frames'] < clip['length']:
+ clip['fade_out'] = min(FADE_FRAMES, clip['frames'] // 2)
+ return dict(ok=True, project=value)
+ jobs.projects._update(identity, lowered)
+ result['music'] = dict(laid=clip_ids, source=source)
+ if skip:
+ result['music']['starts_at_seconds'] = round(skip / comfy_editor.FPS, 2)
+
+
+def _music_power(source):
+ """Mean square of the piece every 10 ms, at 48 kHz, both channels together."""
+ import av
+ import numpy as np
+ from ..editor_export import resolve_source
+ path = resolve_source(source)['audio']
+ if path is None:
+ return None
+ chunks = []
+ with av.open(str(path)) as container:
+ resampler = av.AudioResampler(format='flt', layout='mono', rate=RATE)
+ for frame in container.decode(audio=0):
+ chunks.extend(out.to_ndarray().reshape(-1) for out in resampler.resample(frame))
+ chunks.extend(out.to_ndarray().reshape(-1) for out in resampler.resample(None) or ())
+ samples = np.concatenate(chunks).astype(np.float64) if chunks else np.zeros(0)
+ hop = RATE // 100
+ count = samples.size // hop
+ return np.mean(np.square(samples[:count * hop].reshape(count, hop)), axis=1)
+
+
+def music_offset(power, cut_seconds, film_seconds, *, hop=.01):
+ """Where in a piece longer than the cut to start it (seconds): the start whose own level changes least across
+ the cuts (RMS 0.4 s each side, 10 ms clear, as cut_steps measures), and among the starts within 1 dB of that, the
+ latest, so the piece's own ending comes as close to the film's end as it can."""
+ import numpy as np
+ span = int(round(film_seconds / hop))
+ room = power.size - span
+ if room <= 0 or not cut_seconds:
+ return max(0, room) * hop
+ total = np.concatenate([[0.], np.cumsum(power)])
+ offsets = np.arange(room + 1)
+ gap, size = round(CUT_GAP_SECONDS / hop), round(CUT_SPAN_SECONDS / hop)
+ floor = 10 ** (-70 / 10)
+ worst = np.zeros(offsets.size)
+ for cut in cut_seconds:
+ at = offsets + int(round(cut / hop))
+ low, high = np.clip(at - gap - size, 0, power.size), np.clip(at - gap, 0, power.size)
+ before = (total[high] - total[low]) / size
+ low, high = np.clip(at + gap, 0, power.size), np.clip(at + gap + size, 0, power.size)
+ after = (total[high] - total[low]) / size
+ worst = np.maximum(worst, np.abs(10 * np.log10(np.maximum(after, floor) / np.maximum(before, floor))))
+ return float(offsets[np.nonzero(worst <= worst.min() + 1.)[0][-1]] * hop)
+
+
+def _set_volumes(jobs, identity, factors):
+ """Multiply clip volumes (clip id -> linear factor), capped at MAX_VOLUME; returns the ids that hit the cap."""
+ capped = set()
+
+ def change(project):
+ value = copy.deepcopy(project)
+ for track in value['tracks']:
+ for clip in track['clips']:
+ if clip['id'] in factors and track['kind'] == 'audio':
+ wanted = clip['volume'] * factors[clip['id']]
+ if wanted > MAX_VOLUME:
+ capped.add(clip['id'])
+ clip['volume'] = round(min(MAX_VOLUME, wanted), 4)
+ return dict(ok=True, project=value)
+ jobs.projects._update(identity, change)
+ return capped
+
+
+def _key(clip):
+ return '%s:%s' % (clip['source']['kind'], clip['source']['id'])
+
+
+def _span(block):
+ clip = block['video']
+ return clip['start'] * SAMPLES_PER_FRAME, (clip['start'] + clip['frames']) * SAMPLES_PER_FRAME
+
+
+def _levels(jobs, row, identity, render, result, *, keep_dialogue, fresh=None):
+ """Step 4. fresh: when given, only blocks whose sound clip is in it get their own gain; all count for the target."""
+ from ..audio_loudness import integrated, true_peak
+ project = jobs.projects.get(identity)['project']
+ found = [block for block in blocks(project) if block['sound']]
+ # A split clip is silent on a1 and an unsplit one has nothing on a4, so together they are the dialogue of each
+ # block (with the whole original sound where it was not split).
+ dialogue = render(project, solo=('a1', 'a4'))
+ # The peak of everything but the score: music parts that stayed are effects and move with the ambience.
+ both = render(project, solo=('a1', 'a4', 'a5', 'a6'))
+ _cancelled(row)
+ loudness, peaks = [], []
+ for block in found:
+ a, b = _span(block)
+ loudness.append(integrated(dialogue[:, a:b], RATE) if b - a >= RATE // 2 else -math.inf)
+ peaks.append(true_peak(both[:, a:b], RATE) if b > a else -math.inf)
+ target, gains = level_gains(loudness, peaks, keep_dialogue=keep_dialogue)
+ factors = {}
+ if fresh is not None:
+ gains = [gain if block['sound']['id'] in fresh else dict(dialogue_db=0., ambience_db=0.)
+ for block, gain in zip(found, gains)]
+ for block, gain in zip(found, gains):
+ speech = block['parts'].get('dialogue') or block['sound']
+ ambience = block['parts'].get('ambience')
+ if speech is not None:
+ factors[speech['id']] = 10 ** (gain['dialogue_db'] / 20)
+ if ambience is not None:
+ factors[ambience['id']] = 10 ** (gain['ambience_db'] / 20)
+ kept = block['parts'].get('music')
+ if kept is not None and kept['volume'] > 0:
+ factors[kept['id']] = 10 ** (gain['ambience_db'] / 20)
+ capped = _set_volumes(jobs, identity, factors)
+ result['blocks'] = [dict(clip_id=block['video']['id'], dialogue_lufs=_round(value), gain_db=round(gain['dialogue_db'], 2),
+ ambience_gain_db=round(gain['ambience_db'], 2))
+ for block, value, gain in zip(found, loudness, gains)]
+ result['dialogue_lufs'] = _round(target)
+ project = jobs.projects.get(identity)['project']
+ music_clips = [clip for clip in _audio_tracks(project).get('a2', {}).get('clips', ()) if clip.get('source')]
+ if music_clips and target is not None:
+ bed = render(project, solo=('a2',))
+ level = integrated(bed, RATE)
+ if math.isfinite(level):
+ gain = target - MUSIC_UNDER_LU - level
+ capped |= _set_volumes(jobs, identity, {clip['id']: 10 ** (gain / 20) for clip in music_clips})
+ result['music_gain_db'] = round(gain, 2)
+ # One factor for every clip, until the whole cut sits at TARGET_LUFS.
+ for _ in range(LEVEL_ROUNDS):
+ _cancelled(row)
+ project = jobs.projects.get(identity)['project']
+ level = integrated(render(project), RATE)
+ if not math.isfinite(level) or abs(level - TARGET_LUFS) <= .2:
+ break
+ factor = 10 ** ((TARGET_LUFS - level) / 20)
+ everything = {clip['id']: factor for track in project['tracks'] if track['kind'] == 'audio'
+ for clip in track['clips'] if clip.get('source') and clip['volume'] > 0}
+ capped |= _set_volumes(jobs, identity, everything)
+
+ def ceiling(project):
+ return dict(ok=True, project=dict(copy.deepcopy(project), master_peak_db=PEAK_DB))
+ jobs.projects._update(identity, ceiling)
+ if capped:
+ result['volume_capped'] = sorted(capped)
+
+
+def _fill(jobs, row, identity, render, port, folder, result, made):
+ """Step 5: room tone into each block's ambience file wherever dialogue and ambience fall to digital silence.
+
+ The tone is the block's own; a block with no steady stretch of it borrows the nearest block's."""
+ import numpy as np
+ from ..editor_export import resolve_source
+ from ..editor_speed import changed as speed_changed
+ from ..stems.separate import SAMPLE_RATE, read_audio, write_mp4
+ project = jobs.projects.get(identity)['project']
+ found = [block for block in blocks(project) if block['parts'].get('ambience') and block['parts'].get('dialogue')]
+ both = render(project, solo=('a4', 'a6'))
+ runs_of = [holes(both, *_span(block)) for block in found]
+ read, tones = {}, {}
+
+ def material(index):
+ """The block's ambience and dialogue sources, its range in them and the timeline-to-source mapping."""
+ if index not in read:
+ block = found[index]
+ ambience, dialogue = block['parts']['ambience'], block['parts']['dialogue']
+ read[index] = None
+ if speed_changed(ambience) or ambience.get('reverse'):
+ return None
+ try:
+ tone_source, _ = read_audio(resolve_source(_key(ambience))['audio'])
+ speech_source, _ = read_audio(resolve_source(_key(dialogue))['audio'])
+ except Exception: # noqa: BLE001 - an unreadable part keeps its silence; the report counts it
+ return None
+
+ def source_at(sample, ambience=ambience):
+ # Timeline sample -> source sample at the separation rate.
+ return round((ambience['in'] * SAMPLES_PER_FRAME + sample - ambience['start'] * SAMPLES_PER_FRAME)
+ * SAMPLE_RATE / RATE)
+ a, b = _span(block)
+ first = source_at(max(a, ambience['start'] * SAMPLES_PER_FRAME))
+ last = min(source_at(min(b, (ambience['start'] + ambience['frames']) * SAMPLES_PER_FRAME)),
+ tone_source.shape[1], speech_source.shape[1])
+ read[index] = dict(tone=tone_source, speech=speech_source, first=first, last=last, at=source_at)
+ return read[index]
+
+ def tone(index):
+ if index not in tones:
+ tones[index] = None
+ found_ = material(index)
+ if found_ is not None and found_['last'] > found_['first']:
+ hop = SAMPLE_RATE // 100
+ quiet = np.zeros((found_['last'] - found_['first']) // hop + 1, bool)
+ for x, y in runs_of[index]:
+ quiet[max(0, (found_['at'](x) - found_['first']) // hop):max(0, (found_['at'](y) - found_['first']) // hop) + 1] = True
+ tones[index] = room_tone(found_['tone'], found_['speech'], SAMPLE_RATE, found_['first'], found_['last'], quiet)
+ return tones[index]
+
+ filled, seconds, unfilled = [], 0., []
+ for index, (block, runs) in enumerate(zip(found, runs_of)):
+ _cancelled(row)
+ if not runs:
+ continue
+ own = material(index)
+ donor = index if own is not None and tone(index) is not None else None
+ if own is not None and donor is None:
+ for distance in range(1, len(found)):
+ donor = next((j for j in (index - distance, index + distance) if 0 <= j < len(found) and tone(j) is not None), None)
+ if donor is not None:
+ break
+ if donor is None:
+ unfilled.append(dict(clip_id=block['video']['id'], holes=len(runs)))
+ continue
+ patched = own['tone'].astype(np.float64)
+ for x, y in runs:
+ p, q = max(own['first'], own['at'](x)), min(own['last'], own['at'](y))
+ if q > p:
+ patched[:, p:q] += tile(tones[donor], q - p, SAMPLE_RATE)
+ seconds += (y - x) / RATE
+ ambience = block['parts']['ambience']
+ relative = '%s/%s.mp4' % (folder, ambience['id'])
+ made.append(relative)
+ write_mp4(port.output_path(relative, create=True), patched.astype(np.float32))
+ jobs.projects.replace_voice_source(identity, ambience['id'], 'generated:' + relative)
+ filled.append(dict(clip_id=block['video']['id'], holes=len(runs), **(
+ {} if donor == index else dict(tone_from=found[donor]['video']['id']))))
+ after = render(jobs.projects.get(identity)['project'], solo=('a4', 'a6'))
+ left = sum(len(holes(after, *_span(block))) for block in found)
+ result['room_tone'] = dict(blocks=filled, seconds=round(seconds, 2), holes_left=left, not_filled=unfilled)
+
+
+def _measure(render, project, result):
+ """The finished cut as exported: loudness, true peak and the steps at each cut."""
+ from ..audio_loudness import integrated, true_peak
+ finished = render(project, master=True)
+ result['loudness_lufs'] = _round(integrated(finished, RATE))
+ result['true_peak_db'] = _round(true_peak(finished, RATE))
+ music = None
+ if any(clip.get('source') for clip in _audio_tracks(project).get('a2', {}).get('clips', ())):
+ # The music as it sounds in the cut, lowered during speech: the mix less the mix without it.
+ music = render(project) - render(project, muted=('a2',))
+ rows = cut_steps(finished, music, cuts(blocks(project)))
+ result['cuts'] = rows
+ result['cut_steps'] = dict(mix_lu=summary(rows, 'mix_step_lu'), music_db=summary(rows, 'music_step_db'))
diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.py
index ef3200a1..241a161c 100644
--- a/freevideo_engine/agent/spec.py
+++ b/freevideo_engine/agent/spec.py
@@ -300,9 +300,14 @@ def normalize(value, *, port=None):
elif kind == 'stems':
result.update(editor_project=value['editor_project'], revision=value['revision'], clip_id=value['clip_id'],
lay=value.get('lay', True))
+ elif kind == 'finish_sound':
+ result.update(editor_project=value['editor_project'], revision=value['revision'],
+ keep_dialogue_volume=bool(value.get('keep_dialogue_volume', False)),
+ **({'music': dict(value['music'])} if 'music' in value else {}))
elif kind == 'voice_lines':
result.update(editor_project=value['editor_project'], revision=value['revision'], selection=dict(value['selection']),
- language=value.get('language', 'auto'), separate_first=bool(value.get('separate_first', False)))
+ language=value.get('language', 'auto'), separate_first=bool(value.get('separate_first', False)),
+ **({'script': [dict(entry) for entry in value['script']]} if value.get('script') else {}))
elif kind == 'voice':
result.update(editor_project=value['editor_project'], revision=value['revision'], selection=dict(value['selection']),
lines=[dict(line) for line in value['lines']], language=value.get('language', 'auto'),
diff --git a/freevideo_engine/agent/take_qa.py b/freevideo_engine/agent/take_qa.py
index 57a420d8..a6abf96e 100644
--- a/freevideo_engine/agent/take_qa.py
+++ b/freevideo_engine/agent/take_qa.py
@@ -16,6 +16,11 @@
BLACK_SPREAD = 10.0 # a dark but detailed scene is not a black frame
FREEZE_DIFF = 0.05 # 0-255 mean change between consecutive frames
FREEZE_SECONDS = 1.5
+TAIL_NOTE_SECONDS = 0.5 # a take that stops moving this long before its end is worth a look
+TAIL_FREEZE_SECONDS = 1.0 # ...and this long it ended early (a held last beat is shorter)
+FLICKER_JUMP = 10.0 # 0-255 mean change that is a cut, not motion
+FLICKER_RATIO = 3.0 # ...and this many times larger than the motion that follows
+FLICKER_LOOK = 6 # frames after the jump that set "the motion that follows"
SILENT_DB = -45.0 # whole-take RMS below this is near silence
GAP_DB = -50.0 # 50 ms windows below this are a gap
GAP_SECONDS = 2.0
@@ -51,6 +56,32 @@ def _pictures(path):
return means, small, times
+def recognise(path):
+ """The words heard in a take, Chinese or English (the language is identified from the sound)."""
+ from .. import subtitle_ears
+ from .. import subtitle_sherpa # noqa: F401 registers the recogniser
+ engine = subtitle_ears.engine_for('transcribe')
+ audio = subtitle_ears.decode_mono(path)
+ if len(audio) <= subtitle_ears.RATE * .2:
+ return ''
+ heard = engine.transcribe(audio, 'auto', lambda fraction: None, lambda: False)
+ return ' '.join(word['text'] for line in heard['lines'] for word in line['words'])
+
+
+def _edge_flicker(changes, times):
+ """Seconds of a single odd frame at the start or the end: a jump to or from the neighbour that the
+ motion beside it does not explain."""
+ found = []
+ if len(changes) > FLICKER_LOOK:
+ after = max(changes[1:1 + FLICKER_LOOK])
+ if changes[0] >= FLICKER_JUMP and changes[0] >= FLICKER_RATIO * max(after, 1e-3):
+ found.append(('start', round(times[0], 2)))
+ before = max(changes[-1 - FLICKER_LOOK:-1])
+ if changes[-1] >= FLICKER_JUMP and changes[-1] >= FLICKER_RATIO * max(before, 1e-3):
+ found.append(('end', round(times[-1], 2)))
+ return found
+
+
def _samples(path):
import av
with media._reading(media._file(path)) as container:
@@ -103,11 +134,26 @@ def check(path, *, seconds=None, size=None, lines=None, transcribe=None):
if float(np.abs(small[index] - small[index - 1]).mean()) <= FREEZE_DIFF:
flags[index] = flags[index - 1] = True
frozen = _runs(flags, times, FREEZE_SECONDS)
+ changes = [float(np.abs(small[i] - small[i - 1]).mean()) for i in range(1, len(small))]
+ flicker = _edge_flicker(changes, times[1:]) if changes else []
+ # The tail: still for a while at the very end although the take moved before (shorter than a freeze).
+ tail = 0
+ while tail < len(changes) and changes[-1 - tail] <= FREEZE_DIFF:
+ tail += 1
+ tail_seconds = times[-1] - times[-1 - tail] if tail else 0.0
+ ended = (round(times[-1 - tail], 2), round(times[-1], 2)) \
+ if FREEZE_SECONDS > tail_seconds >= TAIL_NOTE_SECONDS and len(changes) > tail and max(changes[:-tail]) > FREEZE_DIFF * 20 else None
if black:
problems.append('black picture at %s' % ', '.join('%.1f-%.1f s' % s for s in black))
if frozen:
problems.append('frozen picture at %s' % ', '.join('%.1f-%.1f s' % s for s in frozen))
- result.update(black=black, frozen=frozen)
+ if ended and ended[1] - ended[0] < TAIL_FREEZE_SECONDS:
+ notes.append('picture still for the last %.1f s' % (ended[1] - ended[0]))
+ elif ended:
+ problems.append('picture stops at %.1f s, %.1f s before the end' % (ended[0], ended[1] - ended[0]))
+ if flicker:
+ notes.append('single odd frame at the %s (trim %s)' % (' and '.join(w for w, _ in flicker), ' and '.join('the first frame' if w == 'start' else 'the last frame' for w, _ in flicker)))
+ result.update(black=black, frozen=frozen, flicker=[w for w, _ in flicker], stopped=ended)
if not info['has_audio']:
problems.append('no audio track')
@@ -132,7 +178,11 @@ def check(path, *, seconds=None, size=None, lines=None, transcribe=None):
result['gaps'] = gaps
if lines and transcribe is not None and info['has_audio']:
- heard = _words(transcribe(path))
+ try:
+ heard = _words(transcribe(path))
+ except Exception as error: # no speech model is no reason to drop the take
+ notes.append('lines not checked (%s)' % (getattr(error, 'reason', None) or type(error).__name__))
+ return dict(result, problems=problems, notes=notes, ok=not problems)
missing = []
for line in lines:
wanted = _words(line)
diff --git a/freevideo_engine/agent/voice.py b/freevideo_engine/agent/voice.py
index ffd31741..173a34e9 100644
--- a/freevideo_engine/agent/voice.py
+++ b/freevideo_engine/agent/voice.py
@@ -31,6 +31,7 @@
PREEMPTIONS = 8
BUSY_WAITS = 8
SPEED = dict(zh=4.5, en=2.8) # units per second of a voice, when the line gave no rate of its own
+RECOGNITION_CODES = {'no_sound': 'invalid_request', 'too_long': 'invalid_request', 'language': 'speech_language'}
DUB_CODES = {'out_of_memory': 'out_of_memory', 'model_missing': 'voice_model_missing',
'invalid_request': 'invalid_request', 'disk_space': 'storage_unavailable'}
SELECTION_MESSAGE = 'The selection must be a sound clip played at normal speed, forwards'
@@ -304,7 +305,7 @@ def run_voice_lines(jobs, row):
"""The lines of a selection: where each one is (from the voice) and what it says (from the recogniser)."""
from .. import subtitle_ears
from .. import subtitle_sherpa # noqa: F401 registers the recogniser
- from ..voice import sentences
+ from ..voice import script, sentences, speakers
spec = row['spec']
ctx = context(jobs, spec)
sources = None
@@ -334,14 +335,26 @@ def progress(fraction):
except subtitle_ears.Cancelled as error:
raise JobError('cancelled', 'Cancelled') from error
except subtitle_ears.EarsError as error:
- raise JobError(('invalid_request' if error.reason in ('no_sound', 'too_long') else 'voice_failed'),
+ raise JobError(RECOGNITION_CODES.get(error.reason, 'voice_failed'),
'The selection could not be recognised (%s)' % error.reason) from error
words = [word for line in heard['lines'] for word in line['words']]
rows = sentences.lines(words, audio, rate)
+ known = spec.get('script') or []
+ entries = script.match(rows, known) if known else [None] * len(rows)
+ for line, entry in zip(rows, entries):
+ if entry is not None:
+ # the text is known: it is what the line says, and the recogniser only gives the times
+ line['words'] = script.retime(line['words'], entry['text'], line['start'], line['end']) or line['words']
+ line['text'] = sentences._join(line['words'])
+ slices = [audio[int(line['start'] * rate):int(line['end'] * rate)] for line in rows]
+ numbers = speakers.label(slices, speakers.embedder())
+ if known:
+ numbers = speakers.by_role(numbers, [None if entry is None or not (entry.get('speaker') or '').strip()
+ else script.roles(known)[entry['speaker'].strip()] for entry in entries])
lines = []
for index, line in enumerate(rows, 1):
lines.append(dict(id='l%03d' % index, start=round(line['start'], 3), end=round(line['end'], 3), text=line['text'],
- speaker=0, words=line['words'],
+ speaker=numbers[index - 1], words=line['words'],
rate=round(max(len(line['words']), 1) / max(line['end'] - line['start'], .1), 2)))
kind = 'dialogue' if ctx['separated'] or sources else 'a1'
return dict(editor_project=spec['editor_project'], revision=spec['revision'], separated=ctx['separated'],
@@ -471,6 +484,14 @@ def _measure(audio, first, last):
return (low + span[0]) / RATE, (low + span[1]) / RATE
+def _level(audio, first, last):
+ """RMS of the sound in [first, last) of a mono 44.1 kHz file: the loudness the new voice is to come back with."""
+ import numpy as np
+ low, high = max(0, int(first * RATE)), min(len(audio), int(last * RATE))
+ part = np.asarray(audio[low:high], dtype=np.float64)
+ return round(float(np.sqrt(np.mean(np.square(part)))), 6) if len(part) else None
+
+
def _windows(every, todo, spans, base, limit):
"""Where each changed line's new voice goes: the original line plus a little quiet each side, and how far it may reach."""
windows, failures = {}, {}
@@ -566,11 +587,18 @@ def run_voice_replace(jobs, row, *, poll=.25):
reference, own = _reference(jobs, spec, voice, every, original_wav, base)
seed = _seed(spec, todo)
pace = Pace(work)
+ # a text that cannot be said in the room its line has, even at the fastest tempo, is known before anything is made
+ expected = {found['id']: found['overrun'] for found in estimate(every, ctx['end'])['lines'] if found['overrun']}
+ if expected:
+ _phase(row, 'preparing', too_long=sorted(expected))
def item(line, length, offset):
window = windows[line['id']]
made = dict(key=line['id'], text=line['text'].strip(), seconds=round(length, 3),
onset=round(window['onset'] - window['first'], 3), offset=round(offset - window['first'], 3))
+ level = _level(mono, windows[line['id']]['onset'], windows[line['id']]['offset'])
+ if level:
+ made['level'] = level
if own and line['end'] - line['start'] >= 1. and line['original_text'].strip():
made['reference'] = dict(file=str(original_wav), text=line['original_text'].strip(),
start=base + line['start'], end=base + line['end'])
@@ -615,6 +643,8 @@ def item(line, length, offset):
if made is None:
result['code'] = 'cancelled'
continue
+ if line['id'] in expected:
+ result['expected_overrun'] = expected[line['id']]
result.update(available=round((window['deadline'] if line['id'] in second_round else window['last']) - window['onset'], 3),
speed=made.get('speed'), stretch=made.get('stretch'), gain_db=made.get('gain_db'),
overrun=round(float(made.get('overrun') or 0.), 3), warning=made.get('warning'))
diff --git a/freevideo_engine/audio_loudness.py b/freevideo_engine/audio_loudness.py
new file mode 100644
index 00000000..983577ca
--- /dev/null
+++ b/freevideo_engine/audio_loudness.py
@@ -0,0 +1,163 @@
+"""Loudness of a whole cut (ITU-R BS.1770-4 / EBU R128) and a true-peak limiter for export.
+
+Planar float arrays [channels, samples]; channels all weigh 1 (mono and stereo, which is all the editor writes).
+"""
+import math
+
+import numpy as np
+
+ABSOLUTE_GATE = -70.
+RELATIVE_GATE = -10.
+BLOCK_SECONDS, STEP_SECONDS = .4, .1 # 400 ms blocks with 75 % overlap
+SHORT_SECONDS = 3.
+OVERSAMPLE = 4
+
+
+def _k_filters(rate):
+ """The two K-weighting stages (high shelf, then high pass) for any sample rate, as (b, a) pairs."""
+ # Bilinear form of the BS.1770 prototypes (as in libebur128), which reproduces the 48 kHz table exactly.
+ gain, q, fc = 3.999843853973347, .7071752369554196, 1681.974450955533
+ k = math.tan(math.pi * fc / rate)
+ vh = 10 ** (gain / 20)
+ vb = vh ** .4996667741545416
+ shelf_b = [vh + vb * k / q + k * k, 2 * (k * k - vh), vh - vb * k / q + k * k]
+ shelf_a = [1 + k / q + k * k, 2 * (k * k - 1), 1 - k / q + k * k]
+ q, fc = .5003270373238773, 38.13547087602444
+ k = math.tan(math.pi * fc / rate)
+ pass_b = [1., -2., 1.]
+ pass_a = [1., 2 * (k * k - 1) / (1 + k / q + k * k), (1 - k / q + k * k) / (1 + k / q + k * k)]
+ return [(np.array(shelf_b) / shelf_a[0], np.array(shelf_a) / shelf_a[0]), (np.array(pass_b), np.array(pass_a))]
+
+
+def _planar(samples):
+ samples = np.asarray(samples, dtype=np.float64)
+ if samples.ndim == 1:
+ samples = samples[None, :]
+ if samples.ndim != 2 or not np.isfinite(samples).all():
+ raise ValueError('Expected finite planar audio')
+ return samples
+
+
+def _block_powers(samples, rate, seconds, step=STEP_SECONDS):
+ """Mean square of the K-weighted signal, summed over channels, for each window of `seconds` every `step`."""
+ from scipy.signal import lfilter
+ weighted = _planar(samples)
+ for b, a in _k_filters(rate):
+ weighted = lfilter(b, a, weighted, axis=1)
+ size, hop = round(seconds * rate), round(step * rate)
+ if weighted.shape[1] < size:
+ return np.zeros(0)
+ energy = np.concatenate([np.zeros(1), np.cumsum(np.square(weighted).sum(axis=0))])
+ starts = np.arange(0, weighted.shape[1] - size + 1, hop)
+ return (energy[starts + size] - energy[starts]) / size
+
+
+def _lufs(power):
+ return -.691 + 10 * np.log10(np.maximum(power, 1e-20))
+
+
+def integrated(samples, rate):
+ """Gated integrated loudness in LUFS; -inf when nothing is over the absolute gate."""
+ powers = _block_powers(samples, rate, BLOCK_SECONDS)
+ loud = powers[_lufs(powers) > ABSOLUTE_GATE]
+ if not loud.size:
+ return -math.inf
+ gate = _lufs(loud.mean()) + RELATIVE_GATE
+ kept = loud[_lufs(loud) > gate]
+ return float(_lufs(kept.mean()))
+
+
+def short_term(samples, rate, step=STEP_SECONDS):
+ """Short-term loudness (3 s windows) every `step` seconds; value k covers [k*step, k*step + 3 s)."""
+ return _lufs(_block_powers(samples, rate, SHORT_SECONDS, step))
+
+
+def true_peak(samples, rate):
+ """Highest absolute sample after 4x oversampling, in dBTP (-inf for silence)."""
+ from scipy.signal import resample_poly
+ samples = _planar(samples)
+ if not samples.size:
+ return -math.inf
+ peak = float(np.max(np.abs(resample_poly(samples, OVERSAMPLE, 1, axis=1))))
+ return 20 * math.log10(peak) if peak > 0 else -math.inf
+
+
+class Limiter:
+ """Streaming true-peak limiter: the output stays under `ceiling_db` (4x oversampled) and quiet passages pass unchanged.
+
+ `push` returns the samples that are ready and `flush` the rest; together they return exactly the input length, with
+ the output lagging the input inside the limiter. The gain ramps down over the look-ahead before a peak (no step, so
+ no click) and comes back over `release_seconds`."""
+
+ CONTEXT = 16 # input samples each side that the 4x interpolation filter reaches
+
+ def __init__(self, ceiling_db, rate, channels=2, *, lookahead_seconds=.005, release_seconds=.1):
+ # Aim 0.1 dB under the ceiling: the gain moves inside the interpolation filter's reach, which can add a few mdB.
+ self.ceiling = 10 ** ((ceiling_db - .1) / 20)
+ self.window = max(1, round(lookahead_seconds * rate)) | 1
+ self.log_release = -1 / max(1., release_seconds * rate)
+ self.past = np.zeros((channels, self.CONTEXT))
+ self.pending = np.zeros((channels, 0))
+ self.required = np.zeros(0)
+ self.ramp = None # the last window-1 look-ahead minimums; at the start, the first one repeated
+ self.gain = 1.
+
+ def _measure(self, final):
+ """Extend `required` (the gain each pending sample needs on its own) as far as the filter context allows."""
+ from scipy.signal import resample_poly
+ start, size, edge = self.required.size, self.pending.shape[1], self.CONTEXT
+ end = size if final else size - edge
+ if end <= start:
+ return
+ context = np.concatenate([self.past, self.pending], axis=1)[:, start:end + 2 * edge]
+ up = np.abs(resample_poly(context, OVERSAMPLE, 1, axis=1)).max(axis=0)
+ up = np.pad(up, (0, max(0, (end - start + edge) * OVERSAMPLE - up.size)))
+ peaks = up[edge * OVERSAMPLE:(end - start + edge) * OVERSAMPLE].reshape(end - start, OVERSAMPLE).max(axis=1)
+ peaks = np.maximum(peaks, np.abs(self.pending[:, start:end]).max(axis=0))
+ self.required = np.concatenate([self.required, np.minimum(1., self.ceiling / np.maximum(peaks, 1e-12))])
+
+ def push(self, block):
+ block = _planar(block)
+ self.pending = np.concatenate([self.pending, block], axis=1)
+ self._measure(False)
+ return self._emit(max(0, self.required.size - (self.window - 1)))
+
+ def flush(self):
+ self._measure(True)
+ return self._emit(self.pending.shape[1])
+
+ def _release(self, target):
+ """g[i] = min(target[i], 1 - (1 - g[i-1]) * r), in closed form chunk by chunk."""
+ out = np.empty_like(target)
+ held = 1 - self.gain
+ chunk = max(1, int(30 / -self.log_release))
+ for start in range(0, target.size, chunk):
+ dip = np.maximum(1 - target[start:start + chunk], 0.)
+ steps = np.arange(dip.size) * self.log_release
+ with np.errstate(divide='ignore'):
+ depth = np.exp(np.maximum.accumulate(np.log(dip) - steps) + steps)
+ depth = np.maximum(depth, held * np.exp(steps + self.log_release))
+ out[start:start + dip.size] = 1 - depth
+ held = depth[-1]
+ self.gain = 1 - held
+ return out
+
+ def _emit(self, count):
+ from scipy.ndimage import minimum_filter1d
+ channels = self.pending.shape[0]
+ if count <= 0:
+ return np.zeros((channels, 0))
+ half = self.window // 2
+ need = np.pad(self.required, (0, max(0, count + self.window - 1 - self.required.size)), constant_values=1.)
+ lowest = minimum_filter1d(need[:count + self.window - 1], self.window, mode='nearest')[half:half + count]
+ if self.ramp is None:
+ self.ramp = np.full(self.window - 1, lowest[0])
+ joined = np.concatenate([self.ramp, lowest])
+ total = np.concatenate([[0.], np.cumsum(joined)])
+ target = (total[self.window:] - total[:-self.window])[:count] / self.window
+ self.ramp = joined[-(self.window - 1):] if self.window > 1 else self.ramp
+ out = self.pending[:, :count] * self._release(target)[None, :]
+ self.past = np.concatenate([self.past, self.pending[:, :count]], axis=1)[:, -self.CONTEXT:]
+ self.pending = self.pending[:, count:]
+ self.required = self.required[count:]
+ return out
diff --git a/freevideo_engine/chain_plan.py b/freevideo_engine/chain_plan.py
index 8e87173d..03d5be87 100644
--- a/freevideo_engine/chain_plan.py
+++ b/freevideo_engine/chain_plan.py
@@ -101,6 +101,14 @@ def _lengths(total, first, following, least, continued):
return lengths
+def single_seconds(max_frames=None):
+ """Whole seconds of the longest first block a device with this frame limit generates at once (unknown: 15 s)."""
+ if max_frames is None:
+ return BLOCK_SECONDS
+ limit = _number(max_frames, 'The frame limit', whole=True)
+ return (limit - (limit - 5) % UNIT) // FPS if limit >= 39 else 0
+
+
def plan(seconds, single=False, max_frames=None, single_limit_seconds=None, block_seconds=None, continued=False,
max_parts=None, first_frame=False):
"""`max_frames`: the most frames this device generates at once for the current settings (unknown: 15 s).
diff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.py
index 31dcf634..6f3dbcd9 100644
--- a/freevideo_engine/comfy_bridge.py
+++ b/freevideo_engine/comfy_bridge.py
@@ -301,7 +301,7 @@ def _late_history(self):
vars(self).clear()
vars(self).update(saved)
- def configure(self, *, plan=None, media=None, conditioning=False, placement=None, sampled=None):
+ def configure(self, *, plan=None, media=None, conditioning=False, placement=None, sampled=None, reference=None):
"""Use facts already obtained by the bridge; never inspect media/models."""
self._late_history()
from .media_request import task_for
@@ -346,6 +346,14 @@ def configure(self, *, plan=None, media=None, conditioning=False, placement=None
sum(bool(media.get(k)) for k in ('first', 'last')))
if conditioning:
self.cold_flags['encode_cached'] = True
+ if reference:
+ # The prepared references: their rows enlarge every sampling step
+ # and their length sets the encoding time, neither known from the request.
+ rows, seconds = reference.get('reference_video_tokens'), reference.get('reference_seconds')
+ if type(rows) is int and rows > 0:
+ self.request['reference_video_tokens'] = rows
+ if type(seconds) in (int, float) and math.isfinite(seconds) and seconds > 0:
+ self.request['reference_seconds'] = float(seconds)
self._reforecast()
def _reforecast(self, *, reset_source=False):
@@ -542,6 +550,14 @@ def _counts_and_live(self, message, event, now):
# LoRA groups describe a substage, not completion of the whole load.
if type(done) is int and type(total) is int and total > 0 and message.get('unit') != 'bytes':
self._set_count(done, total)
+ elif phase == 'encode' and message.get('stage') == 'media':
+ # Reference video encoding costs far more than the stage's fixed
+ # estimate on a modest GPU; once a chunk is done its rate gives the
+ # rest. Only the stage under way is rescaled.
+ elapsed = _timing_seconds(message.get('elapsed_seconds'))
+ if type(done) is int and type(total) is int and 0 < done <= total and elapsed:
+ self._set_count(done, total)
+ self.weights[phase] = now - self.phase_started + elapsed * (total - done) / done
elif phase == 'decode_save' and message.get('stage') == 'video':
elapsed = _timing_seconds(message.get('elapsed_seconds'))
if type(done) is int and type(total) is int and 0 <= done <= total and total > 0:
@@ -764,6 +780,8 @@ def annotate(self, message, *, event=None):
self._retry({}, now)
if event.get('event') in ('video_start', 'decode_resume'):
self.configure(placement=event)
+ if event.get('event') == 'reference_media':
+ self.configure(reference=event)
phase = self._phase_for(message, event)
if phase:
self._enter(phase, now)
@@ -970,8 +988,11 @@ def events(self):
def preview_state(event):
"""The few facts the page needs to show the newest frame preview, or None."""
values = {key: event.get(key) for key in ('serial', 'step', 'total', 'frames', 'frame_width', 'frame_height')}
- if any(type(value) is not int or value < 1 or value > 100000 for value in values.values()) or values['frames'] > 8:
+ if any(type(value) is not int or value < 1 or value > 100000 for value in values.values()) or values['frames'] > 16:
return None
+ duration = event.get('duration_ms') # optional: the playback speed follows the video's length
+ if type(duration) is int and 0 < duration <= 86400000:
+ values['duration_ms'] = duration
return values
@@ -1102,6 +1123,13 @@ def progress_message(event):
elapsed_seconds=event.get('elapsed_seconds'))
if name == 'media_encode_phase':
return {'label': str(event.get('phase', 'Encoding input media')), 'timing_phase': 'encoding'}
+ if name == 'media_encode_progress':
+ done, total = event.get('done'), event.get('total')
+ result = dict(label='Encoding image/video references', timing_phase='encoding')
+ if type(done) is int and type(total) is int and 0 <= done <= total and total > 0:
+ result.update(phase='encoding', stage='media', done=done, total=total, unit='frames',
+ elapsed_seconds=event.get('elapsed_seconds'))
+ return result
if name == 'reference_trimmed':
return {'label': 'Preparing reference media', 'timing_phase': 'encoding',
'reference_trimmed': {key: event.get(key) for key in ('kind', 'number', 'seconds', 'used_seconds')}}
@@ -1271,6 +1299,12 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,
width, height = previous['geometry']['width'], previous['geometry']['height']
seconds, seed = previous['geometry']['seconds'], previous['seed']
two_pass, preview = True, False
+ # A continued preview's second pass continues the same video the same way.
+ continue_from = argument(arguments, '--continue-from')
+ continue_from = None if continue_from is None else Path(continue_from)
+ frames = int(argument(arguments, '--frames')) if '--frames' in arguments else None
+ first_frame_reference = '--first-frame-reference' in arguments
+ voice_reference = '--voice-reference' in arguments
base_steps = int(argument(arguments, '--base-steps', 8))
refine_steps = int(argument(arguments, '--refine-steps', 3))
# The second pass never uses a step cache; the preview's own command keeps its flag.
@@ -1293,7 +1327,7 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,
def check_ending():
from .continuation_tail import check
from .two_pass import sampling_canvas
- check(continue_from, sampling_canvas(canvas)[0])
+ check(continue_from, sampling_canvas(canvas, staged=bool(preview or upscale))[0])
if step_cache == 'fbcache' and machine.get('device_backend') == 'mps':
# Only the audio-video EasyCache is ported to Mac: the request samples every step,
@@ -1301,24 +1335,31 @@ def check_ending():
logging.warning('[FreeVideo] The first-block cache is not available on Mac yet; sampling every step.')
step_cache = None
if continue_from is not None:
- validate_route(backend=machine.get('device_backend', 'cuda'))
- from .continuation import MISSING, prepare, summary
+ validate_route(preview=preview, refine_from=refine_from, backend=machine.get('device_backend', 'cuda'))
+ from .continuation import MISSING, prepare, restore_snapshot, summary
from .two_pass import sampling_canvas
- # A continuation takes no media or preview, so its sampled size is known here; the snapshot
+ # A continuation takes no media, so its sampled size is known here; the snapshot
# of the previous segment was written for that canvas, not for the requested size.
- try:
- continuation = prepare(continue_from, sampling_canvas(canvas)[0], directory=root / 'continuations')
- except ValueError as error:
- if str(error) != MISSING:
- raise
- # A video with no stored context (single-pass, an earlier version, a replaced file) is
- # continued from its own ending, which the engine reads and encodes. Only a video whose
- # ending cannot be used stops here, before anything is loaded.
- check_ending()
+ started = sampling_canvas(canvas, staged=bool(preview or upscale))[0]
+ preview_encoded = False
+ if upscale:
+ from .preview import continuation_snapshot, source as preview_source
+ snapshot, digest, preview_encoded = continuation_snapshot(preview_source(upscale[0]), continue_from)
+ continuation = restore_snapshot(snapshot, digest, started)
+ else:
+ try:
+ continuation = prepare(continue_from, started, directory=root / 'continuations')
+ except ValueError as error:
+ if str(error) != MISSING:
+ raise
+ # A video with no stored context (single-pass, an earlier version, a replaced file) is
+ # continued from its own ending, which the engine reads and encodes. Only a video whose
+ # ending cannot be used stops here, before anything is loaded.
+ check_ending()
# The engine decides the same way from the same record: whether the film's first frame is there to use.
# An ending encoded from the file has no first frame of the film to give.
anchored = False
- if continue_from is not None and continuation is not None and first_frame_reference:
+ if continue_from is not None and continuation is not None and first_frame_reference and not preview_encoded:
from .continuation import chain_first_frame_picture
try:
anchored = chain_first_frame_picture(continuation, continue_from) is not None
@@ -1362,7 +1403,7 @@ def check_ending():
command.append('--first-frame-reference')
if voice_reference:
command.append('--voice-reference')
- if continuation is not None:
+ if continuation is not None and not upscale:
from .continuation import persist_snapshot
continuation = persist_snapshot(continuation, run / 'continuation-source.safetensors')
command += ['--continuation-snapshot', continuation['snapshot'],
@@ -1375,7 +1416,7 @@ def check_ending():
command += ['--' + field.replace('_', '-'), str(reserve)]
if preview:
command.append('--preview')
- if hide_tail:
+ if hide_tail and '--hide-tail' not in command:
command.append('--hide-tail')
state = {'status': 'starting', 'geometry': canvas, 'seed': seed, 'installation': str(root),
'command': command, 'output': str(output), 'source': str(source), 'resources': resources}
@@ -1741,6 +1782,13 @@ def store_result(check):
# another generation; cancellation should release its memory.
from .resident_process import OWNER
OWNER.release('cancelled')
+ try:
+ # Nothing will read a stopped request's decoded frames or reference frames; the
+ # sampling result and conditioning stay for the failure report.
+ from .comfy_library import release_decoded
+ state['decoded_release'] = release_decoded(run)
+ except Exception:
+ pass
raise
finally:
from .resident_process import OWNER
diff --git a/freevideo_engine/comfy_chain.py b/freevideo_engine/comfy_chain.py
index 5d15cfbc..5d1207fa 100644
--- a/freevideo_engine/comfy_chain.py
+++ b/freevideo_engine/comfy_chain.py
@@ -274,6 +274,12 @@ def continuation_supported():
return supported()[0]
+def continuation_preview():
+ """Whether a part that continues can be made as a preview first, on this device."""
+ from .continuation import previews_supported
+ return continuation_supported() and previews_supported()
+
+
def continue_seconds():
"""The length a new continuation starts with where this device sets it: short parts on Apple silicon, where
a full part takes long and often does not fit; None elsewhere, where the page goes by the measured limits."""
@@ -470,7 +476,7 @@ def asking(request):
@server.routes.get('/freevideo/chains')
async def listing(request):
value = {'chains': runner.list(asking(request)), 'continuation': continuation_available(),
- 'continue_seconds': continue_seconds()}
+ 'continue_seconds': continue_seconds(), 'continuation_preview': continuation_preview()}
if fake:
value['fake'] = True
return response(value)
diff --git a/freevideo_engine/comfy_editor.py b/freevideo_engine/comfy_editor.py
index 767d5987..f862d6b3 100644
--- a/freevideo_engine/comfy_editor.py
+++ b/freevideo_engine/comfy_editor.py
@@ -334,6 +334,12 @@ def bounded(value, depth=0):
raise ValueError('Invalid empty edit retention')
if record['keep_empty']:
result['keep_empty'] = True
+ if 'master_peak_db' in record:
+ # The export's true-peak ceiling, set by the finishing step; without it the export only clips at full scale.
+ peak = record['master_peak_db']
+ if type(peak) not in (int, float) or not math.isfinite(peak) or not -12 <= peak <= 0:
+ raise ValueError('Invalid master peak')
+ result['master_peak_db'] = peak
result.update(marker_fields(record))
if 'replacements' in record:
replacements = record['replacements']
diff --git a/freevideo_engine/comfy_library.py b/freevideo_engine/comfy_library.py
index c3eef4fa..f0f0002c 100644
--- a/freevideo_engine/comfy_library.py
+++ b/freevideo_engine/comfy_library.py
@@ -103,17 +103,36 @@ def _prompt_digest(folder):
return hashlib.sha256(text.encode('utf-8')).hexdigest() if text else None
+# The same prompt links videos only while each is made within this time of the one before it and from the same
+# pictures, clips and audio, across midnight too; the same words hours later, or on other pictures, are another task.
+_SITTING_SECONDS = 2 * 3600
+_INPUT_FILE = re.compile(r'(?:reference-\d{2}\.(?:png|mp4|wav)|(?:first|last)\.png)')
+
+
+def _inputs_digest(folder):
+ """What the run was given besides words: its reference pictures, clips and audio and its first and last frame."""
+ parts = []
+ try:
+ for entry in sorted(os.scandir(folder), key=lambda item: item.name):
+ if _INPUT_FILE.fullmatch(entry.name) and entry.is_file(follow_symlinks=False):
+ with open(entry.path, 'rb') as stream:
+ parts.append('%s:%d:%s' % (entry.name, entry.stat().st_size, hashlib.sha256(stream.read(2**20)).hexdigest()))
+ except OSError:
+ return None
+ return hashlib.sha256('\n'.join(parts).encode('utf-8')).hexdigest()[:16]
+
+
def _tasks(root, rows, folders, reports):
"""Name each row's task: the earliest video connected to it by a preview and its upscale, a continuation,
- one agent job, or the same prompt text on the same day. A video with no such link is its own task."""
+ one agent job, or the same prompt text and inputs within two hours of the one before. A video with no such link is its own task."""
parent = {row['id']: row['id'] for row in rows}
def find(item):
while parent[item] != item:
parent[item] = parent[parent[item]]
item = parent[item]
return item
- seen = {}
- for row in rows:
+ seen, latest = {}, {}
+ for row in sorted(rows, key=lambda row: (row['created_at'], row['id'])):
identity = row['id']
tokens = ['id:' + identity]
source = _source_identity(root, folders[identity], reports[identity])
@@ -123,12 +142,15 @@ def find(item):
tokens.append('job:' + row['agent']['job'])
digest = _prompt_digest(folders[identity])
if digest:
- tokens.append('prompt:%s:%s' % (identity.split('/', 1)[0], digest))
+ tokens.append('prompt:%s:%s' % (digest, _inputs_digest(folders[identity])))
+ moment = datetime.fromisoformat(row['created_at']).timestamp()
for token in tokens:
- if token in seen:
+ # Ascending order: a prompt token far from the video before it starts a new sitting.
+ if token in seen and not (token.startswith('prompt:') and moment - latest[token] > _SITTING_SECONDS):
parent[find(identity)] = find(seen[token])
else:
seen[token] = identity
+ latest[token] = moment
groups = {}
for row in rows:
groups.setdefault(find(row['id']), []).append(row)
@@ -353,7 +375,7 @@ def video_download(output_directory, identity):
# holds anything else, or any link, is not deleted at all.
_OWNED_FILE = re.compile(r'(?:video\.[A-Za-z0-9._-]+|prompt\.txt|workflow\.json|comfy-request\.json|generate\.log'
r'|continuation-source\.safetensors'
- r'|media\.json|prompt-rewrite\.json|input-conditioning\.pt|(?:first|last)\.png'
+ r'|frame-preview\.jpg|media\.json|prompt-rewrite\.json|input-conditioning\.pt|(?:first|last)\.png'
r'|reference-\d{2}\.(?:png|mp4|wav)'
r'|\.(?:share|editor-export)-[0-9a-f]{32}\.mp4)(?:\.[a-z0-9_]{8}\.tmp)?')
_OWNED_FOLDER = 'video.artifacts'
@@ -460,6 +482,9 @@ def _remove(folder):
# workflow read the MP4, and an upscale reads the preview's first pass. The
# sampling result, conditioning and first pass stay.
_DECODED = ('rgb.npy', 'audio.npy', 'audio.wav')
+# The normalized frames of reference videos and pictures: only the encoder reads
+# them, once, to build the conditioning. About 1.3 GB per reference video.
+_REFERENCE_FRAMES = 'media'
_ATTEMPT = re.compile(r'\d{2}-[0-9A-Za-z_-]+')
_RELEASED = '.decoded-released.json'
# The startup release, a generation re-arming it and a library delete take
@@ -490,8 +515,13 @@ def release_decoded(folder):
except OSError:
left.append(_OWNED_FOLDER + '/attempts') # retried at a later start
for place in places:
- for name in _DECODED:
- path = place / name
+ paths = [place / name for name in _DECODED]
+ try:
+ if (place / _REFERENCE_FRAMES).is_dir() and _plain(place / _REFERENCE_FRAMES):
+ paths += sorted((place / _REFERENCE_FRAMES).glob('*.npy'))
+ except OSError:
+ left.append((place / _REFERENCE_FRAMES).relative_to(folder).as_posix())
+ for path in paths:
try:
if not _plain(path):
continue
@@ -601,6 +631,23 @@ def release_earlier(output_directory):
totals['freed_bytes'] += result['freed_bytes']
totals['left'] += len(result['left'])
totals['skipped'] += result.get('skipped') == 'foreign-files'
+ # Stopped requests have no MP4 to find them by; their request record says they are over.
+ for path in sorted(root.glob('*/*/comfy-request.json')):
+ folder = path.parent
+ try:
+ if (not _ID.fullmatch(folder.relative_to(root).as_posix()) or (folder / 'video.mp4').exists()
+ or not _plain(folder.parent) or not _plain(folder) or path.stat().st_size > 1024 * 1024):
+ continue
+ state = json.loads(path.read_text(encoding='utf-8'))
+ if not isinstance(state, dict) or state.get('status') not in ('cancelled', 'failed'):
+ continue
+ except (OSError, ValueError, TypeError):
+ continue
+ result = release_decoded(folder)
+ totals['videos'] += result['freed_bytes'] > 0
+ totals['freed_bytes'] += result['freed_bytes']
+ totals['left'] += len(result['left'])
+ totals['skipped'] += result.get('skipped') == 'foreign-files'
save(marker, dict(schema=1, pending=totals['left'] > 0, released_at=time.time(), **totals))
for path in staged:
try:
diff --git a/freevideo_engine/comfy_nodes.py b/freevideo_engine/comfy_nodes.py
index 97331251..bffdce0b 100644
--- a/freevideo_engine/comfy_nodes.py
+++ b/freevideo_engine/comfy_nodes.py
@@ -177,10 +177,11 @@ def validate_inputs(cls, text, width, height, seconds, seed, continue_from='', f
try:
options = request_options(continue_from, node_frames(frames))
if options.get('continue_from'):
- # A continued part turns on two passes, skips the preview and takes keyframes as pictures (execute).
- two_pass, preview = True, False
+ # A continued part turns on two passes and takes keyframes as pictures; only NVIDIA keeps its preview (execute).
+ from .continuation import previews_supported
+ two_pass, preview = True, preview is True and previews_supported()
base_steps, refine_steps = part_steps(base_steps, refine_steps, step_cache)[:2]
- options.update(two_pass=True, preview=False, refine_from=refine_from or None,
+ options.update(two_pass=True, preview=preview, refine_from=refine_from or None,
media=dict(first=False, last=False, conditioning_info=False))
comfy_bridge.validate_request(text, width, height, seconds, seed, **options,
allow_empty_prompt=allow_empty_prompt is True)
@@ -231,8 +232,10 @@ def execute(cls, text, width, height, seconds, seed, first=None, last=None,
two_pass = True
notes.append('Continuing a video needs "Two-pass acceleration", so it is on for this part.')
if options.get('continue_from') and preview:
- preview = False
- notes.append('A continued part is made in full, without stopping at the preview.')
+ from .continuation import previews_supported
+ if not previews_supported():
+ preview = False
+ notes.append('A continued part is made in full, without stopping at the preview.')
from .two_pass import validate_steps
if not refine_from:
validate_steps(base_steps, refine_steps, two_pass)
diff --git a/freevideo_engine/continuation.py b/freevideo_engine/continuation.py
index c268e918..c47273d4 100644
--- a/freevideo_engine/continuation.py
+++ b/freevideo_engine/continuation.py
@@ -86,8 +86,7 @@ def audio_context():
return validate_audio_context(os.environ.get('FREEVIDEO_CONTINUATION_AUDIO', AUDIO_CONTEXT_DEFAULT))
-def supported():
- """Platform capability, without loading Torch or probing a GPU."""
+def _machine_backend():
from .paths import data_root
try:
machine = json.loads((data_root() / 'machine.json').read_text(encoding='utf-8'))
@@ -95,11 +94,21 @@ def supported():
machine = {}
if not isinstance(machine, dict):
machine = {}
- if machine.get('device_backend', 'cuda') not in ('cuda', 'mps'):
+ return machine.get('device_backend', 'cuda')
+
+
+def supported():
+ """Platform capability, without loading Torch or probing a GPU."""
+ if _machine_backend() not in ('cuda', 'mps'):
return False, 'unsupported_device'
return True, None
+def previews_supported(backend=None):
+ """A continued part can stop at its first pass and be finished from it: NVIDIA only."""
+ return (backend or _machine_backend()) == 'cuda'
+
+
def validate_frames(frames):
if type(frames) is not int or frames < 39 or (frames - 5) % 17:
raise ValueError(INVALID_FRAMES)
@@ -131,9 +140,9 @@ def validate_route(*, task='t2va', two_pass=True, preview=False, refine_from=Non
valid_task = not sound and task in (('ref2va', 'ref2va_av') if anchor else ('t2va', 'ref2va', 'ref2va_audio', 'ref2va_av'))
if not supported()[0] or backend not in (None, 'cuda', 'mps'):
raise ValueError(UNSUPPORTED_DEVICE)
- if refine_from:
+ if refine_from and not previews_supported(backend):
raise ValueError(UNSUPPORTED_REFINE)
- if not two_pass or preview:
+ if not two_pass or (preview and not previews_supported(backend)):
raise ValueError(UNSUPPORTED_PASSES)
# Keyframes reach a continued part as pictures (comfy_media.continued); here they are a request error.
if (not valid_task or any(media.get(key) for key in ('first', 'last', 'conditioning_info'))
diff --git a/freevideo_engine/dubbing/align.py b/freevideo_engine/dubbing/align.py
index 5c3ec0d2..355b04c8 100644
--- a/freevideo_engine/dubbing/align.py
+++ b/freevideo_engine/dubbing/align.py
@@ -180,14 +180,20 @@ def run(speed):
tried.append((speed, voice, rate, seconds(voice, rate), edges))
return tried[-1]
+ # A line that is too long is first closed by the tempo change, which keeps the voice natural; the model
+ # speed only takes what the tempo change cannot (a faster model speed costs naturalness quickly).
+ # A line that is too short is slowed by the tempo change in the same way, inside the same range.
+ def aim(length):
+ return min(max(length, wanted / (max_stretch * .96)), wanted * max_stretch * .96) # inside this range no model speed is needed
+
def off(entry):
- return abs(entry[3] - wanted) / wanted
+ return abs(entry[3] - aim(entry[3])) / wanted
speed = 1.0
entry = run(speed)
while off(entry) > tolerance and len(tried) < 3:
# the length scales with 1/speed, so the next speed is the measured ratio
- speed = min(max(speed * entry[3] / wanted, low), high)
+ speed = min(max(speed * entry[3] / aim(entry[3]), low), high)
if speed == tried[-1][0]:
break
entry = run(speed)
diff --git a/freevideo_engine/dubbing/service.py b/freevideo_engine/dubbing/service.py
index a0ae7b23..0f569cf7 100644
--- a/freevideo_engine/dubbing/service.py
+++ b/freevideo_engine/dubbing/service.py
@@ -90,6 +90,11 @@ def request(value):
raise ValueError('invalid_request')
if line.get('reference') is not None:
row['reference'] = reference_of(line['reference'], required=True)
+ level = line.get('level')
+ if level is not None:
+ if isinstance(level, bool) or not isinstance(level, (int, float)) or not 0 < level <= 1:
+ raise ValueError('invalid_request')
+ row['level'] = float(level)
instruction = line.get('instruction')
if instruction not in (None, ''):
if not isinstance(instruction, str) or len(instruction) > 200:
@@ -141,9 +146,27 @@ def info(self, wait=True):
state['ready'] = pack.settle(root)
if state['ready']:
state['bytes'] = 0
+ # The voice change reads the lines with the speech model of the subtitle pack, so one download brings both.
+ speech = self.speech_state()
+ if speech and not speech[0]:
+ state['ready'], state['bytes'] = False, state['bytes'] + speech[1]
return dict(ready=state['ready'], model=pack.NAME, license=state['license'], bytes=state['bytes'],
published=state['published'], supported=state['supported'], busy=self.busy(), backends=backends.names())
+ @staticmethod
+ def speech_state():
+ """(all here, bytes still to download) of the speech pack (runtime, speech, language), or None where it cannot
+ be had. The pack lives where the recogniser looks for it (the model folder), not under the dubbing root."""
+ from .. import subtitle_pack
+ try:
+ state = subtitle_pack.status()
+ if not state['installable']:
+ return None
+ groups = [state['groups'][group] for group in subtitle_pack.GROUPS]
+ return all(group['ready'] for group in groups), sum(group['bytes'] for group in groups)
+ except Exception:
+ return None
+
def busy(self):
return self.thread is not None and self.thread.is_alive()
@@ -259,7 +282,36 @@ def output_folder(self, key_id):
folder.mkdir()
return folder
+ def install(self, root):
+ """The dubbing model, then the speech model; the progress counts both as one download."""
+ from .. import subtitle_pack
+ speech = self.speech_state()
+ missing = bool(speech and not speech[0])
+ extra = speech[1] if missing else 0
+ mine = []
+
+ def counted(offset, whole):
+ def report(row):
+ if 'total' in row:
+ mine.append(row['total'])
+ row = dict(row, done=min(row.get('done', 0) + offset, whole), total=whole)
+ self.progress(row)
+ return report
+ pack.install(root, self.first_segment(extra, mine), self.check)
+ if missing:
+ first = mine[-1] if mine else 0
+ subtitle_pack.install(subtitle_pack.GROUPS, None, counted(first, first + extra), self.check)
+
+ def first_segment(self, extra, mine):
+ def report(row):
+ if 'total' in row:
+ mine.append(row['total'])
+ row = dict(row, total=row['total'] + extra)
+ self.progress(row)
+ return report
+
def work(self, root, machine, action, value):
+ from .. import subtitle_pack
from ..locking import runtime_lock
from ..monitoring import save
started = time.monotonic()
@@ -267,7 +319,7 @@ def work(self, root, machine, action, value):
if action == 'install':
with runtime_lock(Path(root) / 'dubbing-install.lock', inherit=False):
self.check()
- pack.install(root, self.progress, self.check)
+ self.install(root)
self.check()
self.progress(dict(phase='complete'))
else:
@@ -279,7 +331,7 @@ def work(self, root, machine, action, value):
self.progress(dict(phase='cancelled'))
except BlockingIOError:
self.progress(dict(phase='failed', error='busy'))
- except pack.PackError as error:
+ except (pack.PackError, subtitle_pack.PackError) as error:
self.progress(dict(phase='failed', error=error.reason))
except Exception as error:
reason = str(error)
@@ -506,7 +558,7 @@ async def action(request):
result = service.cancel(value.get('job'))
else:
running, pending = server.prompt_queue.get_current_queue()
- if running or pending:
+ if name == 'dub' and (running or pending):
raise ValueError('busy')
if name == 'dub':
await asyncio.to_thread(pack.settle, service.installation()[0])
diff --git a/freevideo_engine/dubbing/worker.py b/freevideo_engine/dubbing/worker.py
index 8bca3594..7ce212ec 100644
--- a/freevideo_engine/dubbing/worker.py
+++ b/freevideo_engine/dubbing/worker.py
@@ -181,11 +181,13 @@ def prepare(source, key):
used = []
match = {} # per line with its own reference: the voiced RMS the line is to come back with
for index, line in enumerate(lines):
+ if line.get('level'):
+ match[index] = float(line['level'])
own = line.get('reference')
if own:
try:
used.append(('line',) + prepare(own, 'line%d' % index) + (cut['reference-line%d.wav' % index], None))
- match[index] = levels.get('line%d' % index)
+ match.setdefault(index, levels.get('line%d' % index))
continue
except ValueError as error:
if str(error) != 'reference_short' or job_ref is None:
diff --git a/freevideo_engine/editor_export.py b/freevideo_engine/editor_export.py
index b85ba971..0401f06a 100644
--- a/freevideo_engine/editor_export.py
+++ b/freevideo_engine/editor_export.py
@@ -27,6 +27,7 @@
from .editor_limits import PROJECT_BYTES
from .editor_text import is_text_track, has_text_data, validate_text_project, paint_project, check_export_text, MissingGlyphs
from . import editor_export_options
+from .audio_loudness import Limiter
from .editor_speed import validate_clip, changed as speed_changed, audio_changed, independent_join, VideoReader, AudioReader
from .editor_audio import (clip_fields, track_fields, clip_audible, track_audible, crossfade_reads, apply_track_audio,
duck_ranges, duck_gain, duck_sources)
@@ -663,6 +664,9 @@ def _render_once(project, resolve, target, *, progress, cancelled, options, repa
try:
width, height, total, tracks = _layout(project)
settings = editor_export_options.effective(options, width, height, total)
+ peak = project.get('master_peak_db')
+ if peak is not None and (type(peak) not in (int, float) or not math.isfinite(peak) or not -12 <= peak <= 0):
+ raise ExportError()
width, height, total = settings.width, settings.height, settings.frames
if settings.only == 'sound':
tracks = {key: clips for key, clips in tracks.items() if key != 'v1'}
@@ -737,7 +741,32 @@ def save_lead(lead):
readers.callback(lambda: [decoder.close() for decoder in decoders.values()])
black = np.zeros((height, width, 3), dtype=np.uint8) if video is not None else None
written = 0
+
+ def encode_sound(block, number):
+ samples, sample_format = editor_export_options.sound_samples(block, settings)
+ sound = _audio_frame(samples, format=sample_format, layout='stereo')
+ sound.sample_rate = SAMPLE_RATE
+ sound.pts, sound.time_base = (number - settings.start) * SAMPLES_PER_FRAME, Fraction(1, SAMPLE_RATE)
+ for packet in audio.encode(sound):
+ output.mux(packet)
+ if video is None and progress:
+ progress((number - settings.start + 1) / total * .99)
+
+ # With a master ceiling the mix goes through a true-peak limiter, whose output lags its input;
+ # its frames are encoded by their own timeline position.
+ limiter = Limiter(peak, SAMPLE_RATE) if peak is not None else None
+ limited, limited_frame = np.zeros((2, 0)), None
+
+ def drain(final):
+ nonlocal limited, limited_frame
+ while limited.shape[1] >= SAMPLES_PER_FRAME or (final and limited.shape[1]):
+ block = np.clip(limited[:, :SAMPLES_PER_FRAME], -1., 1.).astype(np.float32)
+ limited = limited[:, SAMPLES_PER_FRAME:]
+ if limited_frame >= settings.start:
+ encode_sound(block, limited_frame)
+ limited_frame += 1
lead_start = editor_export_options.audio_lead_start(settings.start, tracks)
+ limited_frame = lead_start
for number in range(lead_start, settings.start + total):
_check(cancelled)
for track, clips in editor_export_options.frame_tracks(tracks, settings.start, number):
@@ -813,17 +842,17 @@ def save_lead(lead):
mixed += apply_track_audio(track_block, number * SAMPLES_PER_FRAME, audio_tracks[track],
project['tracks'], extras=extras)
del extras
+ if limiter is not None:
+ limited = np.concatenate([limited, limiter.push(mixed)], axis=1)
+ drain(False)
+ continue
np.clip(mixed, -1., 1., out=mixed)
if number < settings.start:
continue
- samples, sample_format = editor_export_options.sound_samples(mixed, settings)
- sound = _audio_frame(samples, format=sample_format, layout='stereo')
- sound.sample_rate = SAMPLE_RATE
- sound.pts, sound.time_base = (number - settings.start) * SAMPLES_PER_FRAME, Fraction(1, SAMPLE_RATE)
- for packet in audio.encode(sound):
- output.mux(packet)
- if video is None and progress:
- progress((number - settings.start + 1) / total * .99)
+ encode_sound(mixed, number)
+ if limiter is not None:
+ limited = np.concatenate([limited, limiter.flush()], axis=1)
+ drain(True)
_check(cancelled)
for stream in (video, audio):
if stream is None:
@@ -969,6 +998,77 @@ def resolve_source(key):
return dict(video=path, audio=path)
+class FrameError(ValueError):
+ """Why one frame cannot be saved as a picture: no_picture (nothing is shown there), generating (its part is not made yet) or range."""
+ def __init__(self, reason):
+ super().__init__(reason)
+ self.reason = reason
+
+
+def render_frame(project, resolve, number, *, cancelled=None):
+ """One timeline frame as the export draws it (same placement, fit, picture edits and text), without encoding a video."""
+ width, height, total, tracks = _layout(project)
+ if not _whole(number) or number >= total:
+ raise FrameError('range')
+ clip = next((c for c in tracks['v1'] if c['start'] <= number < c['start'] + c['frames']), None)
+ if clip is None:
+ raise FrameError('no_picture')
+ path = _media(resolve, clip, 'video')
+ if path is None:
+ raise FrameError('generating')
+ wanted = clip['in'] + number - clip['start']
+ # A plain clip seeks straight to the wanted frame; a changed speed or direction keeps the export's own reader.
+ reader = (VideoReader(path, clip, width, height, cancelled) if speed_changed(clip)
+ else _VideoReader(path, wanted, width, height, cancelled, clip))
+ try:
+ pixels = reader.pixels(wanted)
+ return paint_project(pixels, project, number, width, height)
+ finally:
+ reader.close()
+
+
+def _picture_folder():
+ """The folder of pictures made here (the image editor's history), created on first use and never a link."""
+ import folder_paths
+ base = Path(folder_paths.get_input_directory()).resolve()
+ folder = base / 'freevideo-image'
+ if folder.is_symlink() or folder.resolve().parent != base:
+ raise ExportError()
+ folder.mkdir(exist_ok=True)
+ return folder
+
+
+def store_picture(pixels):
+ """Write a frame as a lossless PNG named like the pictures the image editor lists (image-YYYYMMDD-HHMMSS-NN.png)."""
+ from PIL import Image
+ from .file_ops import publish
+ folder = _picture_folder()
+ temporary = folder / ('.frame-' + uuid.uuid4().hex + '.partial')
+ try:
+ with temporary.open('xb') as stream:
+ Image.fromarray(pixels, 'RGB').save(stream, 'PNG')
+ stream.flush()
+ os.fsync(stream.fileno())
+ stamp = time.strftime('image-%Y%m%d-%H%M%S')
+ for number in range(1, 100):
+ target = folder / ('%s-%02d.png' % (stamp, number))
+ try:
+ publish(temporary, target)
+ except FileExistsError:
+ continue
+ return dict(file='freevideo-image/' + target.name, name=target.name,
+ width=int(pixels.shape[1]), height=int(pixels.shape[0]))
+ raise ExportError()
+ finally:
+ temporary.unlink(missing_ok=True)
+
+
+def save_frame(identity, revision, number, *, cancelled=None):
+ """Render one frame of a saved project revision and keep it as a picture."""
+ project = load_project(identity, revision)
+ return store_picture(render_frame(project, resolve_source, number, cancelled=cancelled))
+
+
def _save_report(target, project, seconds, *, options=None):
"""Completion marker after media closes; only video.mp4 enters the library."""
frames = _layout(project)[2]
@@ -1130,7 +1230,7 @@ def progress(fraction):
def register():
- """Register the three endpoints once on the existing ComfyUI server."""
+ """Register the export and frame endpoints once on the existing ComfyUI server."""
from aiohttp import web
from server import PromptServer
server = PromptServer.instance
@@ -1182,3 +1282,27 @@ async def cancel(request):
return web.json_response({'reason': 'failed'}, status=404)
except (ValueError, TypeError):
return web.json_response({'reason': 'failed'}, status=400)
+
+ frame_lock = asyncio.Lock()
+
+ @server.routes.post('/freevideo/editor/frame')
+ async def frame(request):
+ """Keep one frame of a saved project revision, as the export would draw it, in the pictures made here."""
+ try:
+ body = await request.json()
+ if (not isinstance(body, dict) or set(body) != {'id', 'revision', 'frame'}
+ or not _whole(body['revision']) or not _whole(body['frame'])):
+ raise ValueError('Invalid request')
+ async with frame_lock:
+ value = await asyncio.to_thread(save_frame, body['id'], body['revision'], body['frame'])
+ return web.json_response({'picture': value}, headers={'Cache-Control': 'no-store'})
+ except FrameError as error:
+ return web.json_response({'reason': error.reason}, status=422)
+ except ExportConflict as error:
+ return web.json_response({'revision': error.revision}, status=409, headers={'Cache-Control': 'no-store'})
+ except FileNotFoundError:
+ return web.json_response({'reason': 'source_missing'}, status=404)
+ except (ValueError, TypeError, KeyError):
+ return web.json_response({'reason': 'failed'}, status=400)
+ except Exception as error:
+ return web.json_response({'reason': _reason(error)}, status=503)
diff --git a/freevideo_engine/failure_details.py b/freevideo_engine/failure_details.py
index b4359135..3849589d 100644
--- a/freevideo_engine/failure_details.py
+++ b/freevideo_engine/failure_details.py
@@ -550,8 +550,10 @@ def _launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_re
if found:
action = ('请改选“%s”文件夹。' if zh else 'Select the "%s" folder instead.') % found
else:
- action = ('请选择包含 main.py 的 ComfyUI 文件夹，或包含它的便携包文件夹。' if zh else
- 'Select the ComfyUI folder (it contains main.py), or the portable folder that contains it.')
+ # The button name is the one on the launcher's ComfyUI page (Main.qml, newComfyMethod).
+ action = ('请选择包含 main.py 的 ComfyUI 文件夹，或包含它的便携包文件夹。尚未安装 ComfyUI 时，请选择“帮我安装 ComfyUI”。' if zh else
+ 'Select the ComfyUI folder (it contains main.py), or the portable folder that contains it. '
+ 'If ComfyUI is not installed yet, choose "Install ComfyUI".')
return dict(title='这不是 ComfyUI 文件夹' if zh else "This isn't a ComfyUI folder", detail='',
action=action, kind='comfy-folder')
for pattern, kind, title, action in PLACE_RULES:
@@ -687,7 +689,7 @@ def _launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_re
'请点“重试”。仍然停在这里时，可以导出报告。')),
(r'ComfyUI is already running\. Restart it once', 'comfy-restart',
('Restart ComfyUI to load FreeVideo', '需要重启 ComfyUI 才能加载 FreeVideo'),
- ('Close the ComfyUI that is running now, then click Retry.', '请关闭正在运行的 ComfyUI，然后点击重试。')),
+ ('Close the ComfyUI that is running now, then click "Connect again".', '请关闭正在运行的 ComfyUI，然后点击“重新连接”。')),
# web/health.js lists the failed files and what to do on the page itself.
(r'The browser page did not load FreeVideo completely|浏览器页面没有完整加载 FreeVideo', 'page',
('FreeVideo did not load completely in the browser', '浏览器里的 FreeVideo 没有完整加载'),
@@ -864,6 +866,11 @@ def _launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_re
# Validation messages are already actionable. Show the cause, not a stack
# trace or a guessed hardware diagnosis, and leave long details expandable.
lines = [line.strip() for line in text.splitlines() if line.strip()]
+ if lines and all(re.fullmatch(INSPECTION_SUMMARY, line) for line in lines):
+ # The check printed nothing but its own summary line (once per place that kept it); that line
+ # is for the report, not the card. The next step goes under it, as for an unexpected failure.
+ return dict(title='暂时未能完成' if zh else 'Couldn’t complete this step',
+ detail=CHECK_UNFINISHED[zh] + '\n' + UNKNOWN_NEXT_STEP[zh], action='', kind='unknown')
causes = [line for line in lines if re.match(r'^(?:[\w.]+(?:Error|Exception):|ERROR[: ]|Failed step:|Setup failed:)', line)]
detail = causes[-1] if causes else next((line for line in reversed(lines)
if not line.startswith(('File ', 'Traceback', '^', 'All files retained.', 'Existing files are retained.', 'Rerun the same setup command'))), '')
@@ -889,6 +896,9 @@ def _launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_re
return dict(title=title, detail=detail, action='', kind='unknown')
+# comfy_setup.inspection_failed: the first line of a failed check.
+INSPECTION_SUMMARY = r'Setup inspection failed(?: \([^()\r\n]*(?:\([^()\r\n]*\)[^()\r\n]*)*\))?'
+CHECK_UNFINISHED = ('The check did not finish.', '这次检查没有完成。')
UNKNOWN_NEXT_STEP = ('Try again. If it fails again, export the report, then use "Report a problem" and attach it.',
'请重试。仍然失败时，请先导出报告，再点“提交问题”并附上报告。')
diff --git a/freevideo_engine/frame_preview.py b/freevideo_engine/frame_preview.py
index f59c6053..ba47d8f8 100644
--- a/freevideo_engine/frame_preview.py
+++ b/freevideo_engine/frame_preview.py
@@ -2,7 +2,8 @@
At each step the scheduler already holds the model's prediction of the clean
latents. A fixed per-pixel affine map from the 24 latent channels to RGB turns
-four of those latent frames into a small picture without running the VAE. The
+up to sixteen evenly spaced latent frames into one small strip without running
+the VAE, which the page plays as a loop. The
observer only reads tensors the sampler already produced and never writes to
them, so the sampled latents are identical with it on or off. Any failure
silently turns it off for the rest of the request.
@@ -17,7 +18,7 @@
MARKER = 'frame-preview-off'
ENV = 'FREEVIDEO_FRAME_PREVIEW'
FIRST_STEP = 3
-FRAMES = 4
+FRAMES = 16 # evenly spaced latent frames in the strip; the page guards use the same bound
QUALITY = 82
MAX_FRAME_WIDTH = 336
CHANNELS = 24
@@ -110,10 +111,11 @@ def _fail():
class _Preview:
- def __init__(self, latent_frames, frame_h, frame_w, step_offset, total):
+ def __init__(self, latent_frames, frame_h, frame_w, step_offset, total, seconds=None):
self.rows = latent_frames * frame_h * frame_w
self.shape = (latent_frames, frame_h, frame_w)
self.step_offset, self.total = step_offset, total
+ self.duration_ms = round(seconds * 1000) if isinstance(seconds, (int, float)) and 0 < seconds < 86400 else None
self.calls = 0
count = min(FRAMES, latent_frames)
self.indices = sorted({round(i * (latent_frames - 1) / max(1, count - 1)) for i in range(count)})
@@ -164,18 +166,19 @@ def observe(self, model_output, timestep, sample):
_serial = max(_serial + 1, step)
n = len(self.indices)
emit(dict(event='frame_preview', step=step, total=self.total, serial=_serial, frames=n,
- frame_width=strip.shape[1] // n, frame_height=strip.shape[0]))
+ frame_width=strip.shape[1] // n, frame_height=strip.shape[0],
+ **({'duration_ms': self.duration_ms} if self.duration_ms else {})))
@contextmanager
-def observing(*, latent_frames, frame_h, frame_w, step_offset=0, total=None):
+def observing(*, latent_frames, frame_h, frame_w, step_offset=0, total=None, seconds=None):
"""Observe the video scheduler of one sampling pass; no-op unless configured."""
global _active
previous = _active
_active = None
if _sink is not None and not _failed:
try:
- _active = _Preview(latent_frames, frame_h, frame_w, step_offset, total)
+ _active = _Preview(latent_frames, frame_h, frame_w, step_offset, total, seconds)
except Exception:
_active = None
_fail()
diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.py
index b91e6f4d..19c2debb 100644
--- a/freevideo_engine/generate.py
+++ b/freevideo_engine/generate.py
@@ -540,7 +540,12 @@ def usable_continuation(value, *check):
preview=preview, refine_from=getattr(args, 'refine_from', None), media=media,
original_audio=original_audio is not None and 'segment' in original_audio)
# Reject absent/mismatched context before text encoding or model loading.
- if getattr(args, 'continuation_snapshot', None):
+ preview_encoded = False
+ if refine_source:
+ from .preview import continuation_snapshot
+ snapshot, digest, preview_encoded = continuation_snapshot(refine_source, continue_from)
+ continuation = restore_snapshot(snapshot, digest, canvas)
+ elif getattr(args, 'continuation_snapshot', None):
continuation = restore_snapshot(args.continuation_snapshot,
getattr(args, 'continuation_snapshot_sha256', None), canvas)
else:
@@ -566,7 +571,8 @@ def usable_continuation(value, *check):
from .continuation import anchor_media, chain_first_frame_picture
try:
# A context encoded from the file carries no first frame; whoever planned this run saw none either.
- kept = None if 'tail' in continuation else chain_first_frame_picture(continuation, continue_from)
+ kept = (None if 'tail' in continuation or preview_encoded
+ else chain_first_frame_picture(continuation, continue_from))
except ValueError:
kept = None
first_frame['reason'] = 'unreadable'
@@ -577,9 +583,15 @@ def usable_continuation(value, *check):
if args.out.resolve().exists():
raise FileExistsError(args.out.resolve())
args.out.resolve().parent.mkdir(parents=True, exist_ok=True)
- continue_anchor = args.out.resolve().with_suffix('.first-frame.png')
picture, picture_width, picture_height = kept
- continue_anchor.write_bytes(picture)
+ # A second pass names the same picture its preview was made with.
+ continue_anchor = None
+ if refine_source:
+ from .preview import continuation_anchor
+ continue_anchor = continuation_anchor(refine_source)
+ if continue_anchor is None:
+ continue_anchor = args.out.resolve().with_suffix('.first-frame.png')
+ continue_anchor.write_bytes(picture)
media = anchor_media(media, continue_anchor)
validate_route(task=getattr(args, 'task', None) or 'ref2va', anchor=True, two_pass=two_pass,
preview=preview, refine_from=getattr(args, 'refine_from', None), media=media)
@@ -667,7 +679,7 @@ def usable_continuation(value, *check):
validate_route(task=profile['engine'].get('task', 't2va'), anchor=anchored,
original_audio=original_audio is not None)
planned = continuation_plan(canvas, two_pass, base_steps=base_steps, refine_steps=refine_steps,
- scale=upscale_factor(args))
+ staged=bool(preview or refine_source), scale=upscale_factor(args))
if two_pass and not planned['enabled'] and base_steps != 8:
raise ValueError(planned['reason'] + ' A continuation needs two-pass generation.')
continuation_first = planned['first']
@@ -939,6 +951,12 @@ def interrupted(signum, frame):
tuning.save_state(state)
except (OSError, ValueError) as error:
report['tuning']['cache_note'] = str(error)
+ # The encoder is done with the reference frames it normalized (over 1 GB per reference video).
+ for frames in (temporary / 'media').glob('*.npy'):
+ try:
+ frames.unlink(missing_ok=True)
+ except OSError:
+ pass # Windows keeps a file open; the next start's release frees it
info = report.get('encoding', {}).get('conditioning_info') or media.get('conditioning_info')
if info:
if any(info.get(k) != canvas[k] for k in ('width', 'height', 'frames')):
diff --git a/freevideo_engine/launcher/Main.qml b/freevideo_engine/launcher/Main.qml
index 154194b4..7d2cd200 100644
--- a/freevideo_engine/launcher/Main.qml
+++ b/freevideo_engine/launcher/Main.qml
@@ -479,7 +479,7 @@ ApplicationWindow {
text: t("Download the latest ↗", "下载最新版 ↗"); onClicked: backend.link("https://github.com/FlashML-org/FreeVideo/releases/latest") }
FButton { objectName: "copyError"; visible: !diskFailure && !notAFault; text: t("Copy details", "复制详情"); onClicked: backend.copy(s.error) }
FButton { objectName: "showError"; visible: !notAFault; text: errorDetailsOpen ? t("Hide details", "收起详情") : t("Show details", "查看详情"); flat: true; onClicked: errorDetailsOpen = !errorDetailsOpen }
- FButton { objectName: "reportProblem"; visible: !diskFailure && !notAFault && s.failure.kind !== "port-taken"; text: t("Report a problem", "提交问题"); onClicked: backend.reportProblem(s.error, s.failure.title || "") }
+ FButton { objectName: "reportProblem"; visible: !diskFailure && !notAFault && s.failure.kind !== "port-taken" && s.failure.kind !== "comfy-restart" && s.failure.kind !== "comfy-folder"; text: t("Report a problem", "提交问题"); onClicked: backend.reportProblem(s.error, s.failure.title || "") }
FButton { objectName: "exportError"; visible: !diskFailure && !notAFault; text: t("Export report", "导出报告"); enabled: s.report.status !== "running"; onClicked: backend.exportReport() }
}
ScrollView {
@@ -1485,7 +1485,7 @@ ApplicationWindow {
Layout.fillWidth: true; spacing: 8
FButton { text: t("Copy details", "复制详情"); implicitHeight: theme.heightSm; font.pixelSize: theme.micro + 1; onClicked: backend.copy(s.error) }
FButton { text: t("Export report", "导出报告"); implicitHeight: theme.heightSm; font.pixelSize: theme.micro + 1; enabled: s.report.status !== "running"; onClicked: backend.exportReport() }
- FButton { objectName: "settingsReportProblem"; text: t("Report a problem", "提交问题"); implicitHeight: theme.heightSm; font.pixelSize: theme.micro + 1; onClicked: backend.reportProblem(s.error, s.failure.title || "") }
+ FButton { objectName: "settingsReportProblem"; visible: s.failure.kind !== "comfy-folder"; text: t("Report a problem", "提交问题"); implicitHeight: theme.heightSm; font.pixelSize: theme.micro + 1; onClicked: backend.reportProblem(s.error, s.failure.title || "") }
}
FText {
visible: s.report.status !== "idle"; Layout.fillWidth: true; font.pixelSize: theme.micro; color: theme.muted
@@ -1644,20 +1644,24 @@ ApplicationWindow {
}
FText { objectName: "downloadActionNotice"; visible: !!s.action_notice && s.action_notice !== s.busy_reason; text: s.action_notice || ""; color: theme.muted; Layout.fillWidth: true }
FText { objectName: "downloadListError"; visible: text.length > 0; text: s.download_list.error; color: theme.danger; Layout.fillWidth: true; wrapMode: Text.WordWrap; font.pixelSize: theme.micro + 1 }
- FText { objectName: "downloadListRawError"; visible: s.error.length > 0 && !s.download_list.inspection_error; text: s.error; color: theme.danger; Layout.fillWidth: true; maximumLineCount: 2; elide: Text.ElideRight; font.pixelSize: theme.micro }
+ FText { objectName: "downloadListRawError"; visible: s.error.length > 0 && !s.download_list.inspection_failed; text: s.error; color: theme.danger; Layout.fillWidth: true; maximumLineCount: 2; elide: Text.ElideRight; font.pixelSize: theme.micro }
}
}
FText { objectName: "downloadBusyReason"; visible: !s.can_manage && (!s.action_notice || s.action_notice === s.busy_reason); text: s.busy_reason; color: theme.muted; Layout.fillWidth: true }
- RowLayout {
- objectName: "downloadListActions"; Layout.fillWidth: true; Layout.minimumWidth: 0; spacing: 8
- // The actions share what the Close button leaves, so a long label wraps instead of leaving the dialog.
+ GridLayout {
+ id: downloadListActions; objectName: "downloadListActions"; Layout.fillWidth: true; Layout.minimumWidth: 0
+ columnSpacing: 8; rowSpacing: 8
+ // Wide: one row, Close on the left, every label on one line. Narrow, or wherever the row would not
+ // hold every label on one line: one full-width column, the actions first and Close last.
+ readonly property real rowWidth: closeDownloadList.implicitWidth + (checkDownloadList.visible ? checkDownloadList.implicitWidth : 0)
+ + (useDownloadedFolder.visible ? useDownloadedFolder.implicitWidth : 0) + exportDownloadList.implicitWidth + (actionCount + 1) * columnSpacing
+ readonly property bool stacked: win.narrow || rowWidth > width
readonly property int actionCount: (checkDownloadList.visible ? 1 : 0) + (downloads.scope === "image" ? 1 : 2)
- readonly property real actionWidth: Math.max(80, Math.floor((width - closeDownloadList.implicitWidth - (actionCount + 1) * spacing) / actionCount))
- FButton { id: closeDownloadList; text: downloads.copy.close; flat: true; onClicked: modelInfoOpen = false }
- Item { Layout.fillWidth: true }
- FButton { id: checkDownloadList; objectName: "checkDownloadList"; Layout.maximumWidth: parent.actionWidth; visible: downloads.scope !== "image" && (!downloads.available || s.download_list.needs_check); text: downloads.copy.check; enabled: !s.busy; onClicked: backend.prepareDownloadList() }
- FButton { objectName: "useDownloadedFolder"; visible: downloads.scope !== "image"; Layout.maximumWidth: parent.actionWidth; text: downloads.copy.use_folder; enabled: s.can_manage; onClicked: backend.useDownloadedFolder() }
- FButton { objectName: "exportDownloadList"; Layout.maximumWidth: parent.actionWidth; text: downloads.copy.export; primary: true; enabled: downloads.available && (downloads.scope === "image" || !s.busy); onClicked: backend.exportDownloadList() }
+ FButton { id: closeDownloadList; objectName: "closeDownloadList"; Layout.row: downloadListActions.stacked ? 3 : 0; Layout.column: 0; Layout.fillWidth: downloadListActions.stacked; text: downloads.copy.close; flat: true; onClicked: modelInfoOpen = false }
+ Item { visible: !downloadListActions.stacked; Layout.row: 0; Layout.column: 1; Layout.fillWidth: true }
+ FButton { id: checkDownloadList; objectName: "checkDownloadList"; Layout.row: 0; Layout.column: downloadListActions.stacked ? 0 : 2; Layout.fillWidth: downloadListActions.stacked; visible: downloads.scope !== "image" && (!downloads.available || s.download_list.needs_check); text: downloads.copy.check; enabled: !s.busy; onClicked: backend.prepareDownloadList() }
+ FButton { id: useDownloadedFolder; objectName: "useDownloadedFolder"; visible: downloads.scope !== "image"; Layout.row: downloadListActions.stacked ? 1 : 0; Layout.column: downloadListActions.stacked ? 0 : 3; Layout.fillWidth: downloadListActions.stacked; text: downloads.copy.use_folder; enabled: s.can_manage; onClicked: backend.useDownloadedFolder() }
+ FButton { id: exportDownloadList; objectName: "exportDownloadList"; Layout.row: downloadListActions.stacked ? 2 : 0; Layout.column: downloadListActions.stacked ? 0 : 4; Layout.fillWidth: downloadListActions.stacked; text: downloads.copy.export; primary: true; enabled: downloads.available && (downloads.scope === "image" || !s.busy); onClicked: backend.exportDownloadList() }
}
}
}
diff --git a/freevideo_engine/launcher_session.py b/freevideo_engine/launcher_session.py
index 66b25595..578b1be9 100644
--- a/freevideo_engine/launcher_session.py
+++ b/freevideo_engine/launcher_session.py
@@ -2424,6 +2424,8 @@ def snapshot(self):
download_state = dict(checking=bool(self.controller.busy), error=self.download_error,
needs_check=self.download_error_key in ('unchecked', 'changed'), folder=self.download_export,
inspection_error=inspection_read_error,
+ # Any failed check: the list's own line asks for Check again, so the raw error stays in the details.
+ inspection_failed=row.get('status') == 'failed' and row.get('action') == 'inspect',
notice=downloads['copy']['exported'].format(folder=self.download_export) if self.download_export else '')
if inspection_read_error:
download_state.update(error=self.t(*DOWNLOAD_COPY['error_read']), needs_check=True)
diff --git a/freevideo_engine/lora_online_cache.py b/freevideo_engine/lora_online_cache.py
index c00096fa..733cc083 100644
--- a/freevideo_engine/lora_online_cache.py
+++ b/freevideo_engine/lora_online_cache.py
@@ -13,6 +13,7 @@
from .lora_cache import INT8_UNMERGEABLE, MODULATION, _targets, index_adapter, prepare as prepare_fused
from .media_request import digest, verify_files
from .monitoring import save
+from . import online_lora_folders
from .storage import fingerprint
from .tensor_io import open_tensors
@@ -33,6 +34,17 @@ def _supported(adapters, linears, skip_modulation=False):
return True
+def _tidy(output, identity):
+ """Record the use, then drop folders nothing can select again or nobody has used for long."""
+ online_lora_folders.touch(output)
+ current = {key: identity[key] for key in ('source_manifest_sha256', 'implementation_sha256', 'index_sha256', 'names_sha256')}
+ try:
+ for folder, reason, freed in online_lora_folders.prune(output.parent, current, keep=output):
+ print(json.dumps(dict(event='lora_folder_removed', folder=Path(folder).name, reason=reason, bytes=freed)), flush=True)
+ except OSError:
+ pass
+
+
@torch.no_grad()
def prepare(cache, adapters, output_root=None):
cache = Path(cache).resolve()
@@ -80,6 +92,7 @@ def prepare(cache, adapters, output_root=None):
# Another combination can change a shared inode's link count/ctime.
# Recheck it once, rather than rehashing the model on every use.
save(marker, saved)
+ _tidy(output, identity)
return output, dict(enabled=True, mode='online', cache_hit=True, **saved['online_lora']['summary'])
targets = {name: dict(shape=spec['weight_shape']) for name, spec in manifest['linears'].items()}
groups, skipped = {}, []
@@ -153,5 +166,6 @@ def prepare(cache, adapters, output_root=None):
save(marker, dict(manifest, source_id=key, groups=records, online_lora=online,
lora_identity=identity, lora_file_stamps={r['file']: fingerprint(output / r['file']) for r in records},
total_bytes=manifest['total_bytes'] + sum(r['bytes'] for r in records if r['group'].startswith('lora/'))))
+ _tidy(output, identity)
return output, dict(enabled=True, mode='online', cache_hit=False,
prepare_seconds=time.perf_counter() - started, **summary)
diff --git a/freevideo_engine/macos_runtime.py b/freevideo_engine/macos_runtime.py
index 19a41851..6bce3c53 100644
--- a/freevideo_engine/macos_runtime.py
+++ b/freevideo_engine/macos_runtime.py
@@ -602,7 +602,8 @@ def host_guard(module, inputs):
progress.layers(self.model.transformer_blocks), \
frame_preview.observing(latent_frames=canvas['latent_frames'], frame_h=height // 32,
frame_w=width // 32, step_offset=progress_offset + start,
- total=progress_total if progress_total is not None else active_steps):
+ total=progress_total if progress_total is not None else active_steps,
+ seconds=canvas.get('seconds')):
self.residency = residency
from .step_cache import begin_pass, end_pass
step_cache = None
diff --git a/freevideo_engine/media_encoding.py b/freevideo_engine/media_encoding.py
index 2e34f3c3..58ba90c9 100644
--- a/freevideo_engine/media_encoding.py
+++ b/freevideo_engine/media_encoding.py
@@ -14,7 +14,6 @@
REFERENCE_FRAMES = 360 # 15 s at 24 fps, the longest H3 generation
-VOICE_ONLY_CANVAS = {'width': 256, 'height': 144} # a clip kept only for its voice is encoded this small
def reference_frames(canvas):
@@ -35,11 +34,12 @@ def reference_size(width, height, canvas):
return max(32, round(width * scale / 32) * 32), max(32, round(height * scale / 32) * 32)
-def encode_visual(vae, pixels, *, device='cuda', posterior='sample'):
+def encode_visual(vae, pixels, *, device='cuda', posterior='sample', progress=None):
"""The pinned VAE chunk/padding/posterior recipe with one live pixel chunk.
Posterior sampling still occurs once on the original device with seed 42;
sampling separately per chunk would change the conditioning noise.
+ `progress(frames_done)` is told after each temporal chunk; it never touches the tensors.
"""
import numpy as np
import torch
@@ -56,6 +56,8 @@ def encode_visual(vae, pixels, *, device='cuda', posterior='sample'):
current = torch.cat([current, current[:, :, -1:].repeat(1, 1, clip - current.shape[2], 1, 1)], dim=2)
moments.append(vae._encode_clip(current).cpu())
del current
+ if progress is not None:
+ progress(min(len(pixels), begin + clip))
moments = torch.cat(moments, dim=2).to(device)
if len(pixels) > 1 and vae.config.token_drop:
moments = moments[:, :, :-vae.config.token_drop]
@@ -132,6 +134,25 @@ def _audio(path, destination, seconds):
return str(destination), count > limit + 1600, length
+def reference_load(rows):
+ """(packed video rows, encoding seconds) the visual references add, as capabilities.admit counts them.
+
+ A still is one patch grid and five legacy encoding seconds; a clip adds five
+ rows of patches per 17 frames plus two, and its own length.
+ """
+ video, seconds = 0, 0.
+ for row in rows:
+ if row.get('kind') not in ('image', 'video') or not row.get('pixels'):
+ continue
+ patches = (int(row.get('height', 0)) // 32) * (int(row.get('width', 0)) // 32)
+ frames = int(row.get('frames', 1))
+ if frames == 1:
+ video, seconds = video + patches, seconds + 5.
+ else:
+ video, seconds = video + max(0, (frames - 5) // 17 * 5 + 2) * patches, seconds + frames / 24.
+ return video, seconds
+
+
def prepare(media, canvas, directory):
import numpy as np
import torch
@@ -161,6 +182,9 @@ def prepare(media, canvas, directory):
'width': canvas['width'], 'height': canvas['height'], 'crop_mode': 'fit'})
for index, row in enumerate(media.get('references', [])):
kind, path = row['kind'], row['path']
+ if kind == 'video' and row.get('voice_only'):
+ # Only the sound is wanted, so the clip is read as an audio reference and no picture is encoded.
+ kind = 'audio'
if row.get('use') == 'original':
from .original_audio import model_audio, inspect
numbers[kind] += 1
@@ -198,8 +222,7 @@ def prepare(media, canvas, directory):
if not container.streams.video:
raise ValueError('Reference video has no video stream')
stream = container.streams.video[0]
- voice_only = bool(row.get('voice_only'))
- width, height = reference_size(stream.width, stream.height, VOICE_ONLY_CANVAS if voice_only else canvas)
+ width, height = reference_size(stream.width, stream.height, canvas)
length = clip_seconds(container, stream)
pixels = np.lib.format.open_memmap(target, mode='w+', dtype=np.uint8, shape=(limit, height, width, 3))
origin = None
@@ -224,23 +247,12 @@ def prepare(media, canvas, directory):
count += 1
previous = image
del previous
- step = 1
- if voice_only:
- # Every third frame (8 fps), or every second (12 fps) when that would leave too few frames.
- step = next((n for n in (3, 2) if count // n >= 22), 1)
- kept = np.array(pixels[:count:step])
- del pixels
- pixels, count = kept, len(kept)
count = count - (count - 5) % 17
if count < 22:
raise ValueError('The reference video is too short; it needs to be at least about 1 s long.')
- if voice_only:
- np.save(target, pixels[:count], allow_pickle=False)
- else:
- pixels.flush()
- ref.update(pixels=str(target), frames=count, normalized_fps=24 // step, width=width, height=height,
- crop_mode='voice_%dfps' % (24 // step) if voice_only else 'video_24fps')
- used = count * step / 24.
+ pixels.flush()
+ ref.update(pixels=str(target), frames=count, normalized_fps=24, width=width, height=height, crop_mode='video_24fps')
+ used = count / 24.
# The clip's own sound is a reference too, unless the request turned it off.
soundtrack, longer_audio, _ = (None, False, None) if row.get('sound') is False or row.get('guide') else \
_audio(path, directory / ('reference-%02d-audio.npy' % index), limit / 24.)
@@ -248,7 +260,7 @@ def prepare(media, canvas, directory):
if soundtrack:
ref['audio'] = soundtrack
ref_items.append({'type': 'audio'})
- selected = list(range(0, count, 12 // step))
+ selected = list(range(0, count, 12))
sampled = torch.from_numpy(np.array(pixels[selected])).float().div_(255)
if not row.get('guide'):
# A guide only supplies motion latents; it is not described to the
@@ -258,7 +270,8 @@ def prepare(media, canvas, directory):
else:
ref['audio'], longer, length = _audio(path, target, limit / 24.)
if ref['audio'] is None:
- raise ValueError('Reference audio file has no audio stream')
+ raise ValueError('The voice-only reference video has no audio stream' if row['kind'] == 'video'
+ else 'Reference audio file has no audio stream')
ref.update(width=0, height=0, frames=0, crop_mode='audio_32khz')
ref_items.append({'type': 'audio'})
if longer:
@@ -268,6 +281,12 @@ def prepare(media, canvas, directory):
print(json.dumps(dict(event='reference_trimmed', **ref['trimmed'])), flush=True)
result.append(ref)
(directory / 'normalization.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
+ rows, seconds = reference_load(result)
+ if rows:
+ # Known long before the first encoded chunk, and what the time estimate
+ # needs: reference rows enlarge every sampling step.
+ print(json.dumps({'event': 'reference_media', 'reference_video_tokens': rows,
+ 'reference_seconds': seconds}), flush=True)
return result, {'images': key_images} if key_images else {'minimax_ref_items': ref_items} if ref_items else {}
@@ -386,12 +405,12 @@ def get_latent_cache_key_dict(**kwargs):
return make_latent_cache_key(**kwargs)[1]
-def _latent_cache(root, base, device):
- """Per-picture latent receipts; another encoder, recipe or device never matches."""
+def _latent_cache(root, base, device, *, folder='vae', recipe='reference-latent-v2', cache_kind='reference-latent'):
+ """Per-input latent receipts; another encoder, recipe or device never matches."""
import torch
from .input_cache import InputCache, digest
from .paths import data_root
- directory = Path(base) / 'vae'
+ directory = Path(base) / folder
files = sorted(directory.glob('*.safetensors'))
if not files:
return None
@@ -402,13 +421,35 @@ def _latent_cache(root, base, device):
except Exception:
diffusers = None
from . import __version__
- identity = {'recipe': 'reference-latent-v2', 'engine': __version__, 'diffusers': diffusers,
+ identity = {'recipe': recipe, 'engine': __version__, 'diffusers': diffusers,
'torch': getattr(torch, '__version__', 'unknown'), 'device': kind,
'gpu': torch.cuda.get_device_name() if (kind == 'cuda' and hasattr(torch, 'cuda') and hasattr(torch.cuda, 'is_available') and torch.cuda.is_available()) else None,
'config': digest(directory / 'config.json') if (directory / 'config.json').exists() else None,
'weights': [[f.name, f.stat().st_size, f.stat().st_mtime_ns] for f in files]}
target_root = root or (data_root() / 'input-cache')
- return InputCache(identity, root=target_root, kind='reference-latent')
+ return InputCache(identity, root=target_root, kind=cache_kind)
+
+
+def _audio_key(path):
+ """The samples with their shape, so the length counts; every normalized reference is 32 kHz."""
+ import numpy as np
+ mapped = np.load(path, mmap_mode='r', allow_pickle=False)
+ try:
+ return 'audio-32000-' + _pixels_key(mapped)
+ finally:
+ del mapped
+
+
+def _use_audio_latent(row, ref, latent, canvas):
+ """Attach a normalized latent; cached ones are whole, so a clip used as is trims here."""
+ ref['audio_latent'] = latent
+ if row.get('use') == 'original':
+ ticks = round(canvas['frames'] / 24 * 40)
+ sound = latent.reshape(2, -1, 32)
+ if sound.shape[1] < ticks:
+ raise ValueError('Original audio encoder returned too few latent frames')
+ ref.update(use='original', original_audio=row['original_audio'],
+ audio_latent=sound[:, :ticks].reshape(-1, 32).contiguous())
def _pixels_key(mapped):
@@ -465,6 +506,36 @@ def _remember_latent(cache, key, latent, directory=None):
path.unlink(missing_ok=True)
+class _encode_progress:
+ """Tells the caller how many reference frames are encoded, once per temporal chunk.
+
+ Reference video encoding is the longest silent stage of a request on a
+ modest GPU, so each finished chunk prints one `media_encode_progress` row
+ with the elapsed time, from which the remaining time follows. The count only
+ moves forward, also when a stage is retried.
+ """
+
+ def __init__(self, rows, clock=None):
+ import time
+ self.total = sum(int(frames) for frames in rows)
+ self.clock = clock or time.perf_counter
+ self.started = self.clock()
+ self.before = self.done = 0
+
+ def __call__(self, frames):
+ done = min(self.total, self.before + int(frames))
+ if done <= self.done or not self.total:
+ return
+ self.done = done
+ print(json.dumps({'event': 'media_encode_progress', 'done': done, 'total': self.total,
+ 'elapsed_seconds': round(self.clock() - self.started, 3)}), flush=True)
+
+ def finish(self, frames):
+ """One reference is complete: the next one counts on top of it."""
+ self.before += int(frames)
+ self.done = max(self.done, min(self.total, self.before))
+
+
def encode_latents(normalized, value, base, canvas, *, resident=None, device='cuda', cache_root=None):
import gc
import numpy as np
@@ -525,6 +596,8 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cu
if on_mps:
from .backends.mps_vae_encode import bounded_encoder
lifetime = bounded_encoder(vae, device=device)
+ report = _encode_progress([row['frames'] for index, row in enumerate(normalized)
+ if row.get('pixels') and index not in hits])
with lifetime as memory:
for index, (row, ref) in enumerate(zip(normalized, refs)):
if row.get('pixels') and index not in hits:
@@ -534,9 +607,12 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cu
ref['latent'] = _retry_in_host_memory(memory, lambda: encode_vae_condition(
vae, pixels, PIXEL_MEAN, PIXEL_STD, 42)).cpu()
del pixels
+ report(1)
else:
ref['latent'] = _retry_in_host_memory(
- memory, lambda: encode_visual(vae, mapped[:row['frames']], device=device))
+ memory, lambda: encode_visual(vae, mapped[:row['frames']], device=device,
+ progress=report))
+ report.finish(row['frames'])
del mapped
if cache is not None and index in keys:
_remember_latent(cache, keys[index], ref['latent'], Path(row['pixels']).parent)
@@ -547,32 +623,46 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cu
del vae
gc.collect()
empty_cache()
- if any(row.get('audio') for row in normalized):
- print(json.dumps({'event': 'media_encode_phase', 'phase': 'Encoding reference audio'}), flush=True)
- from .vae_weights import load_audio_vae
- vae = (resident.input_audio_encoder(base, canvas, device=device)
- if resident is not None and torch.device(device).type != 'mps' else
- load_audio_vae(base, device=device)[0])
- mean = torch.tensor(vae.config.latents_mean).view(1, 1, -1)
- std = torch.tensor(vae.config.latents_std).view(1, 1, -1)
- if vae.config.sampling_rate != 32000:
- raise ValueError('Reference audio normalization requires the pinned 32 kHz H3 VAE')
- for row, ref in zip(normalized, refs):
- if row.get('audio'):
- audio = torch.from_numpy(np.load(row['audio'], allow_pickle=False)).to(device)
- latent = vae.encode(audio[:, None], return_dict=False)[0].mode().float().cpu().transpose(1, 2)
- ref['audio_latent'] = ((latent - mean) / std).reshape(-1, 32).contiguous()
- if row.get('use') == 'original':
- ticks = round(canvas['frames'] / 24 * 40)
- sound = ref['audio_latent'].reshape(2, -1, 32)
- if sound.shape[1] < ticks:
- raise ValueError('Original audio encoder returned too few latent frames')
- ref.update(use='original', original_audio=row['original_audio'],
- audio_latent=sound[:, :ticks].reshape(-1, 32).contiguous())
- del audio, latent
- del vae
- gc.collect()
- empty_cache()
+ audio_rows = [index for index, row in enumerate(normalized) if row.get('audio')]
+ if audio_rows:
+ # The same voice with another prompt reuses its latent and skips loading the audio VAE.
+ audio_cache, audio_keys, audio_latents = None, {}, {}
+ try:
+ audio_cache = _latent_cache(cache_root, base, device, folder='audio_vae',
+ recipe='reference-audio-latent-v1', cache_kind='reference-audio-latent')
+ for index in audio_rows if audio_cache is not None else ():
+ audio_keys[index] = _audio_key(normalized[index]['audio'])
+ latent = _cached_latent(audio_cache, audio_keys[index], Path(normalized[index]['audio']).parent)
+ if latent is not None:
+ audio_latents[index] = latent
+ except (OSError, ValueError, RuntimeError, KeyError, TypeError):
+ audio_cache = None
+ print(json.dumps({'event': 'reference_audio_latent_cache', 'hits': len(audio_latents),
+ 'misses': len(audio_rows) - len(audio_latents)}), flush=True)
+ if len(audio_latents) < len(audio_rows):
+ print(json.dumps({'event': 'media_encode_phase', 'phase': 'Encoding reference audio'}), flush=True)
+ from .vae_weights import load_audio_vae
+ vae = (resident.input_audio_encoder(base, canvas, device=device)
+ if resident is not None and torch.device(device).type != 'mps' else
+ load_audio_vae(base, device=device)[0])
+ mean = torch.tensor(vae.config.latents_mean).view(1, 1, -1)
+ std = torch.tensor(vae.config.latents_std).view(1, 1, -1)
+ if vae.config.sampling_rate != 32000:
+ raise ValueError('Reference audio normalization requires the pinned 32 kHz H3 VAE')
+ for index in audio_rows:
+ if index not in audio_latents:
+ audio = torch.from_numpy(np.load(normalized[index]['audio'], allow_pickle=False)).to(device)
+ latent = vae.encode(audio[:, None], return_dict=False)[0].mode().float().cpu().transpose(1, 2)
+ audio_latents[index] = ((latent - mean) / std).reshape(-1, 32).contiguous()
+ if audio_cache is not None:
+ _remember_latent(audio_cache, audio_keys[index], audio_latents[index],
+ Path(normalized[index]['audio']).parent)
+ del audio, latent
+ del vae
+ gc.collect()
+ empty_cache()
+ for index in audio_rows:
+ _use_audio_latent(normalized[index], refs[index], audio_latents[index], canvas)
if any(row.get('anchor') for row in normalized):
value.update(keyframe_anchors=[row['anchor'] for row in normalized],
condition_latents=[row['latent'] for row in refs])
diff --git a/freevideo_engine/media_request.py b/freevideo_engine/media_request.py
index 4bc1dd02..c19fca24 100644
--- a/freevideo_engine/media_request.py
+++ b/freevideo_engine/media_request.py
@@ -154,7 +154,7 @@ def validate_guide(row):
def validate_voice_only(row):
- """A voice-only video supplies its sound; its picture is kept small and slow because only the voice is wanted."""
+ """A voice-only video supplies only its sound track, read as an audio reference; its picture is not encoded."""
if row.get('kind') != 'video':
raise ValueError('voice_only is only allowed on video references')
if type(row['voice_only']) is not bool:
diff --git a/freevideo_engine/online_lora_folders.py b/freevideo_engine/online_lora_folders.py
new file mode 100644
index 00000000..0df7fdbe
--- /dev/null
+++ b/freevideo_engine/online_lora_folders.py
@@ -0,0 +1,190 @@
+"""The online LoRA folders this installer builds beside a prepared model, and when they can go.
+
+prepare() in lora_online_cache names a folder after the digest of its identity
+(model manifest, the code that builds it, every adapter and strength), so each
+new combination, strength or code update leaves another folder. Each holds the
+adapter factors it built and hard links to the model's unchanged groups.
+
+A folder is ours only when its name is the digest it was built under, its
+manifest names it, and every file is regular and listed by that manifest.
+Removal takes exactly those files: a link only drops its count, so the model
+it points at is never touched, and unknown files stay.
+
+A folder is stale once nothing can select it again - its model manifest or
+the code that built it has changed - or nobody has used it for a long time.
+Use is recorded as the manifest's modification time.
+"""
+import json
+import os
+from pathlib import Path
+import re
+import stat
+import time
+
+from .long_paths import io_path, plain, plain_errors
+
+NAME = re.compile(r'vdn-online-lora-[0-9a-f]{24}')
+LORA_FILE = re.compile(r'lora/(root|\d{2})\.(safetensors|partial)')
+ADALN_FILE = re.compile(r'adaln-tables-v2-[0-9a-f]{24}/(identity|producer)\.json|adaln-tables-v2-[0-9a-f]{24}/\d{2}\.(safetensors|json)')
+MARKER = 'manifest.json'
+MARKER_LIMIT = 64 << 20
+IDLE_DAYS = 30
+EXPIRED_IDLE_DAYS = 1
+REPARSE_POINT = 0x400
+
+
+def redirected(info):
+ return stat.S_ISLNK(info.st_mode) or bool(getattr(info, 'st_file_attributes', 0) & REPARSE_POINT)
+
+
+def safe_relative(value):
+ return (isinstance(value, str) and value and '\\' not in value and ':' not in value
+ and not value.startswith('/') and all(part not in ('', '.', '..') for part in value.split('/')))
+
+
+@plain_errors
+def listing(beside):
+ """Real folders of this naming below beside, never through a link."""
+ beside = io_path(beside)
+ try:
+ found = sorted(path for path in beside.iterdir() if NAME.fullmatch(path.name))
+ except OSError:
+ return []
+ return [Path(plain(path)) for path in found if not redirected(path.lstat()) and path.is_dir()]
+
+
+@plain_errors
+def manifest(folder):
+ """The folder's manifest when it names this folder as its own, else None."""
+ marker = io_path(folder) / MARKER
+ try:
+ info = marker.lstat()
+ if redirected(info) or not stat.S_ISREG(info.st_mode) or info.st_size > MARKER_LIMIT:
+ return None
+ value = json.loads(marker.read_text(encoding='utf-8'))
+ except (OSError, ValueError):
+ return None
+ if (not isinstance(value, dict) or not isinstance(value.get('lora_identity'), dict)
+ or not isinstance(value.get('online_lora'), dict)
+ or str(value.get('source_id', ''))[:24] != Path(folder).name[len('vdn-online-lora-'):]):
+ return None
+ return value
+
+
+def listed_names(value):
+ """Relative names the manifest of a folder lists: groups, shared AdaLN tables and itself."""
+ names = {MARKER}
+ for group in value.get('groups', []):
+ if isinstance(group, dict) and safe_relative(group.get('file')):
+ names.add(group['file'])
+ for table in value.get('adaln_tables', []):
+ for row in table.get('files', []) if isinstance(table, dict) else []:
+ if isinstance(row, dict) and safe_relative(row.get('file')):
+ names.add(row['file'])
+ return names
+
+
+@plain_errors
+def owned_files(folder):
+ """Regular files of one folder that are ours, or [] when the folder is not ours."""
+ value = manifest(folder)
+ if value is None:
+ return []
+ names = listed_names(value)
+ folder = io_path(folder)
+ found = []
+ for current, directories, files in os.walk(folder, followlinks=False):
+ for name in list(directories):
+ if redirected((Path(current) / name).lstat()):
+ directories.remove(name)
+ for name in files:
+ path = Path(current) / name
+ relative = path.relative_to(folder).as_posix()
+ try:
+ info = path.lstat()
+ except OSError:
+ continue
+ if redirected(info) or not stat.S_ISREG(info.st_mode):
+ continue
+ if relative in names or LORA_FILE.fullmatch(relative):
+ found.append(Path(plain(path)))
+ return found
+
+
+def permitted(relative, group_names):
+ """A planned name below an online folder: its manifest, an adapter file, a model group or an AdaLN table."""
+ return (relative == MARKER or LORA_FILE.fullmatch(relative) is not None
+ or ADALN_FILE.fullmatch(relative) is not None or relative in group_names)
+
+
+def home(path):
+ """The online folder a path lies in, or None."""
+ return next((parent for parent in Path(path).parents if NAME.fullmatch(parent.name)), None)
+
+
+def last_used(folder):
+ try:
+ return (io_path(folder) / MARKER).stat().st_mtime
+ except OSError:
+ return None
+
+
+def touch(folder):
+ """Record a use; never an error."""
+ try:
+ os.utime(io_path(folder) / MARKER)
+ except OSError:
+ pass
+
+
+def stale(folder, current, now=None):
+ """Why this folder can go, or None. current holds the digests a folder built today would carry."""
+ value = manifest(folder)
+ used = last_used(folder)
+ if value is None or used is None:
+ return None
+ idle = ((now if now is not None else time.time()) - used) / 86400
+ identity = value['lora_identity']
+ expired = any(identity.get(key) != digest for key, digest in current.items())
+ if expired and idle >= EXPIRED_IDLE_DAYS:
+ return 'code or model changed'
+ if idle >= IDLE_DAYS:
+ return 'unused for %d days' % IDLE_DAYS
+ return None
+
+
+@plain_errors
+def remove(folder):
+ """Remove one owned folder's files, then its empty directories. Bytes released by files that had no other link."""
+ folder = io_path(folder)
+ freed, left = 0, False
+ # The manifest is what makes the folder ours, so it goes last, and only when every other file went.
+ for path in sorted(owned_files(folder), key=lambda p: p.relative_to(Path(plain(folder))).as_posix() == MARKER):
+ if path.name == MARKER and path.parent == Path(plain(folder)) and left:
+ continue
+ try:
+ info = io_path(path).lstat()
+ io_path(path).unlink()
+ except OSError:
+ left = True
+ continue
+ freed += info.st_size if info.st_nlink == 1 else 0
+ for current, directories, files in os.walk(folder, topdown=False, followlinks=False):
+ try:
+ Path(current).rmdir()
+ except OSError:
+ pass
+ return freed
+
+
+@plain_errors
+def prune(beside, current, keep=None, now=None):
+ """Remove stale folders beside a prepared model, except keep. Returns (folder, reason, bytes) rows."""
+ removed = []
+ for folder in listing(beside):
+ if keep is not None and Path(plain(folder)) == Path(plain(keep)):
+ continue
+ reason = stale(folder, current, now)
+ if reason:
+ removed.append((plain(folder), reason, remove(folder)))
+ return removed
diff --git a/freevideo_engine/policy.py b/freevideo_engine/policy.py
index 673b462d..0ab19354 100644
--- a/freevideo_engine/policy.py
+++ b/freevideo_engine/policy.py
@@ -664,8 +664,9 @@ def feasible_alternatives(hardware, canvas, **options):
"""Smaller requests this machine can plan now, for a refusal to name.
The longest video at the requested size, and the requested length at the
- largest of the panel's smaller pixel targets that fits; each one admitted
- by this same planner, never estimated separately.
+ largest of the smaller pixel targets (1.5, 1.0, 0.75 and 0.5 megapixels)
+ that fits; each one admitted by this same planner, never estimated
+ separately.
"""
from .geometry import geometry
keep = {key: canvas[key] for key in ('task', 'steps', 'reference_video_tokens', 'reference_audio_tokens', 'early_reference')
@@ -698,7 +699,7 @@ def fits(width, height, frames):
if best is not None:
result.append(dict(kind='shorter', width=width, height=height, frames=best, seconds=round(best / 24, 1)))
ratio = width / height
- for megapixels in (.75, .5):
+ for megapixels in (1.5, 1., .75, .5):
area = megapixels * 1024 * 1024
smaller_width = max(256, int(round((area * ratio) ** .5 / 32)) * 32)
smaller_height = max(256, int(round((area / ratio) ** .5 / 32)) * 32)
diff --git a/freevideo_engine/preview.py b/freevideo_engine/preview.py
index 028405c2..f868bd1f 100644
--- a/freevideo_engine/preview.py
+++ b/freevideo_engine/preview.py
@@ -4,6 +4,7 @@
same latents: the learned latent upscale, the refinement steps and a full
decode, as the uninterrupted request would have run them. Torch-free.
"""
+import hashlib
import json
from pathlib import Path
@@ -51,6 +52,31 @@ def source(output, native=False):
return result
+def continuation_snapshot(source_paths, continue_from=None):
+ """The ending a continued preview was made from: (path, sha256, encoded from the file).
+
+ Its second pass continues that same record, whatever the store holds now."""
+ request = json.loads(Path(source_paths['request']).read_text(encoding='utf-8'))
+ value = request.get('continuation')
+ if not isinstance(value, dict):
+ raise ValueError('This preview does not continue a video. Generate the preview again.')
+ if continue_from is not None and Path(value.get('source_video', '')).resolve() != Path(continue_from).resolve():
+ raise ValueError('This preview continues another video. Generate the preview again.')
+ tail = value.get('tail')
+ path = Path((tail if isinstance(tail, dict) else value).get('snapshot') or '')
+ if not path.is_file():
+ raise ValueError('This preview no longer has the video ending it continues. Generate the preview again.')
+ return str(path), hashlib.sha256(path.read_bytes()).hexdigest(), isinstance(tail, dict)
+
+
+def continuation_anchor(source_paths):
+ """The first-frame picture a continued preview was made with, when it was and is still there."""
+ request = json.loads(Path(source_paths['request']).read_text(encoding='utf-8'))
+ anchor = (request.get('continuation') or {}).get('anchor')
+ path = Path(anchor['path']) if isinstance(anchor, dict) and anchor.get('path') else None
+ return path if path is not None and path.is_file() else None
+
+
def upscale_reads(cache, video, upscaler_checkpoint):
"""Files the upscale of this preview reads from disk, in the order it needs them.
diff --git a/freevideo_engine/release_notes.json b/freevideo_engine/release_notes.json
index 6e465075..9793e96a 100644
--- a/freevideo_engine/release_notes.json
+++ b/freevideo_engine/release_notes.json
@@ -1,37 +1,74 @@
{
"schema": 1,
- "product_version": "0.4.33",
+ "product_version": "0.4.34",
"en": {
- "summary": "Prompt enhancement works with less free graphics memory, and Setup or Repair now waits for a video that is still generating.",
+ "summary": "Agent can finish the sound of a whole editor project in one job, a line of a clip can be respoken in the original voice, and the Edit tab can save the current frame as a picture.",
"changes": [
- "Prompts no longer receive the added sentence about natural sounds that 0.4.32 introduced.",
- "Audio references: an mp4 file that only contains sound is accepted. An Agent that needs only the voice of a video can send it at a small picture size, which makes generation much faster.",
- "Agent: choosing a take cancels the takes of that shot that are still queued or running.",
- "Agent: when a voice reference is uploaded, the first six seconds are checked for speech, and the Agent is told when there is too little, such as in a clip that opens with music, so another clip can be chosen.",
- "Setup or Repair that is stopped by a video still generating now waits for it and continues by itself, instead of showing a general failure.",
- "Prompt enhancement: rewriting now works with less free graphics memory. When other programs hold the memory it needs, it waits and starts by itself once there is room, and lists those programs when their names can be read.",
- "Audio processing in videos no longer differs depending on when FreeVideo was installed.",
- "With several graphics cards, a new generation waits when its card is still briefly held by the previous request, instead of failing within a few seconds.",
- "The Autopilot run bar stays fully visible in the top bar on narrow windows.",
- "Settings: on narrow screens, the three Getting started buttons no longer break their labels across lines."
+ "Agent: a new job finishes the sound of an editor project in one step. It separates each clip's sound, removes sustained music that came with the clips while keeping effects such as gunshots and crowds, lays one continuous music track across the cut, evens out loudness and fills silent gaps with room tone.",
+ "Edit only some of the lines in a clip: those lines are respoken in the original speaker's voice at the original timing, and the rest stay as they were. Dubbed lines keep the original pace and loudness. The dubbing model (about 4.9 GB) is downloaded on first use.",
+ "A reference video marked as voice only is now read as an audio reference: only its sound track is used and no picture is encoded. A voice reference used across several shots is no longer encoded again for each one.",
+ "Agent: take checks listen for the scripted lines (Chinese and English), notice a single odd frame at the start or end and a picture that stops before the take ends, and do not pick such takes. The Agent skill text also submits storyboards as one batch and resumes a retryable failure once.",
+ "Short dramas: long shots submitted right after startup are no longer refused while this computer's generation limit is still being worked out.",
+ "The Edit tab can save the frame under the playhead as a picture. It is drawn as the export would draw it and kept under Edit image > Generate with AI > History.",
+ "Continued parts in the editor can now use Preview first, and Run second pass replaces the preview in place.",
+ "The preview shown while a video is generated plays as a loop of up to sixteen frames instead of a still sheet of four, and no longer flashes black when its stage changes size. Recent videos and Creations fold videos with the same prompt together only when they were made within two hours of each other from the same references.",
+ "Reference video encoding now shows progress, and the time estimate counts the reference video from the start.",
+ "When a request is refused for lack of GPU memory, the suggested size is now the largest smaller one that fits, instead of one of only two fixed smaller sizes.",
+ "Reference-video frames are removed after encoding and stopped requests free their leftovers, so a generation with reference videos no longer leaves over 1 GB per reference in its folder. Changing the prepared model also removes the online LoRA folders that pinned the old model, and unused ones are cleaned up automatically.",
+ "Launcher and Settings: the Restart ComfyUI step and the wrong-folder message no longer offer to report a problem, the wrong-folder message points to Install ComfyUI when none is found nearby, and the placement test button is disabled while an installation check or installation is running."
]
},
"zh": {
- "summary": "提示词增强在可用显存较少时也能改写；安装或修复遇到还在生成的视频时会等待并自动继续。",
+ "summary": "Agent 可以一步完成整个剪辑工程的声音；可以只改一段里的几句台词，用原说话人的声音重说；编辑页可以把当前帧存成图片。",
"changes": [
- "提示词不再被补写 0.4.32 加入的自然声音描述。",
- "音频参考：只有声音的 mp4 文件也可以使用。Agent 只需要视频里的声音时，可用小画面传入，生成明显更快。",
- "Agent：选定一版后，同一镜头还在排队或生成的其他版本自动撤下。",
- "Agent：上传声音参考时检查前 6 秒的说话时长，说话太少（例如以音乐开头）时告知 Agent，便于换一段。",
- "安装或修复被还在生成的视频拦下时，启动器现在会等视频结束后自动继续，不再显示通用的失败提示。",
- "提示词增强：可用显存较少时也能改写；显存被其他程序占用时会等待，空出后自动改写，能读取到程序名时一并列出。",
- "视频声音的处理结果不再因安装时间不同而有差异。",
- "多张显卡时，新的生成遇到显卡还被上一个请求短暂占用会等待，不再几秒内失败。",
- "AI 全自动运行时，窄窗口顶栏的运行条完整显示。",
- "设置：窄屏上“新手指引”的三个按钮不再折行断字。"
+ "Agent：新增一个任务，一步完成剪辑工程的全片声音：分离各段音轨，去掉持续的旧配乐（枪声、人群声等效果声保留），铺一条贯穿全片的配乐，统一全片响度，并在静音处补上环境声。",
+ "可以只改其中几句台词：改过的句子用原说话人的声音重说，时间位置不变，其余原声不动。配音的语速与响度保持与原句一致。配音模型约 4.9 GB，第一次使用时下载。",
+ "标为只用声音的参考视频，现在按音频参考读取：只取其声音，不再编码画面。同一段声音参考用于多个镜头时，不再重复编码。",
+ "Agent：镜头质检会听出每条候选是否说出了剧本台词（中英文均可），并发现开头或结尾单帧闪烁和画面提前停住，有这些问题的候选不会被选用。Agent 技能文本也改为把分镜作为一批提交，可重试的失败只重试一次。",
+ "短剧：在服务刚启动时提交长镜头，不再因本机的生成能力尚未算完而被拒绝。",
+ "编辑页可以把播放头所在的一帧存成图片，画面与导出一致，图片放在“编辑图片 > AI 生成 > 历史”里。",
+ "剪辑页的续写段现在可以使用“先出预览”，看过后用“继续二采”原位生成正式版。",
+ "生成中的预览改为循环播放最多 16 帧，取代原来 4 帧的静态拼图，画面尺寸变化时也不再闪黑。最近视频和作品库中，提示词相同的视频只有在相隔不超过两小时、且参考内容相同时才折叠在一起。",
+ "参考视频编码现在会显示进度，预计时间从一开始就把参考视频算进去。",
+ "因显存不足而拒绝请求时，建议的尺寸改为能放行的最大一档，不再只在两个固定的较小尺寸里选。",
+ "视频参考的帧文件在编码后即删除，取消或失败的请求也会释放残留文件，带视频参考的生成不再在文件夹里留下每个参考超过 1 GB 的文件。更换模型后会一并清理占住旧模型的在线 LoRA 文件夹，不再使用的也会自动清理。",
+ "启动器和设置：“需要重启 ComfyUI”这一步和“这不是 ComfyUI 文件夹”的提示不再显示“提交问题”；后者在附近找不到 ComfyUI 时会提示选择“帮我安装 ComfyUI”；安装检查或安装进行时，放置测试按钮置灰。"
]
},
"history": [
+ {
+ "product_version": "0.4.33",
+ "en": {
+ "summary": "Prompt enhancement works with less free graphics memory, and Setup or Repair now waits for a video that is still generating.",
+ "changes": [
+ "Prompts no longer receive the added sentence about natural sounds that 0.4.32 introduced.",
+ "Audio references: an mp4 file that only contains sound is accepted. An Agent that needs only the voice of a video can send it at a small picture size, which makes generation much faster.",
+ "Agent: choosing a take cancels the takes of that shot that are still queued or running.",
+ "Agent: when a voice reference is uploaded, the first six seconds are checked for speech, and the Agent is told when there is too little, such as in a clip that opens with music, so another clip can be chosen.",
+ "Setup or Repair that is stopped by a video still generating now waits for it and continues by itself, instead of showing a general failure.",
+ "Prompt enhancement: rewriting now works with less free graphics memory. When other programs hold the memory it needs, it waits and starts by itself once there is room, and lists those programs when their names can be read.",
+ "Audio processing in videos no longer differs depending on when FreeVideo was installed.",
+ "With several graphics cards, a new generation waits when its card is still briefly held by the previous request, instead of failing within a few seconds.",
+ "The Autopilot run bar stays fully visible in the top bar on narrow windows.",
+ "Settings: on narrow screens, the three Getting started buttons no longer break their labels across lines."
+ ]
+ },
+ "zh": {
+ "summary": "提示词增强在可用显存较少时也能改写；安装或修复遇到还在生成的视频时会等待并自动继续。",
+ "changes": [
+ "提示词不再被补写 0.4.32 加入的自然声音描述。",
+ "音频参考：只有声音的 mp4 文件也可以使用。Agent 只需要视频里的声音时，可用小画面传入，生成明显更快。",
+ "Agent：选定一版后，同一镜头还在排队或生成的其他版本自动撤下。",
+ "Agent：上传声音参考时检查前 6 秒的说话时长，说话太少（例如以音乐开头）时告知 Agent，便于换一段。",
+ "安装或修复被还在生成的视频拦下时，启动器现在会等视频结束后自动继续，不再显示通用的失败提示。",
+ "提示词增强：可用显存较少时也能改写；显存被其他程序占用时会等待，空出后自动改写，能读取到程序名时一并列出。",
+ "视频声音的处理结果不再因安装时间不同而有差异。",
+ "多张显卡时，新的生成遇到显卡还被上一个请求短暂占用会等待，不再几秒内失败。",
+ "AI 全自动运行时，窄窗口顶栏的运行条完整显示。",
+ "设置：窄屏上“新手指引”的三个按钮不再折行断字。"
+ ]
+ }
+ },
{
"product_version": "0.4.32",
"en": {
@@ -374,23 +411,6 @@
"诊断报告：页面检查会写明创作面板已打开却看不到的原因。Windows 上请求因内存不足被拒绝时，报告会写出已用内存中 FreeVideo 自己占用多少。"
]
}
- },
- {
- "product_version": "0.4.21",
- "en": {
- "summary": "Agent image tasks can place characters in a scene by position, size and facing; the page check no longer reports a covered toolbar behind ComfyUI dialogs.",
- "changes": [
- "Agent images: an image task can take a scene picture and one to three characters (character card or picture) with a position, a height, a full-body or half-body framing and a facing direction. The picture is generated with the scene as the first reference instead of a pasted collage, and the result can be used directly as a first frame.",
- "Page check: the FreeVideo toolbar is no longer reported as covered when ComfyUI shows a dimming or click-away layer over it, such as the Templates dialog on a first start or the Help panel; the details now name what covers the toolbar."
- ]
- },
- "zh": {
- "summary": "Agent 图片任务可按位置、大小和朝向把人物放进场景；页面检查不再因 ComfyUI 的对话框遮罩报告工具栏被遮挡。",
- "changes": [
- "Agent 图片：图片任务可以指定一张场景图和一至三个人物（角色卡或图片），并设定位置、人物高度、全身或半身景别和朝向。图片以场景为第一参考图生成，不再拼贴粘图，结果可直接作首帧。",
- "页面检查：ComfyUI 的调暗或点击遮罩（首次启动的模板对话框、帮助面板等）盖在 FreeVideo 工具栏上时，不再报告工具栏被遮挡；诊断详情会写出遮挡工具栏的元素。"
- ]
- }
}
]
}
diff --git a/freevideo_engine/runtime.py b/freevideo_engine/runtime.py
index 49226a75..5d087500 100644
--- a/freevideo_engine/runtime.py
+++ b/freevideo_engine/runtime.py
@@ -862,7 +862,8 @@ def append(self, value):
(watch.layers(self.transformer.transformer_blocks) if watch is not None else nullcontext()), \
frame_preview.observing(latent_frames=canvas.get('latent_frames'), frame_h=height // 32,
frame_w=width // 32, step_offset=progress_offset,
- total=progress_total if progress_total is not None else active_steps):
+ total=progress_total if progress_total is not None else active_steps,
+ seconds=canvas.get('seconds')):
from .step_cache import begin_pass, end_pass
if not refining:
disabled = ('residual_offload' if self.config.get('residual_offload') and self.config.get('step_cache') == 'fbcache' else
diff --git a/freevideo_engine/sampling_assets.py b/freevideo_engine/sampling_assets.py
index 4911f5a0..06bdf4b4 100644
--- a/freevideo_engine/sampling_assets.py
+++ b/freevideo_engine/sampling_assets.py
@@ -177,7 +177,8 @@ def engine_task(media, base=None):
A reference video counts as audio when it has an audio stream, exactly as
the encoder decides, unless the request turned that video's sound off;
- tables therefore match before the request starts.
+ tables therefore match before the request starts. A voice-only reference
+ video is read as audio and never counts as a picture.
"""
from .media_request import task_for
task = task_for(media)
@@ -185,8 +186,9 @@ def engine_task(media, base=None):
return task
visual = audio = False
for ref in media.get('references', []):
- visual |= ref['kind'] in ('image', 'video')
- audio |= ref['kind'] == 'audio'
+ voice = ref['kind'] == 'video' and ref.get('voice_only')
+ visual |= ref['kind'] in ('image', 'video') and not voice
+ audio |= ref['kind'] == 'audio' or bool(voice)
if ref['kind'] == 'video' and not audio and ref.get('sound') is not False and not ref.get('guide'):
path = Path(ref['path'])
if base is not None and not path.is_absolute():
diff --git a/freevideo_engine/variant_cleanup.py b/freevideo_engine/variant_cleanup.py
index 19b0a7c1..5b0f8443 100644
--- a/freevideo_engine/variant_cleanup.py
+++ b/freevideo_engine/variant_cleanup.py
@@ -14,6 +14,10 @@
the folder's name is the digest of its identity marker and it holds nothing
but table files.
+* the online LoRA folders beside its cache that this installer built (see
+ online_lora_folders): they hardlink the model's groups, so they are removed
+ first and the model files then count only their own link;
+
A variant with fused LoRA variants beside its cache is kept whole: int8 cannot
merge those adapters yet, so the old model is still the only way to use them.
@@ -32,6 +36,7 @@
import time
from .long_paths import io_path, plain, plain_errors
+from . import online_lora_folders
from .locking import runtime_lock
from .storage import fingerprint
from .runtime_alias import checked_real_path, AliasUnavailable
@@ -70,7 +75,8 @@ def inside(base, path):
@plain_errors
-def unshared_file(path):
+def unshared_file(path, links=1):
+ """The file's stat when it is a regular file with exactly this many links (its own, by default)."""
path = io_path(path)
if '.replaced-' in path.name:
return None
@@ -78,7 +84,7 @@ def unshared_file(path):
info = path.lstat()
except OSError:
return None
- if redirected(info) or not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
+ if redirected(info) or not stat.S_ISREG(info.st_mode) or info.st_nlink != links:
return None
return info
@@ -162,17 +168,18 @@ def _stage(folder, prepared, stage):
@plain_errors
-def _generated_tables(prepared, table):
+def _generated_tables(prepared, table, links=None):
"""Files of one engine-generated AdaLN table folder, or None unless it is bound to its identity."""
from .adaln_assets import FORMAT, directory
prepared, table = io_path(prepared), io_path(table)
if not ADALN_DIRECTORY.fullmatch(table.name) or not inside(prepared, table) or not table.is_dir():
return None
entries = list(table.iterdir())
- if any(not ADALN_FILE.fullmatch(entry.name) or unshared_file(entry) is None for entry in entries):
+ links = links or (lambda path: 1)
+ if any(not ADALN_FILE.fullmatch(entry.name) or unshared_file(entry, links(entry)) is None for entry in entries):
return None
marker = table / 'identity.json'
- info = unshared_file(marker)
+ info = unshared_file(marker, links(marker))
if info is None or info.st_size > ADALN_IDENTITY_LIMIT:
return None
try:
@@ -185,9 +192,29 @@ def _generated_tables(prepared, table):
@plain_errors
-def _plan_file(path, kind, entries):
+def _regular(path):
+ try:
+ info = io_path(path).lstat()
+ except OSError:
+ return False
+ return not redirected(info) and stat.S_ISREG(info.st_mode)
+
+
+@plain_errors
+def _plan_file(path, kind, entries, linked=False):
path = io_path(path)
entries.append(dict(path=plain(path), kind=kind, bytes=path.lstat().st_size, stamp=fingerprint(path)))
+ if linked:
+ entries[-1]['linked'] = True
+
+
+def _same(entry, now):
+ """The file is as planned. Removing another link to the same data changes its change time, which a linked file may do."""
+ if not entry.get('linked'):
+ return now == entry['stamp']
+ ignored = ('ctime_ns', 'change_time_ns')
+ return ({k: v for k, v in now.items() if k not in ignored}
+ == {k: v for k, v in entry['stamp'].items() if k not in ignored})
@plain_errors
@@ -201,12 +228,43 @@ def retire(root, catalog, name):
return dict(variant=name, folder=plain(folder), files=entries, kept=kept, bytes=0, kept_bytes=0,
stage_directories=stage_directories, present=folder.exists() or folder.is_symlink())
stages = folder / STAGES
+ # Online LoRA folders go first; each hard link they hold on a model file is
+ # one more link that file may carry and still count as unshared.
+ peers = {}
+ beside = (folder / catalog['variants'][name]['cache_prefix']).parent
+ online_files = []
+ if inside(prepared, beside):
+ for online in online_lora_folders.listing(beside):
+ for path in online_lora_folders.owned_files(online):
+ try:
+ info = io_path(path).lstat()
+ except OSError:
+ continue
+ if info.st_ino:
+ peers[(info.st_dev, info.st_ino)] = peers.get((info.st_dev, info.st_ino), 0) + 1
+ online_files.append((path, info))
+ # Each folder's manifest last: it is what makes the folder ours, so it must outlive any file that stays.
+ online_files.sort(key=lambda row: (str(online_lora_folders.home(row[0])), row[0] == online_lora_folders.home(row[0]) / online_lora_folders.MARKER))
+ for path, info in online_files:
+ _plan_file(path, 'online-lora', entries, linked=True)
+ # A link to a model file frees nothing of its own: the model file is counted once, under its own name.
+ if info.st_ino and info.st_nlink > peers[(info.st_dev, info.st_ino)]:
+ entries[-1]['bytes'] = 0
+ planned.add(io_path(path))
+
+ def links(path):
+ """Links a file may carry: its own, plus the planned online LoRA links to the same data."""
+ try:
+ info = io_path(path).lstat()
+ except OSError:
+ return 1
+ return 1 + peers.get((info.st_dev, info.st_ino), 0) if info.st_ino else 1
for row in catalog['variants'][name]['files']:
path = folder / row['file']
if path.exists() or path.is_symlink():
- info = unshared_file(path)
+ info = unshared_file(path, links(path))
if info is not None and info.st_size == row['bytes'] and inside(prepared, path):
- _plan_file(path, 'catalog', entries)
+ _plan_file(path, 'catalog', entries, linked=links(path) > 1)
planned.add(path)
sidecar = path.with_name(path.name + SIDECAR)
if sidecar.exists() and inside(prepared, sidecar) and _sidecar_matches(sidecar, row):
@@ -238,11 +296,11 @@ def retire(root, catalog, name):
cache = folder / catalog['variants'][name]['cache_prefix']
if cache.is_dir() and inside(prepared, cache):
for table in sorted(cache.iterdir()):
- found = _generated_tables(prepared, table)
+ found = _generated_tables(prepared, table, links)
for path in found or ():
path = io_path(path)
if path not in planned:
- _plan_file(path, 'adaln', entries)
+ _plan_file(path, 'adaln', entries, linked=links(path) > 1)
planned.add(path)
kept_bytes = 0
for current, names, files in os.walk(folder, followlinks=False):
@@ -358,6 +416,19 @@ def permitted(path, kind, folder, rows, cache_prefix=None):
if row_kind == 'catalog' and key.startswith(name) and PARTIAL.fullmatch(key[len(name):]):
return Path(name).parent == path.parent
return False
+ if kind == 'online-lora':
+ if cache_prefix is None:
+ return False
+ beside = folder / Path(cache_prefix).parent
+ try:
+ relative = path.relative_to(beside)
+ except ValueError:
+ return False
+ parts = relative.parts
+ groups = {row_file[len(cache_prefix) + 1:] for row_file in (r['file'] for _, r in rows.values())
+ if row_file.startswith(cache_prefix + '/')}
+ return (len(parts) >= 2 and online_lora_folders.NAME.fullmatch(parts[0]) is not None
+ and online_lora_folders.permitted('/'.join(parts[1:]), groups))
if kind == 'adaln':
if cache_prefix is None:
return False
@@ -437,26 +508,37 @@ def checked(value):
if plain(folder) != plain(retired['folder']) or folder == variant_folder(root, catalog, active):
raise ValueError('Retired model folder changed; check storage again.')
directories = set()
+ held = set()
for entry in retired['files']:
path = Path(entry['path'])
reason = None
- if not permitted(path, entry['kind'], Path(plain(folder)), rows, catalog['variants'][name]['cache_prefix']):
+ if entry['kind'] == 'online-lora' and online_lora_folders.home(path) in held and path == online_lora_folders.home(path) / online_lora_folders.MARKER:
+ reason = 'folder still holds files'
+ elif not permitted(path, entry['kind'], Path(plain(folder)), rows, catalog['variants'][name]['cache_prefix']):
reason = 'not a planned name'
- elif not inside(prepared, path) or unshared_file(path) is None:
+ elif entry['kind'] == 'online-lora' and not inside(prepared, path):
+ reason = 'link, junction, shared or missing'
+ elif entry['kind'] == 'online-lora' and not _regular(path):
+ reason = 'link, junction, shared or missing'
+ elif entry['kind'] != 'online-lora' and (not inside(prepared, path) or unshared_file(path) is None):
reason = 'link, junction, shared or missing'
else:
try:
- if fingerprint(path) != entry['stamp']:
+ if not _same(entry, fingerprint(path)):
reason = 'changed since the scan'
except OSError:
reason = 'missing'
if reason:
skipped.append(dict(path=plain(path), reason=reason))
+ if entry['kind'] == 'online-lora':
+ held.add(online_lora_folders.home(path))
continue
try:
io_path(path).unlink()
except OSError as error:
skipped.append(dict(path=plain(path), reason=type(error).__name__))
+ if entry['kind'] == 'online-lora':
+ held.add(online_lora_folders.home(path))
continue
removed.append(dict(path=plain(path), bytes=entry['bytes'], kind=entry['kind']))
released += entry['bytes']
diff --git a/freevideo_engine/voice/script.py b/freevideo_engine/voice/script.py
new file mode 100644
index 00000000..7547f49b
--- /dev/null
+++ b/freevideo_engine/voice/script.py
@@ -0,0 +1,97 @@
+"""A script given with a selection: what each line is to say, and who says it.
+
+The recogniser misspells and misses (a name, a number, a word shorter than a third of a second). When the text of a line
+is known, it replaces the recognised text and keeps the recogniser only for the times: the letters of the known text are
+laid on the letters that were heard, and the ones that were not heard are spread between their neighbours.
+"""
+import re
+from difflib import SequenceMatcher
+
+FLOOR = .55 # least likeness of a recognised line to a script line for them to be the same line
+MINIMUM = .06 # no word is given less time than this, seconds
+_CJK = '-ヿ㐀-鿿가-'
+_PIECE = re.compile('[%s]|[^\\W_%s]+(?:\'[^\\W_%s]+)?' % (_CJK, _CJK, _CJK))
+
+
+def letters(text):
+ return ''.join(char for char in text.lower() if char.isalnum())
+
+
+def pieces(text):
+ """The words of a text (a character each for Chinese, Japanese and Korean)."""
+ return [found.group(0) for found in _PIECE.finditer(text)]
+
+
+def match(lines, script):
+ """For each recognised line, the script entry it is (or None), in order; an entry is used once.
+
+ `lines` are [{text}], `script` [{text, speaker?}]."""
+ wanted = [letters(entry['text']) for entry in script]
+ used, found = set(), []
+ for line in lines:
+ heard = letters(line['text'])
+ best, likeness = None, FLOOR
+ for index, text in enumerate(wanted):
+ if index in used or not heard or not text:
+ continue
+ ratio = SequenceMatcher(None, heard, text, autojunk=False).ratio()
+ if ratio >= likeness:
+ best, likeness = index, ratio
+ if best is not None:
+ used.add(best)
+ found.append(None if best is None else script[best])
+ return found
+
+
+def retime(words, text, start, end):
+ """The words of `text` with times taken from the recognised `words` ([{text, start, end}]) of the same line.
+
+ Letters that were heard keep their time; the others share what lies between the heard ones, in proportion to their
+ count, inside [start, end]."""
+ heard, spans = [], []
+ for word in words:
+ chars = letters(word['text'])
+ for index, char in enumerate(chars):
+ share = (word['end'] - word['start']) / max(len(chars), 1)
+ heard.append(char)
+ spans.append((word['start'] + index * share, word['start'] + (index + 1) * share))
+ given = pieces(text)
+ chars = []
+ for index, piece in enumerate(given):
+ chars.extend((char, index) for char in letters(piece))
+ times = [None] * len(chars)
+ for block in SequenceMatcher(None, ''.join(heard), ''.join(char for char, _ in chars), autojunk=False).get_matching_blocks():
+ for offset in range(block.size):
+ times[block.b + offset] = spans[block.a + offset]
+ anchors = [-1] + [index for index, found in enumerate(times) if found] + [len(chars)]
+ for before, after in zip(anchors, anchors[1:]):
+ if after - before <= 1:
+ continue
+ first = times[before][1] if before >= 0 else start
+ last = times[after][0] if after < len(chars) else end
+ last = max(last, first)
+ for offset in range(before + 1, after):
+ step = (last - first) / (after - before - 1)
+ times[offset] = (first + (offset - before - 1) * step, first + (offset - before) * step)
+ result = []
+ for index, piece in enumerate(given):
+ own = [times[at] for at, (_, word) in enumerate(chars) if word == index]
+ if not own:
+ continue
+ result.append(dict(text=piece, start=round(min(span[0] for span in own), 3), end=round(max(span[1] for span in own), 3)))
+ previous = start
+ for word in result:
+ word['start'] = round(max(word['start'], previous), 3)
+ word['end'] = round(max(word['end'], word['start'] + MINIMUM), 3)
+ previous = word['start']
+ return result
+
+
+def roles(script):
+ """Role name -> number, in order of first appearance."""
+ numbers = {}
+ for entry in script:
+ name = (entry.get('speaker') or '').strip()
+ if name and name not in numbers:
+ numbers[name] = len(numbers)
+ return numbers
diff --git a/freevideo_engine/voice/speakers.py b/freevideo_engine/voice/speakers.py
new file mode 100644
index 00000000..b8767bee
--- /dev/null
+++ b/freevideo_engine/voice/speakers.py
@@ -0,0 +1,115 @@
+"""Who says each line: lines that sound like one person get one number.
+
+A speaker embedding of the line (the model of the dubbing pack) is compared with the average of each person met so
+far. Without the model or its runtime every line is speaker 0, as before.
+"""
+import logging
+from collections import Counter
+
+import numpy as np
+
+_LOG = logging.getLogger(__name__)
+SAME = .8 # cosine likeness from which two lines are one person (measured on film dialogue, see the card)
+SHORTEST = .8 # a shorter line has too little voice to be told apart; it follows the line before it
+RATE = 16000
+
+
+def embedder(root=None):
+ """A function from mono 16 kHz samples to a vector, or None when the model or its runtime is not here."""
+ try:
+ from ..dubbing import pack
+ file = pack.directory(root) / 'campplus.onnx'
+ if not file.is_file():
+ return None
+ try:
+ import onnxruntime
+ except ImportError:
+ if not pack.activate(pack.runtime_path(root)):
+ return None
+ import onnxruntime
+ import torch
+ import torchaudio.compliance.kaldi as kaldi
+ options = onnxruntime.SessionOptions()
+ options.intra_op_num_threads = 1
+ session = onnxruntime.InferenceSession(str(file), sess_options=options, providers=['CPUExecutionProvider'])
+ except Exception as error: # a missing optional part must never fail the lines
+ _LOG.info('Speakers are not told apart: %s', error)
+ return None
+ name = session.get_inputs()[0].name
+
+ def embed(samples):
+ feat = kaldi.fbank(torch.from_numpy(np.ascontiguousarray(samples, dtype=np.float32))[None], num_mel_bins=80,
+ dither=0, sample_frequency=RATE)
+ feat = feat - feat.mean(dim=0, keepdim=True)
+ return np.asarray(session.run(None, {name: feat.unsqueeze(0).numpy()})[0]).flatten()
+ return embed
+
+
+def cluster(vectors, same=SAME):
+ """Numbers for vectors ([array | None]) in order: a vector joins the person it is most like, if alike enough, else is a new one.
+
+ None (a line too short to tell) takes the number of the line before it, or of the first one that has a vector."""
+ centres, counts, numbers = [], [], []
+ for vector in vectors:
+ if vector is None:
+ numbers.append(None)
+ continue
+ vector = np.asarray(vector, dtype=np.float64)
+ vector = vector / (np.linalg.norm(vector) or 1.)
+ best, likeness = None, same
+ for index, centre in enumerate(centres):
+ value = float(np.dot(vector, centre / (np.linalg.norm(centre) or 1.)))
+ if value >= likeness:
+ best, likeness = index, value
+ if best is None:
+ centres.append(vector.copy())
+ counts.append(1)
+ best = len(centres) - 1
+ else:
+ centres[best] = (centres[best] * counts[best] + vector) / (counts[best] + 1)
+ counts[best] += 1
+ numbers.append(best)
+ known = [number for number in numbers if number is not None]
+ fill = known[0] if known else 0
+ for index, number in enumerate(numbers):
+ if number is None:
+ numbers[index] = numbers[index - 1] if index else fill
+ return numbers
+
+
+def label(slices, embed):
+ """Speaker numbers for the mono 16 kHz `slices` of the lines, in order (all 0 without an embedder)."""
+ if embed is None:
+ return [0] * len(slices)
+ vectors = []
+ for samples in slices:
+ try:
+ vectors.append(embed(samples) if len(samples) >= SHORTEST * RATE else None)
+ except Exception as error:
+ _LOG.info('A line could not be told apart: %s', error)
+ vectors.append(None)
+ return cluster(vectors)
+
+
+def by_role(numbers, roles):
+ """Speaker numbers where the lines that match a script say who they are.
+
+ `numbers`: the cluster of each line; `roles`: the role number of each line or None. A cluster takes the role most of
+ its matched lines have; a line without a cluster match keeps the role of the nearest line that has one. Without any
+ role the clusters stand as they are."""
+ if all(role is None for role in roles):
+ return list(numbers)
+ votes = {}
+ for number, role in zip(numbers, roles):
+ if role is not None:
+ votes.setdefault(number, Counter())[role] += 1
+ result = []
+ for number, role in zip(numbers, roles):
+ if role is None and number in votes:
+ role = votes[number].most_common(1)[0][0]
+ result.append(role)
+ known = [index for index, role in enumerate(result) if role is not None]
+ for index, role in enumerate(result):
+ if role is None:
+ result[index] = result[min(known, key=lambda at: abs(at - index))] if known else 0
+ return result
diff --git a/freevideo_source_bootstrap.py b/freevideo_source_bootstrap.py
index 0a5cd05a..11d23113 100644
--- a/freevideo_source_bootstrap.py
+++ b/freevideo_source_bootstrap.py
@@ -111,6 +111,26 @@ def root():
'web/assets/tour/c12-s1.webp',
'web/assets/tour/c13-s1.mp4',
'web/assets/tour/c13-s1.webp',
+ 'web/assets/tour/cb-ambience.mp4',
+ 'web/assets/tour/cb-ambience.webp',
+ 'web/assets/tour/cb-dialogue.mp4',
+ 'web/assets/tour/cb-dialogue.webp',
+ 'web/assets/tour/cb-music.mp4',
+ 'web/assets/tour/cb-music.webp',
+ 'web/assets/tour/cb-source.mp4',
+ 'web/assets/tour/cb-source.webp',
+ 'web/assets/tour/ca-music.mp4',
+ 'web/assets/tour/ca-music.webp',
+ 'web/assets/tour/ca-music-wave.webp',
+ 'web/assets/tour/ca-dub.mp4',
+ 'web/assets/tour/ca-dub.webp',
+ 'web/assets/tour/ca-dub-wave.webp',
+ 'web/assets/tour/ca-record.mp4',
+ 'web/assets/tour/ca-record.webp',
+ 'web/assets/tour/ca-record-wave.webp',
+ 'web/assets/tour/cc-speech.mp4',
+ 'web/assets/tour/cc-speech.webp',
+ 'web/assets/tour/cc-speech-wave.webp',
'web/assets/tour/c10-source.webp',
'web/assets/tour/fox-sheet.webp',
))
diff --git a/scripts/test_freevideo_entry.mjs b/scripts/test_freevideo_entry.mjs
index b9aacb79..41221e9f 100644
--- a/scripts/test_freevideo_entry.mjs
+++ b/scripts/test_freevideo_entry.mjs
@@ -43,7 +43,7 @@ globalThis.__permissionAsks = [];
test('main browser entry registers both views and preserves node hooks', async t => {
const extensions = [], opened = [], installed = [], events = new EventTarget();
const issueURLs = [];
- let refreshed = 0, compatibilityChecks = 0, updateChecks = 0, doneNotifyStarts = 0, rememberedView = null;
+ let refreshed = 0, compatibilityChecks = 0, updateChecks = 0, doneNotifyStarts = 0, whatsNewStarts = 0, rememberedView = null;
const openingEvents = [];
let restoreReleases = 0;
let launchReady = null;
@@ -89,7 +89,8 @@ test('main browser entry registers both views and preserves node hooks', async t
encoderFilePath: file => file?.offline_installation ? file.path || '' : '',
createProgressConnection: () => ({start() {}, refresh() {}, reset() {}}),
notifyCompatibility: async () => { compatibilityChecks++; },
- startUpdateChecks: () => { updateChecks++; },
+ startUpdateChecks: () => { updateChecks++; }, productVersion: () => '0.5.0',
+ startWhatsNew: options => { whatsNewStarts++; },
startWhiteboxWorker: () => ({stop() {}}),
startDoneNotify: options => { doneNotifyStarts++; doneNotifyOptions = options; },
installNavigation: callback => installed.push(callback),
@@ -126,7 +127,8 @@ test('main browser entry registers both views and preserves node hooks', async t
.replace("import { createProgressConnection } from './progress_connection.js';", 'const {createProgressConnection} = globalThis.__freevideoEntryTest;')
.replace("import { notifyCompatibility } from './compatibility.js';", 'const {notifyCompatibility} = globalThis.__freevideoEntryTest;')
.replace("import { startWhiteboxWorker } from './whitebox_worker.js';", 'const {startWhiteboxWorker} = globalThis.__freevideoEntryTest;')
- .replace("import { startUpdateChecks } from './updates.js';", 'const {startUpdateChecks} = globalThis.__freevideoEntryTest;')
+ .replace("import { startUpdateChecks, productVersion } from './updates.js';", 'const {startUpdateChecks,productVersion} = globalThis.__freevideoEntryTest;')
+ .replace("import { startWhatsNew } from './whatsnew.js';", 'const {startWhatsNew} = globalThis.__freevideoEntryTest;')
.replace("import { startDoneNotify, askDesktopPermission, DESKTOP_SETTING } from './done_notify.js';", 'const {startDoneNotify} = globalThis.__freevideoEntryTest; const askDesktopPermission = (...args) => globalThis.__permissionAsks.push(args); const DESKTOP_SETTING = "FreeVideo.DoneNotify.Desktop";')
.replace("import { attachReferencePicker, referenceItems, syncReferencePrompt, referenceInputs, mediaRows, mediaNodeOf } from './prompt_references.js';", 'const {attachReferencePicker,referenceItems,syncReferencePrompt,referenceInputs,mediaRows} = globalThis.__freevideoEntryTest; const mediaNodeOf = () => null;')
.replace("import { timedText } from './timed_prompt.js';", 'const timedText = text => text;')
@@ -159,6 +161,7 @@ test('main browser entry registers both views and preserves node hooks', async t
assert.equal(installed.length, 1, 'the Create / Nodes navigation must be installed');
assert.equal(compatibilityChecks, 1);
assert.equal(updateChecks, 1);
+ assert.equal(whatsNewStarts, 1);
assert.equal(doneNotifyStarts, 1);
await t.test('desktop notification uses the registered default without overwriting stored opt-outs', () => {
diff --git a/web/assets/tour/ca-dub-wave.webp b/web/assets/tour/ca-dub-wave.webp
new file mode 100644
index 00000000..addda33f
Binary files /dev/null and b/web/assets/tour/ca-dub-wave.webp differ
diff --git a/web/assets/tour/ca-dub.mp4 b/web/assets/tour/ca-dub.mp4
new file mode 100644
index 00000000..0ea43149
Binary files /dev/null and b/web/assets/tour/ca-dub.mp4 differ
diff --git a/web/assets/tour/ca-dub.webp b/web/assets/tour/ca-dub.webp
new file mode 100644
index 00000000..15372788
Binary files /dev/null and b/web/assets/tour/ca-dub.webp differ
diff --git a/web/assets/tour/ca-music-wave.webp b/web/assets/tour/ca-music-wave.webp
new file mode 100644
index 00000000..ec163ed4
Binary files /dev/null and b/web/assets/tour/ca-music-wave.webp differ
diff --git a/web/assets/tour/ca-music.mp4 b/web/assets/tour/ca-music.mp4
new file mode 100644
index 00000000..97571281
Binary files /dev/null and b/web/assets/tour/ca-music.mp4 differ
diff --git a/web/assets/tour/ca-music.webp b/web/assets/tour/ca-music.webp
new file mode 100644
index 00000000..47f65515
Binary files /dev/null and b/web/assets/tour/ca-music.webp differ
diff --git a/web/assets/tour/ca-record-wave.webp b/web/assets/tour/ca-record-wave.webp
new file mode 100644
index 00000000..8715eaca
Binary files /dev/null and b/web/assets/tour/ca-record-wave.webp differ
diff --git a/web/assets/tour/ca-record.mp4 b/web/assets/tour/ca-record.mp4
new file mode 100644
index 00000000..b79d944b
Binary files /dev/null and b/web/assets/tour/ca-record.mp4 differ
diff --git a/web/assets/tour/ca-record.webp b/web/assets/tour/ca-record.webp
new file mode 100644
index 00000000..840b260c
Binary files /dev/null and b/web/assets/tour/ca-record.webp differ
diff --git a/web/assets/tour/cb-ambience.mp4 b/web/assets/tour/cb-ambience.mp4
new file mode 100644
index 00000000..65501786
Binary files /dev/null and b/web/assets/tour/cb-ambience.mp4 differ
diff --git a/web/assets/tour/cb-ambience.webp b/web/assets/tour/cb-ambience.webp
new file mode 100644
index 00000000..ab466b0c
Binary files /dev/null and b/web/assets/tour/cb-ambience.webp differ
diff --git a/web/assets/tour/cb-dialogue.mp4 b/web/assets/tour/cb-dialogue.mp4
new file mode 100644
index 00000000..0aefa220
Binary files /dev/null and b/web/assets/tour/cb-dialogue.mp4 differ
diff --git a/web/assets/tour/cb-dialogue.webp b/web/assets/tour/cb-dialogue.webp
new file mode 100644
index 00000000..a310338b
Binary files /dev/null and b/web/assets/tour/cb-dialogue.webp differ
diff --git a/web/assets/tour/cb-music.mp4 b/web/assets/tour/cb-music.mp4
new file mode 100644
index 00000000..4388dc38
Binary files /dev/null and b/web/assets/tour/cb-music.mp4 differ
diff --git a/web/assets/tour/cb-music.webp b/web/assets/tour/cb-music.webp
new file mode 100644
index 00000000..9098c23a
Binary files /dev/null and b/web/assets/tour/cb-music.webp differ
diff --git a/web/assets/tour/cb-source.mp4 b/web/assets/tour/cb-source.mp4
new file mode 100644
index 00000000..bd70a79b
Binary files /dev/null and b/web/assets/tour/cb-source.mp4 differ
diff --git a/web/assets/tour/cb-source.webp b/web/assets/tour/cb-source.webp
new file mode 100644
index 00000000..72558e39
Binary files /dev/null and b/web/assets/tour/cb-source.webp differ
diff --git a/web/assets/tour/cc-speech-wave.webp b/web/assets/tour/cc-speech-wave.webp
new file mode 100644
index 00000000..feb46b1d
Binary files /dev/null and b/web/assets/tour/cc-speech-wave.webp differ
diff --git a/web/assets/tour/cc-speech.mp4 b/web/assets/tour/cc-speech.mp4
new file mode 100644
index 00000000..cc0fb034
Binary files /dev/null and b/web/assets/tour/cc-speech.mp4 differ
diff --git a/web/assets/tour/cc-speech.webp b/web/assets/tour/cc-speech.webp
new file mode 100644
index 00000000..f78b0a64
Binary files /dev/null and b/web/assets/tour/cc-speech.webp differ
diff --git a/web/editor.js b/web/editor.js
index 37885935..3dedf099 100644
--- a/web/editor.js
+++ b/web/editor.js
@@ -30,7 +30,9 @@ import {createEditorGenbar, loadEditorComponents, editorBlockStates, blockMessag
import {createEditorMusic} from './editor_music.js';
import {createEditorRecord, RECORD_ICON} from './editor_record.js';
import {createEditorVoice} from './editor_voice.js';
-import {createVoiceStore, voiceAvailable} from './editor_voice_store.js';
+import {createVoiceStore, voiceFeatures} from './editor_voice_store.js';
+import {createVoiceInstall} from './editor_voice_install.js';
+import {voiceTarget} from './editor_voice_model.js';
import {MAX_MUSIC_TEXT as MUSIC_TEXT_LIMIT} from './editor_music_model.js';
import {createProgressConnection} from './progress_connection.js';
import {createSelectionActions, deletionStatus, isMultiple, isTyping, playheadIds, removableGaps, removeGaps} from './editor_select.js';
@@ -547,7 +549,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given
// Music on the second sound track goes through the same bar; the bar shows a chosen piece's own job.
const music = createEditorMusic({state, store, t, completeSources, onStatus: () => genbar?.update(barValue)});
// Dubbing lines of a stretch of speech.
- let voiceReady = false;
+ let voiceReady = false, voiceInstallable = false;
const generatedSource = value => { const match = /^generated:(.+)$/.exec(String(value || '')); return match ? {kind: 'generated', id: match[1]} : null; };
// Playing one line: the playhead goes to its start and the picture plays up to its end, then stops.
let lineStop = null;
@@ -560,7 +562,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given
const unwatch = state.subscribe(event => { if ((event.type ?? event) !== 'playhead') return; if (state.playhead >= to) { player.pause(); finish(); } else if (!player.playing) finish(); });
lineStop = finish;
}
- const voice = createEditorVoice({state, store: createVoiceStore({api, state, queue}), t, onChange: () => genbar?.update(barValue), available: () => voiceReady, playLine: playVoiceLine, comparePlayback: async (clipId, name) => {
+ const voice = createEditorVoice({state, store: createVoiceStore({api, state, queue}), t, onChange: () => genbar?.update(barValue), available: () => voiceReady, recheck: readVoice, playLine: playVoiceLine, comparePlayback: async (clipId, name) => {
// Hearing the other take changes nothing in the edit: only the preview swaps the sound.
const source = generatedSource(name);
if (!source) { player.previewClipSource(clipId, null); return; }
@@ -602,6 +604,39 @@ export function openEditor({api, app, from = null, panel = {}, components: given
if (queue.dirty) { notify(saveErrorText(queue.failure, t), EDITOR_ISSUE, failureKind('save', {reason: saveReason(queue.failure)})); return false; }
return !!state.record.id;
}
+ // Keeps the frame under the playhead as a picture in the image history, drawn as the export would draw it, from the saved project.
+ let savingFrame = false;
+ async function saveFrame() {
+ if (savingFrame) return;
+ if (initializing || switching) { notify(t('Opening the edit…', '正在打开…')); return; }
+ if (!model.exportFrames(state.record)) { notify(t('The timeline is empty. Add a clip first.', '时间线是空的，请先放入素材。')); return; }
+ const epoch = projectEpoch, shown = Math.min(state.playhead, Math.max(0, model.duration(state.record) - 1));
+ savingFrame = true;
+ try {
+ if (!await saveForExport(epoch)) return;
+ let picture = null;
+ for (let attempt = 0; ; attempt++) {
+ try { picture = await store.saveFrame(state.record.id, state.record.revision, shown); break; }
+ catch (value) {
+ if (value.status !== 409 || attempt >= 2) throw value;
+ queue.enqueue(state.record);
+ if (!await saveForExport(epoch)) return;
+ }
+ }
+ if (disposed || closing || epoch !== projectEpoch) return;
+ if (picture) notify(t('Saved the current frame to Edit image > Generate with AI > History.',
+ '已保存当前帧，位于“编辑图片 > AI 生成 > 历史”。'));
+ } catch (value) {
+ if (disposed || epoch !== projectEpoch) return;
+ const said = {no_picture: t('There is no picture at this position. Move the playhead onto a clip.', '这个位置没有画面，请把播放头移到片段上。'),
+ range: t('There is no picture at this position. Move the playhead onto a clip.', '这个位置没有画面，请把播放头移到片段上。'),
+ generating: t('This part is still being generated. Save the frame when it is done.', '这一段还在生成，完成后再保存。'),
+ unreadable: t('A source file cannot be read. Import it again from the library.', '有素材读不出来，请在素材库重新导入。'),
+ not_writable: t('The picture folder cannot be written to. Check that the folder can be written to, then save the frame again.', '图片文件夹不可写。请检查这个文件夹的写入权限后再保存。'),
+ disk_full: t('The disk is full. Free some space, then save the frame again.', '磁盘空间不足，请清理出空间后再保存。')}[value?.reason];
+ if (said) notify(said); else error(value, 'frame');
+ } finally { savingFrame = false; }
+ }
function updateTop() {
const name = state.record.name || new Date().toLocaleDateString();
if (title.firstElementChild?.textContent !== name) { title.replaceChildren(el('span', '', name), icon('m4 6 4 4 4-4')); headerFit.check(); }
@@ -1044,30 +1079,30 @@ export function openEditor({api, app, from = null, panel = {}, components: given
else genbar?.update(mostRecentJob ? jobStatuses.get(mostRecentJob) ?? null : null);
},
onPass: listener => panel.passMode?.subscribe(listener) || (() => {}),
- // A part made as a half-resolution preview: only one that stands alone, as the next part continues from
- // this one's last frames, which the second pass changes.
+ // A part made as a half-resolution preview: one part only, as the next part continues from this one's last
+ // frames, which the second pass changes. It may continue the part before it where the engine can.
previewFirst(selection = state.selection) {
if (actions.passMode()?.mode !== 'preview' || music.target(selection)) return false;
if ((state.record.settings?.base_steps ?? 8) !== 8) return false;
const run = partRun(selection), chosen = run ? clipChoice(run) : selection, range = selectionRange(chosen); if (!range) return false;
- if (continuesEarlier(range)) return false;
+ if (continuesEarlier(range) && !store.continuationPreview) return false;
try {
// The parts Generate would send, planned as it plans them.
const selected = selectionClips(chosen), planned = selected.length && selected.every(clip => clip.pending && !clip.source);
const next = planned ? state.record : model.planRange(state.record, {...range, target: 'both', maxSeconds: capability.maxSeconds, maxFollowSeconds: capability.maxFollowSeconds});
const ids = planned ? selected.map(clip => clip.id) : next.tracks[0].clips.filter(clip => clip.start >= range.start && clip.start < range.start + range.frames
&& clip.pending && !['queued', 'running'].includes(clip.pending)).map(clip => clip.id);
- return ids.length === 1 && !model.generationBefore(model.prepare(next, ids), ids).some(Boolean);
+ return ids.length === 1 && (store.continuationPreview || !model.generationBefore(model.prepare(next, ids), ids).some(Boolean));
} catch (value) { if (value instanceof model.EditError) return false; throw value; }
},
- // The chosen preview part, shown from its first frame, whose second pass can replace it in place.
+ // The chosen preview part, shown from its first new frame, whose second pass can replace it in place.
secondPassTarget(selection = state.selection) {
if (selection?.kind !== 'clip' || !store.generationAvailable) return null;
const clips = selectionClips(selection);
if (clips.length !== 1) return null;
const clip = clips[0], mate = clip.link ? model.find(state.record, clip.link) : null;
// From the start of the preview: the second pass makes the same frames again, so a part cut short at its end still shows the same moment.
- if (!clip.source || clip.pending || mate?.pending || clip.in !== 0 || clip.frames > clip.length) return null;
+ if (!clip.source || clip.pending || mate?.pending || clip.in !== (clip.splice?.frames ?? 0) || clip.frames > clip.length) return null;
const metadata = store.sources.get(sourceKey(clip.source));
return metadata?.preview === true && metadata.path ? {clip, path: metadata.path} : null;
},
@@ -1155,8 +1190,8 @@ export function openEditor({api, app, from = null, panel = {}, components: given
}
const follow = continuedSize(beforeKeys, sizes), edit = capability.generationSize || generationSize(state.record);
Object.assign(frozen.prompt[frozen.node].inputs, follow || {});
- // A preview is one part that continues nothing; any other part samples both passes, whatever the panel shows.
- const inputs = frozen.prompt[frozen.node].inputs, alone = request.blocks.length === 1 && !beforeKeys.some(Boolean);
+ // A preview is one part; any other part samples both passes, whatever the panel shows.
+ const inputs = frozen.prompt[frozen.node].inputs, alone = request.blocks.length === 1 && (store.continuationPreview || !beforeKeys.some(Boolean));
if (secondPass) {
if (!alone) throw new model.EditError('unsupported');
Object.assign(inputs, {two_pass: true, preview: false, refine_from: secondPass});
@@ -1429,8 +1464,19 @@ export function openEditor({api, app, from = null, panel = {}, components: given
return running;
}
- voiceAvailable(api).then(value => { voiceReady = value; if (!disposed) voice.refresh(); });
- actions.voice = {controller: voice};
+ // The dubbing model is downloaded from the bar where the lines would be; once it is there, the stretch chosen is looked at.
+ async function readVoice() {
+ const found = await voiceFeatures(api);
+ // An unanswered question changes nothing: the lines stay as they were, and a failed recognition says so.
+ if (found.unknown) return voiceReady;
+ voiceReady = found.ready; voiceInstallable = found.install;
+ if (!disposed) { voice.refresh(); if (found.install) voiceInstall.init(); genbar?.update(barValue); }
+ return found.ready;
+ }
+ const voiceInstall = createVoiceInstall({api, available: () => voiceInstallable && !voiceReady, target: selection => !!state.record && !!voiceTarget(state.record, selection),
+ onChange: () => { if (!disposed) genbar?.update(barValue); }, onReady: readVoice});
+ readVoice();
+ actions.voice = {controller: voice, install: voiceInstall};
actions.music = {target: selection => music.target(selection), preset: selection => music.preset(selection), choose: value => music.choose(value),
styles: music.styles, moods: music.moods, downloadBytes: music.downloadBytes};
genbar = createEditorGenbar(genHost, {state, t, actions, text: ''}); settleService();
@@ -1498,7 +1544,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given
historyTools.append(toolUndo, toolRedo); editTools.append(split, remove, closeGap, previous, recordTool);
const pictureTools = createPictureTools({state, sources: store.sources, t, capabilities: () => capability, error});
toolbar.append(historyTools, editTools, zoomHost); element.insertBefore(toolbar, tracksHost);
- const transportControls = createEditorTransport({element, preview, transport, toolbar, editTools, zoomHost, state, player, t, notify});
+ const transportControls = createEditorTransport({element, preview, transport, toolbar, editTools, zoomHost, state, player, t, notify, saveFrame});
editTools.append(secondPassButton);
toolbar.append(adjust.toggleButton);
const textTools = createEditorText({element, state, t, actions, adjust, player, toolbar, exportBox, tracksHost});
@@ -1714,7 +1760,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given
}
}
await store.activateProject(null).catch(() => {});
- rememberView(); disposed = true; music.dispose(); voice.dispose(); guide?.dispose(); transportControls.dispose(); toolbarMore.dispose(); player.dispose();
+ rememberView(); disposed = true; music.dispose(); voice.dispose(); voiceInstall.dispose(); guide?.dispose(); transportControls.dispose(); toolbarMore.dispose(); player.dispose();
textTools.dispose(); adjust.dispose(); notes.dispose(); pictureEdit.dispose();
audio.dispose(); stems?.dispose(); recording.dispose(); headerFit.dispose();
listeners.splice(0).forEach(remove => remove());
diff --git a/web/editor_genbar.js b/web/editor_genbar.js
index 00e2c141..e42a740f 100644
--- a/web/editor_genbar.js
+++ b/web/editor_genbar.js
@@ -10,6 +10,7 @@ import {showEmptyPrompt} from './empty_prompt_notice.js';
import {segmented} from './editor_adjust.js';
import {createPassMode} from './pass_mode.js';
import {createVoicePanel} from './editor_voice.js';
+import {createInstallPanel} from './editor_voice_install.js';
import * as dubbing from './editor_voice_model.js';
const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; };
export const timeEstimate = (seconds, t) => seconds == null ? '' : seconds < 60
@@ -123,6 +124,14 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) {
// Dubbing: the lines of a chosen stretch of speech take the description box's place.
const voice = actions.voice || null, voicePanel = voice ? createVoicePanel(host, voice.controller, {t}) : null;
if (voicePanel) description.before(voicePanel.element);
+ // Where the dubbing model is missing and the engine offers it, the same place offers its download. The panel exists only
+ // while it is offered; with the model there, or on a machine that cannot change a voice, nothing of it is in the bar.
+ let installPanel = null;
+ const syncInstall = () => {
+ const offered = !!voice?.install?.offered?.() && !disposed;
+ if (offered && !installPanel) { installPanel = createInstallPanel(host, voice.install, {t, selection: () => previewSelection || state.selection}); description.before(installPanel.element); }
+ else if (!offered && installPanel) { installPanel.dispose(); installPanel = null; }
+ };
const failure = el('div', 'fv-ed-gen-failure fv-result-links'), failureBar = el('div', 'fv-ed-failure-actions fv-actions');
const failureReason = el('span', 'fv-ed-failure-reason'); failureReason.setAttribute('role', 'status');
const report = el('button', 'fv-ed-failure-report fv-action fv-action-secondary', t('Download diagnostic report', '下载诊断报告')); report.type = 'button';
@@ -183,6 +192,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) {
const musical = !!music?.target(selection);
const dub = !previewSelection && voice?.controller.target(selection) ? voice.controller.session(voice.controller.target(selection)) : null;
const voiced = !!dub && dub.phase !== 'none' && dub.phase !== 'error';
+ const installing = !previewSelection && !!voice?.install?.visible(selection);
const selected = !!selection && (!!previewSelection || musical || (actions.supportedSelection?.() ?? true));
// While a part generates, choosing another place offers Generate again; it is queued behind.
// A press is being sent for the parts in `pressing`; only that same selection waits for it.
@@ -195,11 +205,11 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) {
const opening = (starting || !!actions.selectionSending?.()) && !held;
const stopped = ['failed', 'cancelled', 'interrupted'].includes(status?.state);
host.dataset.mode = busy ? 'running' : selected ? 'selected' : 'idle';
- host.dataset.voice = String(voiced); voicePanel?.update();
+ host.dataset.voice = voiced || installing ? 'true' : dub?.phase === 'error' ? 'note' : 'false'; voicePanel?.update(); syncInstall(); installPanel?.update();
hint.hidden = !busy && !other;
// The description and Generate stay where they are while a part is made, greyed, so nothing around them moves;
// music being made shows only its progress and Cancel.
- input.hidden = false; generate.hidden = musical && busy;
+ input.hidden = false; generate.hidden = musical && busy || installing;
// A choice that will move to an empty spot can still be described first.
const writable = selected || !previewSelection && !!selection && !!actions.selectionMovable?.();
input.disabled = !writable || busy;
@@ -276,7 +286,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) {
generate.title = '';
if (assets) { if (selected && !busy && !musical) assets.refresh(); else assets.element.hidden = true; }
showMusicAssets(selected && !busy && musical && !previewSelection);
- if (selected && !busy && !voiced) {
+ if (selected && !busy && !voiced && !installing) {
const key = JSON.stringify(selection);
if (estimateCache?.key === key) generate.textContent += estimateCache.label;
const seconds = await actions.estimate(selection);
@@ -385,7 +395,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) {
if (!value && note.contains(retryService)) { retryService.remove(); note.hidden = true; }
showService();
}, getText() { expand.commit(); return input.value; }, update(value) { status = value; render(); }, preview(selection) { previewSelection = selection; render(); }, setText(value) { input.value = value || ''; state.setRangeText(input.value); if (bound && input.value !== bound.text) { dirty = true; written = input.value; } render(); },
- dispose() { disposed = true; ++serial; clearTimeout(noteTimer); stopPass?.(); passControl?.dispose(); document.removeEventListener('pointerdown', closePass, true); clearTimeout(refNoticeTimer); duration.dispose(); unsubscribe(); resize?.disconnect(); expand.dispose(); references?.dispose(); voicePanel?.dispose(); assets?.dispose(); Object.values(pickers).forEach(picker => picker.dispose()); }};
+ dispose() { disposed = true; ++serial; clearTimeout(noteTimer); stopPass?.(); passControl?.dispose(); document.removeEventListener('pointerdown', closePass, true); clearTimeout(refNoticeTimer); duration.dispose(); unsubscribe(); resize?.disconnect(); expand.dispose(); references?.dispose(); voicePanel?.dispose(); installPanel?.dispose(); assets?.dispose(); Object.values(pickers).forEach(picker => picker.dispose()); }};
}
// A missing optional module leaves the editor's existing progress in place.
diff --git a/web/editor_model.js b/web/editor_model.js
index 76b2a2c2..3a78f1d3 100644
--- a/web/editor_model.js
+++ b/web/editor_model.js
@@ -219,6 +219,11 @@ export function parse(value) {
result.size_mode = value.size_mode ?? 'first';
check(value.keep_empty === undefined || typeof value.keep_empty === 'boolean', 'invalid');
if (value.keep_empty) result.keep_empty = true;
+ // The export's true-peak ceiling (dBTP), written by the finishing step.
+ if (value.master_peak_db !== undefined) {
+ check(typeof value.master_peak_db === 'number' && Number.isFinite(value.master_peak_db) && value.master_peak_db >= -12 && value.master_peak_db <= 0, 'invalid');
+ result.master_peak_db = value.master_peak_db;
+ }
Object.assign(result, markerFields(value));
if (value.replacements !== undefined) {
check(Array.isArray(value.replacements) && value.replacements.length > 0 && value.replacements.length <= RULES.maxClips, 'invalid');
diff --git a/web/editor_stems.js b/web/editor_stems.js
index b99facc3..f93d9a9f 100644
--- a/web/editor_stems.js
+++ b/web/editor_stems.js
@@ -2,6 +2,7 @@ import * as model from './editor_model.js';
import {registerMenuItem} from './editor_menu.js';
import {agentGet, agentPost, retryAgentPost} from './agent_session.js';
import {sourceKey} from './editor_store.js';
+import {showSoundBubble, markSoundBubbleSeen, isBubbleRemover} from './sound_bubble.js';
const ROOT = '/freevideo/agent/v1';
const ACTIVE = ['waiting', 'queued', 'running', 'cancelling'];
@@ -59,14 +60,19 @@ export function createStemsIntegration({state, api, queue, store, t, error, noti
.flatMap(track => track.clips.filter(clip => selection.ids.includes(clip.id)).map(clip => ({clip, track})));
return sound.length === 1 && !!sound[0].clip.source && sound[0].clip.stem_of === undefined ? sound[0] : null;
};
- let button = null, holder = null;
+ let button = null, holder = null, hideBubble = () => {};
const sync = () => {
if (!button) return;
const found = available && target();
button.hidden = holder.hidden = !found;
- if (!found) return;
+ if (!found) { hideBubble(); return; }
const {clip, track} = found;
button.disabled = !!running || !!clip.pending || track.locked || children(state.record, clip.id).length > 0;
+ if (!button.disabled) {
+ // A bubble that is already up returns nothing to remove; keep the remover that owns it.
+ const remove = showSoundBubble(button, {cn: t('', 'zh') === 'zh'});
+ if (isBubbleRemover(remove)) hideBubble = remove;
+ }
};
// The entries exist only where the engine offers the separation, so other computers keep the menu they had.
let stops = [];
@@ -109,7 +115,7 @@ export function createStemsIntegration({state, api, queue, store, t, error, noti
}
async function separate(clipId) {
if (running) { notify(t('A sound is already being separated.', '正在分离另一段声音。')); return false; }
- const tool = running = {cancelled: false, job: null}; sync();
+ const tool = running = {cancelled: false, job: null}; markSoundBubbleSeen(); sync();
try {
if (!await ready()) return false;
const project = state.record.id, ref = globalThis.crypto?.randomUUID?.() || `${Date.now()}-${Math.random()}`;
@@ -164,5 +170,5 @@ export function createStemsIntegration({state, api, queue, store, t, error, noti
} catch (value) { say(value, 'stems_failed'); return false; }
}
return {separate, cancel, exportStems, active: () => !!running,
- dispose() { disposed = true; stops.forEach(stop => stop()); stops = []; unsubscribe?.(); holder?.remove(); button = holder = null; }};
+ dispose() { disposed = true; hideBubble(); stops.forEach(stop => stop()); stops = []; unsubscribe?.(); holder?.remove(); button = holder = null; }};
}
diff --git a/web/editor_store.js b/web/editor_store.js
index 30974eb6..950ae1e7 100644
--- a/web/editor_store.js
+++ b/web/editor_store.js
@@ -178,6 +178,8 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener
let recovered = null, capabilityPoll = null;
// The length a new continuation starts with where the engine names one (a device that continues in short parts).
let continueSeconds = null;
+ // Whether a part that continues can be made as a preview first, once the engine has said so.
+ let continuationPreview = false;
let generationAvailable = false, generationGone = false, generationProbe = null, generationRetry = null, generationAttempts = 0, generationStalled = false, generationAsking = false;
let disposed = false; const uploads = new Set();
const session = holdSession();
@@ -264,6 +266,7 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener
if (activeProject === id) { activeProject = null; clock.clearTimeout(sessionTimer); sessionTimer = null; }
},
get continueSeconds() { return standIn ? standIn.continueSeconds ?? null : continueSeconds; },
+ get continuationPreview() { return standIn ? standIn.continuationPreview === true : continuationPreview; },
get generationAvailable() { return standIn ? standIn.generationAvailable ?? true : generationAvailable; },
// The server said for certain it cannot generate (not just a missed answer that is asked again).
get generationGone() { return !standIn && generationGone; },
@@ -301,6 +304,7 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener
if (!response.ok) return null;
const body = await response.json();
if (Number.isInteger(body?.continue_seconds) && body.continue_seconds > 0) continueSeconds = body.continue_seconds;
+ continuationPreview = body?.continuation_preview === true;
return Array.isArray(body?.chains) ? true : null;
})
.catch(() => null), timeout]);
@@ -533,6 +537,14 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener
},
watchExport(id, fn, lost) { return watch(async () => (await json('/freevideo/editor/export?' + new URLSearchParams({id})))?.export || null, fn, lost); },
cancelExport: id => json('/freevideo/editor/export/cancel', post({id})),
+ // One frame of the saved project, as the export draws it, kept as a picture in the image history; null where the server is older.
+ async saveFrame(id, revision, frame) {
+ const response = await request('/freevideo/editor/frame', post({id, revision, frame}));
+ const value = await response.json().catch(() => null);
+ if (!response.ok && (response.status !== 404 || value?.reason)) throw Object.assign(errorFor(response), {reason: value?.reason});
+ if (response.status === 404 || !value) { notice('frame', 'To save a frame, update FreeVideo and restart it.', '请更新 FreeVideo 并重新启动后再保存画面。'); return null; }
+ return value.picture || null;
+ },
// Subtitles from speech. What this computer can recognise; null where it cannot or the server is older.
async ears() { return (await json('/freevideo/editor/ears'))?.available || null; },
async startEars(id, revision) { return (await json('/freevideo/editor/ears', post({kind: 'transcribe', id, revision})))?.job || null; },
diff --git a/web/editor_transport.js b/web/editor_transport.js
index 537d308f..ae85e745 100644
--- a/web/editor_transport.js
+++ b/web/editor_transport.js
@@ -67,9 +67,10 @@ const icons = {
loop: 'M4 6h12l-3-3m3 3-3 3M16 14H4l3 3m-3-3 3-3',
'play-range': 'M3 3v14m14-14v14M7 6l6 4-6 4z', fullscreen: 'M7 3H3v4m10-4h4v4M3 13v4h4m10-4v4h-4',
snapping: 'M4 3v8a6 6 0 0 0 12 0V3h-4v8a2 2 0 0 1-4 0V3zM4 7h4m4 0h4',
+ 'save-frame': 'M3 6h3l1.5-2h5L14 6h3v10H3zM10 13.5a2.5 2.5 0 1 0 0-5 2.5 2.5 0 0 0 0 5z',
marker: 'm10 3 6 7-6 7-6-7z', shortcuts: 'M2 5h16v10H2zM5 8h1m3 0h1m3 0h1M5 11h1m3 0h5',
};
-export function createEditorTransport({element, preview, transport, toolbar, editTools, zoomHost, state, player, t, notify}) {
+export function createEditorTransport({element, preview, transport, toolbar, editTools, zoomHost, state, player, t, notify, saveFrame = null}) {
let tracks = null, align = null, layer = null, markerSelection = null, markerStamp = null, drag = null, kHeld = false, disposed = false;
const clock = player.playback, listeners = [], nodes = [];
const on = (target, type, callback, options) => { target.addEventListener(type, callback, options); listeners.push(() => target.removeEventListener(type, callback, options)); };
@@ -88,7 +89,15 @@ export function createEditorTransport({element, preview, transport, toolbar, edi
const loop = button('loop', t('Loop (Shift + L)', '循环播放（Shift + L）'));
const range = button('play-range', t('Play selected range (Shift + Space)', '播放选中范围（Shift + 空格）'));
const fullscreen = button('fullscreen', t('Full screen (F)', '全屏（F）'));
- tools.append(loop, range, fullscreen); transport.append(tools); nodes.push(tools);
+ // Keeps the frame under the playhead as a picture; the button is off on an empty timeline and while the editor saves it.
+ const frame = saveFrame && button('save-frame', t('Save current frame', '保存当前帧'));
+ let savingFrame = false;
+ if (frame) on(frame, 'click', async () => {
+ if (frame.disabled) return;
+ savingFrame = true; frame.disabled = true; frame.setAttribute('aria-busy', 'true');
+ try { await saveFrame(); } finally { savingFrame = false; frame.removeAttribute('aria-busy'); refresh(); }
+ });
+ tools.append(loop, range, ...(frame ? [frame] : []), fullscreen); transport.append(tools); nodes.push(tools);
const sheet = el('div', 'fv-ed-shortcut-sheet'); sheet.hidden = true; sheet.setAttribute('role', 'region'); sheet.setAttribute('aria-label', t('Shortcuts', '快捷键'));
element.append(sheet); nodes.push(sheet);
const fullHost = el('div', 'fv-ed-fullscreen-host'); preview.parentNode.insertBefore(fullHost, preview); fullHost.append(preview, transport); nodes.push(fullHost);
@@ -108,6 +117,7 @@ export function createEditorTransport({element, preview, transport, toolbar, edi
const bounds = selection(); range.disabled = !bounds;
range.title = t('Play selected range (Shift + Space)', '播放选中范围（Shift + 空格）');
range.setAttribute('aria-label', range.title); range.setAttribute('aria-pressed', String(!!clock.range));
+ if (frame) frame.disabled = savingFrame || !model.exportFrames(state.record);
fullscreen.setAttribute('aria-pressed', String(document.fullscreenElement === fullHost));
transport.dataset.rate = String(player.playing ? clock.rate : 0);
if (layer && !drag) renderMarkers();
diff --git a/web/editor_voice.css b/web/editor_voice.css
index e670c91d..ec8dfe40 100644
--- a/web/editor_voice.css
+++ b/web/editor_voice.css
@@ -1,5 +1,5 @@
/* Dubbing lines in the generation bar. */
-.fv-ed-genbar[data-voice=true]{flex-wrap:wrap;align-items:center}
+.fv-ed-genbar[data-voice=true],.fv-ed-genbar[data-voice=note]{flex-wrap:wrap;align-items:center}
.fv-ed-genbar[data-voice=true] .fv-ed-description-box,.fv-ed-genbar[data-voice=true] .fv-ed-first-frame{display:none}
.fv-ed-genbar[data-voice=true] .fv-ed-gen-hint{flex:1}
.fv-ed-genbar[data-voice=true] .fv-ed-generate{margin-left:auto}
@@ -34,4 +34,19 @@
.fv-editor .fv-ed-voice textarea.fv-ed-voice-text{font-size:16px}
.fv-ed-voice-extra{padding-left:60px;flex-wrap:wrap}
}
-.fv-ed-voice-note:has(.fv-issue-button){display:flex;flex-wrap:wrap;align-items:center;gap:6px 8px}
+.fv-ed-voice-note:has(.fv-issue-button),.fv-ed-voice-install-note:has(.fv-issue-button){display:flex;flex-wrap:wrap;align-items:center;gap:6px 8px}
+/* The download of the dubbing model, in the place of the lines. */
+.fv-ed-voice-install{order:-1;flex:1 1 100%;min-width:0;display:flex;flex-wrap:wrap;align-items:center;gap:8px 12px}
+.fv-ed-voice-install[hidden]{display:none}
+/* While a part generates, its own line comes first and the download follows it. */
+.fv-ed-genbar[data-mode=running] .fv-ed-voice-install{order:3}
+.fv-ed-voice-install .fv-ed-voice-install-note{flex:1 1 220px;min-width:0;font-size:13px;color:var(--fv-ink)}
+.fv-ed-voice-install .fv-ed-voice-install-note[hidden]{display:none}
+.fv-ed-voice-install .fv-actions{flex:none}
+.fv-ed-voice-install .fv-actions [hidden]{display:none}
+.fv-ed-voice-bar{order:3;flex:1 1 100%;height:4px;border-radius:2px;background:var(--fv-border);overflow:hidden}
+.fv-ed-voice-bar[hidden]{display:none}
+.fv-ed-voice-bar>span{display:block;height:100%;width:0;border-radius:2px;background:var(--fv-accent,#6db8fa);transition:width .3s}
+.fv-ed-voice-bar[data-indeterminate=true]>span{width:35%;animation:fv-ed-voice-slide 1.4s ease-in-out infinite}
+@keyframes fv-ed-voice-slide{from{margin-left:-35%}to{margin-left:100%}}
+@media (prefers-reduced-motion:reduce){.fv-ed-voice-bar>span{transition:none}.fv-ed-voice-bar[data-indeterminate=true]>span{animation:none;width:100%;opacity:.5}}
diff --git a/web/editor_voice.js b/web/editor_voice.js
index 94319ec2..36c3f27c 100644
--- a/web/editor_voice.js
+++ b/web/editor_voice.js
@@ -9,17 +9,21 @@ import {repairAction} from './repair_hint.js';
const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; };
// What a failed run says: the lines the user typed are kept, whatever went wrong; the code never reaches the words.
-export function failureCopy(error, t) {
+export function failureCopy(error, t, recognising = false) {
if (isNetworkTypeError(error)) return {text: connectionLostText(t), report: false};
switch (error?.code) {
case 'out_of_memory': return {report: true, text: t('Not enough memory to finish dubbing. The edited lines are kept. Close other programs using the GPU, then redub.', '显存或内存不足，配音未完成，所改台词已保留。请关闭其他占用显卡的程序后重新配音。')};
case 'storage_unavailable': return {report: false, text: t('Could not write to disk, so dubbing did not finish. The edited lines are kept. Check the free disk space, then redub.', '无法写入磁盘，配音未完成，所改台词已保留。请检查磁盘空间后重新配音。')};
+ case 'no_speech_model': return {report: true, text: t('The speech model could not be used, so the lines could not be recognized.', '语音识别模型无法使用，未能识别台词。')};
+ case 'speech_language': return {report: false, text: t('The language of this part is not supported for dubbing yet.', '这一段的语言暂不支持配音。')};
case 'voice_model_missing': return {report: false, text: t('The dubbing model is not fully installed. ', '配音模型没有装全。') + t(...repairAction())};
- default: return {report: true, text: t('Dubbing did not finish. The edited lines are kept. Redub them to try again.', '配音未完成，所改台词已保留，请重新配音。')};
+ default:
+ if (recognising) return {report: true, text: t('The lines could not be recognized. Select this part again.', '未能识别台词。请重新选中这一段再试。')};
+ return {report: true, text: t('Dubbing did not finish. The edited lines are kept. Redub them to try again.', '配音未完成，所改台词已保留，请重新配音。')};
}
}
-export function createEditorVoice({state, store, t, onChange = () => {}, place = async () => {}, available = () => true, playLine = () => {}, comparePlayback = () => {}}) {
+export function createEditorVoice({state, store, t, onChange = () => {}, place = async () => {}, available = () => true, recheck = async () => true, playLine = () => {}, comparePlayback = () => {}}) {
const sessions = new Map();
let disposed = false, placing = false;
const target = (selection = state.selection) => available() && state.record && voice.voiceTarget(state.record, selection);
@@ -32,7 +36,7 @@ export function createEditorVoice({state, store, t, onChange = () => {}, place =
if (!found) return null;
const key = keyOf(found);
let current = sessions.get(key);
- if (current) return current;
+ if (current && current.phase !== 'error') return current;
current = {key, found, phase: 'finding', lines: [], texts: new Map(), results: new Map(), shortened: new Map(), compare: 'new',
active: null, index: 0, count: 0, remaining: null, failed: 0, revision: state.record.revision};
sessions.set(key, current); emit();
@@ -43,9 +47,14 @@ export function createEditorVoice({state, store, t, onChange = () => {}, place =
current.lines = result.lines; current.separated = result.separated; current.recognition = result.recognition;
current.rate = new Map(result.lines.map(line => [line.id, line.rate]));
} catch (value) {
- // A failed recognition is not kept: choosing the stretch again asks again.
- current.phase = 'error'; current.error = value;
- if (sessions.get(key) === current) sessions.delete(key);
+ // A failed recognition says why in the panel; choosing the stretch again asks again.
+ // A model that went missing after the check: the download row comes back and recognition starts again once it is there.
+ if (value?.code === 'no_speech_model' && !disposed && sessions.get(key) === current && !(await recheck().catch(() => true))) {
+ sessions.delete(key); emit(); return null;
+ }
+ // Cancelled: nothing to say, so the panel goes; choosing the stretch again asks again.
+ if (value?.code === 'cancelled') { if (sessions.get(key) === current) sessions.delete(key); emit(); return null; }
+ current.phase = 'error'; current.error = value; current.failure = value;
if (!disposed) onChange(); return current;
}
emit(); return current;
@@ -172,12 +181,12 @@ export function createVoicePanel(host, controller, {t}) {
const dubbing = current.phase === 'dubbing', ready = ['ready', 'done', 'dubbing'].includes(current.phase);
compare.hidden = !current.sources;
for (const button of choices) button.setAttribute('aria-pressed', String(button.dataset.value === current.compare));
- voiceName.hidden = !ready;
- const failure = current.failure && failureCopy(current.failure, t);
+ voiceName.hidden = !ready; top.hidden = !ready && compare.hidden;
+ const failure = current.failure && failureCopy(current.failure, t, current.phase === 'error');
note.replaceChildren();
if (failure) {
note.append(failure.text);
- if (failure.report) note.append(issueButton(t, () => ({where: ['Editor generation', '剪辑器生成'], message: failure.text, kind: current.failure?.code || null})));
+ if (failure.report) note.append(issueButton(t, () => ({where: ['Editor generation', '剪辑器生成'], message: failure.text + (current.failure?.detail ? `\n${current.failure.detail}` : ''), kind: current.failure?.code || null})));
} else note.textContent = current.notice === 'changed' ? t('This part changed during dubbing, so the new voice was not placed. Redub it.', '这一段在配音期间有改动，新声未放入。请重新配音。') : current.phase === 'none' ? t('No lines were found in this part.', '这一段没有识别到台词。') : current.phase === 'done' && current.failed ? voice.failedText(current.failed, t) : '';
note.hidden = !note.textContent;
if (current.phase === 'finding') {
diff --git a/web/editor_voice_install.js b/web/editor_voice_install.js
new file mode 100644
index 00000000..1322a776
--- /dev/null
+++ b/web/editor_voice_install.js
@@ -0,0 +1,185 @@
+// The dubbing model that changes a voice, downloaded from the place where the voice would be changed: a stretch of
+// speech is chosen, the model is not there, the bar offers "Download" and then shows the progress. The engine does the
+// download (resumable, cancellable); this keeps the page's side of it: what to say, the progress and the one job.
+// The first half is pure (sentences, percent); the second half talks to the engine's /freevideo/dubbing routes.
+import {issueButton} from './issue_button.js';
+
+const ROOT = '/freevideo/dubbing';
+const NBSP = ' ';
+const DEFAULT_BYTES = 4.9e9;
+const JOB_KEY = 'fv-dubbing-install-job';
+
+// "about 4.9 GB": the size the engine reports for what is still missing, the full model when it reports nothing.
+export const sizeText = bytes => `${((Number.isFinite(bytes) && bytes > 0 ? bytes : DEFAULT_BYTES) / 1e9).toFixed(1)}${NBSP}GB`;
+
+// 0..99 while the engine works (100 is only the finished model); null when the engine has not said how far it is.
+export function percent(row) {
+ if (!row || !(row.total > 0) || !Number.isFinite(row.done)) return null;
+ return Math.max(0, Math.min(99, Math.floor(row.done / row.total * 100)));
+}
+
+export const offerText = (bytes, t) => t(`Voice change needs the dubbing model (about ${sizeText(bytes)}).`, `换人声需要配音模型，约 ${sizeText(bytes)}。`);
+export const progressText = (value, t) => value == null ? t('Downloading the dubbing model', '正在下载配音模型')
+ : t(`Downloading the dubbing model ${value}%`, `正在下载配音模型 ${value}%`);
+
+// What a failed download says; the code never reaches the words. `report` is whether the problem can be reported.
+export function failureText(code, bytes, t) {
+ switch (code) {
+ case 'disk_space': return {report: true, text: t(`There is not enough disk space. The dubbing model needs about ${sizeText(bytes)}.`, `磁盘空间不足，配音模型需要约 ${sizeText(bytes)}。`)};
+ case 'download_failed': case 'network': return {report: true, text: t('The dubbing model could not be downloaded. Check the connection and try again.', '配音模型下载失败，请检查网络后重试。')};
+ default: return {report: true, text: t('The dubbing model could not be installed. Try again.', '配音模型未能安装，请重试。')};
+ }
+}
+
+const readJob = storage => { try { return storage?.getItem(JOB_KEY) || null; } catch { return null; } };
+const writeJob = (storage, value) => { try { value ? storage?.setItem(JOB_KEY, value) : storage?.removeItem(JOB_KEY); } catch { /* the job is simply not remembered */ } };
+const wait = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+/**
+ * The download of the dubbing model. `available()` says whether the engine offers it (capabilities); `target(selection)`
+ * whether the chosen part is a stretch of speech; `onReady()` asks the page to read the capabilities again and returns
+ * whether the model is there now. A download that runs when the page opens is taken up again: by the job this browser
+ * remembers, else only as "downloading" (the engine does not hand its job to anyone else).
+ */
+export function createVoiceInstall({api, available = () => true, target = () => true, onChange = () => {}, onReady = async () => false,
+ storage = globalThis.localStorage, delay = wait, interval = 1000} = {}) {
+ const view = {phase: 'loading', percent: null, cancelling: false, error: null, bytes: 0};
+ let token = '', job = readJob(storage), disposed = false, polling = false, initial = null;
+ const set = patch => { Object.assign(view, patch); if (!disposed) onChange(); };
+ const read = async response => { try { return await response.json(); } catch { return null; } };
+ const info = async () => {
+ const response = await api.fetchApi(ROOT, {cache: 'no-store'});
+ const value = await read(response);
+ if (!response.ok || !value || value.error) throw new Error(value?.error || 'unavailable');
+ token = value.token || token;
+ return value;
+ };
+ async function call(action, body, again = true) {
+ const response = await api.fetchApi(`${ROOT}/${action}`, {method: 'POST', headers: {'Content-Type': 'application/json', 'X-FreeVideo-Dubbing': token}, body: JSON.stringify(body)});
+ const value = await read(response);
+ // A restarted engine has a new token: ask for it and send the same call once more.
+ if (response.status === 403 && again) { await info(); return call(action, body, false); }
+ if (!response.ok) throw Object.assign(new Error(value?.error || 'failed'), {code: value?.error || 'failed'});
+ return value;
+ }
+ const remember = value => { const moved = job !== value; job = value; writeJob(storage, value); if (moved && !disposed) onChange(); };
+ async function finish() {
+ remember(null);
+ let ready = false;
+ try { ready = await onReady(); } catch { /* the capabilities are read again with the next page */ }
+ if (!disposed) set({phase: ready ? 'ready' : 'idle', percent: null, cancelling: false, error: null});
+ }
+ function fail(code, keep = false) { if (!keep) remember(null); set({phase: 'failed', error: code, cancelling: false, percent: null}); }
+ // The engine's answer for the job; false once the job is over.
+ function apply(row) {
+ switch (row?.phase) {
+ case 'complete': return false;
+ case 'failed': fail(row.error || 'failed'); return null;
+ case 'cancelled': remember(null); set({phase: 'idle', percent: null, cancelling: false}); return null;
+ case 'download': case 'unpack': set({percent: percent(row)}); return true;
+ default: return true;
+ }
+ }
+ async function poll() {
+ if (polling) return;
+ polling = true;
+ let lost = 0;
+ try {
+ while (!disposed && view.phase === 'downloading') {
+ try {
+ if (job) {
+ const outcome = apply(await call('status', {job}));
+ if (outcome === false) return await finish();
+ if (outcome === null) return;
+ } else {
+ // Not this page's job: only whether the engine is still at it.
+ const value = await info();
+ if (!value.busy) { if (value.ready) return await finish(); set({phase: 'idle'}); return; }
+ }
+ lost = 0;
+ } catch (error) {
+ if (error.code === 'unknown_job') { remember(null); continue; }
+ // The engine cannot be reached: the job is kept, so Retry takes up the same download.
+ if (++lost >= 10) return fail('network', true);
+ }
+ await delay(interval);
+ }
+ } finally { polling = false; }
+ }
+ async function load() {
+ try {
+ const value = await info();
+ set({bytes: value.bytes});
+ if (value.ready) { set({phase: 'ready'}); await finish(); return; }
+ if (value.supported === false || value.published === false) { set({phase: 'unsupported'}); return; }
+ if (value.busy) { set({phase: 'downloading', percent: null}); poll(); return; }
+ remember(null); set({phase: 'idle'});
+ } catch { set({phase: 'unsupported'}); }
+ }
+ return {
+ view,
+ init() { return initial || (initial = load()); },
+ // Whether the engine offers the download at all (the panel exists only then).
+ offered() { return !disposed && available() && ['idle', 'downloading', 'failed'].includes(view.phase); },
+ // Whether the bar offers the download for this selection: the engine offers it and a stretch of speech is chosen.
+ visible(selection) { return !disposed && available() && ['idle', 'downloading', 'failed'].includes(view.phase) && !!target(selection); },
+ async start() {
+ if (view.phase === 'downloading') return;
+ set({phase: 'downloading', percent: 0, cancelling: false, error: null});
+ try {
+ if (!token) await info();
+ const value = await call('install', {accept_download: true});
+ remember(value.job);
+ poll();
+ } catch (error) {
+ if (error.code !== 'busy') return fail(error.code || 'network');
+ // The engine says it is busy: it is downloading already.
+ try {
+ const value = await info();
+ if (value.busy) { set({percent: null}); poll(); } else fail('failed');
+ } catch { fail('network'); }
+ }
+ },
+ async cancel() {
+ if (view.phase !== 'downloading' || !job || view.cancelling) return;
+ set({cancelling: true});
+ try { await call('cancel', {job}); } catch (error) { if (error.code === 'unknown_job') { remember(null); set({phase: 'idle', cancelling: false, percent: null}); } else set({cancelling: false}); }
+ },
+ hasJob: () => !!job,
+ dispose() { disposed = true; },
+ };
+}
+
+const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; };
+
+// The panel in the generation bar, where the lines would be: a sentence, and one button; or the progress and Cancel.
+export function createInstallPanel(host, install, {t, selection}) {
+ const element = el('div', 'fv-ed-voice-install'), note = el('div', 'fv-ed-voice-install-note'), bar = el('div', 'fv-ed-voice-bar'), fill = el('span');
+ const actions = el('div', 'fv-actions');
+ const make = (label, onclick, cls = '') => { const b = el('button', `fv-action ${cls}`.trim(), label); b.type = 'button'; b.onclick = onclick; return b; };
+ const download = make(t('Download', '下载'), () => install.start(), 'fv-primary'), retry = make(t('Retry', '重试'), () => install.start(), 'fv-primary');
+ const cancel = make(t('Cancel', '取消'), () => install.cancel());
+ note.setAttribute('role', 'status'); bar.setAttribute('role', 'progressbar'); bar.setAttribute('aria-valuemin', '0'); bar.setAttribute('aria-valuemax', '100');
+ bar.append(fill); actions.append(download, retry, cancel); element.append(note, bar, actions); element.hidden = true; host.append(element);
+ let shownText = null;
+ return {element, shown: () => !element.hidden, update() {
+ const view = install.view, shown = install.visible(selection());
+ element.hidden = !shown;
+ if (!shown) return;
+ const state = view.phase === 'downloading' ? 'downloading' : view.phase === 'failed' ? 'failed' : 'offer';
+ element.dataset.state = state;
+ const failure = state === 'failed' ? failureText(view.error, view.bytes, t) : null;
+ const text = state === 'downloading' ? progressText(view.percent, t) : failure ? failure.text : offerText(view.bytes, t);
+ if (shownText !== text) {
+ note.replaceChildren(text); shownText = text;
+ if (failure?.report) note.append(issueButton(t, () => ({where: ['Dubbing model download', '配音模型下载'], message: failure.text, kind: view.error || null})));
+ }
+ bar.hidden = state !== 'downloading';
+ if (state === 'downloading') {
+ if (view.percent == null) { bar.removeAttribute('aria-valuenow'); bar.dataset.indeterminate = 'true'; fill.style.width = ''; } else { bar.setAttribute('aria-valuenow', String(view.percent)); delete bar.dataset.indeterminate; fill.style.width = view.percent + '%'; }
+ }
+ download.hidden = state !== 'offer'; retry.hidden = state !== 'failed';
+ // Without the job (a download another page started) there is nothing to stop from here.
+ cancel.hidden = state !== 'downloading' || !install.hasJob?.(); cancel.disabled = view.cancelling;
+ }, dispose() { element.remove(); }};
+}
diff --git a/web/editor_voice_store.js b/web/editor_voice_store.js
index 2a654f21..6dff2387 100644
--- a/web/editor_voice_store.js
+++ b/web/editor_voice_store.js
@@ -7,15 +7,21 @@ const ROOT = '/freevideo/agent/v1';
const ACTIVE = ['waiting', 'queued', 'running', 'cancelling'];
const wait = ms => new Promise(resolve => setTimeout(resolve, ms));
const detail = progress => { try { return JSON.parse(progress?.detail || 'null') || {}; } catch { return {}; } };
-const failure = (value, fallback) => Object.assign(new Error(value?.code || value?.error?.code || fallback), {code: value?.code || value?.error?.code || fallback});
+const failure = (value, fallback) => Object.assign(new Error(value?.code || value?.error?.code || fallback), {code: value?.code || value?.error?.code || fallback, detail: String(value?.message || value?.error?.message || '')});
-export async function voiceAvailable(api) {
+// What the engine offers for changing a voice: `ready` once the dubbing model is there, `install` where this machine could
+// change a voice but the model is still to be downloaded.
+export async function voiceFeatures(api) {
try {
- const response = await agentGet(api, `${ROOT}/capabilities`);
- return !!(response.ok && (await response.json())?.features?.['projects.voice']);
- } catch { return false; }
+ const response = await agentGet(api, ROOT);
+ if (!response.ok) return {ready: false, install: false, unknown: true};
+ const features = (await response.json())?.features;
+ return {ready: !!features?.['projects.voice'], install: !!features?.['projects.voice.install']};
+ } catch { return {ready: false, install: false, unknown: true}; }
}
+export async function voiceAvailable(api) { return (await voiceFeatures(api)).ready; }
+
export function createVoiceStore({api, state, queue}) {
let running = null;
const uuid = () => globalThis.crypto?.randomUUID?.() || `${Date.now()}-${Math.random()}`;
diff --git a/web/feature_tour.css b/web/feature_tour.css
index c586682c..83a51902 100644
--- a/web/feature_tour.css
+++ b/web/feature_tour.css
@@ -45,3 +45,11 @@
.fv-tour-close::after{transform:rotate(-45deg)}
.fv-tour-book{background:none;border:0;padding:6px 4px;color:inherit;font:inherit;text-decoration:underline;opacity:.75;cursor:pointer;flex:none;margin-right:auto;margin-left:12px}.fv-tour-book:hover,.fv-tour-book:focus-visible{opacity:1}
@media (max-width:680px){.fv-tour-book{margin:0;align-self:center}}
+.fv-tour-segments{display:flex;gap:6px;flex-wrap:wrap;margin-top:10px}
+.fv-feature-tour .fv-tour-segment{flex:1;min-width:0;padding:6px 10px}
+.fv-feature-tour .fv-tour-segment[aria-pressed=true]{background:var(--fv-accent);color:var(--fv-bg);border-color:var(--fv-accent);font-weight:600}
+@media(max-width:680px){
+.fv-tour-dots{flex-wrap:nowrap;gap:2px}
+.fv-feature-tour .fv-tour-dot{flex:0 1 20px;width:20px;min-width:0}
+.fv-tour-dot::after{inset:10px 6px}
+}
diff --git a/web/feature_tour.js b/web/feature_tour.js
index cf65dce3..4208cbee 100644
--- a/web/feature_tour.js
+++ b/web/feature_tour.js
@@ -8,7 +8,7 @@ export const TOUR_TEXT = {
};
// The chapter of the tutorial book that teaches the same thing as the card.
const BOOK_CHAPTER = {'first-video': 'quick-start', 'text-to-video': 'text-to-video', 'first-last-frame': 'first-last-frame', 'quality-tiers': 'quality-tiers',
- 'reference-image': 'reference-video', 'replace-person': 'replace-person', character: 'cast-sheet', 'edit-view': 'editor', extend: 'extend-video', 'two-pass': 'refine', autopilot: 'autopilot'};
+ 'reference-image': 'reference-video', 'replace-person': 'replace-person', character: 'cast-sheet', 'edit-view': 'editor', 'separate-sound': 'separate-tracks', extend: 'extend-video', 'two-pass': 'refine', autopilot: 'autopilot'};
const asset = name => new URL(`./assets/tour/${name}`, import.meta.url).href;
const el = (tag, cls, text) => { const node = document.createElement(tag); if (cls) node.className = cls; if (text != null) node.textContent = text; return node; };
const contexts = new Map();
@@ -135,11 +135,12 @@ export async function openFeatureTour({api = contextFor()?.api, cn = contextFor(
}
title.textContent = card.title[lang];
content.append(el('p', 'fv-tour-line', card.line[lang]));
+ let waveImage = null;
const pair = el('div', 'fv-tour-pair'), input = el('div', 'fv-tour-input'), output = el('div', 'fv-tour-output');
card.input.forEach(item => {
if (item.text) input.append(el('p', null, item.text[lang]));
else if (item.mp4) input.append(media(videos.at(-1)));
- else { const image = el('img'); image.src = asset(item.src); image.alt = ''; input.append(image); }
+ else { const image = el('img'); image.src = asset(item.src); image.alt = ''; input.append(image); waveImage = image; }
});
const arrow = el('span', 'fv-tour-arrow'); arrow.setAttribute('aria-hidden', 'true');
// A thin CSS line and two strokes keep this decorative arrow out of the reading order.
@@ -149,11 +150,26 @@ export async function openFeatureTour({api = contextFor()?.api, cn = contextFor(
videos[0].onended = () => { for (const video of playing) video.currentTime = 0; play(); };
for (const video of videos) video.loop = false;
} else if (card.sequence) {
+ // The four versions play in turn without sound. A choice plays that one version, with sound, until another is chosen.
+ const picker = el('div', 'fv-tour-segments'), segments = [];
+ picker.setAttribute('role', 'group');
+ const show = (i, chosen) => {
+ stop();
+ if (waveImage && card.sequence[i].wave) waveImage.src = asset(card.sequence[i].wave);
+ videos.forEach((video, j) => { video.hidden = j !== i; segments[j]?.setAttribute('aria-pressed', String(j === i && (chosen || videos.length > 1))); if (j !== i) video.muted = true; });
+ const video = videos[i]; video.currentTime = 0; video.muted = !chosen; video.loop = chosen; playing = [video]; play();
+ };
videos.forEach((video, i) => {
- video.hidden = i !== 0; media(video); video.loop = false;
- video.onended = () => { video.hidden = true; const next = videos[(i + 1) % videos.length]; next.hidden = false; next.currentTime = 0; playing = [next]; next.play().catch(() => {}); };
+ media(video); video.loop = false;
+ video.onended = () => show((i + 1) % videos.length, false);
output.append(video);
+ const label = card.sequence[i].label;
+ if (!label) return;
+ const segment = el('button', 'fv-tour-segment', label[lang]); segment.type = 'button';
+ segment.onclick = () => show(i, true); segments[i] = segment; picker.append(segment);
});
+ if (segments.length) output.append(picker);
+ videos.forEach((video, i) => { video.hidden = i !== 0; segments[i]?.setAttribute('aria-pressed', String(i === 0 && videos.length > 1)); });
playing = [videos[0]];
} else output.append(media(videos[0], card.previewRange));
pair.append(input, arrow, output); content.append(pair);
diff --git a/web/feature_tour_content.js b/web/feature_tour_content.js
index d0e5a140..539611b3 100644
--- a/web/feature_tour_content.js
+++ b/web/feature_tour_content.js
@@ -24,6 +24,15 @@ export const FEATURE_TOUR = [
[{text: copy('@Leo walks along the shore.', '@小林 走在海边。')}], clip('c12-s1'), {status: 'pending-assets'}, {trigger: 'media', added: '0.5.0', goto: ['cast-add', 'prompt-box']}),
card('edit-view', copy('Edit view', '剪辑视图'), copy('Arrange clips on the timeline and adjust picture, sound and speed.', '在时间线上排列片段，调整画面、声音和速度。'),
[{src: 'c12-s1.webp'}], clip('c13-s1'), {status: 'pending-assets', view: 'editor'}, {trigger: 'editor', added: '0.5.0', goto: ['editor-timeline', 'editor-inspector']}),
+ card('separate-sound', copy('Separate and replace sound', '分离与替换声音'), copy('Split the sound into dialogue, music and ambience, then replace each one.', '把声音分成人声、音乐和环境声，再分别替换。'),
+ [{src: 'cb-source.webp'}], clip('cb-source'), {status: 'pending-assets', view: 'editor'},
+ {trigger: 'editor', added: '0.5.0', goto: ['audio-track-button'], sequence: [['source', 'Original', '原声'], ['dialogue', 'Dialogue', '人声'], ['music', 'Music', '音乐'], ['ambience', 'Ambience', '环境声']].map(([name, en, zh]) => ({...clip(`cb-${name}`), label: copy(en, zh)}))}),
+ card('add-sound', copy('Add sound to a video', '给视频加声音'), copy('Drag a range on the sound track and add music, a voice-over or a recording.', '在声音轨上拖一段，加配乐、配音，或录一段声音。'),
+ [{src: 'ca-music-wave.webp'}], clip('ca-music'), {status: 'pending-assets', view: 'editor'},
+ {trigger: 'editor', added: '0.5.0', goto: ['audio-a2-track'], sequence: [['music', 'Music', '配乐'], ['dub', 'Voice-over', '配音'], ['record', 'Recording', '录音']].map(([name, en, zh]) => ({...clip(`ca-${name}`), wave: `ca-${name}-wave.webp`, label: copy(en, zh)}))}),
+ card('sound-to-picture', copy('Generate a picture from sound', '按声音生成画面'), copy('Drag an empty range on the video track above a sound, and the picture follows it.', '在有声音的区间上方的视频轨拖出一段空白，画面随声音生成。'),
+ [{src: 'cc-speech-wave.webp'}], clip('cc-speech'), {status: 'pending-assets', view: 'editor'},
+ {trigger: 'editor', added: '0.5.0', goto: ['video-empty-range'], sequence: [{...clip('cc-speech'), label: copy('Play with sound', '播放声音')}]}),
card('extend', copy('Extend', '接着生成'), copy('Continue from where the last clip ends.', '从上一段的结尾接着生成下一段。'),
[{src: 'c6-s1.webp'}], clip('c6-s1'), {status: 'pending-assets', view: 'editor'}, {trigger: 'editor', previewRange: [2, 8]}),
card('two-pass', copy('Two-pass', '二次采样'), copy('The first pass sets the composition; the second adds detail.', '第一遍定构图，第二遍补细节。'),
diff --git a/web/freevideo.js b/web/freevideo.js
index ce34a9ed..6e413672 100644
--- a/web/freevideo.js
+++ b/web/freevideo.js
@@ -9,7 +9,8 @@ import { openStudio, loraPanel, loraWarning, promptGuide } from "./studio.js";
import { createGenerationProgress, referenceTrimText, thermalText, sharedGpuText, keepSoundText, encoderFileText, encoderFilePath } from './generation_progress.js';
import { createProgressConnection } from './progress_connection.js';
import { notifyCompatibility } from './compatibility.js';
-import { startUpdateChecks } from './updates.js';
+import { startUpdateChecks, productVersion } from './updates.js';
+import { startWhatsNew } from './whatsnew.js';
import { startWhiteboxWorker } from './whitebox_worker.js';
import { startDoneNotify, askDesktopPermission, DESKTOP_SETTING } from './done_notify.js';
import { installNavigation, refreshNavigation, preferredView, collapseWorkspaces } from './view_navigation.js';
@@ -800,7 +801,7 @@ app.registerExtension({
async setup() {
// One failing part must not take the others down; health.js shows what failed.
for (const [where, start] of [['progress', () => progressConnection.start()], ['navigation', () => installNavigation(showStudio, agentInterfaceState(api), {openWorkflow: openFreeVideoWorkflow, recheck: () => agentInterfaceState(api)})],
- ['updates', startUpdateChecks], ['done-notify', () => startDoneNotify({api, text, desktop: desktopNoticeEnabled, isFreeVideoNode,
+ ['updates', startUpdateChecks], ['whatsnew', () => startWhatsNew({api, cn, productVersion})], ['done-notify', () => startDoneNotify({api, text, desktop: desktopNoticeEnabled, isFreeVideoNode,
isFreeVideoWorkflow: () => (app.graph?._nodes || []).some(isGenerate)})],
['whitebox', () => startWhiteboxWorker({api})]]) {
try { start(); } catch (error) { health({type: 'error', where, error}); }
diff --git a/web/generation_progress.css b/web/generation_progress.css
index 2485fd5d..b5407711 100644
--- a/web/generation_progress.css
+++ b/web/generation_progress.css
@@ -25,10 +25,10 @@
.fv-generation-status{box-sizing:border-box;width:100%;flex:none;margin-top:10px;font-size:12px;line-height:18px;color:var(--fv-muted,#a2abba);text-align:center;text-wrap:balance}
.fv-generation-compatibility{box-sizing:border-box;width:100%;flex:none;margin-top:10px;font-size:12px;line-height:18px;color:var(--fv-ink,#e8edf5);text-align:center;text-wrap:balance}
.fv-generation-compatibility[hidden]{display:none!important}
-.fv-generation-frames{display:block;box-sizing:border-box;width:auto;max-width:100%;max-height:min(30vh,200px);margin:12px auto 0;border-radius:var(--fv-r-sm,6px);object-fit:contain;background:var(--fv-raised,#212a37)}
-.fv-studio .fv-stage .fv-generation-frames{width:auto;height:auto}
-.fv-generation-frames[data-wide=true],.fv-studio .fv-stage .fv-generation-frames[data-wide=true]{height:64px}
-.fv-generation-frames[hidden]{display:none!important}
+.fv-generation-frames{display:block;box-sizing:border-box;width:auto;max-width:100%;height:min(var(--fv-frame-height,120px),120px);aspect-ratio:var(--fv-frame-ratio,16 / 9);margin:12px auto 0;border-radius:var(--fv-r-sm,6px);background:var(--fv-raised,#212a37) no-repeat 0 0}
+.fv-generation-frames[data-wide=true],.fv-studio .fv-stage .fv-generation-frames[data-wide=true]{height:128px}
+.fv-generation-frames[hidden],.fv-generation-frames[data-fit=false]{display:none!important}
+.fv-studio .fv-stage .fv-generation-progress[data-tight=true],.fv-studio .fv-stage .fv-generation-progress.fv-generation-compact[data-tight=true]{padding:8px 16px}
.fv-studio .fv-stage[data-short=true] .fv-generation-frames{display:none!important}
.fv-generation-live{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip-path:inset(50%);white-space:nowrap;border:0}
/* Studio shows progress with the preview's rising wave, so its bar stays
diff --git a/web/generation_progress.js b/web/generation_progress.js
index def8c135..83aa711b 100644
--- a/web/generation_progress.js
+++ b/web/generation_progress.js
@@ -449,8 +449,8 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f
const track = el('div', 'fv-generation-track'), fill = el('div', 'fv-generation-fill');
track.setAttribute('role', 'progressbar'); track.setAttribute('aria-valuemin', '0'); track.append(fill);
// The sampler's newest coarse frames; decorative, so it carries no text.
- const frames = el('img', 'fv-generation-frames'); frames.alt = ''; frames.hidden = true;
- frames.decoding = 'async'; frames.setAttribute('aria-hidden', 'true'); frames.draggable = false;
+ // One frame shows in place; the strip of frames plays through it as a loop (see playFrames).
+ const frames = el('div', 'fv-generation-frames'); frames.hidden = true; frames.setAttribute('aria-hidden', 'true');
const times = el('div', 'fv-generation-times');
const elapsed = el('span', ''), eta = el('span', 'fv-generation-eta'); times.append(elapsed, eta);
const late = el('div', 'fv-generation-late'); late.hidden = true;
@@ -517,22 +517,60 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f
const {report, updateReport, fetchReport, downloadReport} = diagnostic;
// The newest frame picture. Any failure leaves the page as it was without one.
// `framesToken` changes whenever a pending fetch stops being wanted, so a late answer is dropped.
- let framesKey = '', framesUrl = null, framesLoading = null, framesToken = 0;
+ let framesKey = '', framesUrl = null, framesLoading = null, framesToken = 0, strip = null, stripTimer = null;
+ function stopPlayback() { if (stripTimer !== null) cancelAnimationFrame(stripTimer); stripTimer = null; }
function clearFrames() {
- framesKey = ''; framesLoading = null; framesToken++; frames.hidden = true;
+ framesKey = ''; framesLoading = null; framesToken++; frames.hidden = true; strip = null; stopPlayback();
if (framesUrl) { URL.revokeObjectURL(framesUrl); framesUrl = null; }
- frames.removeAttribute('src');
+ frames.style.backgroundImage = '';
}
- // In a wide column each frame is 64 px high; a narrow one keeps the picture's own size.
+ // The frame is at most 128 px high in a wide column and 120 px in a narrow one, never above the picture's own
+ // height. In the Studio stage the percentage, step and times come first: the frame takes what is left of the
+ // stage (the padding shrinks first, as in a short stage) and is left out below 48 px.
+ let fitWatch = null;
function fitFrames() {
const width = (element.parentElement || element).clientWidth;
- frames.dataset.wide = width > 480 ? 'true' : 'false';
+ const wide = width > 480;
+ frames.dataset.wide = wide ? 'true' : 'false';
+ const stage = element.closest?.('.fv-stage') ?? null;
+ if (typeof ResizeObserver === 'function') {
+ fitWatch ??= new ResizeObserver(() => fitFrames());
+ fitWatch.observe(element.parentElement || element); if (stage) fitWatch.observe(stage);
+ }
+ if (!strip || frames.hidden) return;
+ let height = Math.min(wide ? 128 : 120, strip.height), tight = false;
+ if (stage && stage.clientHeight > 0) {
+ const room = () => {
+ frames.style.display = 'none';
+ const rest = element.offsetHeight;
+ frames.style.display = '';
+ return Math.floor(stage.clientHeight - rest - 12);
+ };
+ element.dataset.tight = 'false';
+ let left = room();
+ if (left < height) { tight = true; element.dataset.tight = 'true'; left = room(); }
+ height = Math.min(height, left);
+ }
+ if (height < 48) { frames.dataset.fit = 'false'; element.dataset.tight = 'false'; }
+ else { frames.dataset.fit = 'true'; frames.style.height = `${height}px`; }
+ }
+ // Where the loop stands, from 0 up to (not including) 1.
+ const stripPhase = () => strip ? (strip.phase + (now() - strip.start) / 1000 * strip.fps / strip.count) % 1 : 0;
+ // The strip plays forward at its frames per second over the video's length, 4 to 8 a second; with reduced
+ // motion, or a single frame, the last frame stands still.
+ function playFrames() {
+ stripTimer = null;
+ if (!strip || frames.hidden) return;
+ const playing = strip.count > 1 && moving();
+ const index = playing ? Math.min(strip.count - 1, Math.floor(stripPhase() * strip.count)) : strip.count - 1;
+ frames.style.backgroundPosition = strip.count > 1 ? `${100 * index / (strip.count - 1)}% 0` : '0 0';
+ if (playing) stripTimer = requestAnimationFrame(playFrames);
}
function updateFrames(message, id) {
const value = message.frame_preview;
fitFrames();
const usable = value && typeof value === 'object' && Number.isInteger(value.serial) && value.serial > 0
- && Number.isInteger(value.frames) && value.frames > 0 && value.frames <= 8;
+ && Number.isInteger(value.frames) && value.frames > 0 && value.frames <= 16;
if (!usable || !id || ['complete', 'failed', 'cancelled'].includes(message.phase)) { if (framesKey || framesLoading) clearFrames(); return; }
const key = `${id}:${value.serial}`;
if (key === framesKey || key === framesLoading) return;
@@ -540,18 +578,32 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f
const token = ++framesToken;
const path = `/freevideo/frame-preview/${id}`;
(async () => {
+ let url = null;
try {
const response = await (api ? api.fetchApi(path) : fetch(path));
if (!response.ok) throw new Error('No picture');
const blob = await response.blob();
if (framesToken !== token) return;
- const url = URL.createObjectURL(blob);
- const old = framesUrl;
- frames.onload = () => { fitFrames(); if (framesUrl === url) frames.hidden = false; try { onLayout?.(); } catch {} };
- frames.onerror = () => { if (framesUrl === url) clearFrames(); };
- framesUrl = url; framesKey = key; frames.src = url;
+ url = URL.createObjectURL(blob);
+ // The new strip replaces the old one only once it has loaded, and the loop carries on from the same place.
+ const picture = new Image();
+ await new Promise((resolve, reject) => { picture.onload = resolve; picture.onerror = () => reject(new Error('No picture')); picture.src = url; });
+ if (framesToken !== token) { URL.revokeObjectURL(url); return; }
+ const seconds = Number.isInteger(value.duration_ms) && value.duration_ms > 0 ? value.duration_ms / 1000 : null;
+ const fps = Math.min(8, Math.max(4, seconds ? value.frames / seconds : 6));
+ const frameWidth = picture.naturalWidth / value.frames, frameHeight = picture.naturalHeight;
+ const old = framesUrl, phase = stripPhase();
+ strip = {count: value.frames, fps, phase, start: now(), height: frameHeight};
+ frames.style.setProperty('--fv-frame-height', `${frameHeight}px`);
+ frames.style.setProperty('--fv-frame-ratio', `${frameWidth} / ${frameHeight}`);
+ frames.style.backgroundSize = `${value.frames * 100}% 100%`;
+ frames.style.backgroundImage = `url("${url}")`;
+ framesUrl = url; framesKey = key; framesLoading = null;
+ frames.hidden = false; fitFrames(); stopPlayback(); playFrames();
+ try { onLayout?.(); } catch {}
if (old) URL.revokeObjectURL(old);
} catch {
+ if (url && url !== framesUrl) URL.revokeObjectURL(url);
if (framesToken === token) clearFrames();
}
})();
diff --git a/web/health.js b/web/health.js
index 3a515a49..69cd3161 100644
--- a/web/health.js
+++ b/web/health.js
@@ -14,12 +14,12 @@ const t = (en, zh) => cn ? zh : en;
const launchVisit = query.get('freevideo') === 'launch';
// Every script this folder ships; a test keeps the list in step with web/*.js.
export const MODULES = ['audio_original.js', 'chat_data.js', 'phone_model.js', 'phone.js', 'lan_access.js', 'qrcodegen.js', 'agent_activity.js', 'agent_confirm_settings.js', 'agent_connect.js', 'agent_session.js', 'autopilot.js', 'autopilot_chat.js', 'chat_attachments.js', 'autopilot_conversations.js', 'browser_extension.js', 'caret_rect.js', 'chain_support.js', 'chain_rules.js', 'timed_prompt.js', 'capability_view.js', 'cast_generate.js', 'cast_mention.js', 'cast_overlay.js', 'cast_source.js', 'compatibility.js', 'entry_showcase.js', 'error_panel.js', 'first_run_guide.js', 'fonts.js', 'freevideo.js', 'generation_progress.js',
- 'editor.js', 'editor_actions.js', 'editor_lifecycle.js', 'editor_adjust.js', 'editor_adjust_picture.js', 'editor_picture_fields.js', 'editor_picture_session.js', 'editor_preview_picture.js', 'editor_entry.js', 'editor_expand.js', 'editor_export_dialog.js', 'editor_first_frame.js', 'editor_media_references.js', 'editor_references.js', 'editor_genbar.js', 'editor_duration.js', 'editor_geometry.js', 'editor_library.js', 'editor_menu.js', 'editor_stems.js', 'editor_record.js', 'editor_record_model.js', 'editor_voice.js', 'editor_voice_model.js', 'editor_voice_store.js', 'editor_model.js', 'editor_frame.js', 'editor_player.js', 'editor_room.js', 'editor_select.js', 'editor_state.js', 'editor_store.js', 'editor_tracks.js', 'editor_generation.js', 'editor_transport.js', 'editor_transport_model.js', 'editor_audio.js', 'editor_audio_model.js', 'editor_audio_notes.js', 'editor_audio_replace.js', 'editor_audio_rules.js', 'editor_music.js', 'editor_music_model.js', 'editor_speed.js', 'editor_speed_model.js', 'editor_failure.js', 'editor_header_fit.js',
+ 'editor.js', 'editor_actions.js', 'editor_lifecycle.js', 'editor_adjust.js', 'editor_adjust_picture.js', 'editor_picture_fields.js', 'editor_picture_session.js', 'editor_preview_picture.js', 'editor_entry.js', 'editor_expand.js', 'editor_export_dialog.js', 'editor_first_frame.js', 'editor_media_references.js', 'editor_references.js', 'editor_genbar.js', 'editor_duration.js', 'editor_geometry.js', 'editor_library.js', 'editor_menu.js', 'editor_stems.js', 'editor_record.js', 'editor_record_model.js', 'editor_voice.js', 'editor_voice_install.js', 'editor_voice_model.js', 'editor_voice_store.js', 'editor_model.js', 'editor_frame.js', 'editor_player.js', 'editor_room.js', 'editor_select.js', 'editor_state.js', 'editor_store.js', 'editor_tracks.js', 'editor_generation.js', 'editor_transport.js', 'editor_transport_model.js', 'editor_audio.js', 'editor_audio_model.js', 'editor_audio_notes.js', 'editor_audio_replace.js', 'editor_audio_rules.js', 'editor_music.js', 'editor_music_model.js', 'editor_speed.js', 'editor_speed_model.js', 'editor_failure.js', 'editor_header_fit.js',
'editor_loader.js', 'editor_text_coverage.js', 'editor_text.js', 'editor_toolbar_more.js', 'editor_text_model.js', 'editor_text_layout.js', 'editor_notes.js', 'editor_icons.js', 'dialog_keys.js', 'done_notify.js', 'failure_issue.js', 'feedback.js', 'frame_sizes.js', 'health.js', 'image_commands.js', 'image_editor.js', 'image_editor_engine.js', 'image_generation.js', 'image_lora.js', 'launch_visit.js', 'launcher.js', 'length_plan.js', 'lora_triggers.js', 'lora_check.js', 'library.js', 'motion.js', 'native_errors.js', 'translate_guard.js', 'output_download.js', 'port_notice.js', 'preview_scene.js',
'progress_connection.js', 'prompt_draft.js', 'prompt_enhance.js', 'prompt_references.js', 'prompt_sections.js', 'prompt_terms.js', 'request_assets.js', 'repair_hint.js', 'report_issue.js', 'issue_button.js', 'studio_recent.js', 'task_groups.js',
'result_actions.js', 'result_reuse.js', 'sampling_effort.js', 'pass_mode.js','setup.js', 'share.js', 'share_layout.js', 'studio.js', 'studio_chain.js', 'studio_extend.js', 'studio_length.js', 'first_frame_row.js', 'replace_person.js', 'replace_state.js', 'optional_inputs.js', 'other_usage.js', 'studio_queue.js', 'studio_size.js',
- 'toolbar_position.js', 'updates.js', 'view_navigation.js', 'wordmark.js', 'workspace_restore.js', 'cast_library.js', 'media_upload.js', 'whitebox_encode.js', 'whitebox_mp4.js', 'whitebox_recording.js', 'whitebox_render.js', 'whitebox_scenes.js', 'whitebox_worker.js',
- 'empty_prompt_notice.js', 'feature_tour.js', 'feature_tour_content.js', 'howto.js', 'swap_lora.js', 'tour_goto.js', 'tour_try.js', 'tutorial_book.js', 'tutorial_book_content.js'];
+ 'toolbar_position.js', 'updates.js', 'view_navigation.js', 'whatsnew.js', 'wordmark.js', 'workspace_restore.js', 'cast_library.js', 'media_upload.js', 'whitebox_encode.js', 'whitebox_mp4.js', 'whitebox_recording.js', 'whitebox_render.js', 'whitebox_scenes.js', 'whitebox_worker.js',
+ 'empty_prompt_notice.js', 'feature_tour.js', 'feature_tour_content.js', 'howto.js', 'swap_lora.js', 'sound_bubble.js', 'tour_goto.js', 'tour_try.js', 'tutorial_book.js', 'tutorial_book_content.js'];
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
let visibleAt = 0;
async function waitForVisible() {
diff --git a/web/howto.js b/web/howto.js
index 0850b573..9e796c64 100644
--- a/web/howto.js
+++ b/web/howto.js
@@ -44,6 +44,20 @@ const HOWTOS = [
{id: 'editor-refs', anchor: '[data-fv-guide="editor-refs"]', image: 'editor-refs',
en: 'Click + next to the prompt to upload or generate images, up to 3.', zh: '点描述框旁的 +，上传或生成图片，最多 3 张。'},
{id: 'editor-describe', anchor: '[data-fv-guide="editor-describe"]', en: 'Describe it, then choose "Generate".', zh: '填写描述，然后点“生成”。'}]},
+ {id: 'separate-tracks', view: 'editor', title: ['Separate tracks', '分离音轨'],
+ steps: [
+ {id: 'sound-clip', anchor: '[data-tour-id="audio-a1-track"]', en: 'Select a clip with sound.', zh: '请选中带声音的片段。'},
+ {id: 'separate', anchor: '[data-tour-id="audio-track-button"]', en: 'Choose "Separate tracks". Dialogue, music and ambience become three tracks.', zh: '点“分离音轨”，人声、音乐和环境声会分成三条轨道。'},
+ {id: 'parts', anchor: '[data-tour-id="audio-music-stem"]', en: 'Each part can now be replaced on its own.', zh: '每条轨道现在都可以单独替换。'}]},
+ {id: 'replace-music', view: 'editor', title: ['Replace the music', '换一段配乐'],
+ steps: [
+ {id: 'music-track', anchor: '[data-tour-id="audio-music-stem"]', en: 'Select a clip on the music track.', zh: '选中“音乐”轨上的片段。'},
+ {id: 'music-describe', anchor: '[data-fv-guide="editor-describe"]', en: 'Describe the music you want.', zh: '描述想要的配乐。'},
+ {id: 'music-generate', anchor: '[data-fv-guide="editor-generate"]', en: 'Choose "Replace music".', zh: '点“替换配乐”。'}]},
+ {id: 'replace-voice', view: 'editor', title: ['Edit lines one by one', '逐句改台词'],
+ steps: [
+ {id: 'voice-lines', anchor: '[data-tour-id="voice-lines"]', en: 'Select a clip on the dialogue track. Change the words of a line here.', zh: '选中“人声”轨上的片段，在这里修改某一句的文字。'},
+ {id: 'voice-generate', anchor: '[data-fv-guide="editor-generate"]', en: 'Choose "Redub" to speak the changed lines again.', zh: '点“重新配音”，只重配改过的句子。'}]}
];
export const howToList = cn => HOWTOS.map(({id, title}) => ({id, title: title[cn ? 1 : 0]}));
diff --git a/web/phone.js b/web/phone.js
index 0943f353..14341092 100644
--- a/web/phone.js
+++ b/web/phone.js
@@ -138,14 +138,34 @@ compose.onsubmit = async event => {
} catch { notice(t('Could not send. Please try again.', '没有发送成功，请重试。')); }
finally { sending = false; grow(); }
};
-function picture(job, preview = false) {
+// Large previews play their strip forward as a loop, 4 to 8 frames a second over the video's length. The position
+// follows the clock, so a redraw that builds a new image carries on from the same place; with reduced motion, or in the
+// small list picture, the last frame stands still.
+const playing = new Set();
+let playTimer = null;
+const reducedMotion = () => typeof matchMedia === 'function' && matchMedia('(prefers-reduced-motion: reduce)').matches;
+function playStrips() {
+ playTimer = null;
+ for (const item of playing) {
+ if (!item.img.isConnected) { playing.delete(item); continue; }
+ const index = Math.min(item.count - 1, Math.floor((Date.now() / 1000 * item.fps / item.count) % 1 * item.count));
+ item.img.style.transform = `translateX(-${index * 100 / item.count}%)`;
+ }
+ if (playing.size && !reducedMotion() && typeof requestAnimationFrame === 'function') playTimer = requestAnimationFrame(playStrips);
+}
+function picture(job, preview = false, play = false) {
const box = el('div', 'phone-picture');
const p = job.progress || {}, frame = p.frame_preview;
- const path = preview && frame?.serial > 0 && frame.frames > 0 && frame.frames <= 8 && job.node != null
+ const path = preview && frame?.serial > 0 && frame.frames > 0 && frame.frames <= 16 && job.node != null
? `/freevideo/frame-preview/${encodeURIComponent(job.node)}?serial=${frame.serial}` : mediaPath(job.thumbnail, 'image');
if (path) {
const img = el('img'); img.alt = ''; img.loading = 'lazy'; img.src = path; box.append(img);
- if (preview && frame?.frames > 0) { box.classList.add('strip'); img.style.width = `${frame.frames * 100}%`; img.style.transform = `translateX(-${(frame.frames - 1) * 100 / frame.frames}%)`; }
+ if (preview && frame?.frames > 0) { box.classList.add('strip'); img.style.width = `${frame.frames * 100}%`; img.style.transform = `translateX(-${(frame.frames - 1) * 100 / frame.frames}%)`;
+ if (play && frame.frames > 1 && !reducedMotion()) {
+ const seconds = frame.duration_ms > 0 ? frame.duration_ms / 1000 : null;
+ playing.add({img, count: frame.frames, fps: Math.min(8, Math.max(4, seconds ? frame.frames / seconds : 6))});
+ if (playTimer === null && typeof requestAnimationFrame === 'function') playTimer = requestAnimationFrame(playStrips);
+ } }
img.onerror = () => { img.remove(); box.closest('.phone-frame')?.classList.add('no-frame'); };
}
return box;
@@ -206,7 +226,7 @@ function renderJobs() {
if (running.length) generations.append(el('h2', 'phone-section', t('Generating', '正在生成')));
for (const job of running) {
const card = el('article', 'phone-running'), frame = el('div', 'phone-frame'), info = el('div', 'phone-info'); card.dataset.job = job.id;
- const img = picture(job, true); img.style.aspectRatio = `${job.width || 9}/${job.height || 16}`; frame.append(img); frame.classList.toggle('no-frame', !img.firstChild);
+ const img = picture(job, true, true); img.style.aspectRatio = `${job.width || 9}/${job.height || 16}`; frame.append(img); frame.classList.toggle('no-frame', !img.firstChild);
if (!img.firstChild) frame.append(el('span', 'phone-frame-pct', `${Math.floor(fraction(job) * 100)}%`));
const row = el('div', 'phone-job-row'), numbers = el('div'); numbers.append(el('span', 'phone-pct', `${Math.floor(fraction(job) * 100)}%`), document.createTextNode(' '), el('span', 'phone-eta', remainingWords(job.progress?.remaining_seconds, t)));
const stop = button('phone-quiet', t('Stop', '停止'), () => cancel(job, true)); stop.disabled = job.status === 'cancelling'; stop.dataset.action = 'stop-' + job.id; row.append(numbers, stop);
diff --git a/web/preview_scene.js b/web/preview_scene.js
index 2a79ed2c..1b630050 100644
--- a/web/preview_scene.js
+++ b/web/preview_scene.js
@@ -104,7 +104,10 @@ export function createPreviewScene(stage, progress, media) {
width = stage.clientWidth; height = stage.clientHeight;
const scale = Math.min(window.devicePixelRatio || 1, 1.5);
canvas.width = Math.round(width * scale); canvas.height = Math.round(height * scale);
- ctx.setTransform(scale, 0, 0, scale, 0, 0); schedule();
+ ctx.setTransform(scale, 0, 0, scale, 0, 0);
+ // Resizing clears the canvas; paint now so the throttled frame loop cannot leave a black frame.
+ if (!disposed && visible && width && height) paint();
+ schedule();
});
const observer = new MutationObserver(state);
observer.observe(progress, {attributes: true, attributeFilter: ['hidden', 'style', 'data-counted']});
diff --git a/web/setup.js b/web/setup.js
index 8670141f..73692bdf 100644
--- a/web/setup.js
+++ b/web/setup.js
@@ -22,6 +22,7 @@ const SPEED_TEST_FAILED = 'The speed test did not finish. Click "Test source spe
const repairStep = text => text.includes('In the FreeVideo launcher, open Settings and click Repair.')
? t('In the FreeVideo launcher, open Settings and click Repair.', '请在 FreeVideo 启动器的设置里点“修复”。')
: t('Click Detect & review, then Install / repair.', '请先点“检测并预览”，再点“安装／修复”。');
+const WAIT_REFUSAL = /^Wait for (the installation check|the installation|a placement test|a setup task) to finish; its progress is in this panel\.$/;
const SETUP_ERRORS = [
[/^Choose an engine installation folder$/, () => t('Choose a FreeVideo installation folder first.', '请先选择 FreeVideo 安装目录。')],
[/^Use a dedicated FreeVideo folder, not the ComfyUI root or its parent$/,
@@ -32,6 +33,13 @@ const SETUP_ERRORS = [
[/^Finish or cancel queued ComfyUI requests before calibrating$/,
() => t('Generations are running or queued. Run the test after they finish, or cancel them first.', '仍有生成正在进行或排队。请等待完成或先取消，再运行放置测试。')],
[/^Wait for setup to finish$/, () => t('Installation is still running. Try again after it finishes.', '安装仍在进行，请在完成后重试。')],
+ // The placement test refuses to start beside another setup task and names it; that is a
+ // wait, not a fault, so it is shown as plain text without the report button.
+ [WAIT_REFUSAL,
+ text => {
+ const names = {'the installation check': '安装检查', 'the installation': '安装', 'a placement test': '放置测试'};
+ return t(text, `请等待${names[text.match(/^Wait for (.+) to finish/)[1]] ?? '安装任务'}完成，进度见本面板。`);
+ }],
[/^Inspect this installation again before installing$/, () => t('Click Detect & review again, then Install / repair.', '请重新点“检测并预览”，再点“安装／修复”。')],
[/^Review the plan and accept its model\/toolkit licenses first$/, () => t('Tick the agreement above to enable Install / repair.', '勾选上方的同意选项后，可点击“安装／修复”。')],
[/^FreeVideo setup is missing\./, text => t('This folder has no finished FreeVideo installation. ', '该文件夹中没有已安装的 FreeVideo。') + repairStep(text)],
@@ -199,6 +207,7 @@ export async function openSetup() {
dialog.addEventListener('cancel', e => { e.preventDefault(); closeDialog(dialog); });
document.body.append(dialog); dialog.showModal();
let savedFolders = "", savedMissing = false;
+ let calibrateWait = false;
let token, planId, busy = false, probeBusy = false, pollTimer, lastPlanId, planErrors = false, planStale = false, ready = false, waitSince;
// A click names what is happening at once, before the server has answered.
const doing = (en, zh) => { waitSince ??= Date.now(); state.className = ''; state.textContent = t(en, zh); };
@@ -239,9 +248,13 @@ export async function openSetup() {
installation.open = true;
cancel.scrollIntoView({block: 'center'});
} catch (error) {
- calibrateStatus.className = 'fv-failure';
+ // A refusal that names the running task is a wait, not a fault: shown quietly, with the
+ // installation section open and the state polled so the progress it points to is visible.
+ calibrateWait = WAIT_REFUSAL.test(String(error.message || ''));
+ calibrateStatus.className = calibrateWait ? 'fv-muted' : 'fv-failure';
fault(calibrateStatus, error.message);
- } finally { calibrateButton.disabled = false; }
+ if (calibrateWait) { installation.open = true; poll(); }
+ } finally { calibrateButton.disabled = busy; }
};
saveReserve.onclick = async () => {
clearReserveStatus();
@@ -310,7 +323,7 @@ export async function openSetup() {
catch(error) { probeButton.disabled=false; fail(error); }
};
const controls = () => {
- for (const e of [root, extra, copy, samplingCaches, saveFolders, inspect, use, hfToken, applyToken, clearToken]) e.disabled = busy;
+ for (const e of [root, extra, copy, samplingCaches, saveFolders, inspect, use, hfToken, applyToken, clearToken, calibrateButton]) e.disabled = busy;
cancel.hidden = !busy; install.disabled = busy || !planId || !accept.checked; accept.disabled = busy; inspect.classList.toggle("fv-primary", !planId); install.classList.toggle("fv-primary", Boolean(planId));
const why = busy ? "" : planStale ? t("Settings changed. Click Detect & review again.", "设置已更改，请重新点“检测并预览”。")
: planErrors ? t("Fix the problems marked in red above, then click Detect & review again.", "上面有标红的问题，处理后请重新点“检测并预览”。")
@@ -365,6 +378,8 @@ export async function openSetup() {
}
function render(task) {
busy = Boolean(task.busy); if (!busy) waitSince = undefined; else waitSince ??= Date.now();
+ // The wait sentence ends with the task it named.
+ if (!busy && calibrateWait) { calibrateWait = false; calibrateStatus.className = 'fv-muted'; calibrateStatus.textContent = ''; }
ready = Boolean(task.ready) || ready && task.status !== 'failed';
cancel.textContent = task.action === 'calibrate' ? t('Stop', '停止') : t('Stop setup', '停止安装');
inputs.hidden = busy; plan.hidden = busy;
diff --git a/web/sound_bubble.css b/web/sound_bubble.css
new file mode 100644
index 00000000..243684fb
--- /dev/null
+++ b/web/sound_bubble.css
@@ -0,0 +1,8 @@
+.fv-sound-bubble{box-sizing:border-box;position:absolute;z-index:30;display:grid;grid-template-columns:minmax(0,1fr) auto;gap:6px 8px;align-items:center;padding:10px 36px 10px 12px;color:var(--fv-ink);background:var(--fv-raised,#1b2430);border:1px solid var(--fv-accent,#6db8fa);border-radius:var(--fv-r-md,10px);box-shadow:0 8px 28px #0008;font:13px/1.5 var(--fv-font);animation:fv-sound-bubble-in .2s ease-out}
+.fv-sound-bubble::before{content:'';position:absolute;top:-6px;left:calc(var(--fv-sound-arrow,16px) - 5px);width:10px;height:10px;transform:rotate(45deg);background:var(--fv-raised,#1b2430);border-left:1px solid var(--fv-accent,#6db8fa);border-top:1px solid var(--fv-accent,#6db8fa)}
+.fv-sound-bubble p{margin:0;grid-column:1 / -1;max-height:3em;overflow:hidden}
+.fv-sound-bubble button{font:inherit;cursor:pointer;color:var(--fv-ink);background:transparent;border:1px solid var(--fv-border);border-radius:var(--fv-r-sm,6px);padding:3px 10px;min-height:28px}
+.fv-sound-bubble button:hover{background:var(--fv-hover)}
+.fv-sound-bubble .fv-sound-bubble-close{position:absolute;top:4px;right:4px;width:28px;padding:0;border:0;font-size:18px;line-height:1}
+@keyframes fv-sound-bubble-in{from{opacity:0}}
+@media(prefers-reduced-motion:reduce){.fv-sound-bubble{animation:none}}
diff --git a/web/sound_bubble.js b/web/sound_bubble.js
new file mode 100644
index 00000000..0ffac24b
--- /dev/null
+++ b/web/sound_bubble.js
@@ -0,0 +1,45 @@
+// A one-time bubble on the "Separate tracks" button, shown when a clip with sound is first selected.
+// It is seen once the button is used, the chapter is opened, the bubble is closed or any click lands; a bubble that only
+// fades out is shown one more time.
+if (typeof document !== 'undefined' && !document.head.querySelector('link[href$="sound_bubble.css"]')) {
+ const link = document.createElement('link'); link.rel = 'stylesheet'; link.href = new URL('./sound_bubble.css', import.meta.url).href;
+ document.head.append(link);
+}
+const KEY = 'freevideo-sound-bubble', FADE_MS = 8000, OTHER_CARD = '.fv-guide-layer,.fv-first-start,.fv-next-step,body[data-fv-how-to],dialog.fv-feature-tour[open],.fv-tour-ring,dialog.fv-tutorial-book[open]';
+const read = () => { try { return JSON.parse(localStorage.getItem(KEY) || '{}') || {}; } catch { return {}; } };
+const write = value => { try { localStorage.setItem(KEY, JSON.stringify(value)); } catch { /* The bubble then shows again next time. */ } };
+export const soundBubbleSeen = () => read().seen === true;
+export function markSoundBubbleSeen() { write({seen: true}); globalThis.document?.querySelector('.fv-sound-bubble')?.remove(); }
+const TEXT = {line: ['Splits dialogue, music and ambience.', '分离出人声、音乐和环境声。'], view: ['Read the tutorial', '看教程'], close: ['Close', '关闭']};
+const NO_BUBBLE = () => {};
+/** Show the bubble under `button` when it is due. Returns a function that removes it. */
+export const isBubbleRemover = remove => remove !== NO_BUBBLE;
+export function showSoundBubble(button, {cn = false} = {}) {
+ const t = pair => pair[cn ? 1 : 0], state = read();
+ if (state.seen || state.faded > 1 || document.querySelector('.fv-sound-bubble') || document.querySelector(OTHER_CARD) || button.hidden) return NO_BUBBLE;
+ const bubble = document.createElement('div'); bubble.className = 'fv-sound-bubble'; bubble.setAttribute('role', 'status');
+ const line = document.createElement('p'); line.textContent = t(TEXT.line);
+ const view = document.createElement('button'); view.type = 'button'; view.className = 'fv-sound-bubble-view'; view.textContent = t(TEXT.view);
+ const close = document.createElement('button'); close.type = 'button'; close.className = 'fv-sound-bubble-close'; close.textContent = '×'; close.setAttribute('aria-label', t(TEXT.close));
+ bubble.append(line, view, close);
+ const host = button.closest('dialog[open]') || document.body;
+ let timer = 0, gone = false;
+ const place = () => {
+ const r = button.getBoundingClientRect(), panel = host.getBoundingClientRect(), width = Math.min(300, panel.width - 16);
+ const left = Math.max(panel.left + 8, Math.min(r.left, panel.right - 8 - width));
+ Object.assign(bubble.style, {width: `${width}px`, left: `${left - (host === document.body ? 0 : panel.left + host.clientLeft)}px`, top: `${r.bottom + 8 - (host === document.body ? 0 : panel.top + host.clientTop)}px`});
+ bubble.style.setProperty('--fv-sound-arrow', `${Math.max(12, Math.min(r.left + r.width / 2 - left, width - 12))}px`);
+ };
+ const remove = () => {
+ if (gone) return; gone = true; clearTimeout(timer);
+ document.removeEventListener('pointerdown', onPointer, true); removeEventListener('resize', place);
+ bubble.remove();
+ };
+ function onPointer(event) { if (!bubble.contains(event.target)) { markSoundBubbleSeen(); remove(); } }
+ view.onclick = async () => { markSoundBubbleSeen(); remove(); const {openTutorialBook} = await import('./tutorial_book.js'); openTutorialBook({chapter: 'separate-tracks'}); };
+ close.onclick = () => { markSoundBubbleSeen(); remove(); };
+ host.append(bubble); place(); addEventListener('resize', place);
+ document.addEventListener('pointerdown', onPointer, true);
+ timer = setTimeout(() => { write({...read(), faded: (read().faded || 0) + 1}); remove(); }, FADE_MS);
+ return remove;
+}
diff --git a/web/task_groups.js b/web/task_groups.js
index c8e7ead7..8f380ca2 100644
--- a/web/task_groups.js
+++ b/web/task_groups.js
@@ -1,4 +1,4 @@
-// Videos made by one task (a preview and its upscale, a continuation, one agent job, the same prompt on one day)
+// Videos made by one task (a preview and its upscale, a continuation, one agent job, the same prompt within two hours)
// share the `task` the library API gives them. A list shows the newest as the cover; the others fold under it
// until the cover is pressed again. Both lists (Recent videos and Creations) fold with this one rule.
diff --git a/web/theme.css b/web/theme.css
index d8c27790..279fb645 100644
--- a/web/theme.css
+++ b/web/theme.css
@@ -5,7 +5,7 @@
Only Linux faces are named, so other systems keep their own quotes. */
@font-face{font-family:"FreeVideo Quotes";font-weight:100 500;unicode-range:U+2018-2019;src:local("Noto Sans Regular"),local("NotoSans-Regular"),local("DejaVu Sans"),local("Liberation Sans")}
@font-face{font-family:"FreeVideo Quotes";font-weight:600 900;unicode-range:U+2018-2019;src:local("Noto Sans Bold"),local("NotoSans-Bold"),local("DejaVu Sans Bold"),local("Liberation Sans Bold")}
-:where(.fv-phone,.fv-lan,.fv-studio,.fv-setup,.fv-panel,.fv-generation-progress,.fv-view-navigation,.fv-prompt-guide,.fv-failure-dialog,.fv-reference-picker,.fv-release-notes,.fv-image-editor,.fv-cast-dialog,.fv-tutorial-book,.fv-feature-tour){
+:where(.fv-phone,.fv-lan,.fv-studio,.fv-setup,.fv-panel,.fv-generation-progress,.fv-view-navigation,.fv-prompt-guide,.fv-failure-dialog,.fv-reference-picker,.fv-release-notes,.fv-whatsnew,.fv-image-editor,.fv-cast-dialog,.fv-tutorial-book,.fv-feature-tour){
/* Surfaces, from the backdrop up to a hovered control; mirrors branding.py. */
--fv-bg:#111720;--fv-canvas:#161e29;--fv-card:#1a222e;--fv-raised:#212a37;--fv-hover:#2a3341;
--fv-border:#2a3443;--fv-sheen:#35404e;
diff --git a/web/tutorial/replace-music-1.webp b/web/tutorial/replace-music-1.webp
new file mode 100644
index 00000000..0eac9704
Binary files /dev/null and b/web/tutorial/replace-music-1.webp differ
diff --git a/web/tutorial/replace-music-2.webp b/web/tutorial/replace-music-2.webp
new file mode 100644
index 00000000..da930710
Binary files /dev/null and b/web/tutorial/replace-music-2.webp differ
diff --git a/web/tutorial/separate-tracks-1.webp b/web/tutorial/separate-tracks-1.webp
new file mode 100644
index 00000000..2893c62d
Binary files /dev/null and b/web/tutorial/separate-tracks-1.webp differ
diff --git a/web/tutorial/separate-tracks-2.webp b/web/tutorial/separate-tracks-2.webp
new file mode 100644
index 00000000..7b675dee
Binary files /dev/null and b/web/tutorial/separate-tracks-2.webp differ
diff --git a/web/tutorial/sound-to-picture-1.webp b/web/tutorial/sound-to-picture-1.webp
new file mode 100644
index 00000000..5503d943
Binary files /dev/null and b/web/tutorial/sound-to-picture-1.webp differ
diff --git a/web/tutorial/sound-to-picture-2.webp b/web/tutorial/sound-to-picture-2.webp
new file mode 100644
index 00000000..bbf81967
Binary files /dev/null and b/web/tutorial/sound-to-picture-2.webp differ
diff --git a/web/tutorial/voice-recording-1.webp b/web/tutorial/voice-recording-1.webp
new file mode 100644
index 00000000..3badd6ad
Binary files /dev/null and b/web/tutorial/voice-recording-1.webp differ
diff --git a/web/tutorial/voice-recording-2.webp b/web/tutorial/voice-recording-2.webp
new file mode 100644
index 00000000..4ea0f537
Binary files /dev/null and b/web/tutorial/voice-recording-2.webp differ
diff --git a/web/tutorial_book.js b/web/tutorial_book.js
index 98cad791..11b4620a 100644
--- a/web/tutorial_book.js
+++ b/web/tutorial_book.js
@@ -41,6 +41,7 @@ let current = null;
/** Open the bundled manual; chapter is one of the ids in tutorial_book_content.js. */
export function openTutorialBook({chapter = TUTORIAL_CHAPTERS[0].id} = {}) {
+ if (chapter === 'separate-tracks') import('./sound_bubble.js').then(({markSoundBubbleSeen}) => markSoundBubbleSeen(), () => {});
if (current?.dialog.isConnected) {
current.select(chapter);
return current.dialog;
diff --git a/web/tutorial_book_content.js b/web/tutorial_book_content.js
index df730df5..304a66c5 100644
--- a/web/tutorial_book_content.js
+++ b/web/tutorial_book_content.js
@@ -322,6 +322,82 @@ export const TUTORIAL_CHAPTERS = [
en: ['A part that continues the previous one uses up to 3 reference images and no first or last frames.', 'Edit takes images only. Add reference videos and audio under "Media" in the creation panel.', 'The same part limits apply. An NVIDIA graphics card or a Mac is required.'],
},
},
+ {
+ id: 'separate-tracks',
+ goto: ['audio-track-button'],
+ title: {zh: '分离音轨和替换声音', en: 'Separate tracks and replace sound'},
+ summary: {zh: '把声音分成人声、音乐和环境声，并分别替换。', en: 'Split the sound into dialogue, music and ambience and replace each one.'},
+ media: {
+ type: 'image',
+ src: {zh: 'separate-tracks-1.webp', en: 'separate-tracks-2.webp'},
+ alt: {zh: '剪辑视图：选中的声音片段和“分离音轨”按钮已用红框标出。', en: 'The Edit view, with a selected sound clip and the "Separate tracks" button outlined in red.'},
+ },
+ steps: {
+ zh: ['在剪辑视图中选中带声音的片段。', '点“分离音轨”，人声、音乐和环境声会分成三条轨道。', '选中“音乐”轨上的片段，在描述框中写下想要的音乐，点“替换配乐”。', '选中“人声”轨上的片段，在“台词”中修改某一句的文字。', '点“重新配音”，只重配改过的句子。'],
+ en: ['Select a clip with sound in the Edit view.', 'Choose "Separate tracks". Dialogue, music and ambience become three tracks.', 'Select a clip on the music track, describe the music you want and choose "Replace music".', 'Select a clip on the dialogue track and change the words of one line under "Lines".', 'Choose "Redub". The changed lines are spoken again.'],
+ },
+ limits: {
+ zh: ['部分电脑暂不支持分离音轨，此时按钮不会出现。'],
+ en: ['Some computers cannot separate tracks. The button does not appear on them.'],
+ },
+ },
+ {
+ id: 'replace-music',
+ goto: ['audio-a2-track'],
+ title: {zh: '加配乐', en: 'Add music'},
+ summary: {zh: '在声音轨上选一段，描述想要的音乐。', en: 'Select a range on a sound track and describe the music.'},
+ media: {
+ type: 'image',
+ src: {zh: 'replace-music-1.webp', en: 'replace-music-2.webp'},
+ alt: {zh: '剪辑视图：第二条声音轨上选中的一段和风格选项已用红框标出。', en: 'The Edit view, with a range on the second sound track and the style options outlined in red.'},
+ },
+ steps: {
+ zh: ['点“＋ 声音轨”，添加第二条声音轨“声音 2”。', '在“声音 2”轨的空白处拖出一段。', '在“风格”和“情绪”中各选一项。', '在“描述配乐”中写下想要的音乐。', '点“生成”。'],
+ en: ['Choose "+ Audio track" to add a second audio track, "Audio 2".', 'Drag a range on an empty part of the "Audio 2" track.', 'Choose one option under "Style" and one under "Mood".', 'Describe the music under "Describe the music".', 'Choose "Generate".'],
+ },
+ limits: {
+ zh: ['首次生成需要下载配乐模型。', '要换掉分离后的音乐，请选中“音乐”轨上的片段，按钮会变为“替换配乐”。'],
+ en: ['The first run downloads the music model.', 'To replace separated music, select a clip on the music track. The button then reads "Replace music".'],
+ },
+ },
+ {
+ id: 'voice-recording',
+ goto: ['record-entry'],
+ title: {zh: '配音与录音', en: 'Voice-over and recording'},
+ summary: {zh: '直接录音，或对已有的台词重新配音。', en: 'Record your own voice, or redub lines that already exist.'},
+ media: {
+ type: 'image',
+ src: {zh: 'voice-recording-1.webp', en: 'voice-recording-2.webp'},
+ alt: {zh: '剪辑视图：工具栏中的“录音”按钮已用红框标出。', en: 'The Edit view, with the "Record" button in the toolbar outlined in red.'},
+ },
+ steps: {
+ zh: ['点工具栏中的“录音”。', '点“开始录音”，对着麦克风说话。', '点“停止”结束，录音会放到声音轨上。', '已有的人声可逐句改写：选中“人声”轨上的片段，修改“台词”中的文字，点“重新配音”。'],
+ en: ['Choose "Record" in the toolbar.', 'Choose "Start recording" and speak into the microphone.', 'Choose "Stop". The recording is placed on an audio track.', 'Existing dialogue can be changed line by line: select a clip on the dialogue track, edit the words under "Lines" and choose "Redub".'],
+ },
+ limits: {
+ zh: ['录音需要允许使用麦克风。'],
+ en: ['Recording needs permission to use the microphone.'],
+ },
+ },
+ {
+ id: 'sound-to-picture',
+ goto: ['editor-timeline'],
+ title: {zh: '按声音生成画面', en: 'Generate a picture from sound'},
+ summary: {zh: '为只有声音的部分生成匹配的画面。', en: 'Generate a picture that matches a part that only has sound.'},
+ media: {
+ type: 'image',
+ src: {zh: 'sound-to-picture-1.webp', en: 'sound-to-picture-2.webp'},
+ alt: {zh: '剪辑视图：画面轨上选中的一段和描述框已用红框标出。', en: 'The Edit view, with a range on the picture track and the prompt box outlined in red.'},
+ },
+ steps: {
+ zh: ['在剪辑视图中放入一段只有声音的素材。', '在“画面”轨上，对着声音拖出一段。', '在“描述这一段”中写下想看到的画面。', '点“生成”。'],
+ en: ['Put a clip that only has sound into the Edit view.', 'On the picture track, drag a range along the sound.', 'Describe the picture you want under "Describe this part".', 'Choose "Generate".'],
+ },
+ limits: {
+ zh: ['画面时长以所选的一段为准。'],
+ en: ['The picture has the length of the selected range.'],
+ },
+ },
{
id: 'autopilot',
title: {zh: 'AI 全自动', en: 'Autopilot'},
diff --git a/web/updates.js b/web/updates.js
index c9e7081d..05873dc0 100644
--- a/web/updates.js
+++ b/web/updates.js
@@ -209,8 +209,11 @@ function showReleaseNotes(cn) {
const body = document.createElement('div'); body.className = 'fv-release-body';
const close = document.createElement('button'); close.type = 'button'; close.autofocus = true;
close.textContent = t('Close', '关闭'); close.onclick = () => dialog.close();
- dialog.append(title, body, close);
+ const news = document.createElement('button'); news.type = 'button'; news.textContent = t("What's new", '新功能');
+ news.onclick = () => { dialog.close(); import('./whatsnew.js').then(({showWhatsNew}) => showWhatsNew(cn)); };
+ dialog.append(title, body, news, close);
const render = () => {
+ news.hidden = !/^0\.5\./.test(productVersion() || '');
body.replaceChildren();
for (const [label, release] of [[t('Available update', '可用更新'), value?.available],
[t('Current version', '当前版本'), value?.current_release || (value?.current_version ? {version: value.current_version} : null)]]) {
diff --git a/web/whatsnew.css b/web/whatsnew.css
new file mode 100644
index 00000000..8c0083b7
--- /dev/null
+++ b/web/whatsnew.css
@@ -0,0 +1,35 @@
+.fv-whatsnew{box-sizing:border-box;width:min(720px,calc(100vw - 32px));max-height:calc(100dvh - 48px);padding:24px;border:1px solid var(--fv-border);border-radius:var(--fv-r-lg);background:var(--fv-canvas);color:var(--fv-ink);font:14px/1.6 var(--fv-font);overflow:auto}
+.fv-whatsnew::backdrop{background:#0009}
+.fv-whatsnew[open]{animation:fv-dialog-enter .28s cubic-bezier(.2,.7,.2,1)}
+.fv-whatsnew h2{margin:0 0 16px;font-size:20px;font-weight:600}
+.fv-whatsnew-host{border:1px solid var(--fv-border);border-radius:var(--fv-r-md);overflow:hidden;background:var(--fv-bg)}
+.fv-whatsnew-stage .fv-ed-track-view{height:236px}
+.fv-whatsnew-stage .fv-ed-track-scroll,.fv-whatsnew-stage .fv-ed-track-heads{bottom:0}
+.fv-whatsnew-stage .fv-ed-clip{cursor:default;position:absolute}
+.fv-whatsnew-picture{background:linear-gradient(90deg,color-mix(in srgb,var(--fv-accent) 28%,var(--fv-card)),color-mix(in srgb,var(--fv-accent-ramp) 22%,var(--fv-card)) 50%,color-mix(in srgb,var(--fv-accent) 18%,var(--fv-card)))}
+/* Rows and clips change by state only; each change fades. */
+.fv-whatsnew-stage .fv-ed-track-row,.fv-whatsnew-stage .fv-ed-track-head,.fv-whatsnew-stage .fv-ed-clip{transition:opacity .5s var(--fv-ease),translate .5s var(--fv-ease)}
+.fv-whatsnew-stage .fv-ed-progress-fill{transition:width .7s linear}
+.fv-whatsnew-stage[data-split="0"] :is([data-track=a2],[data-track=a3]){opacity:0;translate:0 -6px}
+.fv-whatsnew-stage[data-split="0"] [data-name=voice-old],.fv-whatsnew-stage[data-split="0"] [data-name=voice-new]{opacity:0}
+.fv-whatsnew-stage[data-split="1"] [data-name=mixed]{opacity:0}
+.fv-whatsnew-stage [data-name=voice-new],.fv-whatsnew-stage [data-name=music-new]{opacity:0}
+.fv-whatsnew-stage[data-voice=new] [data-name=voice-old],.fv-whatsnew-stage[data-music=new] [data-name=music-old]{opacity:0}
+.fv-whatsnew-stage[data-split="1"][data-voice=new] [data-name=voice-new],.fv-whatsnew-stage[data-music=new] [data-name=music-new]{opacity:1}
+.fv-whatsnew-stage[data-music=running] [data-name=music-old],.fv-whatsnew-stage[data-voice=running] [data-name=voice-old]{outline:2px solid var(--fv-accent);outline-offset:-2px}
+.fv-whatsnew{position:relative}
+.fv-whatsnew:focus{outline:none}
+.fv-whatsnew-close{position:absolute;top:12px;right:12px;width:var(--fv-h-sm);height:var(--fv-h-sm);padding:0;border:0;border-radius:var(--fv-r-sm);background:transparent;color:var(--fv-muted,var(--fv-ink));font:22px/1 var(--fv-font);cursor:pointer}
+.fv-whatsnew-close:hover{background:var(--fv-hover);color:var(--fv-ink)}
+.fv-whatsnew h2{padding-right:36px}
+.fv-whatsnew-stage .fv-ed-track-view .fv-ed-track-head{justify-content:flex-start;padding-left:12px;box-sizing:border-box;white-space:nowrap}
+.fv-whatsnew-foot{display:flex;align-items:center;gap:8px;margin-top:16px}
+.fv-whatsnew-dots{display:flex;gap:6px;margin-right:auto}
+.fv-whatsnew-dot{width:8px;height:8px;padding:0;border:0;border-radius:50%;background:var(--fv-border);cursor:pointer}
+.fv-whatsnew-dot[aria-current=true]{background:var(--fv-accent)}
+.fv-whatsnew-foot button:not(.fv-whatsnew-dot){min-height:var(--fv-h-sm);padding:4px 16px;border:0;border-radius:var(--fv-r-sm);font:inherit;cursor:pointer;background:var(--fv-hover);color:var(--fv-ink)}
+.fv-whatsnew-next{background:var(--fv-accent)!important;color:var(--fv-bg)!important;font-weight:600}
+.fv-whatsnew-foot[hidden],.fv-whatsnew-take[hidden]{display:none}
+.fv-whatsnew :focus-visible{outline:2px solid var(--fv-accent);outline-offset:2px}
+@media(prefers-reduced-motion:reduce){.fv-whatsnew[open]{animation:none}.fv-whatsnew-stage *{transition:none!important}}
+@media(max-width:520px){.fv-whatsnew{padding:16px}.fv-whatsnew-stage .fv-ed-track-view{--fv-ed-head-width:88px}.fv-whatsnew-stage .fv-ed-track-head>span:first-of-type{display:inline}.fv-whatsnew-stage .fv-ed-track-view .fv-ed-track-head{padding-left:8px;gap:4px;font-size:12px}.fv-whatsnew-stage .fv-ed-track-head svg{width:14px;height:14px}.fv-whatsnew-close{top:8px;right:8px}}
diff --git a/web/whatsnew.js b/web/whatsnew.js
new file mode 100644
index 00000000..1a47663f
--- /dev/null
+++ b/web/whatsnew.js
@@ -0,0 +1,175 @@
+// What's new in 0.5: the editor's own track view drawn with the editor's own classes and played as a short loop.
+// Nothing is recorded; the picture is a placeholder, the tracks, clips, progress and waveforms are the real styles.
+const KEY = 'freevideo.whatsnew-seen';
+const SERIES = '0.5';
+const PAGES = [
+ {id: 'story', title: ['Edit picture and sound, track by track', '画面和声音，逐轨编辑'], steps: [[0, {split: 0, music: 'old', voice: 'old'}], [1800, {split: 1}],
+ [3000, {music: 'running', progress: .3}], [4200, {progress: .8}], [5000, {music: 'new', progress: 0}],
+ [6600, {voice: 'running', progress: .5}], [7600, {progress: .9}], [8300, {voice: 'new', progress: 0}]], length: 11000},
+ {id: 'stems', title: ['Split the sound into three tracks', '把声音拆成三条轨道'], howto: 'separate-tracks',
+ steps: [[0, {split: 0, music: 'old', voice: 'old'}], [1400, {split: 1}]], length: 5200},
+ {id: 'music', title: ['Replace the music', '换一段配乐'], howto: 'replace-music',
+ steps: [[0, {split: 1, music: 'old', voice: 'old'}], [1200, {music: 'running', progress: .35}], [2000, {progress: .8}], [2700, {music: 'new', progress: 0}]], length: 5200},
+ {id: 'voice', title: ['Regenerate a line', '重新生成一句台词'], howto: 'replace-voice',
+ steps: [[0, {split: 1, music: 'old', voice: 'old'}], [1200, {voice: 'running', progress: .5}], [2200, {progress: .9}], [2800, {voice: 'new', progress: 0}]], length: 5200},
+];
+const el = (tag, text, cls) => { const e = document.createElement(tag); if (text != null) e.textContent = text; if (cls) e.className = cls; return e; };
+const svg = (tag, attrs = {}) => { const e = document.createElementNS('http://www.w3.org/2000/svg', tag); for (const k in attrs) e.setAttribute(k, attrs[k]); return e; };
+const ICON = {video: 'M2 4h16v12H2z M4 13l4-5 4 4 2-2 3 4', audio: 'M2 8h4l4-4v12l-4-4H2z M13 6c4 2 4 6 0 8 M13 9c1 1 1 1 0 2'};
+const icon = kind => { const s = svg('svg', {viewBox: '0 0 20 20', 'aria-hidden': 'true'}); s.append(svg('path', {d: ICON[kind]})); return s; };
+
+// A deterministic waveform in the editor's 200x40 box; shape decides how the sound reads.
+function wave(seed, shape) {
+ let x = seed * 9301 + 49297, d = '';
+ const random = () => (x = (x * 9301 + 49297) % 233280) / 233280;
+ for (let i = 0; i < 100; i++) {
+ const at = i / 100;
+ const level = shape === 'speech' ? (Math.sin(at * 23) > -.2 ? .25 + random() * .75 : .06) * (.5 + .5 * Math.sin(at * 7 + 1))
+ : shape === 'music' ? .35 + .4 * Math.sin(at * 9 + seed) ** 2 + random() * .15
+ : shape === 'ambience' ? .12 + random() * .12 : .3 + random() * .7;
+ const h = Math.max(.5, Math.min(1, level) * 17);
+ d += `M${(i + .5) * 2} ${20 - h}v${h * 2}`;
+ }
+ const s = svg('svg', {viewBox: '0 0 200 40', preserveAspectRatio: 'none', 'aria-hidden': 'true'});
+ s.classList.add('fv-ed-waveform'); s.append(svg('path', {d})); return s;
+}
+
+const ROWS = [['v1', 'video', 64], ['a1', 'audio', 40], ['a2', 'audio', 40], ['a3', 'audio', 40]];
+
+export function createStage(cn) {
+ const t = (en, zh) => cn ? zh : en;
+ const root = el('div', null, 'fv-whatsnew-stage'); root.setAttribute('aria-hidden', 'true');
+ const view = el('section', null, 'fv-ed-track-view'); view.dataset.trackSwitches = 'false';
+ const heads = el('div', null, 'fv-ed-track-heads'), scroll = el('div', null, 'fv-ed-track-scroll'), content = el('div', null, 'fv-ed-track-content');
+ const ruler = el('div', null, 'fv-ed-ruler');
+ for (let s = 0; s <= 10; s++) {
+ const tick = el('div', null, 'fv-ed-tick' + (s % 2 ? '' : ' fv-ed-tick-major')); tick.style.left = `${s * 10}%`;
+ if (s % 2 === 0) tick.append(el('span', `0:${String(s).padStart(2, '0')}`));
+ ruler.append(tick);
+ }
+ content.append(ruler);
+ let top = 24;
+ const rows = {}, heading = {};
+ for (const [id, kind, height] of ROWS) {
+ const row = el('div', null, 'fv-ed-track-row'); row.dataset.track = id; row.style.top = `${top}px`; row.style.height = `${height}px`;
+ const head = el('div', null, 'fv-ed-track-head'); head.dataset.track = id; head.style.top = `${top}px`; head.style.height = `${height}px`;
+ const name = el('span'); head.append(icon(kind), name); heads.append(head); content.append(row);
+ rows[id] = row; heading[id] = name; top += height + 6;
+ }
+ const clip = (row, name, node) => { const c = el('div', null, 'fv-ed-clip'); c.dataset.name = name; c.style.left = '0'; c.style.width = '100%';
+ const media = el('span', null, 'fv-ed-clip-media'); if (node) media.append(node); c.append(media);
+ const fill = el('span', null, 'fv-ed-progress-fill'); c.append(fill); row.append(c); return c; };
+ const video = clip(rows.v1, 'video'); video.querySelector('.fv-ed-clip-media').classList.add('fv-whatsnew-picture');
+ const mixed = clip(rows.a1, 'mixed', wave(3, 'mixed'));
+ const voice = [clip(rows.a1, 'voice-old', wave(5, 'speech')), clip(rows.a1, 'voice-new', wave(11, 'speech'))];
+ const music = [clip(rows.a2, 'music-old', wave(7, 'music')), clip(rows.a2, 'music-new', wave(13, 'music'))];
+ clip(rows.a3, 'ambience', wave(9, 'ambience'));
+ for (const c of [video, mixed, ...voice, ...music]) c.tabIndex = -1;
+ scroll.append(content); view.append(heads, scroll); root.append(view);
+ const names = () => {
+ const split = root.dataset.split === '1';
+ heading.v1.textContent = t('Video', '画面');
+ heading.a1.textContent = split ? t('Dialogue', '人声') : t('Audio', '声音');
+ heading.a2.textContent = t('Music', '音乐'); heading.a3.textContent = t('Ambience', '环境声');
+ };
+ const set = state => {
+ for (const [k, v] of Object.entries(state)) { if (k === 'progress') root.style.setProperty('--fv-ed-progress', String(v)); else root.dataset[k] = String(v); }
+ for (const [pair, key] of [[voice, 'voice'], [music, 'music']]) {
+ pair[0].dataset.state = root.dataset[key] === 'running' ? 'running' : 'ready';
+ if (root.dataset[key] === 'running') pair[0].style.setProperty('--fv-ed-progress', root.style.getPropertyValue('--fv-ed-progress') || '0');
+ }
+ names();
+ };
+ set({split: 0, music: 'old', voice: 'old', progress: 0});
+ return {element: root, set};
+}
+
+// Plays a page's steps in a loop; reduced motion shows the finished state once.
+export function play(stage, page, reduce) {
+ let timers = [], dead = false;
+ const run = () => {
+ if (dead) return;
+ for (const [at, state] of page.steps) timers.push(setTimeout(() => !dead && stage.set(state), at));
+ timers.push(setTimeout(run, page.length));
+ };
+ if (reduce) { for (const [, state] of page.steps) stage.set(state); return () => {}; }
+ run();
+ return () => { dead = true; timers.forEach(clearTimeout); };
+}
+
+const seenSeries = () => { try { return localStorage.getItem(KEY) === SERIES; } catch { return false; } };
+const remember = () => { try { localStorage.setItem(KEY, SERIES); } catch { /* Shown again next time. */ } };
+
+export function showWhatsNew(cn, {openedBy = 'user', start} = {}) {
+ const t = (en, zh) => cn ? zh : en;
+ const css = document.getElementById('fv-whatsnew-css') || Object.assign(document.createElement('link'), {id: 'fv-whatsnew-css', rel: 'stylesheet', href: new URL('./whatsnew.css', import.meta.url).href});
+ if (!css.isConnected) document.head.append(css);
+ for (const sheet of ['editor_tracks.css']) {
+ const id = 'fv-whatsnew-' + sheet;
+ if (!document.getElementById(id)) document.head.append(Object.assign(document.createElement('link'), {id, rel: 'stylesheet', href: new URL('./' + sheet, import.meta.url).href}));
+ }
+ const dialog = el('dialog', null, 'fv-whatsnew'); dialog.setAttribute('aria-label', t("What's new", '新功能'));
+ const title = el('h2'), close = el('button', '×', 'fv-whatsnew-close'), stageHost = el('div', null, 'fv-whatsnew-host'), foot = el('div', null, 'fv-whatsnew-foot');
+ const dots = el('div', null, 'fv-whatsnew-dots'), take = el('button', t('Take me there', '带我去'), 'fv-whatsnew-take'), next = el('button', null, 'fv-whatsnew-next');
+ take.type = next.type = close.type = 'button'; take.hidden = true; close.setAttribute('aria-label', t('Close', '关闭'));
+ foot.append(dots, take, next); dialog.append(close, title, stageHost, foot);
+ const reduce = matchMedia('(prefers-reduced-motion: reduce)').matches;
+ let index = 0, stop = () => {}, token = 0;
+ const dotButtons = PAGES.map((page, i) => { const b = el('button', null, 'fv-whatsnew-dot'); b.type = 'button'; b.setAttribute('aria-label', page.title[cn ? 1 : 0]); b.onclick = () => show(i); dots.append(b); return b; });
+ function show(i) {
+ stop(); index = i; const page = PAGES[i], mine = ++token;
+ title.textContent = page.title[cn ? 1 : 0];
+ const stage = createStage(cn); stageHost.replaceChildren(stage.element);
+ stop = play(stage, page, reduce);
+ dotButtons.forEach((b, n) => b.setAttribute('aria-current', String(n === i)));
+ next.textContent = i === PAGES.length - 1 ? t('Done', '完成') : t('Next', '下一页');
+ take.hidden = true;
+ // A page that has no guided tour yet shows no button.
+ if (page.howto) Promise.resolve(window.freevideoHowTo?.available?.(page.howto)).then(ok => { if (ok && token === mine) take.hidden = false; }).catch(() => {});
+ }
+ close.onclick = () => dialog.close();
+ next.onclick = () => index === PAGES.length - 1 ? dialog.close() : show(index + 1);
+ take.onclick = () => { const id = PAGES[index].howto; dialog.close(); void window.freevideoHowTo?.start?.(id); };
+ dialog.addEventListener('keydown', event => {
+ event.stopPropagation();
+ if (event.key === 'ArrowRight' && index < PAGES.length - 1) show(index + 1);
+ if (event.key === 'ArrowLeft' && index > 0) show(index - 1);
+ });
+ dialog.addEventListener('close', () => { stop(); remember(); dialog.remove(); }, {once: true});
+ document.body.append(dialog); dialog.showModal(); show(start ?? 0); dialog.tabIndex = -1; dialog.focus();
+ return () => { if (dialog.open) dialog.close(); };
+}
+
+// The first open after upgrading to 0.5: once, and not while something is being generated.
+export function maybeShowWhatsNew({cn, version, busy}) {
+ if (seenSeries() || !version || !version.startsWith(SERIES + '.')) return false;
+ if (busy() || document.querySelector('dialog[open]:not(.fv-studio)')) return false;
+ showWhatsNew(cn, {openedBy: 'upgrade'});
+ return true;
+}
+
+// A page that holds settings from an earlier version is an upgrade; a first launch has none.
+const usedBefore = () => {
+ try {
+ for (let index = 0; index < localStorage.length; index++) {
+ const key = localStorage.key(index) || '';
+ if (key !== KEY && key !== 'freevideo.page-retry' && (key.startsWith('freevideo.') || key.startsWith('fv-'))) return true;
+ }
+ } catch { /* Unreadable storage: treat as an upgrade rather than hide the news. */ return true; }
+ return false;
+};
+
+// Waits for the version answer and for an idle server, then shows the page once.
+export function startWhatsNew({api, cn, productVersion}) {
+ if (seenSeries()) return;
+ if (!usedBefore()) { remember(); return; }
+ let queued = 0, done = false;
+ const status = event => { queued = event?.detail?.exec_info?.queue_remaining || 0; };
+ try { api.addEventListener('status', status); } catch { /* A page without the event still waits for dialogs. */ }
+ const timer = setInterval(() => {
+ if (done || seenSeries()) { done = true; clearInterval(timer); return; }
+ if (maybeShowWhatsNew({cn, version: productVersion(), busy: () => queued > 0})) { done = true; clearInterval(timer); }
+ }, 5000);
+}
+
+export const hasWhatsNew = version => !!version && version.startsWith(SERIES + '.');