FreeVideo 0.4.34

MANIFEST.in

@@ -74,5 +74,25 @@ include web/assets/tour/c12-s1.mp4
 include web/assets/tour/c12-s1.webp
 include web/assets/tour/c13-s1.mp4
 include web/assets/tour/c13-s1.webp
+include web/assets/tour/cb-ambience.mp4
+include web/assets/tour/cb-ambience.webp
+include web/assets/tour/cb-dialogue.mp4
+include web/assets/tour/cb-dialogue.webp
+include web/assets/tour/cb-music.mp4
+include web/assets/tour/cb-music.webp
+include web/assets/tour/cb-source.mp4
+include web/assets/tour/cb-source.webp
+include web/assets/tour/ca-music.mp4
+include web/assets/tour/ca-music.webp
+include web/assets/tour/ca-music-wave.webp
+include web/assets/tour/ca-dub.mp4
+include web/assets/tour/ca-dub.webp
+include web/assets/tour/ca-dub-wave.webp
+include web/assets/tour/ca-record.mp4
+include web/assets/tour/ca-record.webp
+include web/assets/tour/ca-record-wave.webp
+include web/assets/tour/cc-speech.mp4
+include web/assets/tour/cc-speech.webp
+include web/assets/tour/cc-speech-wave.webp
 include web/assets/tour/c10-source.webp
 include web/assets/tour/fox-sheet.webp

freevideo_engine/agent/errors.py

@@ -82,6 +82,7 @@
 # Preserve that runtime contract while the pure production code keeps its 409.
 PRODUCTION_STATUS = {'not_ready': 409}
 ERRORS.update({
+    'capabilities_computing': (503, True, 'This computer is still working out what it can generate.', 'Wait retry_after_seconds, then retry the request.'),
     'unknown_shot': (404, False, 'The shot does not exist.', 'Use a shot id from the production view.'),
     'unknown_take': (404, False, 'The take does not exist.', 'Use a take number from this shot in the production view.'),
     'take_not_ready': (409, False, 'The take is stale or has not succeeded.', 'Choose a succeeded, non-stale take, or reshoot this shot.'),
@@ -114,6 +115,8 @@
     'export_failed': (409, True, 'The export did not finish.', 'Read the project, then submit the export again with a new ref.'),
     'no_speech_model': (409, False, 'No speech model is installed on this machine.',
                         'Do not retry. Write the subtitle lines yourself with the subtitles edit, or tell the user the model is not installed.'),
+    'speech_language': (409, False, 'The speech model does not cover the language of this sound.',
+                        'Do not retry; tell the user the language is not supported, or write the lines yourself.'),
     'listen_failed': (409, True, 'Listening to the sound did not finish.',
                       'Read the project, then submit the listening job again with a new ref; if it fails again, write the lines yourself.'),
     'music_download_failed': (409, True, 'The music model could not be downloaded.',
@@ -125,6 +128,8 @@
                               'Tell the user the model needs a network connection, then submit the job again with a new ref.'),
     'stems_failed': (409, True, 'Splitting the sound did not finish.',
                      'Read the project, then submit the job again with a new ref; if it fails again, leave the sound as it is.'),
+    'sound_failed': (409, True, 'The sound of the cut was not finished.',
+                     'Read the project, then submit the job again with a new ref; if it fails again, leave the sound as it is.'),
     'voice_failed': (409, True, 'Changing the voice did not finish.',
                      'Read the project, then submit the job again with a new ref; if it fails again, leave the sound as it is.'),
     'voice_model_missing': (409, False, 'The dubbing model is not ready on this machine.',
@@ -247,7 +252,7 @@ def recovery(code, *, kind=None, stage='request', resumable=True):
         return 'Tell the user to free disk space, then submit the picture again with a new ref.'
     if kind == 'music' and code == 'out_of_memory':
         return 'Close other GPU applications, then submit the music job again with a new ref.'
-    if kind in ('stems', 'voice', 'voice_lines') and code == 'out_of_memory':
+    if kind in ('stems', 'sound', 'finish_sound', 'voice', 'voice_lines') and code == 'out_of_memory':
         return 'Close other GPU applications, then submit the job again with a new ref.'
     if kind == 'image' and code == 'no_memory':
         return 'Lower the picture dimensions or use fewer reference pictures, then submit with a new ref.'
@@ -290,6 +295,7 @@ def safe_details(value):
 FAILURE_REASONS_ZH = {
     **LORA_MESSAGE_ZH,
     'lan_unavailable': '局域网设置需要启动器安装记录',
+    'capabilities_computing': '正在计算这台电脑的生成能力，稍后再试',
     'out_of_memory': '生成时显存或内存不足',
     'insufficient_memory': '可用显存或内存不够生成这段内容',
     'no_memory': '可用显存或内存不够生成这张图片',
@@ -341,6 +347,7 @@ def safe_details(value):
 FAILURE_REASONS_EN = {
     **LORA_MESSAGE_EN,
     'lan_unavailable': 'Phone access requires FreeVideo installed with the launcher',
+    'capabilities_computing': 'Working out what this computer can generate; try again shortly',
     'out_of_memory': 'Graphics or system memory ran out during generation',
     'insufficient_memory': 'There is not enough free graphics or system memory for this generation',
     'no_memory': 'There is not enough free graphics or system memory for this picture',
@@ -480,6 +487,8 @@ def build(code, *, details=None, request=None, validation=False, message=None, n
     result = capture_other_usage(result)
     if action_required:
         result['next'] = next_action if validation and next_action else action
+    if code == 'capabilities_computing':
+        result['retry_after_seconds'] = retry_after_seconds or 1
     if code == 'busy':
         result['retry_after_seconds'] = retry_after_seconds or 1
         result['details'].append(dict(owner=owner if owner in ('agent', 'ui') else 'ui'))

freevideo_engine/agent/guide.py

@@ -39,7 +39,7 @@
         'pedestal, arc shot, tracking shot, static shot, zoom, POV, roll; "with small amplitude", "at slow speed".',
     ],
     'dialogue': [
-        'Speakers get stable IDs (S1), (S2) across shots; describe each voice when it first appears.',
+        'Speakers are numbered (S1), (S2) once, in the order they first speak in the video, and keep that number throughout it; describe each voice when it first appears.',
         'Spoken words go inside <d>[Language] ...</d>, verbatim: The woman (S1) says: <d>[Chinese] 我们走吧。</d>',
         'Voiceover: "says in an off-screen voiceover: <d>...</d> while his lips remain completely closed."',
         'Visible text goes in double quotes: a neon sign reading "营业中". Never put spoken words in quotes: '
@@ -74,6 +74,7 @@
         "User message attachments are reference images by default. Use each name directly in media.references, media.first or media.last according to the user's words; if the intended role is unclear, ask one question. Text can be empty when a message has attachments.",
         'Tags are numbered per kind in reference order: <Picture 1>, <Picture 2>, <Video 1>, <Audio 1>. '
         'For keyframes the first frame is <Picture 1> and the last frame the next picture.',
+        'A reference video marked voice_only is an audio reference: its sound is <Audio N> and it has no <Video N> tag.',
         'Name subjects <Subject 1>, <Subject 2> in subject_definitions and reuse those labels in every section.',
         'Clips are cut to the generated length (at most 15 s); the job result lists any cut.',
     ],
@@ -122,7 +123,8 @@
     ],
     'agent_jobs': [
         'GET /capabilities reads the studio cache with the same size/task/material query parameters. '
-        'status=computing includes retry_after_seconds; unknown plan estimates have seconds=null.',
+        'status=computing includes retry_after_seconds; unknown plan estimates have seconds=null. '
+        'A request answered with capabilities_computing waits retry_after_seconds and repeats unchanged.',
         'GET /jobs/{id}?wait=25 waits at most 25 seconds and returns poll_after_ms. Like POST /jobs, the answer '
         'has the job under job (its fields are also at the top level). Poll the same ID until job.status is '
         'succeeded, failed, cancelled or interrupted; disconnecting a wait or SSE does not cancel generation.',
@@ -145,7 +147,8 @@
         'POST /projects/{id}/generate reserves pending parts for clip_ids or a start/frames range and fills them as the returned job finishes. POST /projects/{id}/export renders the revision you read; wait for its job.',
         'Music: POST /projects/{id}/music with the revision you read and a caption; wait for the job, then read the project: result.clip_ids are the new clips on a2. Run it after the cut is joined; a second run needs the earlier score removed from a2.',
         'Split sound: POST /projects/{id}/stems with the revision you read and the clip_id of a sound clip; wait for the job. The clip\'s sound is split into dialogue, music and ambience on tracks a4, a5 and a6 over the clip\'s range, and the clip itself goes silent. result.parts lists each track with its name, clip_id, start, end and frames. Offered only where features lists projects.stems; a clip is split once, and deleting the three parts gives its sound back. To save the three tracks and the original mix as four wav files, POST /projects/{id}/stems/export with the revision and the split clip\'s clip_id; the answer lists folder and files at once.',
-        'Change a voice: POST /projects/{id}/voice/lines with the revision you read and a selection (clip_id, and start and end in seconds if only part) to write down the lines; read result.lines. Change the text of the lines to redo and POST /projects/{id}/voice/replace with the same selection and all lines (original_text unchanged); only the changed lines are spoken again, in the voice of the original, and the rest keep their sound. result.lines gives where each new line starts and ends; a line that cannot fit before the next one keeps its original voice with replaced false and warning too_long. POST /projects/{id}/voice/estimate with the lines answers at once with the seconds the changed lines take. Offered only where features lists projects.voice.',
+        'Finish the sound: after the cut is joined, POST /projects/{id}/finish_sound with the revision you read; wait for the job. In one job it splits each picture clip\'s sound where separation is offered and removes the separated music where it is a sustained score (shots, explosions and crowds filed there stay, at the ambience level), lays one score on a2 for the whole cut (music.caption composes it, music.source lays a file; music already on a2 stays), evens the dialogue, sets the music 16 LU under it, brings the cut to -18 LUFS with a -1 dBTP peak ceiling and fills digital silence with room tone. result gives loudness_lufs, true_peak_db, music_removed (per block: removed or kept, with the levels) and the steps in skipped; skipped.music is too_long for a cut over 600 s, and the rest still runs; then tell the user: The cut is over 10 minutes, so no music was composed. The other steps are done. To add music, run this again with a music file. Run again, only the clips split this time are judged and levelled.',
+        'Change a voice: POST /projects/{id}/voice/lines with the revision you read and a selection (clip_id, and start and end in seconds if only part) to write down the lines; read result.lines. Change the text of the lines to redo and POST /projects/{id}/voice/replace with the same selection and all lines (original_text unchanged); only the changed lines are spoken again, in the voice of the original, and the rest keep their sound. result.lines gives where each new line starts and ends; a line that cannot fit before the next one keeps its original voice with replaced false and warning too_long. POST /projects/{id}/voice/estimate with the lines answers at once with the seconds the changed lines take. Offered only where features lists projects.voice. Where features lists projects.voice.install instead, the dubbing model is not installed yet: the person downloads it (about 4.9 GB) from the voice change bar in the editor, and no agent route does it.',
         'Subtitles: POST /projects/{id}/listen with mode transcribe (writes down what is said) or align (times the lines you give) and the revision you read; wait for the job and read result.lines, each with start and end frames. Then add them with a subtitles operation of cues {start, frames, text}; replace true swaps the whole track. With no speech model the job fails with no_speech_model: write the lines yourself.',
         'The user sees one line under each picture clip: its note, or else the first line of its description. A description is generation text, often long and in English, so after you add, generate or change a clip, write a note (operation note) in the user\'s language saying what is in the picture: the subject first, then the setting, for example "Train through morning mist" (in Chinese "小火车穿过晨雾"). A short clip shows only the first few characters, about four in Chinese, and those must already make sense on their own. Do not write where the clip sits (opening, middle, ending, part 2): the order of the clips shows that. Do not write how far along it is: the clip already shows its progress. Keep it true when you change the clip again.',
         'Reuse ref or Idempotency-Key only for the identical request. Source video/audio/thumbnail URLs require the same Agent token.',
@@ -381,6 +384,8 @@
         '剪辑拼好后先读取 revision，并删去该轨道上原有的配乐；首次使用会下载音乐模型（约 20 GB）',
     ('POST', '/projects/{id}/stems'): '把一个声音片段的声音分成人声、音乐和环境声，放在其下方的 a4、a5、a6 三条轨道上，原片段静音。'
         '先读取 revision 与片段 ID，等待返回的任务完成，结果 parts 给出各轨道的名称、起止帧；仅在 features 含 projects.stems 时可用',
+    ('POST', '/projects/{id}/finish_sound'): '剪辑拼好后，用一个任务完成整条剪辑的声音：分离人声、音乐与环境声，在 a2 铺一段贯穿全片的配乐，'
+        '统一人声响度，整片调至 -18 LUFS，并以环境底噪填补无声处。先读取 revision，等待返回的任务完成',
     ('POST', '/projects/{id}/voice/lines'): '把选中声音片段里说的话逐句写下来，每句带起止秒数；先读取 revision 与片段 ID，等待返回的任务完成，结果 lines 给出各句；仅在 features 含 projects.voice 时可用',
     ('POST', '/projects/{id}/voice/replace'): '把改过文字的句子用原说话人的声音重新说一遍，放回原句的位置，没改的句子保持原声；提交 voice/lines 给出的全部句子，只改 text；结果 lines 给出每句新声的起止，放不进下一句之前的句子保留原声',
     ('POST', '/projects/{id}/voice/shorten'): '按目标秒数改短一句的文字；只返回文字，不保存也不配音',
@@ -626,7 +631,7 @@ def connection_text(value):
     'continue the previous one carry the sheet and the face for a single person, else one picture each (the face, '
     'else the front) for at most 3 people, and keep the voices of the previous shot; POST /drama/plan shows each shot\'s cast. '
     'A shot\'s own references replace the card pictures; references=[] sends none. '
-    'A shot with a carded character is at most 15 s; continue it in the next shot to go longer.',
+    'A shot with a carded character is at most one generation long on this machine (freevideo_capabilities reports it; 15 s when unknown); continue it in the next shot to go longer.',
     'POST /cast/{id}/generate makes reference pictures from the description as a normal job (face=true adds a close-up). '
     'It defaults to the installed image model, otherwise video frames (labels guessed, label_guess). '
     'When generation ends the card gets the pictures and a new revision; a failure '

freevideo_engine/agent/jobs.py

@@ -32,7 +32,7 @@
 ACTIVE = ('waiting', 'queued', 'running', 'cancelling')
 CAST_AGAIN = 'POST /freevideo/agent/v1/cast/%s/generate to make the reference pictures again.'
 DONE = ('succeeded', 'failed', 'cancelled', 'interrupted')
-GPU_KINDS = ('vlm', 'enhance', 'image', 'music', 'stems', 'voice')
+GPU_KINDS = ('vlm', 'enhance', 'image', 'music', 'stems', 'finish_sound', 'voice')
 IMAGE_ERRORS = errors.IMAGE_ERRORS
 IMAGE_WAIT_ERRORS = ('busy', 'busy_video')
 IMAGE_WAIT_NOTICES = {
@@ -1488,6 +1488,9 @@ def run():
             elif row['kind'] == 'stems':
                 from .stems import run_stems
                 result = run_stems(self, row)
+            elif row['kind'] == 'finish_sound':
+                from .sound import run_finish_sound
+                result = run_finish_sound(self, row)
             elif row['kind'] == 'voice_lines':
                 from .voice import run_voice_lines
                 result = run_voice_lines(self, row)

freevideo_engine/agent/music.py

@@ -49,10 +49,10 @@ def phase(name, fraction=None):
                 generate_seconds=numbers.get('generate_s'), load_seconds=numbers.get('load_s'))
 
 
-def _compose(jobs, row, service, out, frames, seed, quiet, phase, poll):
+def _compose(jobs, row, service, out, frames, seed, quiet, phase, poll, *, caption=None):
     for attempt in range(PREEMPTIONS + 1):
         try:
-            return service.compose(out, row['spec']['caption'], frames, seed=seed, quiet=quiet, fps=comfy_editor.FPS,
+            return service.compose(out, row['spec']['caption'] if caption is None else caption, frames, seed=seed, quiet=quiet, fps=comfy_editor.FPS,
                                    server=getattr(jobs.port, 'server', None), cancelled=lambda: bool(row.get('_cancel')),
                                    phase=phase)
         except service.Preempted: