Skip to content

Commit b5cd28b

Browse files
committed
fix(merger): copy processor_config.json for custom processors
AutoProcessor.from_pretrained falls back to the base tokenizer for custom processors (e.g. AeroRealtimeProcessor), so save_pretrained does not emit processor_config.json. Copy it from the source checkpoint if present.
1 parent d1ac27a commit b5cd28b

1 file changed

Lines changed: 8 additions & 0 deletions

File tree

src/lmms_engine/merger/fsdp2.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
"""FSDP2 checkpoint merger implementation."""
22

33
import os
4+
import shutil
45
from concurrent.futures import ThreadPoolExecutor
56
from pathlib import Path
67
from typing import Literal
@@ -199,4 +200,11 @@ def merge(
199200
# Save merged checkpoint
200201
model.save_pretrained(output_path)
201202

203+
# Copy over any extra config files that AutoProcessor may not handle
204+
# (e.g. processor_config.json for custom processors)
205+
for extra_file in ["processor_config.json"]:
206+
src = checkpoint_path / extra_file
207+
if src.exists():
208+
shutil.copy2(src, output_path / extra_file)
209+
202210
return output_path

0 commit comments

Comments
 (0)