[pdf2vqa] minor fix

Author: fatty-belly

dataflow/operators/pdf2vqa/generate/llm_output_parser.py

@@ -67,7 +67,7 @@ def _id_to_text(self, input_ids, input_json, image_prefix="images"):
 
     def _convert_response(self, input_response, input_json_path, image_prefix="images"):
         qa_list = []
-        with open(input_json_path, 'r') as infile:
+        with open(input_json_path, 'r', encoding='utf-8') as infile:
             input_json = list(json.load(infile))
         # 提取title
         for chapter_block in re.findall(r'<chapter>(.*?)</chapter>', input_response, flags=re.DOTALL):
@@ -116,18 +116,18 @@ def run(self, storage: DataFlowStorage,
             qa_list = self._convert_response(response, converted_json_path, image_prefix)
             output_qalist_path = os.path.join(self.output_dir, name, f"extracted_vqa.jsonl")
             os.makedirs(os.path.dirname(output_qalist_path), exist_ok=True)
-            with open(output_qalist_path, 'w') as outfile:
+            with open(output_qalist_path, 'w', encoding='utf-8') as outfile:
                 for qa in qa_list:
                     json.dump(qa, outfile, ensure_ascii=False)
                     outfile.write('\n')
 
             # 复制图片
-            src_dir = converted_json_path.rpartition('/')[0]
+            src_dir = os.path.dirname(converted_json_path)
             src_images = os.path.join(src_dir, 'vlm', 'images')
             if not os.path.exists(src_images):
                 src_images = os.path.join(src_dir, 'images')
             if not os.path.exists(src_images):
-                raise ValueError("Images directory not found! There might be a change in Mineru API!")
+                raise ValueError(f"Images directory {src_images} not found! There might be a change in Mineru API!")
             dst_images = os.path.join(self.output_dir, image_prefix)
 
             try:

dataflow/operators/pdf2vqa/generate/mineru_to_llm_input_operator.py

@@ -26,7 +26,7 @@ def get_desc(lang: str = "zh") -> str:
             )
 
     def _convert_json(self, input_file, output_file):
-        with open(input_file, 'r') as infile:
+        with open(input_file, 'r', encoding="utf-8") as infile:
             data = list(json.load(infile))
 
         new_data = []
@@ -49,7 +49,7 @@ def _convert_json(self, input_file, output_file):
                 new_data.append(item)
                 id += 1
 
-        with open(output_file, 'w') as outfile:
+        with open(output_file, 'w', encoding='utf-8') as outfile:
             json.dump(new_data, outfile, ensure_ascii=False)
 
     def run(self, storage: DataFlowStorage,
@@ -69,7 +69,7 @@ def run(self, storage: DataFlowStorage,
             self._convert_json(input_json_path, converted_path)
             dataframe.at[index, output_converted_layout_key] = converted_path
 
-            with open(converted_path, 'r') as infile:
+            with open(converted_path, 'r', encoding='utf-8') as infile:
                 data = json.load(infile)
                 assert isinstance(data, list), f"Expected list, got {type(data)} for {input_json_path}"
 

dataflow/statics/pipelines/api_pipelines/pdf_vqa_extract_pipeline.py

@@ -1,4 +1,4 @@
-from dataflow.operators.knowledge_cleaning import FileOrURLToMarkdownConverterFlash, FileOrURLToMarkdownConverterLocal, FileOrURLToMarkdownConverterAPI
+from dataflow.operators.knowledge_cleaning import FileOrURLToMarkdownConverterAPI
 
 from dataflow.serving import APILLMServing_request
 from dataflow.utils.storage import FileStorage