[MinerU Ops] integrate 3 mineru operators for clarity (#473)

* integrate 3 mineru operators

* lazy load flash_mineru

* [op] update mineru OPs: update get_desc; update param for __init__; fit flash-mineru v0.0.2

* [statics] revise name of mineru OPs; update all script to fit mineru update.

---------

Co-authored-by: Sunnyhaze <mxch1122@126.com>

Author: Lavender1

dataflow/cli_funcs/pdf2model_pipeline/pdf_to_qa_pipeline.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 from dataflow.operators.knowledge_cleaning import (
     KBCChunkGeneratorBatch,
-    FileOrURLToMarkdownConverterBatch,
+    FileOrURLToMarkdownConverterFlash,
     KBCTextCleanerBatch,
     KBCMultiHopQAGeneratorBatch,
     QAExtractor
@@ -29,7 +29,7 @@ def __init__(self, cache_base="./"):
             cache_type="json",
         )
 
-        self.knowledge_cleaning_step1 = FileOrURLToMarkdownConverterBatch(
+        self.knowledge_cleaning_step1 = FileOrURLToMarkdownConverterFlash(
             intermediate_dir=str(cache_path / ".cache"),
             mineru_backend="vlm-vllm-engine",  # 可选 pipeline, vlm-vllm-engine, vlm-vllm-transformer, vlm-http-client
         )

dataflow/operators/knowledge_cleaning/__init__.py

@@ -5,8 +5,9 @@
     from .generate.kbc_chunk_generator import KBCChunkGenerator
     from .generate.kbc_chunk_generator_batch import KBCChunkGeneratorBatch
     # from .generate.file_or_url_to_markdown_converter import FileOrURLToMarkdownConverter
-    from .generate.file_or_url_to_markdown_converter_batch import FileOrURLToMarkdownConverterBatch
-    from .generate.file_or_url_to_markdown_converter_api import FileOrURLToMarkdownConverterAPI
+    from .generate.mineru_operators import FileOrURLToMarkdownConverterLocal
+    from .generate.mineru_operators import FileOrURLToMarkdownConverterAPI
+    from .generate.mineru_operators import FileOrURLToMarkdownConverterFlash
     from .generate.kbc_text_cleaner import KBCTextCleaner
     from .generate.kbc_text_cleaner_batch import KBCTextCleanerBatch
     from .generate.mathbook_question_extract import MathBookQuestionExtract