Corpus

@app.tool(output="parse_file_path,text_corpus_save_path->None")
async def build_text_corpus(parse_file_path: str, text_corpus_save_path: str) -> None

{"id": "<stem>", "title": "<stem>", "contents": "<全文文本>"}

@app.tool(output="parse_file_path,image_corpus_save_path->None")
async def build_image_corpus(parse_file_path: str, image_corpus_save_path: str) -> None

{"id": 0, "image_id": "paper/page_0.jpg", "image_path": "image/paper/page_0.jpg"}

@app.tool(output="parse_file_path,mineru_dir,mineru_extra_params->None")
async def mineru_parse(parse_file_path: str, mineru_dir: str, mineru_extra_params: Optional[Dict[str, Any]] = None) -> None

@app.tool(output="raw_chunk_path,chunk_backend_configs,chunk_backend,chunk_path,use_title->None")
async def build_mineru_corpus(mineru_dir: str, parse_file_path: str, text_corpus_save_path: str, image_corpus_save_path: str) -> None

{"id": "<stem>", "title": "<stem>", "contents": "<markdown全文>"}

{"id": 0, "image_id": "paper/page_0.jpg", "image_path": "images/paper/page_0.jpg"}

@app.tool(output="raw_chunk_path,chunk_backend_configs,chunk_backend,chunk_path,use_title->None")
async def chunk_documents(
  raw_chunk_path: str,
  chunk_backend_configs: Dict[str, Any],
  chunk_backend: str = "token",
  chunk_path: Optional[str] = None,
  use_title: bool = True,
) -> None

{"id": 0, "doc_id": "paper", "title": "paper", "contents": "切块后的文本"}