Corpus

@app.tool(output="parse_file_path,text_corpus_save_path->None")
async def build_text_corpus(parse_file_path: str, text_corpus_save_path: str) -> None

{"id": "<stem>", "title": "<stem>", "contents": "<全文文本>"}

@app.tool(output="parse_file_path,image_corpus_save_path->None")
async def build_image_corpus(parse_file_path: str, image_corpus_save_path: str) -> None

{"id": 0, "image_id": "paper/page_0.jpg", "image_path": "image/paper/page_0.jpg"}

@app.tool(output="parse_file_path,mineru_dir,mineru_extra_params->None")
async def mineru_parse(
    parse_file_path: str, 
    mineru_dir: str, 
    mineru_extra_params: Optional[Dict[str, Any]] = None
) -> None

@app.tool(output="mineru_dir,parse_file_path,text_corpus_save_path,image_corpus_save_path->None")
async def build_mineru_corpus(
    mineru_dir: str, 
    parse_file_path: str, 
    text_corpus_save_path: str, 
    image_corpus_save_path: str
) -> None

{"id": "<stem>", "title": "<stem>", "contents": "<markdown全文>"}

{"id": 0, "image_id": "paper/page_0.jpg", "image_path": "images/paper/page_0.jpg"}

@app.tool(output="raw_chunk_path,chunk_backend_configs,chunk_backend,tokenizer_or_token_counter,chunk_size,chunk_path,use_title->None")
async def chunk_documents(
    raw_chunk_path: str,
    chunk_backend_configs: Dict[str, Any],
    chunk_backend: str = "token",
    tokenizer_or_token_counter: str = "character",
    chunk_size: int = 256,
    chunk_path: Optional[str] = None,
    use_title: bool = True,
) -> None

{"id": 0, "doc_id": "paper", "title": "paper", "contents": "切块后的文本"}