diff --git a/CLAUDE.md b/CLAUDE.md index 69afb8b..ca1e643 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -64,7 +64,7 @@ The canonical bib is `$KB/references.bib` — inside the KB, beside `INDEX.md`/` `how-to-download-ref` owns `INDEX.md`, `references.bib` (via append), `.raw/`, `.figures/`, and the rendered `_.md` files. `survey` / `know-me-better` / humans own `NOTES.md`. -Acquisition checks DOI/arXiv identity across namespaces, falls back from Semantic Scholar to Crossref for DOI metadata, and optionally uses Unpaywall for PDFs when `SCIBRAIN_CONTACT_EMAIL` is set. Rendered frontmatter `note`, `tags`, and `rating` are human-owned and survive re-renders. `how-to-download-ref/helpers/kb_sync.py` restores clone-local caches without changing tracked text; `kb_doctor.py` checks KB consistency offline and can repair only INDEX.md with `--fix`. Full-text values include `jats`, `latex`, `yes`, and `no`. +Acquisition checks DOI/arXiv identity across namespaces, falls back from Semantic Scholar to Crossref for DOI metadata, and optionally uses Unpaywall for PDFs when `SCIBRAIN_CONTACT_EMAIL` is set. Rendered frontmatter `note`, `tags`, and `rating` are human-owned and survive re-renders. `how-to-download-ref/helpers/kb_sync.py` restores clone-local caches without changing tracked text; `kb_doctor.py` checks KB consistency offline and can repair only INDEX.md with `--fix`. Full-text values include `jats`, `latex`, `mineru`, `yes`, and `no`. Each SKILL.md defines installed-resource resolution. Run helpers by their absolute installed paths while keeping the working directory at the user's project. Shared writing resources live inside the installable `how-to-write-ideas-report/references/` directory (`writing-workflow.md`: context loading, citations, output mechanics); the shared prose rules are the `how-to-technical-writing` skill itself. diff --git a/skills/how-to-download-ref/SKILL.md b/skills/how-to-download-ref/SKILL.md index ac0172b..8af5edb 100644 --- a/skills/how-to-download-ref/SKILL.md +++ b/skills/how-to-download-ref/SKILL.md @@ -174,13 +174,30 @@ python3 "$DOWNLOAD_REF_DIR/helpers/render.py" --kb "$KB" --tex-source No manifest needed — renderer auto-discovers `.raw/{arxiv,doi}/*.json`. Renders new entries; overwrites existing. -**Body priority: JATS > LaTeX > PDF.** A `.jats.xml` in `.raw/doi/` always wins +For math-heavy or two-column papers without JATS or LaTeX, MinerU gives a much +cleaner PDF body: inline and display math as LaTeX, tables as HTML, figures as +local images. Parse first (remote service, no local GPU; about a minute per +batch), then render with `--mineru`: + +```sh +python3 "$DOWNLOAD_REF_DIR/helpers/mineru_parse.py" --kb "$KB" +python3 "$DOWNLOAD_REF_DIR/helpers/render.py" --kb "$KB" --mineru +``` + +`mineru_parse.py` caches `.raw/*/.mineru.md` and skips PDFs already parsed +(`--force` redoes, `--only ` narrows). It sends PDFs to a third-party +service: arXiv papers as public arxiv.org URLs, DOI PDFs as uploads, so confirm +with the user before uploading paywalled DOI PDFs. Setup and options: +[dependencies.md](references/dependencies.md#mineru). + +**Body priority: JATS > LaTeX > MinerU > PDF.** A `.jats.xml` in `.raw/doi/` always wins (`full_text: jats`, plus a `## References` section from the publisher's list). `--tex-source` is the only switch that prefers a flattened `.tex` -(`full_text: latex`); without it every ref renders from its PDF, even when a -`.tex` sits in `.raw/`. Human frontmatter `note`, `tags`, and `rating` survives -re-rendering; generated bodies refresh from source, so keep prose notes in -NOTES.md. PDF backends: [dependencies.md](references/dependencies.md). +(`full_text: latex`), and `--mineru` the only one that prefers a cached +`.mineru.md` (`full_text: mineru`); without them every ref renders from its PDF, +even when a `.tex` or `.mineru.md` sits in `.raw/`. Human frontmatter `note`, +`tags`, and `rating` survives re-rendering; generated bodies refresh from +source, so keep prose notes in NOTES.md. PDF backends: [dependencies.md](references/dependencies.md). `.raw/` and `.figures/` should stay out of git. Append to `.gitignore` if missing. @@ -258,8 +275,8 @@ from source. Keep other prose notes in NOTES.md. ## Completion and handoff -Return cite keys, rendered paths, full-text status (`jats`, `latex`, `yes`, or -`no`), and remaining findings. Remove the temporary manifest. If invoked by +Return cite keys, rendered paths, full-text status (`jats`, `latex`, `mineru`, +`yes`, or `no`), and remaining findings. Remove the temporary manifest. If invoked by `survey`, `know-me-better`, `create-advisor`, or `brainstorm-ideas`, return to that workflow with its scope and authorization intact. Continue to a survey report only when that report is already requested; standalone acquisition ends here. @@ -275,5 +292,5 @@ For unexpected helper output, consult - [ ] One `_.md` per distinct paper at `$KB/` root, with frontmatter - [ ] `$KB/INDEX.md` regenerated, lists each new entry - [ ] `$KB/references.bib` has the new cite key (no duplicate) -- [ ] User told cite keys, file names, and `full_text` jats/latex/yes/no per ref +- [ ] User told cite keys, file names, and `full_text` jats/latex/mineru/yes/no per ref - [ ] If the user requested LaTeX sources: `.raw/arxiv/.tex` exists for every arXiv id, and `.raw/doi/.tex` for every DOI with an arXiv preprint (or the `src-miss` reported) diff --git a/skills/how-to-download-ref/helpers/index.py b/skills/how-to-download-ref/helpers/index.py index c1870d9..6d15d94 100755 --- a/skills/how-to-download-ref/helpers/index.py +++ b/skills/how-to-download-ref/helpers/index.py @@ -114,7 +114,7 @@ def main(argv=None) -> int: if len(au) > 60: au = au.split(",")[0] + " et al." ven = (r["venue"] or "").replace("|", "\\|") - ft = "✅" if r["full_text"] in ("yes", "latex", "jats") else "—" + ft = "✅" if r["full_text"] in ("yes", "latex", "jats", "mineru") else "—" out.append( f"| [{r['file']}]({r['file']}) | {ttl} | {au} | {r['year']} | {ven} | {ft} |" ) diff --git a/skills/how-to-download-ref/helpers/kb_doctor.py b/skills/how-to-download-ref/helpers/kb_doctor.py index 0cf4667..61f4e81 100644 --- a/skills/how-to-download-ref/helpers/kb_doctor.py +++ b/skills/how-to-download-ref/helpers/kb_doctor.py @@ -24,8 +24,8 @@ def add(check, message, level="FAIL"): md_keys = {} for path, meta in records: missing = [key for key in ("source", "type", "canonical_id", "title") if not meta.get(key)] - if meta.get("type") in ("arxiv", "doi") and meta.get("full_text") not in ("yes", "no", "latex", "jats"): - missing.append("full_text (yes/no/latex/jats)") + if meta.get("type") in ("arxiv", "doi") and meta.get("full_text") not in ("yes", "no", "latex", "jats", "mineru"): + missing.append("full_text (yes/no/latex/jats/mineru)") if missing: add("frontmatter-required", f"{path.name}: missing/invalid {', '.join(missing)}") if not isinstance(meta.get("type"), str) or meta["type"] not in TITLES: diff --git a/skills/how-to-download-ref/helpers/kb_sync.py b/skills/how-to-download-ref/helpers/kb_sync.py index 67bb1ae..40cf599 100644 --- a/skills/how-to-download-ref/helpers/kb_sync.py +++ b/skills/how-to-download-ref/helpers/kb_sync.py @@ -123,6 +123,12 @@ def main(argv=None) -> int: warnings.append(f"JATS unavailable ({status})") else: assets.append("jats") + if fm.get("full_text") == "mineru": + if base.with_name(safe + ".mineru.md").exists(): + assets.append("mineru") + else: + all_cached = False + warnings.append("MinerU body/figures missing; run mineru_parse.py, then render.py --mineru") if fm.get("full_text") == "yes" and valid_pdf(pdf): status = restore_pdf_figures(kb, pdf, subdir, path.read_text() if path else "") all_cached &= status == "cached" diff --git a/skills/how-to-download-ref/helpers/mineru_parse.py b/skills/how-to-download-ref/helpers/mineru_parse.py new file mode 100644 index 0000000..50acbbc --- /dev/null +++ b/skills/how-to-download-ref/helpers/mineru_parse.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""Parse KB PDFs with MinerU's v1 parse API into cached markdown bodies. + +For every .raw/{arxiv,doi}/.pdf that sits beside its .json, writes +.raw//.mineru.md and copies the images that markdown links into +.figures/__/. ``render.py --mineru`` then uses the cache as the body +(``full_text: mineru``). MinerU emits display and inline math as LaTeX, tables +as HTML, and keeps the reading order of two-column layouts, which the default +pymupdf4llm path flattens. + +arXiv papers are submitted as https://arxiv.org/pdf/ URL sources, so the +service fetches them itself and nothing is uploaded. DOI PDFs must be uploaded, +which is slow far from the service. All files go into batched jobs that the +service parses in parallel. + +Service: the remote https://mineru.net/api by default, so no local GPU or +models are needed; anonymous access works and MINERU_API_KEY raises the quota. +MINERU_API_URL or --api-url points at a self-hosted ``mineru-api`` instead, +which reads local paths directly when it allows that. + +Stdlib-only. + +Usage: + mineru_parse.py --kb /abs/path/.knowledge + mineru_parse.py --kb ... --only 1106.3375 --force +""" +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import zipfile +from pathlib import Path + +DEFAULT_API_URL = "https://mineru.net/api" +TERMINAL = {"completed", "partial", "failed", "canceled"} +IMAGE_LINK = re.compile(r"""(\]\(|src=["'])images/([^)"'\s]+)""") +# MinerU drops the U+FB00 "ff" ligature glyph ("efect"); fi/fl survive. +FF_STEMS = ("effect", "effici", "effort", "differ", "diffus", "diffract", "suffic", + "suffer", "coeffic", "affect", "offer", "offset", "off-diag", "cutoff", + "tradeoff", "stiffness") +FF_FIXES = [(re.compile(r"\b" + re.escape(s.replace("ff", "f", 1)) + r"(?!f)", re.I), s) + for s in FF_STEMS] + + +class APIError(RuntimeError): + pass + + +class Client: + def __init__(self, api_url: str, api_key: str | None): + self.base = api_url.rstrip("/") + self.key = api_key + host = urllib.parse.urlparse(self.base).hostname or "" + self.local = host in ("localhost", "127.0.0.1", "::1") + self._sources: set[str] | None = None + + def _headers(self, extra: dict | None = None) -> dict: + h = {"User-Agent": "sci-brain/mineru_parse"} + if self.key: + h["Authorization"] = f"Bearer {self.key}" + h.update(extra or {}) + return h + + def request(self, method: str, path: str, body: dict | None = None, timeout: int = 60) -> dict: + data = json.dumps(body).encode() if body is not None else None + headers = self._headers({"Content-Type": "application/json"} if data else None) + req = urllib.request.Request(self.base + path, data=data, method=method, headers=headers) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read() or b"{}") + except urllib.error.HTTPError as e: + detail = e.read().decode(errors="replace")[:500] + raise APIError(f"{method} {path}: HTTP {e.code} {detail}") from None + + def download(self, file_id: str) -> bytes: + req = urllib.request.Request(f"{self.base}/v1/files/{file_id}/content", headers=self._headers()) + with urllib.request.urlopen(req, timeout=300) as r: + return r.read() + + def sources(self) -> set[str]: + if self._sources is None: + health = self.request("GET", "/v1/health") + self._sources = set((health.get("features") or {}).get("sources") or []) + return self._sources + + def upload(self, pdf: Path) -> str: + data = pdf.read_bytes() + resp = self.request("POST", "/v1/uploads", { + "filename": pdf.name, "bytes": len(data), "mime_type": "application/pdf", + "purpose": "parse", "sha256sum": hashlib.sha256(data).hexdigest()}) + if resp.get("status") == "completed": + return resp["file"]["id"] + url = resp["upload_url"] + if url.startswith("/"): + url = self.base + url + headers = dict(resp.get("upload_headers") or {}) + same_origin = urllib.parse.urlparse(url)[:2] == urllib.parse.urlparse(self.base)[:2] + if same_origin and self.key: + # Presigned storage URLs carry their own auth; never leak the key there. + headers["Authorization"] = f"Bearer {self.key}" + req = urllib.request.Request(url, data=data, method="PUT", headers=headers) + # Upstream bandwidth to the service can be a few tens of KB/s. + with urllib.request.urlopen(req, timeout=max(120, len(data) // 20_000)): + pass + return self.request("POST", f"/v1/uploads/{resp['id']}/complete")["file"]["id"] + + def source_for(self, kind: str, ident: str, pdf: Path) -> dict: + if self.local and "local" in self.sources(): + return {"type": "local", "path": str(pdf.resolve())} + if kind == "arxiv" and "url" in self.sources(): + return {"type": "url", "url": f"https://arxiv.org/pdf/{ident}"} + return {"type": "file_id", "file_id": self.upload(pdf)} + + +def postprocess(md: str, fig_rel: str) -> tuple[str, set[str]]: + """Repoint image links at fig_rel/ and repair lost ff ligatures. + + Returns the markdown and the set of image names it references. + """ + used: set[str] = set() + + def relink(m: re.Match) -> str: + used.add(m[2]) + return f"{m[1]}{fig_rel}/{m[2]}" + + md = IMAGE_LINK.sub(relink, md) + for pat, word in FF_FIXES: + md = pat.sub(lambda m, w=word: w[0].upper() + w[1:] if m[0][0].isupper() else w, md) + return md.strip() + "\n", used + + +def save_result(kb: Path, kind: str, stem: str, blob: bytes) -> int: + fig_sub = f"{kind}__{stem}" + with zipfile.ZipFile(io.BytesIO(blob)) as z: + md, used = postprocess(z.read("full.md").decode("utf-8", errors="replace"), f".figures/{fig_sub}") + if used: + fig_dir = kb / ".figures" / fig_sub + fig_dir.mkdir(parents=True, exist_ok=True) + for name in used: + try: + (fig_dir / name).write_bytes(z.read(f"images/{name}")) + except KeyError: + print(f" {kind}:{stem}: image {name} missing from result", file=sys.stderr) + (kb / ".raw" / kind / f"{stem}.mineru.md").write_text(md) + return len(used) + + +def discover(kb: Path, only: set[str], force: bool) -> list[tuple[str, str, str, Path]]: + """Return (kind, stem, ident, pdf) for every PDF that needs parsing.""" + todo = [] + for kind in ("arxiv", "doi"): + for json_path in sorted((kb / ".raw" / kind).glob("*.json")): + stem = json_path.stem + pdf = json_path.with_suffix(".pdf") + if not pdf.exists() or (only and stem not in only): + continue + if not force and json_path.with_suffix(".mineru.md").exists(): + continue + ident = stem + if kind == "arxiv": + meta = json.loads(json_path.read_text()) + ident = (meta.get("externalIds") or {}).get("ArXiv") or stem + if "-" in ident and "/" not in ident: + ident = "/".join(ident.rsplit("-", 1)) + todo.append((kind, stem, ident, pdf)) + return todo + + +def run_batch(client: Client, kb: Path, batch: list, tier: str | None, timeout: int) -> int: + files, entries = [], [] + for kind, stem, ident, pdf in batch: + try: + files.append({"source": client.source_for(kind, ident, pdf)}) + entries.append((kind, stem)) + except (OSError, APIError, KeyError) as e: + print(f"FAIL {kind}:{stem}: upload: {e}") + if not files: + return len(batch) + body = {"files": files, "output_formats": ["zip"]} + if tier: + body["tier"] = tier + job = client.request("POST", "/v1/parse/jobs", body, timeout=120) + deadline = time.time() + timeout + while job.get("status") not in TERMINAL: + if time.time() > deadline: + raise APIError(f"job {job.get('job_id')} still {job.get('status')} after {timeout}s") + time.sleep(3) + job = client.request("GET", f"/v1/parse/jobs/{job['job_id']}") + failures = len(batch) - len(files) + for (kind, stem), result in zip(entries, job.get("files") or []): + zip_ref = (result.get("output_files") or {}).get("zip") or {} + if result.get("status") != "completed" or not zip_ref.get("file_id"): + failures += 1 + print(f"FAIL {kind}:{stem}: {result.get('status')} {result.get('error') or ''}".rstrip()) + continue + try: + n_figs = save_result(kb, kind, stem, client.download(zip_ref["file_id"])) + parse = result.get("parse") or {} + print(f"ok {kind}:{stem} ({result.get('page_range', '?')} pages, " + f"{parse.get('model_used', '?')}, {n_figs} figs)") + except (OSError, KeyError, zipfile.BadZipFile, urllib.error.URLError) as e: + failures += 1 + print(f"FAIL {kind}:{stem}: download: {e}") + return failures + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--kb", required=True, type=Path) + p.add_argument("--only", action="append", default=[], + help="Parse only this .raw stem (e.g. 1106.3375); repeatable") + p.add_argument("--force", action="store_true", help="Re-parse PDFs that already have a .mineru.md") + p.add_argument("--tier", choices=("flash", "basic", "standard"), + help="Parse tier (default: server decides; mineru.net picks its VLM model)") + p.add_argument("--api-url", default=os.environ.get("MINERU_API_URL") or DEFAULT_API_URL) + p.add_argument("--batch", type=int, default=20, help="Files per parse job (service max 200)") + p.add_argument("--timeout", type=int, default=1800, help="Seconds to wait for each job") + args = p.parse_args() + + todo = discover(args.kb, set(args.only), args.force) + if not todo: + print("nothing to parse (every PDF already has a .mineru.md; use --force to redo)") + return 0 + client = Client(args.api_url, os.environ.get("MINERU_API_KEY")) + print(f"parsing {len(todo)} PDF(s) via {client.base}") + failures = 0 + for i in range(0, len(todo), args.batch): + batch = todo[i:i + args.batch] + try: + failures += run_batch(client, args.kb, batch, args.tier, args.timeout) + except (APIError, urllib.error.URLError, TimeoutError) as e: + failures += len(batch) + print(f"FAIL batch of {len(batch)}: {e}") + return int(bool(failures)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/how-to-download-ref/helpers/render.py b/skills/how-to-download-ref/helpers/render.py index 7f4c9b1..154b8c2 100755 --- a/skills/how-to-download-ref/helpers/render.py +++ b/skills/how-to-download-ref/helpers/render.py @@ -13,6 +13,8 @@ version, with exact TeX from MathML and a structured reference list) - .raw/{arxiv,doi}/.tex — flattened arXiv LaTeX source (optional; used as the full-text body ONLY with --tex-source; otherwise PDF->markdown is the default) +- .raw/{arxiv,doi}/.mineru.md — MinerU parse of the PDF from mineru_parse.py + (optional; used as the full-text body ONLY with --mineru, after JATS and LaTeX) - .raw/repos/-/ — shallow clones (rendered as README + ls-tree) - .raw/web/.html or .raw/web// subdir — web page or local source dir - A manifest JSON describing web entries and bib stubs (titles/sources/notes) @@ -26,6 +28,7 @@ Usage: render.py --kb /abs/path/.knowledge --manifest manifest.json render.py --kb /abs/path/.knowledge --tex-source # prefer LaTeX bodies when .tex exists + render.py --kb /abs/path/.knowledge --mineru # prefer MinerU bodies when .mineru.md exists """ from __future__ import annotations @@ -188,7 +191,16 @@ def _to_md(**extra): return text.strip() -def render_arxiv(kb: Path, raw: Path, only_missing: bool = False, use_tex: bool = False) -> int: +def mineru_body(json_path: Path) -> str: + """Return the cached mineru_parse.py body beside json_path, or ''.""" + cached = json_path.with_suffix(".mineru.md") + if cached.exists() and cached.stat().st_size > 100: + return cached.read_text(encoding="utf-8", errors="replace").strip() + return "" + + +def render_arxiv(kb: Path, raw: Path, only_missing: bool = False, use_tex: bool = False, + use_mineru: bool = False) -> int: n = 0 arx_dir = raw / "arxiv" if not arx_dir.exists(): @@ -234,11 +246,16 @@ def render_arxiv(kb: Path, raw: Path, only_missing: bool = False, use_tex: bool body += ["", "## Abstract", "", s2.get("abstract") or "_(abstract unavailable)_"] pdf = json_path.with_suffix(".pdf") tex = json_path.with_suffix(".tex") + mineru = mineru_body(json_path) if use_mineru else "" if use_tex and tex.exists() and tex.stat().st_size > 100: meta["full_text"] = "latex" body[0] = render_frontmatter(meta) body += ["", "## Full Text (LaTeX source)", "", tex.read_text(encoding="utf-8", errors="replace").strip()] + elif mineru: + meta["full_text"] = "mineru" + body[0] = render_frontmatter(meta) + body += ["", "## Full Text (MinerU)", "", mineru] else: full = extract_pdf_text(pdf, kb=kb, fig_subdir=f"arxiv__{json_path.stem}") if pdf.exists() else "" if full: @@ -253,7 +270,8 @@ def render_arxiv(kb: Path, raw: Path, only_missing: bool = False, use_tex: bool return n -def render_doi(kb: Path, raw: Path, only_missing: bool = False, use_tex: bool = False) -> int: +def render_doi(kb: Path, raw: Path, only_missing: bool = False, use_tex: bool = False, + use_mineru: bool = False) -> int: n = 0 doi_dir = raw / "doi" if not doi_dir.exists(): @@ -322,6 +340,10 @@ def render_doi(kb: Path, raw: Path, only_missing: bool = False, use_tex: bool = body[0] = render_frontmatter(meta) body += ["", "## Full Text (LaTeX source)", "", tex.read_text(encoding="utf-8", errors="replace").strip()] + elif use_mineru and (mineru := mineru_body(json_path)): + meta["full_text"] = "mineru" + body[0] = render_frontmatter(meta) + body += ["", "## Full Text (MinerU)", "", mineru] else: full = extract_pdf_text(pdf, kb=kb, fig_subdir=f"doi__{safe}") if pdf.exists() else "" if full: @@ -455,12 +477,16 @@ def main() -> int: p.add_argument("--tex-source", action="store_true", help="Render from flattened arXiv LaTeX source (.raw/*/.tex) when available; " "default is PDF->markdown for every ref") + p.add_argument("--mineru", action="store_true", + help="Render from mineru_parse.py output (.raw/*/.mineru.md) when available; " + "JATS and --tex-source bodies still win") args = p.parse_args() raw = args.kb / ".raw" m = json.loads(args.manifest.read_text()) if args.manifest else {} om = args.only_missing - print(f"arxiv: {render_arxiv(args.kb, raw, only_missing=om, use_tex=args.tex_source)}") - print(f"doi: {render_doi(args.kb, raw, only_missing=om, use_tex=args.tex_source)}") + src = {"use_tex": args.tex_source, "use_mineru": args.mineru} + print(f"arxiv: {render_arxiv(args.kb, raw, only_missing=om, **src)}") + print(f"doi: {render_doi(args.kb, raw, only_missing=om, **src)}") print(f"github: {render_github(args.kb, raw)}") print(f"web: {render_web(args.kb, raw, m)}") print(f"stub: {render_stubs(args.kb, m)}") diff --git a/skills/how-to-download-ref/references/dependencies.md b/skills/how-to-download-ref/references/dependencies.md index c57019e..3591f99 100644 --- a/skills/how-to-download-ref/references/dependencies.md +++ b/skills/how-to-download-ref/references/dependencies.md @@ -54,3 +54,27 @@ Without it, APS refs silently fall back to the arXiv/PDF tiers. For arXiv LaTeX sources (optional, when LaTeX sources are requested), `latexpand` (ships with TeX Live) gives the cleanest flattening; if absent, a built-in Python inliner is used — no action needed either way. + +## MinerU + +`mineru_parse.py` is stdlib-only: it talks to MinerU's v1 parse API over HTTPS, +so nothing needs installing and no local GPU is used. By default it calls the +hosted service at `https://mineru.net/api`, which accepts anonymous jobs +(about 200 pages per file; results are kept 30 days on the service). A key +from raises the quota; the user exports it +themselves, and it must never be written into the KB or the repo: + +```sh +export MINERU_API_KEY=... # optional +``` + +arXiv papers are submitted as `https://arxiv.org/pdf/` URLs, so the service +fetches them itself and the job returns in about a minute for a 60-page paper. +DOI PDFs have to be uploaded to the service's storage, whose upstream bandwidth +can be a few tens of KB/s from outside China, so a large DOI PDF can take minutes. + +`--tier flash|basic|standard` picks the parse model (default: the service +decides, which on mineru.net is its VLM). To run MinerU on your own hardware +instead, start `mineru-api --allow-local-source` (from `uv tool install mineru`) +and point the helper at it with `--api-url http://127.0.0.1:8000` or +`MINERU_API_URL`; it then reads the PDFs straight from `.raw/`. diff --git a/skills/how-to-download-ref/references/troubleshooting.md b/skills/how-to-download-ref/references/troubleshooting.md index a26a4a1..0db946f 100644 --- a/skills/how-to-download-ref/references/troubleshooting.md +++ b/skills/how-to-download-ref/references/troubleshooting.md @@ -16,5 +16,7 @@ paths use the installed `DOWNLOAD_REF_DIR` from SKILL.md. | Expecting `.figures/` images for `full_text: latex` refs to come from the PDF | They come from the source tarball; PDF image extraction runs only on the PDF path. | | Rendered from PDF despite a `.tex` in `.raw/` | PDF is the default. To use LaTeX bodies, pass `--tex-source` in Step 5 (and `--download-arxiv-source` in Step 4). | | APS paper rendered from PDF, math mangled | `pandoc` is missing, or the article is genuinely `closed`. Check with `aps_harvest.py --check `. | +| Rendered from PDF despite a `.mineru.md` in `.raw/` | Pass `--mineru` to `render.py`; the cache is opt-in like `--tex-source`. | +| `mineru_parse.py` DOI upload times out | Upstream bandwidth to mineru.net is slow from abroad. Retry later. arXiv entries never upload (they go as arxiv.org URLs), so only DOI PDFs hit this. | | Reaching for MinerU/Marker on an APS DOI | Try Harvest first — a 401 is the only thing that justifies parsing a PDF at all. | | APS DOI reported `notfound` | Harvest matches the DOI suffix case-sensitively; `aps_harvest.canonical_doi` restores APS's capitalisation before the request. Add the journal to `APS_JOURNAL_TOKENS` if a new title 404s. | diff --git a/tests/test_mineru.py b/tests/test_mineru.py new file mode 100644 index 0000000..f078cfd --- /dev/null +++ b/tests/test_mineru.py @@ -0,0 +1,101 @@ +"""mineru_parse.py caching and render.py --mineru body selection (offline).""" +import importlib.util +import io +import json +import sys +import zipfile +from pathlib import Path + +HELPERS = Path(__file__).resolve().parents[1] / "skills" / "how-to-download-ref" / "helpers" + + +def _load(name): + sys.path.insert(0, str(HELPERS)) + spec = importlib.util.spec_from_file_location(f"{name}_helper", HELPERS / f"{name}.py") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +S2 = {"title": "Test Paper", "authors": [{"name": "A. Author"}], "year": 2026, + "venue": "TestConf", "abstract": "An abstract.", "externalIds": {}} +BODY = "MINERU-BODY-MARKER with $x^2$ inline math. " + "x" * 100 + + +def _kb(tmp_path): + kb = tmp_path / "kb" + (kb / ".raw" / "arxiv").mkdir(parents=True) + (kb / ".raw" / "arxiv" / "2401.00001.json").write_text(json.dumps(S2)) + return kb + + +def test_mineru_body_with_flag(tmp_path): + mod = _load("render") + kb = _kb(tmp_path) + (kb / ".raw" / "arxiv" / "2401.00001.mineru.md").write_text(BODY) + assert mod.render_arxiv(kb, kb / ".raw", use_mineru=True) == 1 + md = (kb / "2401.00001_test-paper.md").read_text() + assert "full_text: mineru" in md + assert "## Full Text (MinerU)" in md + assert "MINERU-BODY-MARKER" in md + + +def test_mineru_ignored_without_flag(tmp_path): + mod = _load("render") + kb = _kb(tmp_path) + (kb / ".raw" / "arxiv" / "2401.00001.mineru.md").write_text(BODY) + mod.render_arxiv(kb, kb / ".raw") + md = (kb / "2401.00001_test-paper.md").read_text() + assert "full_text: no" in md + assert "MINERU-BODY-MARKER" not in md + + +def test_tex_beats_mineru(tmp_path): + mod = _load("render") + kb = _kb(tmp_path) + (kb / ".raw" / "arxiv" / "2401.00001.mineru.md").write_text(BODY) + (kb / ".raw" / "arxiv" / "2401.00001.tex").write_text("TEX-BODY-MARKER " + "x" * 100) + mod.render_arxiv(kb, kb / ".raw", use_tex=True, use_mineru=True) + md = (kb / "2401.00001_test-paper.md").read_text() + assert "full_text: latex" in md + assert "MINERU-BODY-MARKER" not in md + + +def test_postprocess_relinks_images_and_fixes_ff(): + mod = _load("mineru_parse") + md, used = mod.postprocess( + "The efect is Efective, a cutoff, an of-diagonal term.\n" + "![](images/a.jpg)\n\n", ".figures/arxiv__x") + assert "The effect is Effective, a cutoff, an off-diagonal term." in md + assert "![](.figures/arxiv__x/a.jpg)" in md + assert 'src=".figures/arxiv__x/b.jpg"' in md + assert used == {"a.jpg", "b.jpg"} + + +def test_save_result_copies_only_linked_images(tmp_path): + mod = _load("mineru_parse") + kb = _kb(tmp_path) + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as z: + z.writestr("full.md", "Fig. ![](images/fig.jpg)\n") + z.writestr("images/fig.jpg", b"JPEG") + z.writestr("images/equation-crop.jpg", b"JPEG") + assert mod.save_result(kb, "arxiv", "2401.00001", buf.getvalue()) == 1 + assert (kb / ".raw" / "arxiv" / "2401.00001.mineru.md").read_text() == \ + "Fig. ![](.figures/arxiv__2401.00001/fig.jpg)\n" + assert sorted(p.name for p in (kb / ".figures" / "arxiv__2401.00001").iterdir()) == ["fig.jpg"] + + +def test_discover_skips_cached_and_maps_old_style_ids(tmp_path): + mod = _load("mineru_parse") + kb = _kb(tmp_path) + raw = kb / ".raw" / "arxiv" + (raw / "2401.00001.pdf").write_bytes(b"%PDF-") + (raw / "hep-th-9901001.json").write_text(json.dumps(S2)) + (raw / "hep-th-9901001.pdf").write_bytes(b"%PDF-") + todo = mod.discover(kb, set(), force=False) + assert [(k, s, i) for k, s, i, _ in todo] == [ + ("arxiv", "2401.00001", "2401.00001"), ("arxiv", "hep-th-9901001", "hep-th/9901001")] + (raw / "2401.00001.mineru.md").write_text(BODY) + assert [s for _, s, _, _ in mod.discover(kb, set(), force=False)] == ["hep-th-9901001"] + assert len(mod.discover(kb, set(), force=True)) == 2