From 82e11b5b29870ec8bd6729a1214a4b423facc249 Mon Sep 17 00:00:00 2001 From: hz-xiaxz <1806656034@qq.com> Date: Tue, 3 Nov 2026 14:02:16 -0500 Subject: [PATCH] fix(download-ref): don't stall on Semantic Scholar 429s; hand chemistry PDFs to a person Without an API key the shared Semantic Scholar pool often returns 429 for minutes. post_batch backed off 5+10+20+40+80 s per call, and when it gave up, arXiv IDs had no metadata at all and Crossref-only DOIs lost their preprint link. Agents re-ran the fetch and sessions sat for half an hour. - post_batch gives up after ~9 s (two retries) and points at SEMANTIC_SCHOLAR_API_KEY. - arXiv IDs fall back to the arXiv API (title, authors, abstract, DOI, BibTeX; metadata_source "arxiv", accepted by verify_bib). - Crossref-only DOIs look up an arXiv preprint by exact title. - AIP (10.1063), ACS (10.1021) and RSC (10.1039) serve PDFs only after a Cloudflare human check, even to subscribed IPs. Those misses are now printed as links (through SCIBRAIN_PROXY_PREFIX when set), and the new import_pdfs.py matches the PDFs a person saves to the pending DOIs by DOI or exact title and copies them into .raw/doi/. Co-Authored-By: Claude Opus 5.5 --- skills/how-to-download-ref/SKILL.md | 16 +-- .../helpers/fetch_metadata.py | 117 ++++++++++++++++-- .../helpers/import_pdfs.py | 112 +++++++++++++++++ .../how-to-download-ref/helpers/verify_bib.py | 2 +- .../references/acquisition.md | 28 +++++ tests/test_kb_maintenance.py | 106 +++++++++++++++- 6 files changed, 356 insertions(+), 25 deletions(-) create mode 100644 skills/how-to-download-ref/helpers/import_pdfs.py diff --git a/skills/how-to-download-ref/SKILL.md b/skills/how-to-download-ref/SKILL.md index 8af5edb..241b629 100644 --- a/skills/how-to-download-ref/SKILL.md +++ b/skills/how-to-download-ref/SKILL.md @@ -145,13 +145,15 @@ python3 "$DOWNLOAD_REF_DIR/helpers/fetch_metadata.py" \ --kb "$KB" --manifest "$TMP" --download-arxiv-pdfs ``` -Metadata comes from cached JSON, then Semantic Scholar, then Crossref; PDFs from -open-access sources, then the arXiv preprint. APS (`10.1103/*`) publisher JATS -is fetched automatically when the article is open. `--email` / -`SCIBRAIN_CONTACT_EMAIL` enables Unpaywall. - -For source details, APS extras, or DOI misses requiring `scihub_download.py`, -read only the applicable section of [acquisition.md](references/acquisition.md). +Metadata comes from cached JSON, then Semantic Scholar, then the arXiv API or +Crossref when Semantic Scholar is rate-limited; PDFs from open-access sources, +then the arXiv preprint (found by exact title for Crossref-only DOIs). APS +(`10.1103/*`) publisher JATS is fetched automatically when the article is open. +`--email` / `SCIBRAIN_CONTACT_EMAIL` enables Unpaywall. + +For source details, APS extras, chemistry-publisher misses listed for +`import_pdfs.py`, or DOI misses requiring `scihub_download.py`, read only the +applicable section of [acquisition.md](references/acquisition.md). Record unavailable assets and continue the remaining references. ### 5. Render PDF to markdown diff --git a/skills/how-to-download-ref/helpers/fetch_metadata.py b/skills/how-to-download-ref/helpers/fetch_metadata.py index 532817b..cabf477 100755 --- a/skills/how-to-download-ref/helpers/fetch_metadata.py +++ b/skills/how-to-download-ref/helpers/fetch_metadata.py @@ -28,6 +28,7 @@ import urllib.error import urllib.parse import urllib.request +import xml.etree.ElementTree as ET from pathlib import Path from kb_identity import cache_path, find_match, identities, identity_index, normalize, same_record @@ -36,6 +37,13 @@ S2_BATCH_URL = f"https://api.semanticscholar.org/graph/v1/paper/batch?fields={S2_FIELDS}" S2_API_KEY = os.environ.get("SEMANTIC_SCHOLAR_API_KEY", "") +# Chemistry publishers (AIP, ACS, RSC) put article PDFs behind a Cloudflare human +# check even on a subscribed network; misses there are listed for a person to open +# and then picked up by import_pdfs.py. +BOT_CHECKED_PREFIXES = ("10.1063/", "10.1021/", "10.1039/") +PROXY_PREFIX = os.environ.get("SCIBRAIN_PROXY_PREFIX", "") +ARXIV_API = "https://export.arxiv.org/api/query" +ATOM = {"a": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"} def post_batch(ids: list[str]) -> list[dict | None]: @@ -47,13 +55,15 @@ def post_batch(ids: list[str]) -> list[dict | None]: req = urllib.request.Request( S2_BATCH_URL, data=body, method="POST", headers=headers, ) - backoff = 5 - for attempt in range(6): + # The keyless S2 pool is often exhausted for minutes; fail fast and let + # lookup_metadata fall back to arXiv / Crossref instead of sleeping ~2.5 min. + backoff = 3 + for attempt in range(3): try: with urllib.request.urlopen(req, timeout=60) as r: return json.loads(r.read()) except urllib.error.HTTPError as e: - if e.code in (429, 500, 502, 503) and attempt < 5: + if e.code in (429, 500, 502, 503) and attempt < 2: print(f" HTTP {e.code}, sleep {backoff}s", file=sys.stderr) time.sleep(backoff) backoff *= 2 @@ -137,6 +147,72 @@ def escape(value): return None +def arxiv_entries(params: dict) -> list: + url = ARXIV_API + "?" + urllib.parse.urlencode(params) + req = urllib.request.Request(url, headers={"User-Agent": "sci-brain/0.5"}) + with urllib.request.urlopen(req, timeout=30) as r: + return ET.fromstring(r.read()).findall("a:entry", ATOM) + + +def arxiv_record(entry) -> dict | None: + """Normalize one arXiv Atom entry into the S2-shaped record the KB stores.""" + def text(tag): + return " ".join((entry.findtext(tag, "", ATOM) or "").split()) + title, ident = text("a:title"), text("a:id") + if not title or "/abs/" not in ident: + return None + aid = normalize("arxiv", ident.rsplit("/abs/", 1)[1]) + authors = [" ".join((a.findtext("a:name", "", ATOM) or "").split()) for a in entry.findall("a:author", ATOM)] + year = int(text("a:published")[:4]) if text("a:published")[:4].isdigit() else None + doi = text("arxiv:doi").lower() + fields = {"title": title, "author": " and ".join(authors), "year": year, + "eprint": aid, "archivePrefix": "arXiv", "doi": doi} + def escape(value): + return re.sub(r"[\\{}%&#_$]", lambda match: "\\" + match[0], str(value)) + bib = "@misc{arxiv,\n" + "\n".join( + f" {key} = {{{value if key == 'doi' else escape(value)}}}," for key, value in fields.items() if value) + "\n}" + return {"title": title, "authors": [{"name": a} for a in authors], "year": year, "venue": "arXiv", + "externalIds": {"ArXiv": aid, **({"DOI": doi} if doi else {})}, "abstract": text("a:summary"), + "citationStyles": {"bibtex": bib}, "metadata_source": "arxiv"} + + +def arxiv_metadata(ids: list[str]) -> dict[str, dict]: + """Batch arXiv API lookup; one unknown ID empties a batch, so retry those singly.""" + found: dict[str, dict] = {} + def query(batch): + try: + for entry in arxiv_entries({"id_list": ",".join(batch), "max_results": len(batch)}): + meta = arxiv_record(entry) + if meta: + found[meta["externalIds"]["ArXiv"]] = meta + except (OSError, ET.ParseError, ValueError) as e: + print(f" arXiv API miss {','.join(batch)}: {e}", file=sys.stderr) + time.sleep(3) + for start in range(0, len(ids), 100): + query(ids[start:start + 100]) + for aid in [aid for aid in ids if aid not in found and len(ids) > 1]: + query([aid]) + return found + + +def arxiv_preprint(title: str) -> str | None: + """arXiv ID whose title matches exactly, so Crossref-only DOIs still get a free PDF.""" + def key(value): + return re.sub(r"[^a-z0-9]+", " ", value.lower()).strip() + try: + entries = arxiv_entries({"search_query": f'ti:"{key(title)}"', "max_results": 5}) + except (OSError, ET.ParseError) as e: + print(f" arXiv search miss {title[:60]!r}: {e}", file=sys.stderr) + return None + finally: + time.sleep(3) + for entry in entries: + meta = arxiv_record(entry) + if meta and key(meta["title"]) == key(title): + return meta["externalIds"]["ArXiv"] + return None + + def lookup_metadata(refs: list[tuple[str, str]], email: str = "") -> list[dict | None]: results = [] for start in range(0, len(refs), 500): @@ -146,13 +222,23 @@ def lookup_metadata(refs: list[tuple[str, str]], email: str = "") -> list[dict | if len(found) != len(batch): raise ValueError("Semantic Scholar returned an incomplete batch") except (OSError, ValueError) as e: - print(f" Semantic Scholar unavailable: {e}", file=sys.stderr) + hint = "" if S2_API_KEY else " (set SEMANTIC_SCHOLAR_API_KEY to avoid the shared rate limit)" + print(f" Semantic Scholar unavailable: {e}; falling back to arXiv/Crossref{hint}", file=sys.stderr) found = [None] * len(batch) - for (kind, value), meta in zip(batch, found): - if meta is None and kind == "doi": - meta = crossref_metadata(value, email) - time.sleep(2) - results.append(meta) + results.extend(found) + missing = [value for (kind, value), meta in zip(refs, results) if meta is None and kind == "arxiv"] + from_arxiv = arxiv_metadata(missing) if missing else {} + for i, ((kind, value), meta) in enumerate(zip(refs, results)): + if meta is not None: + continue + if kind == "arxiv": + results[i] = from_arxiv.get(value) + continue + meta = crossref_metadata(value, email) + time.sleep(2) + if meta and (preprint := arxiv_preprint(meta["title"])): + meta["externalIds"]["ArXiv"] = preprint + results[i] = meta return results @@ -246,8 +332,8 @@ def main() -> int: accepted.sort(key=lambda ref: ref[0] != "arxiv") for kind, value, meta in accepted: summarize(kind, value, meta) - if meta and meta.get("metadata_source") == "crossref": - print(f"ok {kind}:{value} (crossref)") + if meta and meta.get("metadata_source") in ("crossref", "arxiv"): + print(f"ok {kind}:{value} ({meta['metadata_source']})") arxiv_ids = [value for kind, value, _ in accepted if kind == "arxiv"] dois = [value for kind, value, _ in accepted if kind == "doi"] results = [meta for _, _, meta in accepted] @@ -276,13 +362,22 @@ def main() -> int: else: print(f" FAIL arxiv:{aid}") time.sleep(2) + manual = [] for doi, r in zip(dois, results[len(arxiv_ids):]): if r is None: continue out = cache_path(args.kb, "doi", doi, ".pdf") status = fetch_doi_pdf(doi, r, out, args.email) print(f" {'miss' if status == 'miss' else 'ok '} doi:{doi} ({status})") + if status == "miss" and doi.startswith(BOT_CHECKED_PREFIXES): + manual.append(doi) time.sleep(2) + if manual: + where = "through SCIBRAIN_PROXY_PREFIX" if PROXY_PREFIX else "on your institution's network" + print(f"\n{len(manual)} chemistry-publisher PDF(s) need a human check: open each {where}, " + "save the PDF, then run import_pdfs.py --kb ") + for doi in manual: + print(f" {PROXY_PREFIX}https://doi.org/{doi}") if args.download_arxiv_source: from tex_source import fetch_arxiv_source diff --git a/skills/how-to-download-ref/helpers/import_pdfs.py b/skills/how-to-download-ref/helpers/import_pdfs.py new file mode 100644 index 0000000..784aa1f --- /dev/null +++ b/skills/how-to-download-ref/helpers/import_pdfs.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Import chemistry-publisher PDFs that a person downloaded by hand. + +AIP (10.1063), ACS (10.1021) and RSC (10.1039) serve article PDFs only after a +Cloudflare human check, even on a subscribed network, so fetch_metadata.py lists +those misses as links. Open each link, save the PDF anywhere under --from, then: + + import_pdfs.py --kb /abs/path/.knowledge [--from ~/Downloads] [--days 14] + +Each recent PDF is matched to a pending DOI (a KB DOI under those prefixes with +no PDF yet) by a DOI printed in it, or else by the exact title on its first +pages, and copied to .raw/doi/.pdf. Files matching no pending DOI, or more +than one, are left alone. Re-render afterwards with render.py --kb . +""" +from __future__ import annotations + +import argparse +import json +import re +import shutil +import subprocess +import sys +import time +from pathlib import Path + +from fetch_metadata import BOT_CHECKED_PREFIXES, valid_pdf +from kb_identity import cache_path, normalize + +def title_key(value: str) -> str: + return " ".join(re.sub(r"[^a-z0-9]+", " ", value.lower()).split()) + + +def pending(kb: Path) -> dict[str, str]: + """Chemistry-publisher DOIs in the KB without a PDF, mapped to their titles.""" + out = {} + for path in sorted((kb / ".raw" / "doi").glob("*.json")): + try: + meta = json.loads(path.read_text()) + doi = normalize("doi", (meta.get("externalIds") or {}).get("DOI") or "") + except (OSError, ValueError, AttributeError): + continue + if doi.startswith(BOT_CHECKED_PREFIXES) and not valid_pdf(cache_path(kb, "doi", doi, ".pdf")): + out[doi] = meta.get("title") or "" + return out + + +def first_pages_text(pdf: Path) -> bytes: + """Text of pages 1-2 (pdftotext, else PyMuPDF) plus the raw bytes, which hold XMP metadata.""" + text = b"" + try: + text = subprocess.run(["pdftotext", "-f", "1", "-l", "2", str(pdf), "-"], + capture_output=True, timeout=60).stdout + except (OSError, subprocess.TimeoutExpired): + try: + import pymupdf # type: ignore + with pymupdf.open(pdf) as doc: + text = "".join(doc[i].get_text() for i in range(min(2, len(doc)))).encode() + except Exception: + pass + return text + b"\n" + pdf.read_bytes() + + +def match(pdf: Path, wanted: dict[str, str]) -> str | None: + blob = first_pages_text(pdf) + lower = blob.lower() + # The DOI must end there: 10.1063/1.49440 must not match inside 10.1063/1.4944087. + hits = [doi for doi in wanted + if re.search(re.escape(doi.encode()) + rb"(?![a-z0-9]|[./-][a-z0-9])", lower)] + if len(hits) == 1: + return hits[0] + text = title_key(blob.split(b"\n%PDF", 1)[0].decode("utf-8", "replace")) + hits = [doi for doi, title in wanted.items() if len(title_key(title)) > 20 and title_key(title) in text] + return hits[0] if len(hits) == 1 else None + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--kb", required=True, type=Path, help="Absolute path to /.knowledge/") + p.add_argument("--from", dest="source", type=Path, default=Path.home() / "Downloads", + help="Folder holding the downloaded PDFs (default ~/Downloads)") + p.add_argument("--days", type=float, default=14, help="Only look at PDFs modified in the last N days") + args = p.parse_args() + + wanted = pending(args.kb) + if not wanted: + print("no chemistry-publisher DOI in this KB is waiting for a PDF") + return 0 + cutoff = time.time() - args.days * 86400 + pdfs = sorted(f for f in args.source.expanduser().rglob("*.pdf") + if f.is_file() and f.stat().st_mtime >= cutoff) + imported = 0 + for pdf in pdfs: + if not wanted or not valid_pdf(pdf): + continue + doi = match(pdf, wanted) + if doi is None: + continue + dest = cache_path(args.kb, "doi", doi, ".pdf") + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(pdf, dest) + print(f" ok doi:{doi} <- {pdf}") + del wanted[doi] + imported += 1 + for doi in wanted: + print(f" miss doi:{doi} (no matching PDF in {args.source})") + if imported: + print(f"\n{imported} imported; re-render with: render.py --kb {args.kb}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/how-to-download-ref/helpers/verify_bib.py b/skills/how-to-download-ref/helpers/verify_bib.py index 7e2aa0c..648aec7 100644 --- a/skills/how-to-download-ref/helpers/verify_bib.py +++ b/skills/how-to-download-ref/helpers/verify_bib.py @@ -354,7 +354,7 @@ def _load_cache(path: Path) -> dict[str, Any] | None: data = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return None - if not isinstance(data, dict) or not data.get("title") or not (data.get("paperId") or data.get("metadata_source") == "crossref"): + if not isinstance(data, dict) or not data.get("title") or not (data.get("paperId") or data.get("metadata_source") in ("crossref", "arxiv")): return None return data diff --git a/skills/how-to-download-ref/references/acquisition.md b/skills/how-to-download-ref/references/acquisition.md index 8a493b6..2d4288b 100644 --- a/skills/how-to-download-ref/references/acquisition.md +++ b/skills/how-to-download-ref/references/acquisition.md @@ -38,6 +38,34 @@ names an arXiv preprint (`externalIds.ArXiv`) get the same treatment, into **Tip:** Set `SEMANTIC_SCHOLAR_API_KEY` in your environment to raise the Semantic Scholar rate limit from ~1 req/s to 100 req/s. Get a free key at https://www.semanticscholar.org/product/api#api-key-form. +## Chemistry publishers behind a human check (AIP, ACS, RSC) + +AIP (`10.1063/*`, e.g. JCP), ACS (`10.1021/*`) and RSC (`10.1039/*`) answer +scripts and headless browsers with a Cloudflare "Just a moment..." page even on a +subscribed network, so their PDFs cannot be fetched automatically. APS is +different: it serves PDFs to subscribed IPs directly, and most APS papers have +arXiv preprints. When such a DOI misses, the fetch ends with a block of links: + +``` +2 chemistry-publisher PDF(s) need a human check: open each on your institution's network, ... + https://doi.org/10.1063/1.4944087 +``` + +Give the user those links (off campus, set `SCIBRAIN_PROXY_PREFIX` to the +library proxy, e.g. `https://proxy.library.nyu.edu/login?url=`, so the links go +through it). Do not try to pass the check yourself. Once the user has saved the +PDFs, import and re-render: + +```sh +python3 "$DOWNLOAD_REF_DIR/helpers/import_pdfs.py" --kb "$KB" # --from ~/Downloads --days 14 +python3 "$DOWNLOAD_REF_DIR/helpers/render.py" --kb "$KB" +``` + +`import_pdfs.py` scans recent PDFs under `--from`, matches each to a pending +DOI by a DOI printed in it (or the exact title on its first pages), and copies +it to `$KB/.raw/doi/.pdf`. Unmatched files are left alone. It uses +`pdftotext` (poppler) or PyMuPDF when available. + ## Sci-Hub fallback for paywalled PDFs (script) If the fetch reports `miss` for any DOI (no open-access PDF and no arXiv preprint), diff --git a/tests/test_kb_maintenance.py b/tests/test_kb_maintenance.py index 5781dff..cbb76a5 100644 --- a/tests/test_kb_maintenance.py +++ b/tests/test_kb_maintenance.py @@ -12,6 +12,7 @@ sys.path.insert(0, str(HELPERS)) import append_bibtex as bib import fetch_metadata as fetch +import import_pdfs import index import kb_doctor as doctor import kb_identity as identity @@ -94,16 +95,68 @@ def test_batch_dedup_resolves_aliases_and_allows_explicit_override(tmp_path, mon assert error.value.code == 2 +class Response: + def __init__(self, body): self.body = body + def __enter__(self): return self + def __exit__(self, *args): pass + def read(self): return self.body + + +def atom(*entries): + body = ''.join(f'''http://arxiv.org/abs/{aid}v2{title} + 2013-05-31T00:00:00ZAbstract. + Daniel Kats{f"{doi}" if doi else ""}''' + for aid, title, doi in entries) + return ('' + + body + '').encode() + + +def test_s2_rate_limit_fails_fast(monkeypatch): + sleeps, calls = [], [] + def urlopen(req, **kwargs): + calls.append(req.full_url) + raise fetch.urllib.error.HTTPError(req.full_url, 429, 'Too Many Requests', {}, None) + monkeypatch.setattr(fetch.urllib.request, 'urlopen', urlopen) + monkeypatch.setattr(fetch.time, 'sleep', sleeps.append) + with pytest.raises(fetch.urllib.error.HTTPError): + fetch.post_batch(['DOI:' + DOI]) + assert len(calls) == 3 and sum(sleeps) <= 10 + + +def test_arxiv_api_fallback_when_s2_unavailable(tmp_path, monkeypatch, capsys): + monkeypatch.setattr(fetch, 'post_batch', lambda ids: (_ for _ in ()).throw(OSError('HTTP Error 429'))) + calls = [] + def urlopen(req, **kwargs): + calls.append(req.full_url) + return Response(atom((ARXIV, 'The distinguishable cluster approximation', '10.1063/1.4813481'))) + monkeypatch.setattr(fetch.urllib.request, 'urlopen', urlopen) + invoke_fetch(monkeypatch, tmp_path, {'arxiv': [ARXIV]}) + meta = json.loads((tmp_path / f'.raw/arxiv/{ARXIV}.json').read_text()) + assert meta['metadata_source'] == 'arxiv' and meta['title'] == 'The distinguishable cluster approximation' + assert meta['externalIds'] == {'ArXiv': ARXIV, 'DOI': '10.1063/1.4813481'} and meta['year'] == 2013 + assert 'eprint = {2601.12345}' in meta['citationStyles']['bibtex'] + assert len(calls) == 1 and 'id_list=' in calls[0] + assert f'ok arxiv:{ARXIV} (arxiv)' in capsys.readouterr().out + + +def test_crossref_record_links_arxiv_preprint_by_exact_title(monkeypatch): + monkeypatch.setattr(fetch, 'post_batch', lambda ids: [None] * len(ids)) + monkeypatch.setattr(fetch.time, 'sleep', lambda _: None) + monkeypatch.setattr(fetch, 'crossref_metadata', lambda doi, email: {'title': 'The distinguishable cluster approximation', 'externalIds': {'DOI': doi}}) + monkeypatch.setattr(fetch.urllib.request, 'urlopen', lambda req, **kw: Response(atom( + ('1903.06932', 'On the distinguishable cluster approximation for triple excitations', None), + ('1305.7451', 'The Distinguishable Cluster Approximation', None)))) + [meta] = fetch.lookup_metadata([('doi', '10.1063/1.4813481')]) + assert meta['externalIds'] == {'DOI': '10.1063/1.4813481', 'ArXiv': '1305.7451'} + + def test_crossref_miss_fallback_reaches_render_and_bibliography(tmp_path, monkeypatch, capsys): monkeypatch.setattr(fetch, 'post_batch', lambda ids: [None] * len(ids)) calls = [] - class Response: - def __enter__(self): return self - def __exit__(self, *args): pass - def read(self): return json.dumps({'message': {'DOI': DOI, 'title': ['A & B'], 'author': [{'given': 'Test', 'family': 'Author'}], 'published': {'date-parts': [[2026]]}, 'container-title': ['Chemistry Journal'], 'type': 'journal-article'}}).encode() + crossref = json.dumps({'message': {'DOI': DOI, 'title': ['A & B'], 'author': [{'given': 'Test', 'family': 'Author'}], 'published': {'date-parts': [[2026]]}, 'container-title': ['Chemistry Journal'], 'type': 'journal-article'}}).encode() def urlopen(req, **kwargs): calls.append(req.full_url) - return Response() + return Response(atom() if 'arxiv.org' in req.full_url else crossref) monkeypatch.setattr(fetch.urllib.request, 'urlopen', urlopen) invoke_fetch(monkeypatch, tmp_path, {'doi': [DOI]}, '--email', 'reader@example.org') meta = json.loads((tmp_path / '.raw/doi/10.1234-example.json').read_text()) @@ -111,7 +164,8 @@ def urlopen(req, **kwargs): assert meta['authors'] == [{'name': 'Test Author'}] assert meta['metadata_source'] == 'crossref' assert '(crossref)' in capsys.readouterr().out - assert len(calls) == 1 and 'api.crossref.org/works/' in calls[0] + assert len(calls) == 2 and 'api.crossref.org/works/' in calls[0] and 'search_query=ti' in calls[1] + assert 'ArXiv' not in meta['externalIds'] render.render_doi(tmp_path, tmp_path / '.raw') assert 'full_text: no' in (tmp_path / '10-1234-example.md').read_text() bib.cmd_append(SimpleNamespace(kb=str(tmp_path), type='doi', id=DOI, key='author_2026_test', bib=str(tmp_path / 'references.bib'))) @@ -351,3 +405,43 @@ def test_cache_free_clone_uses_tracked_image_case_not_s2_doi_case(tmp_path): shutil.rmtree(tmp_path / '.raw') assert identity.cache_path(tmp_path, 'doi', mixed) == dest assert identity.cache_path(tmp_path, 'doi', mixed, '.pdf') == dest.with_suffix('.pdf') + + +def test_only_chemistry_publisher_misses_are_listed_for_a_person(tmp_path, monkeypatch, capsys): + chem, aps = '10.1063/1.4944087', '10.1103/physrevb.98.045103' + monkeypatch.setattr(fetch, 'post_batch', lambda ids: [dict(META, externalIds={'DOI': i.split(':', 1)[1]}) for i in ids]) + monkeypatch.setattr(fetch, 'fetch_doi_pdf', lambda *args: 'miss') + monkeypatch.setattr(fetch, 'PROXY_PREFIX', 'https://proxy.example.edu/login?url=') + invoke_fetch(monkeypatch, tmp_path, {'doi': [chem, aps]}, '--download-arxiv-pdfs', '--allow-duplicate') + out = capsys.readouterr().out + assert '1 chemistry-publisher PDF(s) need a human check' in out + assert f'https://proxy.example.edu/login?url=https://doi.org/{chem}' in out + assert f'https://doi.org/{aps}' not in out + + +def test_import_pdfs_matches_by_doi_then_title_and_skips_others(tmp_path, monkeypatch, capsys): + kb, downloads = tmp_path / 'kb', tmp_path / 'Downloads' + downloads.mkdir() + papers = {'10.1063/1.4944087': 'Assessing the distinguishable cluster approximation', + '10.1021/acs.jctc.2c00059': 'FCIQMC-tailored distinguishable cluster approach', + '10.1103/physrevb.98.045103': 'An APS paper that is never imported'} + for doi, title in papers.items(): + fetch.save(identity.cache_path(kb, 'doi', doi), {'title': title, 'externalIds': {'DOI': doi}}) + (downloads / '124117_1_online.pdf').write_bytes(PDF.replace(b'x' * 40, b'doi.org/10.1063/1.4944087.\nxxxxx', 1)) + (downloads / 'acs.pdf').write_bytes(PDF) + (downloads / 'aps.pdf').write_bytes(PDF.replace(b'x' * 30, b'10.1103/PhysRevB.98.045103\nxxx', 1)) + (downloads / 'notes.pdf').write_bytes(PDF) + real = import_pdfs.first_pages_text + def text(pdf): + title = b'FCIQMC-Tailored Distinguishable\nCluster Approach' if pdf.name == 'acs.pdf' else b'' + return title + real(pdf) + monkeypatch.setattr(import_pdfs, 'first_pages_text', text) + monkeypatch.setattr(sys, 'argv', ['import_pdfs.py', '--kb', str(kb), '--from', str(downloads)]) + assert import_pdfs.main() == 0 + raw = kb / '.raw/doi' + assert (raw / '10.1063-1.4944087.pdf').read_bytes() == (downloads / '124117_1_online.pdf').read_bytes() + assert (raw / '10.1021-acs.jctc.2c00059.pdf').exists() + assert not (raw / '10.1103-physrevb.98.045103.pdf').exists() + out = capsys.readouterr().out + assert '2 imported' in out and 'notes.pdf' not in out + assert import_pdfs.pending(kb) == {}