Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions skills/how-to-download-ref/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -145,13 +145,15 @@ python3 "$DOWNLOAD_REF_DIR/helpers/fetch_metadata.py" \
--kb "$KB" --manifest "$TMP" --download-arxiv-pdfs
```

Metadata comes from cached JSON, then Semantic Scholar, then Crossref; PDFs from
open-access sources, then the arXiv preprint. APS (`10.1103/*`) publisher JATS
is fetched automatically when the article is open. `--email` /
`SCIBRAIN_CONTACT_EMAIL` enables Unpaywall.

For source details, APS extras, or DOI misses requiring `scihub_download.py`,
read only the applicable section of [acquisition.md](references/acquisition.md).
Metadata comes from cached JSON, then Semantic Scholar, then the arXiv API or
Crossref when Semantic Scholar is rate-limited; PDFs from open-access sources,
then the arXiv preprint (found by exact title for Crossref-only DOIs). APS
(`10.1103/*`) publisher JATS is fetched automatically when the article is open.
`--email` / `SCIBRAIN_CONTACT_EMAIL` enables Unpaywall.

For source details, APS extras, chemistry-publisher misses listed for
`import_pdfs.py`, or DOI misses requiring `scihub_download.py`, read only the
applicable section of [acquisition.md](references/acquisition.md).
Record unavailable assets and continue the remaining references.

### 5. Render PDF to markdown
Expand Down
117 changes: 106 additions & 11 deletions skills/how-to-download-ref/helpers/fetch_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from pathlib import Path

from kb_identity import cache_path, find_match, identities, identity_index, normalize, same_record
Expand All @@ -36,6 +37,13 @@
S2_BATCH_URL = f"https://api.semanticscholar.org/graph/v1/paper/batch?fields={S2_FIELDS}"

S2_API_KEY = os.environ.get("SEMANTIC_SCHOLAR_API_KEY", "")
# Chemistry publishers (AIP, ACS, RSC) put article PDFs behind a Cloudflare human
# check even on a subscribed network; misses there are listed for a person to open
# and then picked up by import_pdfs.py.
BOT_CHECKED_PREFIXES = ("10.1063/", "10.1021/", "10.1039/")
PROXY_PREFIX = os.environ.get("SCIBRAIN_PROXY_PREFIX", "")
ARXIV_API = "https://export.arxiv.org/api/query"
ATOM = {"a": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"}


def post_batch(ids: list[str]) -> list[dict | None]:
Expand All @@ -47,13 +55,15 @@ def post_batch(ids: list[str]) -> list[dict | None]:
req = urllib.request.Request(
S2_BATCH_URL, data=body, method="POST", headers=headers,
)
backoff = 5
for attempt in range(6):
# The keyless S2 pool is often exhausted for minutes; fail fast and let
# lookup_metadata fall back to arXiv / Crossref instead of sleeping ~2.5 min.
backoff = 3
for attempt in range(3):
try:
with urllib.request.urlopen(req, timeout=60) as r:
return json.loads(r.read())
except urllib.error.HTTPError as e:
if e.code in (429, 500, 502, 503) and attempt < 5:
if e.code in (429, 500, 502, 503) and attempt < 2:
print(f" HTTP {e.code}, sleep {backoff}s", file=sys.stderr)
time.sleep(backoff)
backoff *= 2
Expand Down Expand Up @@ -137,6 +147,72 @@ def escape(value):
return None


def arxiv_entries(params: dict) -> list:
url = ARXIV_API + "?" + urllib.parse.urlencode(params)
req = urllib.request.Request(url, headers={"User-Agent": "sci-brain/0.5"})
with urllib.request.urlopen(req, timeout=30) as r:
return ET.fromstring(r.read()).findall("a:entry", ATOM)


def arxiv_record(entry) -> dict | None:
"""Normalize one arXiv Atom entry into the S2-shaped record the KB stores."""
def text(tag):
return " ".join((entry.findtext(tag, "", ATOM) or "").split())
title, ident = text("a:title"), text("a:id")
if not title or "/abs/" not in ident:
return None
aid = normalize("arxiv", ident.rsplit("/abs/", 1)[1])
authors = [" ".join((a.findtext("a:name", "", ATOM) or "").split()) for a in entry.findall("a:author", ATOM)]
year = int(text("a:published")[:4]) if text("a:published")[:4].isdigit() else None
doi = text("arxiv:doi").lower()
fields = {"title": title, "author": " and ".join(authors), "year": year,
"eprint": aid, "archivePrefix": "arXiv", "doi": doi}
def escape(value):
return re.sub(r"[\\{}%&#_$]", lambda match: "\\" + match[0], str(value))
bib = "@misc{arxiv,\n" + "\n".join(
f" {key} = {{{value if key == 'doi' else escape(value)}}}," for key, value in fields.items() if value) + "\n}"
return {"title": title, "authors": [{"name": a} for a in authors], "year": year, "venue": "arXiv",
"externalIds": {"ArXiv": aid, **({"DOI": doi} if doi else {})}, "abstract": text("a:summary"),
"citationStyles": {"bibtex": bib}, "metadata_source": "arxiv"}


def arxiv_metadata(ids: list[str]) -> dict[str, dict]:
"""Batch arXiv API lookup; one unknown ID empties a batch, so retry those singly."""
found: dict[str, dict] = {}
def query(batch):
try:
for entry in arxiv_entries({"id_list": ",".join(batch), "max_results": len(batch)}):
meta = arxiv_record(entry)
if meta:
found[meta["externalIds"]["ArXiv"]] = meta
except (OSError, ET.ParseError, ValueError) as e:
print(f" arXiv API miss {','.join(batch)}: {e}", file=sys.stderr)
time.sleep(3)
for start in range(0, len(ids), 100):
query(ids[start:start + 100])
for aid in [aid for aid in ids if aid not in found and len(ids) > 1]:
query([aid])
return found


def arxiv_preprint(title: str) -> str | None:
"""arXiv ID whose title matches exactly, so Crossref-only DOIs still get a free PDF."""
def key(value):
return re.sub(r"[^a-z0-9]+", " ", value.lower()).strip()
try:
entries = arxiv_entries({"search_query": f'ti:"{key(title)}"', "max_results": 5})
except (OSError, ET.ParseError) as e:
print(f" arXiv search miss {title[:60]!r}: {e}", file=sys.stderr)
return None
finally:
time.sleep(3)
for entry in entries:
meta = arxiv_record(entry)
if meta and key(meta["title"]) == key(title):
return meta["externalIds"]["ArXiv"]
return None


def lookup_metadata(refs: list[tuple[str, str]], email: str = "") -> list[dict | None]:
results = []
for start in range(0, len(refs), 500):
Expand All @@ -146,13 +222,23 @@ def lookup_metadata(refs: list[tuple[str, str]], email: str = "") -> list[dict |
if len(found) != len(batch):
raise ValueError("Semantic Scholar returned an incomplete batch")
except (OSError, ValueError) as e:
print(f" Semantic Scholar unavailable: {e}", file=sys.stderr)
hint = "" if S2_API_KEY else " (set SEMANTIC_SCHOLAR_API_KEY to avoid the shared rate limit)"
print(f" Semantic Scholar unavailable: {e}; falling back to arXiv/Crossref{hint}", file=sys.stderr)
found = [None] * len(batch)
for (kind, value), meta in zip(batch, found):
if meta is None and kind == "doi":
meta = crossref_metadata(value, email)
time.sleep(2)
results.append(meta)
results.extend(found)
missing = [value for (kind, value), meta in zip(refs, results) if meta is None and kind == "arxiv"]
from_arxiv = arxiv_metadata(missing) if missing else {}
for i, ((kind, value), meta) in enumerate(zip(refs, results)):
if meta is not None:
continue
if kind == "arxiv":
results[i] = from_arxiv.get(value)
continue
meta = crossref_metadata(value, email)
time.sleep(2)
if meta and (preprint := arxiv_preprint(meta["title"])):
meta["externalIds"]["ArXiv"] = preprint
results[i] = meta
return results


Expand Down Expand Up @@ -246,8 +332,8 @@ def main() -> int:
accepted.sort(key=lambda ref: ref[0] != "arxiv")
for kind, value, meta in accepted:
summarize(kind, value, meta)
if meta and meta.get("metadata_source") == "crossref":
print(f"ok {kind}:{value} (crossref)")
if meta and meta.get("metadata_source") in ("crossref", "arxiv"):
print(f"ok {kind}:{value} ({meta['metadata_source']})")
arxiv_ids = [value for kind, value, _ in accepted if kind == "arxiv"]
dois = [value for kind, value, _ in accepted if kind == "doi"]
results = [meta for _, _, meta in accepted]
Expand Down Expand Up @@ -276,13 +362,22 @@ def main() -> int:
else:
print(f" FAIL arxiv:{aid}")
time.sleep(2)
manual = []
for doi, r in zip(dois, results[len(arxiv_ids):]):
if r is None:
continue
out = cache_path(args.kb, "doi", doi, ".pdf")
status = fetch_doi_pdf(doi, r, out, args.email)
print(f" {'miss' if status == 'miss' else 'ok '} doi:{doi} ({status})")
if status == "miss" and doi.startswith(BOT_CHECKED_PREFIXES):
manual.append(doi)
time.sleep(2)
if manual:
where = "through SCIBRAIN_PROXY_PREFIX" if PROXY_PREFIX else "on your institution's network"
print(f"\n{len(manual)} chemistry-publisher PDF(s) need a human check: open each {where}, "
"save the PDF, then run import_pdfs.py --kb <kb>")
for doi in manual:
print(f" {PROXY_PREFIX}https://doi.org/{doi}")

if args.download_arxiv_source:
from tex_source import fetch_arxiv_source
Expand Down
112 changes: 112 additions & 0 deletions skills/how-to-download-ref/helpers/import_pdfs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
#!/usr/bin/env python3
"""Import chemistry-publisher PDFs that a person downloaded by hand.

AIP (10.1063), ACS (10.1021) and RSC (10.1039) serve article PDFs only after a
Cloudflare human check, even on a subscribed network, so fetch_metadata.py lists
those misses as links. Open each link, save the PDF anywhere under --from, then:

import_pdfs.py --kb /abs/path/.knowledge [--from ~/Downloads] [--days 14]

Each recent PDF is matched to a pending DOI (a KB DOI under those prefixes with
no PDF yet) by a DOI printed in it, or else by the exact title on its first
pages, and copied to .raw/doi/<safe>.pdf. Files matching no pending DOI, or more
than one, are left alone. Re-render afterwards with render.py --kb <kb>.
"""
from __future__ import annotations

import argparse
import json
import re
import shutil
import subprocess
import sys
import time
from pathlib import Path

from fetch_metadata import BOT_CHECKED_PREFIXES, valid_pdf
from kb_identity import cache_path, normalize

def title_key(value: str) -> str:
return " ".join(re.sub(r"[^a-z0-9]+", " ", value.lower()).split())


def pending(kb: Path) -> dict[str, str]:
"""Chemistry-publisher DOIs in the KB without a PDF, mapped to their titles."""
out = {}
for path in sorted((kb / ".raw" / "doi").glob("*.json")):
try:
meta = json.loads(path.read_text())
doi = normalize("doi", (meta.get("externalIds") or {}).get("DOI") or "")
except (OSError, ValueError, AttributeError):
continue
if doi.startswith(BOT_CHECKED_PREFIXES) and not valid_pdf(cache_path(kb, "doi", doi, ".pdf")):
out[doi] = meta.get("title") or ""
return out


def first_pages_text(pdf: Path) -> bytes:
"""Text of pages 1-2 (pdftotext, else PyMuPDF) plus the raw bytes, which hold XMP metadata."""
text = b""
try:
text = subprocess.run(["pdftotext", "-f", "1", "-l", "2", str(pdf), "-"],
capture_output=True, timeout=60).stdout
except (OSError, subprocess.TimeoutExpired):
try:
import pymupdf # type: ignore
with pymupdf.open(pdf) as doc:
text = "".join(doc[i].get_text() for i in range(min(2, len(doc)))).encode()
except Exception:
pass
return text + b"\n" + pdf.read_bytes()


def match(pdf: Path, wanted: dict[str, str]) -> str | None:
blob = first_pages_text(pdf)
lower = blob.lower()
# The DOI must end there: 10.1063/1.49440 must not match inside 10.1063/1.4944087.
hits = [doi for doi in wanted
if re.search(re.escape(doi.encode()) + rb"(?![a-z0-9]|[./-][a-z0-9])", lower)]
if len(hits) == 1:
return hits[0]
text = title_key(blob.split(b"\n%PDF", 1)[0].decode("utf-8", "replace"))
hits = [doi for doi, title in wanted.items() if len(title_key(title)) > 20 and title_key(title) in text]
return hits[0] if len(hits) == 1 else None


def main() -> int:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--kb", required=True, type=Path, help="Absolute path to <stream>/.knowledge/")
p.add_argument("--from", dest="source", type=Path, default=Path.home() / "Downloads",
help="Folder holding the downloaded PDFs (default ~/Downloads)")
p.add_argument("--days", type=float, default=14, help="Only look at PDFs modified in the last N days")
args = p.parse_args()

wanted = pending(args.kb)
if not wanted:
print("no chemistry-publisher DOI in this KB is waiting for a PDF")
return 0
cutoff = time.time() - args.days * 86400
pdfs = sorted(f for f in args.source.expanduser().rglob("*.pdf")
if f.is_file() and f.stat().st_mtime >= cutoff)
imported = 0
for pdf in pdfs:
if not wanted or not valid_pdf(pdf):
continue
doi = match(pdf, wanted)
if doi is None:
continue
dest = cache_path(args.kb, "doi", doi, ".pdf")
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(pdf, dest)
print(f" ok doi:{doi} <- {pdf}")
del wanted[doi]
imported += 1
for doi in wanted:
print(f" miss doi:{doi} (no matching PDF in {args.source})")
if imported:
print(f"\n{imported} imported; re-render with: render.py --kb {args.kb}")
return 0


if __name__ == "__main__":
sys.exit(main())
2 changes: 1 addition & 1 deletion skills/how-to-download-ref/helpers/verify_bib.py
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ def _load_cache(path: Path) -> dict[str, Any] | None:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return None
if not isinstance(data, dict) or not data.get("title") or not (data.get("paperId") or data.get("metadata_source") == "crossref"):
if not isinstance(data, dict) or not data.get("title") or not (data.get("paperId") or data.get("metadata_source") in ("crossref", "arxiv")):
return None
return data

Expand Down
28 changes: 28 additions & 0 deletions skills/how-to-download-ref/references/acquisition.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,34 @@ names an arXiv preprint (`externalIds.ArXiv`) get the same treatment, into

**Tip:** Set `SEMANTIC_SCHOLAR_API_KEY` in your environment to raise the Semantic Scholar rate limit from ~1 req/s to 100 req/s. Get a free key at https://www.semanticscholar.org/product/api#api-key-form.

## Chemistry publishers behind a human check (AIP, ACS, RSC)

AIP (`10.1063/*`, e.g. JCP), ACS (`10.1021/*`) and RSC (`10.1039/*`) answer
scripts and headless browsers with a Cloudflare "Just a moment..." page even on a
subscribed network, so their PDFs cannot be fetched automatically. APS is
different: it serves PDFs to subscribed IPs directly, and most APS papers have
arXiv preprints. When such a DOI misses, the fetch ends with a block of links:

```
2 chemistry-publisher PDF(s) need a human check: open each on your institution's network, ...
https://doi.org/10.1063/1.4944087
```

Give the user those links (off campus, set `SCIBRAIN_PROXY_PREFIX` to the
library proxy, e.g. `https://proxy.library.nyu.edu/login?url=`, so the links go
through it). Do not try to pass the check yourself. Once the user has saved the
PDFs, import and re-render:

```sh
python3 "$DOWNLOAD_REF_DIR/helpers/import_pdfs.py" --kb "$KB" # --from ~/Downloads --days 14
python3 "$DOWNLOAD_REF_DIR/helpers/render.py" --kb "$KB"
```

`import_pdfs.py` scans recent PDFs under `--from`, matches each to a pending
DOI by a DOI printed in it (or the exact title on its first pages), and copies
it to `$KB/.raw/doi/<safe>.pdf`. Unmatched files are left alone. It uses
`pdftotext` (poppler) or PyMuPDF when available.

## Sci-Hub fallback for paywalled PDFs (script)

If the fetch reports `miss` for any DOI (no open-access PDF and no arXiv preprint),
Expand Down
Loading
Loading