diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abab5c2..1210360 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,13 +32,23 @@ jobs: python -m pip install flit flit install -s - - name: Test with pytest + - name: Test with pytest (non-web) + if: ${{ matrix.python-version != '3.14' }} run: > pytest -v -ra --doctest-modules --cov=pyabc2 --cov-report xml --cov-report term-missing + -m "not web" -n auto + - name: Test with pytest (full, including web) + if: ${{ matrix.python-version == '3.14' }} + run: > + pytest -v -ra + --doctest-modules + --cov=pyabc2 --cov-report xml --cov-report term-missing + -n auto --dist loadgroup + - name: Check type annotations run: | mypy --non-interactive . @@ -47,6 +57,10 @@ jobs: run: | cd docs/examples for f in *.ipynb; do + if [ "$f" = "sources.ipynb" ] && [ "${{ matrix.python-version }}" != "3.14" ]; then + echo "Skipping $f on Python ${{ matrix.python-version }}" + continue + fi jupyter nbconvert --to notebook --execute $f || exit 1 done diff --git a/docs/changes.md b/docs/changes.md index 95d405f..898d77f 100644 --- a/docs/changes.md +++ b/docs/changes.md @@ -1,5 +1,9 @@ # Release notes +## v0.1.4 (unreleased) + +* Updates for Eskin and Bill Black ({pull}`112`) + ## v0.1.3 (2026-04-17) * Improve styling of abcjs containers, pandas dataframes, and ipywidgets diff --git a/docs/examples/sources.ipynb b/docs/examples/sources.ipynb index 5fbb208..3b2eac4 100644 --- a/docs/examples/sources.ipynb +++ b/docs/examples/sources.ipynb @@ -402,7 +402,7 @@ "metadata": {}, "outputs": [], "source": [ - "Tune(df.query(\"group == 'jigs'\").iloc[0].abc)" + "Tune(df.query(\"group == 'Jig'\").iloc[0].abc)" ] }, { diff --git a/pyabc2/sources/bill_black.py b/pyabc2/sources/bill_black.py index 04cf74c..856e80a 100644 --- a/pyabc2/sources/bill_black.py +++ b/pyabc2/sources/bill_black.py @@ -8,60 +8,105 @@ * `requests `__ """ +from __future__ import annotations + +import functools import logging import re from pathlib import Path +from typing import TYPE_CHECKING from pyabc2._util import get_logger as _get_logger +if TYPE_CHECKING: # pragma: no cover + import requests + logger = _get_logger(__name__) HERE = Path(__file__).parent SAVE_TO = HERE / "_bill-black" TXT_FNS = [ - "a-tunes-1.txt", - "b-tunes-1.txt", - "c-tunes-1.txt", - "d-tunes-1.txt", - "e-tunes-1.txt", - "f-tunes-1.txt", - "g-tunes-1.txt", - "h-tunes-1.txt", - "i-tunes-1.txt", - "j-tunes-1.txt", - "k-tunes-1.txt", - "l-tunes-1.txt", - "m-tunes-1.txt", - "n-tunes-1.txt", - "o-tunes-1.txt", - "pq-tunes-1.txt", - "r-tunes-1.txt", - "s-tunes-2.rtf", - "t-tunes-1.txt", - "uv-tunes-1.txt", - "wz-tunes-1.txt", + "a-tunes.txt", + "b-tunes.txt", + "c-tunes.txt", + "d-tunes.txt", + "e-tunes.txt", + "f-tunes.txt", + "g-tunes.txt", + "h-tunes.txt", + "i-tunes.txt", + "j-tunes.txt", + "k-tunes.txt", + "L-tunes.txt", + "m-tunes.txt", + "n-tunes.txt", + "o-tunes.txt", + "pq-tunes.txt", + "r-tunes.txt", + "s-tunes.txt", + "t-tunes.txt", + "uv-tunes.txt", + "wxyz-tunes.txt", ] +@functools.lru_cache(1) +def _get_session() -> requests.Session: + return _build_session() + + +def _build_session() -> requests.Session: + import requests + from requests.adapters import HTTPAdapter + from urllib3.util import Retry + + session = requests.Session() + session.headers.update({"User-Agent": "pyabc2"}) + retries = Retry( + total=5, + backoff_factor=0.5, + backoff_jitter=0.5, + allowed_methods={"GET", "HEAD"}, + status_forcelist=[403, 429, 500, 502, 503, 504], + # Bill Black seems to sporadically return 403 (forbidden) + # possibly to indicate a temporary server issue or throttling/anti-bot + ) + session.mount("https://", HTTPAdapter(max_retries=retries)) + session.mount("http://", HTTPAdapter(max_retries=retries)) + + return session + + def download() -> None: - """Download the alphabetical text files from https://www.capeirish.com/ittl/alltunes/text/ + """Download the alphabetical text files from https://www.capeirish.com/ittl/alltunes/alltunes-text/ and store them in a compressed archive. """ + import threading import zipfile from concurrent.futures import ThreadPoolExecutor - import requests + thread_local = threading.local() + + def get_worker_session() -> requests.Session: + # One Session per worker thread to avoid cross-thread Session sharing. + try: + return thread_local.session + except AttributeError: + session = _build_session() + thread_local.session = session + return session def download_one(url): - r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) + session = get_worker_session() + r = session.get(url, timeout=5) r.raise_for_status() return r.text with ThreadPoolExecutor(max_workers=4) as executor: futures = [] for fn in TXT_FNS: - url = f"https://www.capeirish.com/ittl/alltunes/text/{fn}" + url = f"https://www.capeirish.com/ittl/alltunes/alltunes-text/{fn}" futures.append(executor.submit(download_one, url)) SAVE_TO.mkdir(exist_ok=True) diff --git a/pyabc2/sources/bill_black_tunefolders.py b/pyabc2/sources/bill_black_tunefolders.py index 7a37944..f60dd79 100644 --- a/pyabc2/sources/bill_black_tunefolders.py +++ b/pyabc2/sources/bill_black_tunefolders.py @@ -4,8 +4,8 @@ https://www.capeirish.com/ittl/ As of the 2025-06-14 update, the "tunefolders" method is deprecated. -Bill Black is now using the Eskin ABC Tools (https://www.capeirish.com/ittl/alltunes/html/), -while also posting ABC text files (https://www.capeirish.com/ittl/alltunes/text/), +Bill Black is now using the Eskin ABC Tools (https://www.capeirish.com/ittl/alltunes/alltunes-html/), +while also posting ABC text files (https://www.capeirish.com/ittl/alltunes/alltunes-text/), both split up alphabetically by tune name. Requires: @@ -19,6 +19,7 @@ from pathlib import Path from pyabc2._util import get_logger as _get_logger +from pyabc2.sources.bill_black import _get_session logger = _get_logger(__name__) @@ -209,7 +210,7 @@ def get_collection(key: str) -> Collection: def download(key: str | Iterable[str] | None = None) -> None: import gzip - import requests + session = _get_session() SAVE_TO.mkdir(exist_ok=True) @@ -224,7 +225,7 @@ def download(key: str | Iterable[str] | None = None) -> None: for url in collection.abc_urls: p = collection.url_to_file(url) logger.info(f"Downloading {url} to {p.relative_to(HERE).as_posix()}") - r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.get(url, timeout=5) r.raise_for_status() # Extract filename from URL and append .gz diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index c2fe38b..effda3a 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -7,8 +7,12 @@ * `requests `__ """ +from __future__ import annotations + +import functools import json import re +from collections import defaultdict from pathlib import Path from typing import TYPE_CHECKING, Literal, NamedTuple from urllib.parse import parse_qs, urlsplit @@ -19,6 +23,7 @@ if TYPE_CHECKING: # pragma: no cover import pandas + import requests logger = _get_logger(__name__) @@ -30,14 +35,16 @@ _CCE_SD = "https://michaeleskin.com/cce_sd" _TUNEBOOK_KEY_TO_URL = { # https://michaeleskin.com/tunebooks.html#websites_irish - "kss": f"{_TBWS}/king_street_sessions_tunebook_17Jan2025.html", - "oflaherty_2025": f"{_TBWS}/oflahertys_2025_retreat_tunes_final.html", - "carp": f"{_TBWS}/carp_celtic_jam_tunebook_17Jan2025.html", + "kss": f"{_TBWS}/king-street-session-tunebook-25jun2026.html", + "oflaherty_2025": f"{_TBWS}/oflaherty-2025-retreat-tunes-played-slowly.html", + "oflaherty_2026": f"{_TBWS}/oflaherty-2026-retreat-tunes-played-slowly.html", + "carp": f"{_TBWS}/carp_celtic_jam_tunebook_29jun2026.html", "hardy_2024": f"{_TBWS}/paul_hardy_2024_8feb2025.html", - "hardy_2025": f"{_TBWS}/paul_hardy_2025_12aug2025.html", - "cce_dublin_2001": f"{_CCE_SD}/cce_dublin_2001_tunebook_17Jan2025.html", + "hardy_2025": f"{_TBWS}/paul_hardy_2025_29jun2026.html", + "hardy_2026": f"{_TBWS}/paul_hardy_2026_1aug2026.html", + "cce_dublin_2001": f"{_CCE_SD}/cce_dublin_2001_tunebook_29jun2026.html", "cce_san_diego_jan2025": f"{_CCE_SD}/cce_san_diego_tunes_31jan2025.html", - "cce_san_diego_nov2025": f"{_CCE_SD}/cce_san_diego_tunes_10nov2025.html", + "cce_san_diego_jun2026": f"{_TBWS}/comhaltas-san-diego-tunebook-24jun2026.html", # https://michaeleskin.com/tunebooks.html#websites_18th_century_collections "playford1": f"{_TBWS}/playford_1_partington_17jan2025.html", "playford2": f"{_TBWS}/playford_2_partington_17jan2025.html", @@ -50,13 +57,107 @@ # Definitive versions _TUNEBOOK_ALIAS = { - "cce_san_diego": "cce_san_diego_nov2025", + "cce_san_diego": "cce_san_diego_jun2026", } for _alias, _target in _TUNEBOOK_ALIAS.items(): _TUNEBOOK_KEY_TO_URL[_alias] = _TUNEBOOK_KEY_TO_URL[_target] _URL_NETLOCS = {"michaeleskin.com", "www.michaeleskin.com"} +_TUNE_TYPE_PATTERN = r"[A-Za-z][A-Za-z &'/-]*" +_TUNE_TYPE_RE = re.compile(rf"^{_TUNE_TYPE_PATTERN}$") + +_GROUP_ALPHA_SPLIT_RE = re.compile( + rf"^(?P{_TUNE_TYPE_PATTERN}?) " + r"(?P[A-Za-z](?:-[A-Za-z])?)" + r"(?: (?P[0-9]+-[0-9]+))?$" +) + +_GROUP_ALPHA_ONLY_RE = re.compile( + r"^(?P[A-Za-z](?:-[A-Za-z])?)(?: (?P[0-9]+-[0-9]+))?$" +) + + +def _is_valid_alpha_split_group_name(s: str, /) -> bool: + """Return whether `s` looks like an Eskin type + alpha split group label.""" + + return _GROUP_ALPHA_SPLIT_RE.fullmatch(s) is not None + + +def _is_valid_alpha_only_group_name(s: str, /) -> bool: + """Return whether `s` looks like an Eskin alpha-only split group label.""" + + return _GROUP_ALPHA_ONLY_RE.fullmatch(s) is not None + + +def _is_valid_tune_type_group_name(s: str, /) -> bool: + """Return whether `s` looks like a standalone tune type group label.""" + + return _TUNE_TYPE_RE.fullmatch(s) is not None + + +def _normalize_tune_type(s: str, /) -> str: + """Normalize tune type labels parsed from Eskin group names.""" + + if s in {"Highland Schottish", "Highland Scottische"}: + return "Highland Schottische" + + if s == "Set dance": + return "Set Dance" + + m = re.fullmatch(r"(.+?) or ([A-Za-z][A-Za-z &'/-]*)", s) + if m is not None: + first, second = m.groups() + return f"{first} or {second.title()}" + + return s + + +def _normalize_group_name(s: str, /) -> str: + """Normalize parsed Eskin group labels to expected style.""" + + s = s.strip() + s = s.replace(" · ", " ") + s = s.replace("–", "-") + + if _is_valid_alpha_split_group_name(s): + # Upstream alpha/number splits can churn; keep only tune type. + m = _GROUP_ALPHA_SPLIT_RE.fullmatch(s) + assert m is not None # for typing + return _normalize_tune_type(m.group("tune_type")) + + if _is_valid_alpha_only_group_name(s): + # Some tunebooks split by alpha range only; collapse to one generic group. + return "tunes" + + if _is_valid_tune_type_group_name(s): + return _normalize_tune_type(s) + + return s + + +@functools.lru_cache(1) +def _get_session() -> requests.Session: + import requests + from requests.adapters import HTTPAdapter + from urllib3.util import Retry + + session = requests.Session() + session.headers.update({"User-Agent": "pyabc2"}) + retries = Retry( + total=5, + backoff_factor=0.5, + backoff_jitter=0.5, + allowed_methods={"GET", "HEAD"}, + status_forcelist=[415, 429, 500, 502, 503, 504], + # Eskin seems to sporadically return 415 (unsupported media type) + # possibly to indicate a temporary server issue or throttling/anti-bot + ) + session.mount("https://", HTTPAdapter(max_retries=retries)) + session.mount("http://", HTTPAdapter(max_retries=retries)) + + return session + def _deflate(s: str, /) -> str: """Use deflate (zlib) to compress and base64-encode `s`.""" @@ -233,22 +334,11 @@ def get_tunebook_info(key: str) -> EskinTunebookInfo: ) -def _download_data(key: str): - """Extract and save the tune data from the tunebook webpage as JSON.""" - import gzip - - import requests - - tb_info = get_tunebook_info(key) - - r = requests.get(tb_info.url, timeout=5) - r.raise_for_status() - html = r.text - +def _extract_data_from_html_2025(html: str, *, key: str): # First find the tune type options by searching for 'tunes = type;' types = sorted(set(re.findall(r"tunes = (.*?);", html))) if types: - pass + pass # pragma: no cover elif "const tunes=[" in html: # no types, just one list of tunes types = ["tunes"] else: @@ -258,13 +348,13 @@ def _download_data(key: str): all_data = {} for type_ in types: m = re.search(rf"const {type_}=\[(.*?)\];", html, flags=re.DOTALL) - if m is None: + if m is None: # pragma: no cover raise RuntimeError(f"Unable to find data for type {type_!r}") s_data = "[" + m.group(1) + "]" try: data = json.loads(s_data) - except json.JSONDecodeError as e: + except json.JSONDecodeError as e: # pragma: no cover w = 25 a = max(0, e.pos - w) b = min(len(s_data), e.pos + w) @@ -281,9 +371,115 @@ def _download_data(key: str): all_data[type_] = data + return all_data + + +def _extract_data_from_html_2026(html: str, *, key: str): + # The ABCs are in ', + html, + flags=re.DOTALL | re.MULTILINE, + ): + tune_id, abc_raw = m.groups() + + abc_lines = [] + for line in abc_raw.splitlines(): + line = line.strip() + if not line or line.startswith("%"): + continue + abc_lines.append(line) + + tunes.append({"id": tune_id, "abc": "\n".join(abc_lines)}) + + # Group tunes + all_data = defaultdict(list) + for tune in tunes: + group_id = group_assignments.get(tune["id"]) + if group_id is None: # pragma: no cover + logger.warning(f"No group assignment found for tune {tune['id']}") + continue + group_name = group_names.get(group_id, f"Unknown group {group_id}") + all_data[group_name].append(tune) + + return all_data + + +def _download_data(key: str): + """Extract and save the tune data from the tunebook webpage as JSON.""" + import gzip + + session = _get_session() + + tb_info = get_tunebook_info(key) + + r = session.get(tb_info.url, timeout=5) + r.raise_for_status() + html = r.content.decode("utf-8") + + try: + data = _extract_data_from_html_2025(html, key=key) + except RuntimeError as e: + if str(e) == "Unable to detect tune types": + # We can be pretty sure it's the new format + logger.debug("Tune groups array not detected, assuming new format.") + data = _extract_data_from_html_2026(html, key=key) + else: + raise + SAVE_TO.mkdir(exist_ok=True) with gzip.open(tb_info.path, "wt") as f: - json.dump(all_data, f, indent=2) + json.dump(data, f, indent=2) def _load_data(key: str): @@ -294,7 +490,7 @@ def _load_data(key: str): return json.load(f) -def load_meta(key: str, *, redownload: bool = False) -> "pandas.DataFrame": +def load_meta(key: str, *, redownload: bool = False) -> pandas.DataFrame: """Load the tunebook data, no parsing. Parameters @@ -314,11 +510,11 @@ def load_meta(key: str, *, redownload: bool = False) -> "pandas.DataFrame": - CCE Dublin 2001 * - ``cce_san_diego`` - CCE San Diego - * - ``hardy_{2024,2025}`` + * - ``hardy_{2024,2025,2026}`` - Paul Hardy's Session Tunebook * - ``kss`` - King Street Sessions Tunebook - * - ``oflaherty_2025`` + * - ``oflaherty_{2025,2026}`` - O'Flaherty's Retreat Tunes * - ``playford{1,2,3}`` - Playford vols. 1--3 diff --git a/pyproject.toml b/pyproject.toml index 76624c2..f51326a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ sources = [ "numpy ==2.*", "pandas ==2.*", "requests ==2.*", + "urllib3 ==2.*", ] abcjs-widget = [ "anywidget", @@ -62,7 +63,10 @@ profile = "black" line_length = 100 [tool.pytest.ini_options] -markers = ["slow"] +markers = [ + "slow", + "web: tests that hit external websites", +] [tool.mypy] exclude = ["^venv"] diff --git a/tests/test_sources.py b/tests/test_sources.py index 425c8c4..b5c7b13 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -44,6 +44,8 @@ def test_example_random(): assert type(tune) is Tune +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_tune_type_file_prefix(): norbeck._maybe_download() all_fps = list(norbeck.SAVE_TO.glob("*.abc")) @@ -58,6 +60,8 @@ def test_norbeck_tune_type_file_prefix(): assert set(all_fps) == all_fps_type_set +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_x_unique(): # X value should be unique within tune type norbeck._maybe_download() @@ -74,6 +78,8 @@ def test_norbeck_x_unique(): assert len(set(xs)) == len(xs), "X unique" +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_x_vals(): # Make sure they match with what file says norbeck._maybe_download() @@ -100,6 +106,8 @@ def test_norbeck_x_vals(): @pytest.mark.slow +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_load(): # NOTE: downloads files if not already present @@ -131,6 +139,8 @@ def test_norbeck_load(): norbeck.load("asdf") +@pytest.mark.web +@pytest.mark.xdist_group("the_session") @pytest.mark.parametrize( "url,title,key,type", [ @@ -149,11 +159,14 @@ def test_the_session_load_url(url, title, key, type): assert tune.header["reference number"] == "1" +@pytest.mark.xdist_group("the_session") def test_the_session_url_check(): with pytest.raises(AssertionError): the_session.load_url("https://www.google.com") +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_load_archive_threaded(): # NOTE: downloads file if not already present with pytest.warns(UserWarning, match=r"The Session tune\(s\) failed to load"): @@ -163,12 +176,15 @@ def test_the_session_load_archive_threaded(): assert tunes1 == tunes2 +@pytest.mark.xdist_group("the_session") def test_the_session_download_invalid(): with pytest.raises(ValueError): _ = the_session.download("asdf") @pytest.mark.slow +@pytest.mark.web +@pytest.mark.xdist_group("the_session") @pytest.mark.parametrize( "which", ["aliases", "events", "recordings", "sessions", "sets", "tune_popularity", "tunes"] ) @@ -203,6 +219,7 @@ def test_the_session_load_meta(which): # in df3, `pd.Float64Dtype()` +@pytest.mark.xdist_group("the_session") def test_the_session_load_meta_invalid(): with pytest.raises(ValueError): _ = the_session.load_meta("asdf") @@ -211,6 +228,7 @@ def test_the_session_load_meta_invalid(): _ = the_session.load_meta("sessions", format="asdf") +@pytest.mark.xdist_group("the_session") def test_the_session_load_meta_doc_consistency(): s_options = ", ".join(repr(x) for x in sorted(the_session._META_ALLOWED)) expected_line = f"which : {{{s_options}}}" @@ -246,12 +264,16 @@ def test_int_downcast(): assert s3.dtype == expected_dtype_ext +@pytest.mark.web +@pytest.mark.xdist_group("the_session") @pytest.mark.parametrize("netloc", sorted(the_session._URL_NETLOCS)) def test_load_url_the_session(netloc): tune = load_url(f"https://{netloc}/tunes/10000") assert tune.title == "Brian Quinn's" +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") @pytest.mark.parametrize("netloc", sorted(norbeck._URL_NETLOCS)) def test_load_url_norbeck(netloc): import requests @@ -265,6 +287,7 @@ def test_load_url_norbeck(netloc): assert tune.title == "For The Love Of Music" +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("netloc", sorted(eskin._URL_NETLOCS)) @pytest.mark.parametrize("param", list(ESKIN_COMPRESSED_ABC_DATA)) def test_load_url_eskin(netloc, param): @@ -283,14 +306,17 @@ def test_load_url_invalid_domain(): _ = load_url("https://www.google.com") +@pytest.mark.web +@pytest.mark.xdist_group("eskin") +@pytest.mark.xfail(reason="now it does 404", strict=False) def test_eskin_tunebook_bad_url_redirects(): - import requests + session = eskin._get_session() # Bad URL (2025 -> 3025) # Redirects to the home page. # Nothing in `r.history`. `allow_redirects=False` has no impact. url = "https://michaeleskin.com/cce_sd/cce_san_diego_tunes_10nov3025.html" - r = requests.head(url, timeout=5) + r = session.head(url, timeout=5) r.raise_for_status() assert r.status_code == 302 @@ -299,12 +325,14 @@ def test_eskin_tunebook_bad_url_redirects(): assert r.is_redirect +@pytest.mark.web +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("key", eskin._TUNEBOOK_KEY_TO_URL) def test_eskin_tunebook_url_exist(key): - import requests + session = eskin._get_session() url = eskin._TUNEBOOK_KEY_TO_URL[key] - r = requests.head(url, timeout=5) + r = session.head(url, timeout=5) r.raise_for_status() # Bad URLs seem to just redirect to his homepage, # so we need to check the final URL @@ -316,11 +344,13 @@ def test_eskin_tunebook_url_exist(key): raise ValueError(f"{key!r} URL {url} redirects to homepage") +@pytest.mark.web +@pytest.mark.xdist_group("eskin") def test_eskin_tunebook_url_current(): - import requests + session = eskin._get_session() url = "https://michaeleskin.com/tunebooks.html" - r = requests.get(url, timeout=5) + r = session.get(url, timeout=5) r.raise_for_status() if ( r.status_code == 302 @@ -332,6 +362,7 @@ def test_eskin_tunebook_url_current(): old_keys = { "cce_san_diego_jan2025", "hardy_2024", + "hardy_2025", } for key, tb_url in eskin._TUNEBOOK_KEY_TO_URL.items(): m = re.search(rf'href=["\']({tb_url})["\']', html) @@ -342,11 +373,16 @@ def test_eskin_tunebook_url_current(): raise ValueError(f"Could not find link for tunebook {key!r} in tunebooks page.") +@pytest.mark.web +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("key", eskin._TUNEBOOK_KEY_TO_URL) def test_eskin_tunebook_data_load(key): df = eskin.load_meta(key) + assert len(df) >= 20 + tune_group_keys = { + # Old KSS "airs_songs", "hornpipes", "jigs", @@ -361,14 +397,44 @@ def test_eskin_tunebook_data_load(key): "slipjigs", "strathspeys", "waltzes", + # New KSS + "Air", + "Barndance", + "Bourree", + "Fling", + "Highland Schottische", + "Hornpipe", + "Jig", + "Jig or March", + "Long Dance", + "March", + "Mazurka", + "Other", + "Pipe Reel", + "Polka", + "Polka or March", + "Polka or Reel", + "Quadrille", + "Reel", + "Schottische", + "Set Dance", + "Single Jig", + "Slide", + "Slip Jig", + "Slow Air", + "Song", + "Strathspey", + "Waltz", } + unique_groups = set(df.group.unique()) if key in {"kss"}: - assert set(df.group.unique()) <= tune_group_keys + assert unique_groups <= tune_group_keys else: - assert df.group.unique().tolist() == ["tunes"] + assert unique_groups == {"tunes"} +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_parsing(): # From https://michaeleskin.com/cce_sd/cce_san_diego_tunes_10nov2025.html url = "https://michaeleskin.com/abctools/abctools.html?lzw=BoLgjAUApFAuCWsA2BTAZgewHawAQAUBDJQhLDXMADmigGcBXAIwWXWzyJLIrAGZa8LJkw4CxUkN4CYAB0IAnWHVGcJPSjLgoAtrIyrx3KZtqwUAD1iGuk8qYAqIBwAsUuAIJMmKAJ64IABlwAHoaAFkQABYQqIgARRAAJliAXgBOAAYIACUQHJQUJGg6AHchAHNcTIA6SABpEABxaEImAGMAKzoAfToMBiwAE0M0UiYMX1pwgEkAERncWQUMCoVCHWrp+cWmQjo6ZdWtmFmF3HaXDAUho6rs053cPYOANwwkXAA2OMfzy+uQ3enx+rSGQx6xCQPVkJF8e3aAGsekghIi6BAAEQeHSYzx8XAAIU8SVwTQAorgAD6eDxNDy4TFNPGE8HEmnY3G0jzEunkgBi1MZzLJTXpRLZ1KxOLxHlJPJJZMpNI8dIZTJZko5MsVCr5go5IrF4tZQyqVKp0q5KAqttwhFJeyFGtwFTaVUIFRQQ2dOptdodrsIzuZTDdaFd7iGpMtnLx-o9FTDIcxnuTnu9vutto9pLdKbDhAjXqG7IAukA&format=noten&ssp=10&name=The_Abbey&play=1" @@ -387,6 +453,7 @@ def test_eskin_abc_url_parsing(): assert sum(line.startswith(r"%%") for line in abc_no_rm.splitlines()) > 0 +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_missing_param(): url = "https://michaeleskin.com/abctools/abctools.html?" with pytest.raises( @@ -396,18 +463,21 @@ def test_eskin_abc_url_missing_param(): _ = eskin.abctools_url_to_abc(url) +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_bad_param(): url = "https://michaeleskin.com/abctools/abctools.html?lzw=hi" with pytest.raises(RuntimeError, match="Failed to decompress LZString data"): _ = eskin.abctools_url_to_abc(url) +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_bad_param_def(): url = "https://michaeleskin.com/abctools/abctools.html?def=hi" with pytest.raises(RuntimeError, match="Failed to decompress deflate data"): _ = eskin.abctools_url_to_abc(url) +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_bad(caplog): url = "https://michaeleski.com/deftools/abctools.html?lzw=BoLgjAUApFAuCWsA2BTAZgewHawAQAUBDJQhLDXMADmigGcBXAIwWXWzyJLIrAGZa8LJkw4CxUkN4CYAB0IAnWHVGcJPSjLgoAtrIyrx3KZtqwUAD1iGuk8qYAqIBwAsUuAIJMmKAJ64IABlwAHoaAFkQABYQqIgARRAAJliAXgBOAAYIACUQHJQUJGg6AHchAHNcTIA6SABpEABxaEImAGMAKzoAfToMBiwAE0M0UiYMX1pwgEkAERncWQUMCoVCHWrp+cWmQjo6ZdWtmFmF3HaXDAUho6rs053cPYOANwwkXAA2OMfzy+uQ3enx+rSGQx6xCQPVkJF8e3aAGsekghIi6BAAEQeHSYzx8XAAIU8SVwTQAorgAD6eDxNDy4TFNPGE8HEmnY3G0jzEunkgBi1MZzLJTXpRLZ1KxOLxHlJPJJZMpNI8dIZTJZko5MsVCr5go5IrF4tZQyqVKp0q5KAqttwhFJeyFGtwFTaVUIFRQQ2dOptdodrsIzuZTDdaFd7iGpMtnLx-o9FTDIcxnuTnu9vutto9pLdKbDhAjXqG7IAukA&format=noten&ssp=10&name=The_Abbey&play=1" with caplog.at_level("DEBUG"): @@ -419,6 +489,8 @@ def test_eskin_abc_url_bad(caplog): ] +@pytest.mark.web +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("use_lzw", [True, False]) def test_eskin_abc_url_creation(use_lzw): import requests @@ -435,11 +507,13 @@ def test_eskin_abc_url_creation(use_lzw): raise ValueError(f"URL {url} redirects to homepage") +@pytest.mark.xdist_group("eskin") def test_eskin_invalid_tunebook_key(): with pytest.raises(ValueError, match="Unknown Eskin tunebook key: 'asdf'"): _ = eskin.get_tunebook_info("asdf") +@pytest.mark.xdist_group("eskin") def test_eskin_inflate_invalid_length(): s = "eJyFjbEKwkAQRPv9iv2DQyvd7jbGK0wQJIVtktucJ4FIghZyH-abcdefg" with pytest.raises( @@ -449,64 +523,167 @@ def test_eskin_inflate_invalid_length(): _ = eskin._inflate(s) +@pytest.mark.xdist_group("eskin") def test_eskin_inflate_pad_3(): s = "abc" assert eskin._inflate(eskin._deflate(s)) == s +@pytest.mark.xdist_group("eskin") +def test_eskin_normalize_group_name(): + assert eskin._normalize_group_name("Reel M · 1–50") == "Reel" + assert eskin._normalize_group_name("Reel A-E") == "Reel" + assert eskin._normalize_group_name("Highland Schottish") == "Highland Schottische" + assert eskin._normalize_group_name("Highland Scottische") == "Highland Schottische" + assert eskin._normalize_group_name("Set dance") == "Set Dance" + assert eskin._normalize_group_name("Jig or march") == "Jig or March" + assert eskin._normalize_group_name("Polka or march") == "Polka or March" + + +@pytest.mark.parametrize( + "s", + [ + "Reel M 51-56", + "Reel C", + "Jig C-E", + "Reel L", + "Reel T-Y", + "Hornpipe O-W", + "Reel S", + "Reel G-I", + "Reel J-K", + "Reel M 1-50", + "Jig I-L", + "Reel N-P", + "Reel D-F", + "Reel A-B", + "Jig A-B", + "Jig M-R", + "Jig F-H", + "Jig S-Y", + "Reel R", + "Hornpipe A-N", + ], +) +def test_eskin_alpha_split_group_validator(s): + assert eskin._is_valid_alpha_split_group_name(s) + + +@pytest.mark.parametrize( + "s, expected", + [ + ("Reel M 51-56", "Reel"), + ("Jig C-E", "Jig"), + ("Hornpipe O-W", "Hornpipe"), + ], +) +def test_eskin_alpha_split_group_normalization(s, expected): + assert eskin._normalize_group_name(s) == expected + + +@pytest.mark.parametrize( + "s", + [ + "T-V", + "A", + "D-E", + "J-L", + "S 51-80", + "C", + "N-P", + "W", + "M 51-54", + "F-G", + "M 1-50", + "B 1-50", + "S 1-50", + "B 51-51", + "R", + "H-I", + ], +) +def test_eskin_alpha_only_group_validator(s): + assert eskin._is_valid_alpha_only_group_name(s) + + +@pytest.mark.parametrize( + "s", + [ + "T-V", + "S 51-80", + "M 1-50", + ], +) +def test_eskin_alpha_only_group_normalization(s): + assert eskin._normalize_group_name(s) == "tunes" + + +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") def test_bill_black_https(): - import requests + session = bill_black._get_session() url = "http://www.capeirish.com/ittl/tunefolders/" url_https = url.replace("http://", "https://") - r = requests.head(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.head(url, timeout=5) r.raise_for_status() assert "Strict-Transport-Security" not in r.headers - r = requests.head(url_https, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.head(url_https, timeout=5) r.raise_for_status() assert "Strict-Transport-Security" in r.headers @pytest.mark.xfail(reason="Bill Black tunefolders are currently in flux", strict=False) +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") @pytest.mark.parametrize("key", list(bill_black_tunefolders._KEY_TO_COLLECTION)) def test_bill_black_tunefolders(key): lst = bill_black_tunefolders.load_meta(key, redownload=True) assert len(lst) > 0 +@pytest.mark.xdist_group("bill_black") def test_bill_black_tunefolders_invalid_key(): with pytest.raises(ValueError, match="Unknown collection key: 'asdf'"): _ = bill_black_tunefolders.get_collection("asdf") +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") def test_bill_black_text_fns(): - import requests + session = bill_black._get_session() - url = "http://www.capeirish.com/ittl/alltunes/text/" - r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) + url = "http://www.capeirish.com/ittl/alltunes/alltunes-text/" + r = session.get(url, timeout=5) r.raise_for_status() - fns_web = sorted(re.findall(r'href=["\']([a-z0-9\-]+\.(?:txt|rtf))["\']', r.text)) - if "s-tunes-1.txt" in fns_web: - # We're using s-tunes-2, not both - fns_web.remove("s-tunes-1.txt") + fns_web = sorted( + re.findall(r'href=["\']([a-zA-Z0-9\-]+\.(?:txt|rtf))["\']', r.text), + key=lambda s: s.lower(), + ) assert bill_black.TXT_FNS == fns_web +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") def test_bill_black_load(): lst = bill_black.load_meta() assert len(lst) > 0 assert lst[0].startswith("X:") +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_tune_collections(): df = the_session.get_tune_collections(1) # Cooley's assert not df.empty +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_member_set(): tunes = the_session.get_member_set(65013, 106212) assert len(tunes) == 3 @@ -516,6 +693,8 @@ def test_the_session_get_member_set(): assert d["setting_id"] == 31341 +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_member_sets(): sets = the_session.get_member_sets(65013) assert len(sets) >= 1 @@ -525,6 +704,8 @@ def test_the_session_get_member_sets(): assert d["setting_id"] == 31341 +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_member_sets_multipage(): sets = the_session.get_member_sets(1, pages=3, size=2, max_threads=2, orderby="oldest") assert len(sets) == 6 @@ -534,6 +715,7 @@ def test_the_session_get_member_sets_multipage(): assert d["tune_id"] == 560 +@pytest.mark.xdist_group("the_session") def test_the_session_consume_validation(): f = the_session.get_member_sets @@ -547,11 +729,15 @@ def test_the_session_consume_validation(): _ = f(1, max_threads=0) +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_consume_auto_leading_slash(): (d,) = the_session._consume("tunes/22878") assert d["name"] == "Jack Farrell's" +@pytest.mark.web +@pytest.mark.xdist_group("hardy") @pytest.mark.parametrize("key", list(hardy._TUNEBOOK_KEY_TO_URL)) def test_hardy_load_meta(key): abcs = hardy.load_meta(key) @@ -563,6 +749,8 @@ def test_hardy_load_meta(key): assert "\n\n" not in abc, "no empty lines within a tune block" +@pytest.mark.web +@pytest.mark.xdist_group("hardy") def test_hardy_load_meta_remove_prefs(): # Default: no % lines key = "session" @@ -576,16 +764,20 @@ def test_hardy_load_meta_remove_prefs(): assert sum(line.lstrip().startswith("%") for abc in abcs_raw for line in abc.splitlines()) > 1 +@pytest.mark.xdist_group("hardy") def test_hardy_bad_key(): with pytest.raises(ValueError, match="Unknown Hardy tunebook key"): _ = hardy.load_meta("asdf") +@pytest.mark.xdist_group("hardy") def test_hardy_download_bad_key(): with pytest.raises(ValueError, match="Unknown Hardy tunebook key"): _ = hardy.download("asdf") +@pytest.mark.web +@pytest.mark.xdist_group("hardy") def test_hardy_annex_is_latest(): """Confirm the hardcoded 'annex' URL points to the current (non-superseded) annex file.""" import requests