From a6c1e9519e6e4ac93d9a560ae24819144101f4e2 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 10:21:53 -0500 Subject: [PATCH 01/22] Update Eskin URLs but note the new tunebook websites have the ABCs stored differently (in script tags) --- pyabc2/sources/eskin.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index c2fe38b..9c379ea 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -30,14 +30,16 @@ _CCE_SD = "https://michaeleskin.com/cce_sd" _TUNEBOOK_KEY_TO_URL = { # https://michaeleskin.com/tunebooks.html#websites_irish - "kss": f"{_TBWS}/king_street_sessions_tunebook_17Jan2025.html", - "oflaherty_2025": f"{_TBWS}/oflahertys_2025_retreat_tunes_final.html", - "carp": f"{_TBWS}/carp_celtic_jam_tunebook_17Jan2025.html", + "kss": f"{_TBWS}/king-street-session-tunebook-25jun2026.html", + "oflaherty_2025": f"{_TBWS}/oflaherty-2025-retreat-tunes-played-slowly.html", + "oflaherty_2026": f"{_TBWS}/oflaherty-2026-retreat-tunes-played-slowly.html", + "carp": f"{_TBWS}/carp_celtic_jam_tunebook_29jun2026.html", "hardy_2024": f"{_TBWS}/paul_hardy_2024_8feb2025.html", - "hardy_2025": f"{_TBWS}/paul_hardy_2025_12aug2025.html", - "cce_dublin_2001": f"{_CCE_SD}/cce_dublin_2001_tunebook_17Jan2025.html", + "hardy_2025": f"{_TBWS}/paul_hardy_2025_29jun2026.html", + "cce_dublin_2001": f"{_CCE_SD}/cce_dublin_2001_tunebook_29jun2026.html", "cce_san_diego_jan2025": f"{_CCE_SD}/cce_san_diego_tunes_31jan2025.html", "cce_san_diego_nov2025": f"{_CCE_SD}/cce_san_diego_tunes_10nov2025.html", + "cce_san_diego_jun2026": f"{_TBWS}/comhaltas-san-diego-tunebook-24jun2026.html", # https://michaeleskin.com/tunebooks.html#websites_18th_century_collections "playford1": f"{_TBWS}/playford_1_partington_17jan2025.html", "playford2": f"{_TBWS}/playford_2_partington_17jan2025.html", @@ -50,7 +52,7 @@ # Definitive versions _TUNEBOOK_ALIAS = { - "cce_san_diego": "cce_san_diego_nov2025", + "cce_san_diego": "cce_san_diego_jun2026", } for _alias, _target in _TUNEBOOK_ALIAS.items(): _TUNEBOOK_KEY_TO_URL[_alias] = _TUNEBOOK_KEY_TO_URL[_target] @@ -318,7 +320,7 @@ def load_meta(key: str, *, redownload: bool = False) -> "pandas.DataFrame": - Paul Hardy's Session Tunebook * - ``kss`` - King Street Sessions Tunebook - * - ``oflaherty_2025`` + * - ``oflaherty_{2025,2026}`` - O'Flaherty's Retreat Tunes * - ``playford{1,2,3}`` - Playford vols. 1--3 From e684125c0a9fd02471594287515b46654372a18e Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 11:45:39 -0500 Subject: [PATCH 02/22] Extract tunes and groups from new Eskin tbws --- pyabc2/sources/eskin.py | 117 +++++++++++++++++++++++++++++++++++----- tests/test_sources.py | 46 +++++++++++++++- 2 files changed, 147 insertions(+), 16 deletions(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index 9c379ea..44d871c 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -9,6 +9,7 @@ import json import re +from collections import defaultdict from pathlib import Path from typing import TYPE_CHECKING, Literal, NamedTuple from urllib.parse import parse_qs, urlsplit @@ -38,7 +39,6 @@ "hardy_2025": f"{_TBWS}/paul_hardy_2025_29jun2026.html", "cce_dublin_2001": f"{_CCE_SD}/cce_dublin_2001_tunebook_29jun2026.html", "cce_san_diego_jan2025": f"{_CCE_SD}/cce_san_diego_tunes_31jan2025.html", - "cce_san_diego_nov2025": f"{_CCE_SD}/cce_san_diego_tunes_10nov2025.html", "cce_san_diego_jun2026": f"{_TBWS}/comhaltas-san-diego-tunebook-24jun2026.html", # https://michaeleskin.com/tunebooks.html#websites_18th_century_collections "playford1": f"{_TBWS}/playford_1_partington_17jan2025.html", @@ -235,18 +235,7 @@ def get_tunebook_info(key: str) -> EskinTunebookInfo: ) -def _download_data(key: str): - """Extract and save the tune data from the tunebook webpage as JSON.""" - import gzip - - import requests - - tb_info = get_tunebook_info(key) - - r = requests.get(tb_info.url, timeout=5) - r.raise_for_status() - html = r.text - +def _extract_data_from_html_2025(html: str, *, key: str): # First find the tune type options by searching for 'tunes = type;' types = sorted(set(re.findall(r"tunes = (.*?);", html))) if types: @@ -283,9 +272,109 @@ def _download_data(key: str): all_data[type_] = data + return all_data + + +def _extract_data_from_html_2026(html: str, *, key: str): + # The ABCs are in ', + html, + flags=re.DOTALL | re.MULTILINE, + ): + tune_id, abc_raw = m.groups() + + abc_lines = [] + for line in abc_raw.splitlines(): + line = line.strip() + if line.startswith("%"): + continue + abc_lines.append(line) + + tunes.append({"id": tune_id, "abc": "\n".join(abc_lines)}) + + # Group tunes + all_data = defaultdict(list) + for tune in tunes: + group_id = group_assignments.get(tune["id"]) + if group_id is None: + logger.warning(f"No group assignment found for tune {tune['id']}") + continue + group_name = group_names.get(group_id, f"Unknown group {group_id}") + all_data[group_name].append(tune) + + return all_data + + +def _download_data(key: str): + """Extract and save the tune data from the tunebook webpage as JSON.""" + import gzip + + import requests + + tb_info = get_tunebook_info(key) + + r = requests.get(tb_info.url, timeout=5) + r.raise_for_status() + html = r.text + + try: + data = _extract_data_from_html_2025(html, key=key) + except RuntimeError: + data = _extract_data_from_html_2026(html, key=key) + SAVE_TO.mkdir(exist_ok=True) with gzip.open(tb_info.path, "wt") as f: - json.dump(all_data, f, indent=2) + json.dump(data, f, indent=2) def _load_data(key: str): diff --git a/tests/test_sources.py b/tests/test_sources.py index 425c8c4..f4e9e6d 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -346,7 +346,10 @@ def test_eskin_tunebook_url_current(): def test_eskin_tunebook_data_load(key): df = eskin.load_meta(key) + assert len(df) >= 20 + tune_group_keys = { + # Old KSS "airs_songs", "hornpipes", "jigs", @@ -361,12 +364,51 @@ def test_eskin_tunebook_data_load(key): "slipjigs", "strathspeys", "waltzes", + # New KSS + "Long Dance", + "Barndance", + "Reel O-R", + "Reel F-K", + "Set Dance", + "Reel A-E", + "Single Jig", + "Set dance", + "Jig or march", + "Quadrille", + "Highland Schottish", # XXX: "Highland Schottische" + "Pipe Reel", + "Strathspey", + "Hornpipe", + "Reel S-Y", + "Reel L-N", + "March", + "Waltz", + "Slip Jig", + "Jig A-G", + "Slide", + "Highland Scottische", # XXX: "Highland Schottische" + "Other", + "Polka or Reel", + "Bourree", + "Schottische", + "Air", + "Polka or march", + "Song", + "Fling", + "Slow Air", + "Mazurka", + "Jig Q-Y", + "Polka", + "Jig H-P", } + unique_groups = set(df.group.unique()) if key in {"kss"}: - assert set(df.group.unique()) <= tune_group_keys + assert unique_groups <= tune_group_keys else: - assert df.group.unique().tolist() == ["tunes"] + assert unique_groups == {"tunes"} or all( + re.fullmatch(r"^[A-Z\-]+$", g) is not None for g in unique_groups + ) def test_eskin_abc_url_parsing(): From 4b3305fc8babfb96f135216b63fb2678cbe1d186 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 11:48:51 -0500 Subject: [PATCH 03/22] Normalize "Highland Schottische" --- pyabc2/sources/eskin.py | 4 ++++ tests/test_sources.py | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index 44d871c..f12a906 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -320,6 +320,10 @@ def _extract_data_from_html_2026(html: str, *, key: str): group_id, group_name_raw = m.groups() group_name = re.sub(r" *\([0-9]+\)$", "", unescape(group_name_raw)) logger.debug(f"Found group name: {group_id=}, {group_name=}") + if group_name in {"Highland Schottish", "Highland Scottische"}: + group_name_norm = "Highland Schottische" + logger.debug(f"Normalizing group name {group_name!r} to {group_name_norm!r}") + group_name = group_name_norm group_names[group_id] = group_name if not group_names: logger.debug("No group names found in HTML") diff --git a/tests/test_sources.py b/tests/test_sources.py index f4e9e6d..09aa4b1 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -375,7 +375,7 @@ def test_eskin_tunebook_data_load(key): "Set dance", "Jig or march", "Quadrille", - "Highland Schottish", # XXX: "Highland Schottische" + "Highland Schottische", "Pipe Reel", "Strathspey", "Hornpipe", @@ -386,7 +386,6 @@ def test_eskin_tunebook_data_load(key): "Slip Jig", "Jig A-G", "Slide", - "Highland Scottische", # XXX: "Highland Schottische" "Other", "Polka or Reel", "Bourree", From 876040a0ac5ef7d9873bf98452196bdfe13b2a0e Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 12:01:08 -0500 Subject: [PATCH 04/22] Update Bill Black alltunes --- pyabc2/sources/bill_black.py | 46 ++++++++++++------------ pyabc2/sources/bill_black_tunefolders.py | 4 +-- tests/test_sources.py | 10 +++--- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/pyabc2/sources/bill_black.py b/pyabc2/sources/bill_black.py index 04cf74c..6558e37 100644 --- a/pyabc2/sources/bill_black.py +++ b/pyabc2/sources/bill_black.py @@ -20,32 +20,32 @@ SAVE_TO = HERE / "_bill-black" TXT_FNS = [ - "a-tunes-1.txt", - "b-tunes-1.txt", - "c-tunes-1.txt", - "d-tunes-1.txt", - "e-tunes-1.txt", - "f-tunes-1.txt", - "g-tunes-1.txt", - "h-tunes-1.txt", - "i-tunes-1.txt", - "j-tunes-1.txt", - "k-tunes-1.txt", - "l-tunes-1.txt", - "m-tunes-1.txt", - "n-tunes-1.txt", - "o-tunes-1.txt", - "pq-tunes-1.txt", - "r-tunes-1.txt", - "s-tunes-2.rtf", - "t-tunes-1.txt", - "uv-tunes-1.txt", - "wz-tunes-1.txt", + "a-tunes.txt", + "b-tunes.txt", + "c-tunes.txt", + "d-tunes.txt", + "e-tunes.txt", + "f-tunes.txt", + "g-tunes.txt", + "h-tunes.txt", + "i-tunes.txt", + "j-tunes.txt", + "k-tunes.txt", + "L-tunes.txt", + "m-tunes.txt", + "n-tunes.txt", + "o-tunes.txt", + "pq-tunes.txt", + "r-tunes.txt", + "s-tunes.txt", + "t-tunes.txt", + "uv-tunes.txt", + "wxyz-tunes.txt", ] def download() -> None: - """Download the alphabetical text files from https://www.capeirish.com/ittl/alltunes/text/ + """Download the alphabetical text files from https://www.capeirish.com/ittl/alltunes/alltunes-text/ and store them in a compressed archive. """ import zipfile @@ -61,7 +61,7 @@ def download_one(url): with ThreadPoolExecutor(max_workers=4) as executor: futures = [] for fn in TXT_FNS: - url = f"https://www.capeirish.com/ittl/alltunes/text/{fn}" + url = f"https://www.capeirish.com/ittl/alltunes/alltunes-text/{fn}" futures.append(executor.submit(download_one, url)) SAVE_TO.mkdir(exist_ok=True) diff --git a/pyabc2/sources/bill_black_tunefolders.py b/pyabc2/sources/bill_black_tunefolders.py index 7a37944..324eb5c 100644 --- a/pyabc2/sources/bill_black_tunefolders.py +++ b/pyabc2/sources/bill_black_tunefolders.py @@ -4,8 +4,8 @@ https://www.capeirish.com/ittl/ As of the 2025-06-14 update, the "tunefolders" method is deprecated. -Bill Black is now using the Eskin ABC Tools (https://www.capeirish.com/ittl/alltunes/html/), -while also posting ABC text files (https://www.capeirish.com/ittl/alltunes/text/), +Bill Black is now using the Eskin ABC Tools (https://www.capeirish.com/ittl/alltunes/alltunes-html/), +while also posting ABC text files (https://www.capeirish.com/ittl/alltunes/alltunes-text/), both split up alphabetically by tune name. Requires: diff --git a/tests/test_sources.py b/tests/test_sources.py index 09aa4b1..e785a0a 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -525,14 +525,14 @@ def test_bill_black_tunefolders_invalid_key(): def test_bill_black_text_fns(): import requests - url = "http://www.capeirish.com/ittl/alltunes/text/" + url = "http://www.capeirish.com/ittl/alltunes/alltunes-text/" r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) r.raise_for_status() - fns_web = sorted(re.findall(r'href=["\']([a-z0-9\-]+\.(?:txt|rtf))["\']', r.text)) - if "s-tunes-1.txt" in fns_web: - # We're using s-tunes-2, not both - fns_web.remove("s-tunes-1.txt") + fns_web = sorted( + re.findall(r'href=["\']([a-zA-Z0-9\-]+\.(?:txt|rtf))["\']', r.text), + key=lambda s: s.lower(), + ) assert bill_black.TXT_FNS == fns_web From 97baaccdd5666b051e21230497a8a400218a2c01 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 12:21:32 -0500 Subject: [PATCH 05/22] Update KSS group query for new groups --- docs/examples/sources.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/sources.ipynb b/docs/examples/sources.ipynb index 5fbb208..483b806 100644 --- a/docs/examples/sources.ipynb +++ b/docs/examples/sources.ipynb @@ -402,7 +402,7 @@ "metadata": {}, "outputs": [], "source": [ - "Tune(df.query(\"group == 'jigs'\").iloc[0].abc)" + "Tune(df.query(\"group == 'Jig A-G'\").iloc[0].abc)" ] }, { From d43b7d2c60a21bcf230aec00ea2b25bb7d58ce79 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 12:54:14 -0500 Subject: [PATCH 06/22] Add User-Agent header for Eskin requests Getting 415 errors in the Python 3.10 CI run (though the 3.14 is fine, and my 3.13 local testing too) --- pyabc2/sources/eskin.py | 2 +- tests/test_sources.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index f12a906..6dbd3d7 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -367,7 +367,7 @@ def _download_data(key: str): tb_info = get_tunebook_info(key) - r = requests.get(tb_info.url, timeout=5) + r = requests.get(tb_info.url, headers={"User-Agent": "pyabc2"}, timeout=5) r.raise_for_status() html = r.text diff --git a/tests/test_sources.py b/tests/test_sources.py index e785a0a..f502380 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -290,7 +290,7 @@ def test_eskin_tunebook_bad_url_redirects(): # Redirects to the home page. # Nothing in `r.history`. `allow_redirects=False` has no impact. url = "https://michaeleskin.com/cce_sd/cce_san_diego_tunes_10nov3025.html" - r = requests.head(url, timeout=5) + r = requests.head(url, headers={"User-Agent": "pyabc2"}, timeout=5) r.raise_for_status() assert r.status_code == 302 @@ -304,7 +304,7 @@ def test_eskin_tunebook_url_exist(key): import requests url = eskin._TUNEBOOK_KEY_TO_URL[key] - r = requests.head(url, timeout=5) + r = requests.head(url, headers={"User-Agent": "pyabc2"}, timeout=5) r.raise_for_status() # Bad URLs seem to just redirect to his homepage, # so we need to check the final URL @@ -320,7 +320,7 @@ def test_eskin_tunebook_url_current(): import requests url = "https://michaeleskin.com/tunebooks.html" - r = requests.get(url, timeout=5) + r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) r.raise_for_status() if ( r.status_code == 302 From ae652b1de8c84cec38b9507e88b26ae9f889f9ac Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 12:57:59 -0500 Subject: [PATCH 07/22] cov --- pyabc2/sources/eskin.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index 6dbd3d7..b5cd3d9 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -239,7 +239,7 @@ def _extract_data_from_html_2025(html: str, *, key: str): # First find the tune type options by searching for 'tunes = type;' types = sorted(set(re.findall(r"tunes = (.*?);", html))) if types: - pass + pass # pragma: no cover elif "const tunes=[" in html: # no types, just one list of tunes types = ["tunes"] else: @@ -249,13 +249,13 @@ def _extract_data_from_html_2025(html: str, *, key: str): all_data = {} for type_ in types: m = re.search(rf"const {type_}=\[(.*?)\];", html, flags=re.DOTALL) - if m is None: + if m is None: # pragma: no cover raise RuntimeError(f"Unable to find data for type {type_!r}") s_data = "[" + m.group(1) + "]" try: data = json.loads(s_data) - except json.JSONDecodeError as e: + except json.JSONDecodeError as e: # pragma: no cover w = 25 a = max(0, e.pos - w) b = min(len(s_data), e.pos + w) @@ -302,7 +302,7 @@ def _extract_data_from_html_2026(html: str, *, key: str): tune_name = unescape(tune_name) logger.debug(f"Found group assignment: {tune_id=}, {group_id=}, {tune_name=}") group_assignments[tune_id] = group_id - if not group_assignments: + if not group_assignments: # pragma: no cover logger.debug("No group assignments found in HTML") # Find group names @@ -325,7 +325,7 @@ def _extract_data_from_html_2026(html: str, *, key: str): logger.debug(f"Normalizing group name {group_name!r} to {group_name_norm!r}") group_name = group_name_norm group_names[group_id] = group_name - if not group_names: + if not group_names: # pragma: no cover logger.debug("No group names found in HTML") # Find tunes @@ -350,7 +350,7 @@ def _extract_data_from_html_2026(html: str, *, key: str): all_data = defaultdict(list) for tune in tunes: group_id = group_assignments.get(tune["id"]) - if group_id is None: + if group_id is None: # pragma: no cover logger.warning(f"No group assignment found for tune {tune['id']}") continue group_name = group_names.get(group_id, f"Unknown group {group_id}") From 9eda0da5c2bf7d76e345bfb40504e3e75fb506f1 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 12:58:46 -0500 Subject: [PATCH 08/22] changelog --- docs/changes.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changes.md b/docs/changes.md index 95d405f..898d77f 100644 --- a/docs/changes.md +++ b/docs/changes.md @@ -1,5 +1,9 @@ # Release notes +## v0.1.4 (unreleased) + +* Updates for Eskin and Bill Black ({pull}`112`) + ## v0.1.3 (2026-04-17) * Improve styling of abcjs containers, pandas dataframes, and ipywidgets From 372b0435fbe727a384e45e20a5f85351bbbfc971 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 13:16:52 -0500 Subject: [PATCH 09/22] Skip blank line (e.g. leading) --- pyabc2/sources/eskin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index b5cd3d9..f90b3a8 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -340,7 +340,7 @@ def _extract_data_from_html_2026(html: str, *, key: str): abc_lines = [] for line in abc_raw.splitlines(): line = line.strip() - if line.startswith("%"): + if not line or line.startswith("%"): continue abc_lines.append(line) From 2e8264ef1a801d3c4e856a0d31030f43a76cd4e8 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 13:50:10 -0500 Subject: [PATCH 10/22] Add retries for Eskin and BB using urllib3 Retry --- pyabc2/sources/bill_black.py | 27 ++++++++++++++++++-- pyabc2/sources/bill_black_tunefolders.py | 5 ++-- pyabc2/sources/eskin.py | 32 +++++++++++++++++++++--- pyproject.toml | 1 + tests/test_sources.py | 22 ++++++++-------- 5 files changed, 69 insertions(+), 18 deletions(-) diff --git a/pyabc2/sources/bill_black.py b/pyabc2/sources/bill_black.py index 6558e37..5f3872d 100644 --- a/pyabc2/sources/bill_black.py +++ b/pyabc2/sources/bill_black.py @@ -8,6 +8,7 @@ * `requests `__ """ +import functools import logging import re from pathlib import Path @@ -44,6 +45,28 @@ ] +@functools.lru_cache(1) +def _get_session(): + import requests + from requests.adapters import HTTPAdapter + from urllib3.util import Retry + + session = requests.Session() + session.headers.update({"User-Agent": "pyabc2"}) + retries = Retry( + total=10, + backoff_factor=1.0, + backoff_jitter=0.5, + allowed_methods={"GET", "HEAD"}, + status_forcelist=[403, 429, 500, 502, 503, 504], + # Bill Black seems to sporadically return 403 (forbidden) + # possibly to indicate a temporary server issue or throttling/anti-bot + ) + session.mount("https://", HTTPAdapter(max_retries=retries)) + + return session + + def download() -> None: """Download the alphabetical text files from https://www.capeirish.com/ittl/alltunes/alltunes-text/ and store them in a compressed archive. @@ -51,10 +74,10 @@ def download() -> None: import zipfile from concurrent.futures import ThreadPoolExecutor - import requests + session = _get_session() def download_one(url): - r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.get(url, timeout=5) r.raise_for_status() return r.text diff --git a/pyabc2/sources/bill_black_tunefolders.py b/pyabc2/sources/bill_black_tunefolders.py index 324eb5c..f60dd79 100644 --- a/pyabc2/sources/bill_black_tunefolders.py +++ b/pyabc2/sources/bill_black_tunefolders.py @@ -19,6 +19,7 @@ from pathlib import Path from pyabc2._util import get_logger as _get_logger +from pyabc2.sources.bill_black import _get_session logger = _get_logger(__name__) @@ -209,7 +210,7 @@ def get_collection(key: str) -> Collection: def download(key: str | Iterable[str] | None = None) -> None: import gzip - import requests + session = _get_session() SAVE_TO.mkdir(exist_ok=True) @@ -224,7 +225,7 @@ def download(key: str | Iterable[str] | None = None) -> None: for url in collection.abc_urls: p = collection.url_to_file(url) logger.info(f"Downloading {url} to {p.relative_to(HERE).as_posix()}") - r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.get(url, timeout=5) r.raise_for_status() # Extract filename from URL and append .gz diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index f90b3a8..1065d08 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -7,6 +7,9 @@ * `requests `__ """ +from __future__ import annotations + +import functools import json import re from collections import defaultdict @@ -20,6 +23,7 @@ if TYPE_CHECKING: # pragma: no cover import pandas + import requests logger = _get_logger(__name__) @@ -60,6 +64,28 @@ _URL_NETLOCS = {"michaeleskin.com", "www.michaeleskin.com"} +@functools.lru_cache(1) +def _get_session() -> requests.Session: + import requests + from requests.adapters import HTTPAdapter + from urllib3.util import Retry + + session = requests.Session() + session.headers.update({"User-Agent": "pyabc2"}) + retries = Retry( + total=5, + backoff_factor=0.5, + backoff_jitter=0.5, + allowed_methods={"GET", "HEAD"}, + status_forcelist=[415, 429, 500, 502, 503, 504], + # Eskin seems to sporadically return 415 (unsupported media type) + # possibly to indicate a temporary server issue or throttling/anti-bot + ) + session.mount("https://", HTTPAdapter(max_retries=retries)) + + return session + + def _deflate(s: str, /) -> str: """Use deflate (zlib) to compress and base64-encode `s`.""" import base64 @@ -363,11 +389,11 @@ def _download_data(key: str): """Extract and save the tune data from the tunebook webpage as JSON.""" import gzip - import requests + session = _get_session() tb_info = get_tunebook_info(key) - r = requests.get(tb_info.url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.get(tb_info.url, timeout=5) r.raise_for_status() html = r.text @@ -389,7 +415,7 @@ def _load_data(key: str): return json.load(f) -def load_meta(key: str, *, redownload: bool = False) -> "pandas.DataFrame": +def load_meta(key: str, *, redownload: bool = False) -> pandas.DataFrame: """Load the tunebook data, no parsing. Parameters diff --git a/pyproject.toml b/pyproject.toml index 76624c2..eea9e44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ sources = [ "numpy ==2.*", "pandas ==2.*", "requests ==2.*", + "urllib3 ==2.*", ] abcjs-widget = [ "anywidget", diff --git a/tests/test_sources.py b/tests/test_sources.py index f502380..0b81512 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -284,13 +284,13 @@ def test_load_url_invalid_domain(): def test_eskin_tunebook_bad_url_redirects(): - import requests + session = eskin._get_session() # Bad URL (2025 -> 3025) # Redirects to the home page. # Nothing in `r.history`. `allow_redirects=False` has no impact. url = "https://michaeleskin.com/cce_sd/cce_san_diego_tunes_10nov3025.html" - r = requests.head(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.head(url, timeout=5) r.raise_for_status() assert r.status_code == 302 @@ -301,10 +301,10 @@ def test_eskin_tunebook_bad_url_redirects(): @pytest.mark.parametrize("key", eskin._TUNEBOOK_KEY_TO_URL) def test_eskin_tunebook_url_exist(key): - import requests + session = eskin._get_session() url = eskin._TUNEBOOK_KEY_TO_URL[key] - r = requests.head(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.head(url, timeout=5) r.raise_for_status() # Bad URLs seem to just redirect to his homepage, # so we need to check the final URL @@ -317,10 +317,10 @@ def test_eskin_tunebook_url_exist(key): def test_eskin_tunebook_url_current(): - import requests + session = eskin._get_session() url = "https://michaeleskin.com/tunebooks.html" - r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.get(url, timeout=5) r.raise_for_status() if ( r.status_code == 302 @@ -496,16 +496,16 @@ def test_eskin_inflate_pad_3(): def test_bill_black_https(): - import requests + session = bill_black._get_session() url = "http://www.capeirish.com/ittl/tunefolders/" url_https = url.replace("http://", "https://") - r = requests.head(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.head(url, timeout=5) r.raise_for_status() assert "Strict-Transport-Security" not in r.headers - r = requests.head(url_https, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.head(url_https, timeout=5) r.raise_for_status() assert "Strict-Transport-Security" in r.headers @@ -523,10 +523,10 @@ def test_bill_black_tunefolders_invalid_key(): def test_bill_black_text_fns(): - import requests + session = bill_black._get_session() url = "http://www.capeirish.com/ittl/alltunes/alltunes-text/" - r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.get(url, timeout=5) r.raise_for_status() fns_web = sorted( From 2acd9d45e50864850a9a02565b46e103977a2819 Mon Sep 17 00:00:00 2001 From: Zachary Moon Date: Fri, 3 Jul 2026 14:11:29 -0500 Subject: [PATCH 11/22] Configure Eskin and BB to retry for `http://` too might as well Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- pyabc2/sources/bill_black.py | 1 + pyabc2/sources/eskin.py | 1 + 2 files changed, 2 insertions(+) diff --git a/pyabc2/sources/bill_black.py b/pyabc2/sources/bill_black.py index 5f3872d..fb912a7 100644 --- a/pyabc2/sources/bill_black.py +++ b/pyabc2/sources/bill_black.py @@ -63,6 +63,7 @@ def _get_session(): # possibly to indicate a temporary server issue or throttling/anti-bot ) session.mount("https://", HTTPAdapter(max_retries=retries)) + session.mount("http://", HTTPAdapter(max_retries=retries)) return session diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index 1065d08..91b4b42 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -82,6 +82,7 @@ def _get_session() -> requests.Session: # possibly to indicate a temporary server issue or throttling/anti-bot ) session.mount("https://", HTTPAdapter(max_retries=retries)) + session.mount("http://", HTTPAdapter(max_retries=retries)) return session From 3cc64b97b49a1c14b6075dc56ee3d926bba7ae7f Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 14:12:55 -0500 Subject: [PATCH 12/22] Type requests session for BB --- pyabc2/sources/bill_black.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pyabc2/sources/bill_black.py b/pyabc2/sources/bill_black.py index fb912a7..95ddb1c 100644 --- a/pyabc2/sources/bill_black.py +++ b/pyabc2/sources/bill_black.py @@ -8,13 +8,19 @@ * `requests `__ """ +from __future__ import annotations + import functools import logging import re from pathlib import Path +from typing import TYPE_CHECKING from pyabc2._util import get_logger as _get_logger +if TYPE_CHECKING: # pragma: no cover + import requests + logger = _get_logger(__name__) HERE = Path(__file__).parent @@ -46,7 +52,7 @@ @functools.lru_cache(1) -def _get_session(): +def _get_session() -> requests.Session: import requests from requests.adapters import HTTPAdapter from urllib3.util import Retry From b27e7b6cf8c17c6649aa911d1395fe2e16c4a2f8 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 14:21:56 -0500 Subject: [PATCH 13/22] Switch to new parser only for specific message --- pyabc2/sources/eskin.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index 91b4b42..94414e7 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -400,8 +400,13 @@ def _download_data(key: str): try: data = _extract_data_from_html_2025(html, key=key) - except RuntimeError: - data = _extract_data_from_html_2026(html, key=key) + except RuntimeError as e: + if str(e) == "Unable to detect tune types": + # We can be pretty sure it's the new format + logger.debug("Tune groups array not detected, assuming new format.") + data = _extract_data_from_html_2026(html, key=key) + else: + raise SAVE_TO.mkdir(exist_ok=True) with gzip.open(tb_info.path, "wt") as f: From 6453cd34532dc64587b0e8727dced1d898da7aeb Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 14:32:05 -0500 Subject: [PATCH 14/22] BB one Session per thread --- pyabc2/sources/bill_black.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pyabc2/sources/bill_black.py b/pyabc2/sources/bill_black.py index 95ddb1c..7049edd 100644 --- a/pyabc2/sources/bill_black.py +++ b/pyabc2/sources/bill_black.py @@ -53,6 +53,10 @@ @functools.lru_cache(1) def _get_session() -> requests.Session: + return _build_session() + + +def _build_session() -> requests.Session: import requests from requests.adapters import HTTPAdapter from urllib3.util import Retry @@ -78,12 +82,23 @@ def download() -> None: """Download the alphabetical text files from https://www.capeirish.com/ittl/alltunes/alltunes-text/ and store them in a compressed archive. """ + import threading import zipfile from concurrent.futures import ThreadPoolExecutor - session = _get_session() + thread_local = threading.local() + + def get_worker_session() -> requests.Session: + # One Session per worker thread to avoid cross-thread Session sharing. + try: + return thread_local.session + except AttributeError: + session = _build_session() + thread_local.session = session + return session def download_one(url): + session = get_worker_session() r = session.get(url, timeout=5) r.raise_for_status() return r.text From 440d3c0230885ed503f4c88a304ac3e0d7f4026a Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 14:32:58 -0500 Subject: [PATCH 15/22] Try harder with Eskin just got 415's again in a CI run --- pyabc2/sources/eskin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index 94414e7..c52ef3c 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -73,8 +73,8 @@ def _get_session() -> requests.Session: session = requests.Session() session.headers.update({"User-Agent": "pyabc2"}) retries = Retry( - total=5, - backoff_factor=0.5, + total=10, + backoff_factor=1.0, backoff_jitter=0.5, allowed_methods={"GET", "HEAD"}, status_forcelist=[415, 429, 500, 502, 503, 504], From 77446a0d51a8bad8d4928b7f7fe88cc7907f8d36 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 18:31:04 -0500 Subject: [PATCH 16/22] Try less hard Eskin ended up making a CI run take 149m and still fail --- pyabc2/sources/bill_black.py | 4 ++-- pyabc2/sources/eskin.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyabc2/sources/bill_black.py b/pyabc2/sources/bill_black.py index 7049edd..856e80a 100644 --- a/pyabc2/sources/bill_black.py +++ b/pyabc2/sources/bill_black.py @@ -64,8 +64,8 @@ def _build_session() -> requests.Session: session = requests.Session() session.headers.update({"User-Agent": "pyabc2"}) retries = Retry( - total=10, - backoff_factor=1.0, + total=5, + backoff_factor=0.5, backoff_jitter=0.5, allowed_methods={"GET", "HEAD"}, status_forcelist=[403, 429, 500, 502, 503, 504], diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index c52ef3c..94414e7 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -73,8 +73,8 @@ def _get_session() -> requests.Session: session = requests.Session() session.headers.update({"User-Agent": "pyabc2"}) retries = Retry( - total=10, - backoff_factor=1.0, + total=5, + backoff_factor=0.5, backoff_jitter=0.5, allowed_methods={"GET", "HEAD"}, status_forcelist=[415, 429, 500, 502, 503, 504], From c4223a77909415a0b6ee5f2d1e11dd47597c662e Mon Sep 17 00:00:00 2001 From: zmoon Date: Sat, 4 Jul 2026 14:25:26 -0500 Subject: [PATCH 17/22] Only run web tests in one CI run and use xdist groups --- .github/workflows/ci.yml | 12 ++++++- pyproject.toml | 5 ++- tests/test_sources.py | 69 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abab5c2..eb30da3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,13 +32,23 @@ jobs: python -m pip install flit flit install -s - - name: Test with pytest + - name: Test with pytest (non-web) + if: ${{ matrix.python-version != '3.14' }} run: > pytest -v -ra --doctest-modules --cov=pyabc2 --cov-report xml --cov-report term-missing + -m "not web" -n auto + - name: Test with pytest (full, including web) + if: ${{ matrix.python-version == '3.14' }} + run: > + pytest -v -ra + --doctest-modules + --cov=pyabc2 --cov-report xml --cov-report term-missing + -n auto --dist loadgroup + - name: Check type annotations run: | mypy --non-interactive . diff --git a/pyproject.toml b/pyproject.toml index eea9e44..f51326a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,10 @@ profile = "black" line_length = 100 [tool.pytest.ini_options] -markers = ["slow"] +markers = [ + "slow", + "web: tests that hit external websites", +] [tool.mypy] exclude = ["^venv"] diff --git a/tests/test_sources.py b/tests/test_sources.py index 0b81512..3d97792 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -44,6 +44,8 @@ def test_example_random(): assert type(tune) is Tune +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_tune_type_file_prefix(): norbeck._maybe_download() all_fps = list(norbeck.SAVE_TO.glob("*.abc")) @@ -58,6 +60,8 @@ def test_norbeck_tune_type_file_prefix(): assert set(all_fps) == all_fps_type_set +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_x_unique(): # X value should be unique within tune type norbeck._maybe_download() @@ -74,6 +78,8 @@ def test_norbeck_x_unique(): assert len(set(xs)) == len(xs), "X unique" +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_x_vals(): # Make sure they match with what file says norbeck._maybe_download() @@ -100,6 +106,8 @@ def test_norbeck_x_vals(): @pytest.mark.slow +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_load(): # NOTE: downloads files if not already present @@ -131,6 +139,8 @@ def test_norbeck_load(): norbeck.load("asdf") +@pytest.mark.web +@pytest.mark.xdist_group("the_session") @pytest.mark.parametrize( "url,title,key,type", [ @@ -149,11 +159,14 @@ def test_the_session_load_url(url, title, key, type): assert tune.header["reference number"] == "1" +@pytest.mark.xdist_group("the_session") def test_the_session_url_check(): with pytest.raises(AssertionError): the_session.load_url("https://www.google.com") +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_load_archive_threaded(): # NOTE: downloads file if not already present with pytest.warns(UserWarning, match=r"The Session tune\(s\) failed to load"): @@ -163,12 +176,15 @@ def test_the_session_load_archive_threaded(): assert tunes1 == tunes2 +@pytest.mark.xdist_group("the_session") def test_the_session_download_invalid(): with pytest.raises(ValueError): _ = the_session.download("asdf") @pytest.mark.slow +@pytest.mark.web +@pytest.mark.xdist_group("the_session") @pytest.mark.parametrize( "which", ["aliases", "events", "recordings", "sessions", "sets", "tune_popularity", "tunes"] ) @@ -203,6 +219,7 @@ def test_the_session_load_meta(which): # in df3, `pd.Float64Dtype()` +@pytest.mark.xdist_group("the_session") def test_the_session_load_meta_invalid(): with pytest.raises(ValueError): _ = the_session.load_meta("asdf") @@ -211,6 +228,7 @@ def test_the_session_load_meta_invalid(): _ = the_session.load_meta("sessions", format="asdf") +@pytest.mark.xdist_group("the_session") def test_the_session_load_meta_doc_consistency(): s_options = ", ".join(repr(x) for x in sorted(the_session._META_ALLOWED)) expected_line = f"which : {{{s_options}}}" @@ -246,12 +264,16 @@ def test_int_downcast(): assert s3.dtype == expected_dtype_ext +@pytest.mark.web +@pytest.mark.xdist_group("the_session") @pytest.mark.parametrize("netloc", sorted(the_session._URL_NETLOCS)) def test_load_url_the_session(netloc): tune = load_url(f"https://{netloc}/tunes/10000") assert tune.title == "Brian Quinn's" +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") @pytest.mark.parametrize("netloc", sorted(norbeck._URL_NETLOCS)) def test_load_url_norbeck(netloc): import requests @@ -265,6 +287,7 @@ def test_load_url_norbeck(netloc): assert tune.title == "For The Love Of Music" +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("netloc", sorted(eskin._URL_NETLOCS)) @pytest.mark.parametrize("param", list(ESKIN_COMPRESSED_ABC_DATA)) def test_load_url_eskin(netloc, param): @@ -283,6 +306,8 @@ def test_load_url_invalid_domain(): _ = load_url("https://www.google.com") +@pytest.mark.web +@pytest.mark.xdist_group("eskin") def test_eskin_tunebook_bad_url_redirects(): session = eskin._get_session() @@ -299,6 +324,8 @@ def test_eskin_tunebook_bad_url_redirects(): assert r.is_redirect +@pytest.mark.web +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("key", eskin._TUNEBOOK_KEY_TO_URL) def test_eskin_tunebook_url_exist(key): session = eskin._get_session() @@ -316,6 +343,8 @@ def test_eskin_tunebook_url_exist(key): raise ValueError(f"{key!r} URL {url} redirects to homepage") +@pytest.mark.web +@pytest.mark.xdist_group("eskin") def test_eskin_tunebook_url_current(): session = eskin._get_session() @@ -342,6 +371,8 @@ def test_eskin_tunebook_url_current(): raise ValueError(f"Could not find link for tunebook {key!r} in tunebooks page.") +@pytest.mark.web +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("key", eskin._TUNEBOOK_KEY_TO_URL) def test_eskin_tunebook_data_load(key): df = eskin.load_meta(key) @@ -410,6 +441,7 @@ def test_eskin_tunebook_data_load(key): ) +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_parsing(): # From https://michaeleskin.com/cce_sd/cce_san_diego_tunes_10nov2025.html url = "https://michaeleskin.com/abctools/abctools.html?lzw=BoLgjAUApFAuCWsA2BTAZgewHawAQAUBDJQhLDXMADmigGcBXAIwWXWzyJLIrAGZa8LJkw4CxUkN4CYAB0IAnWHVGcJPSjLgoAtrIyrx3KZtqwUAD1iGuk8qYAqIBwAsUuAIJMmKAJ64IABlwAHoaAFkQABYQqIgARRAAJliAXgBOAAYIACUQHJQUJGg6AHchAHNcTIA6SABpEABxaEImAGMAKzoAfToMBiwAE0M0UiYMX1pwgEkAERncWQUMCoVCHWrp+cWmQjo6ZdWtmFmF3HaXDAUho6rs053cPYOANwwkXAA2OMfzy+uQ3enx+rSGQx6xCQPVkJF8e3aAGsekghIi6BAAEQeHSYzx8XAAIU8SVwTQAorgAD6eDxNDy4TFNPGE8HEmnY3G0jzEunkgBi1MZzLJTXpRLZ1KxOLxHlJPJJZMpNI8dIZTJZko5MsVCr5go5IrF4tZQyqVKp0q5KAqttwhFJeyFGtwFTaVUIFRQQ2dOptdodrsIzuZTDdaFd7iGpMtnLx-o9FTDIcxnuTnu9vutto9pLdKbDhAjXqG7IAukA&format=noten&ssp=10&name=The_Abbey&play=1" @@ -428,6 +460,7 @@ def test_eskin_abc_url_parsing(): assert sum(line.startswith(r"%%") for line in abc_no_rm.splitlines()) > 0 +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_missing_param(): url = "https://michaeleskin.com/abctools/abctools.html?" with pytest.raises( @@ -437,18 +470,21 @@ def test_eskin_abc_url_missing_param(): _ = eskin.abctools_url_to_abc(url) +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_bad_param(): url = "https://michaeleskin.com/abctools/abctools.html?lzw=hi" with pytest.raises(RuntimeError, match="Failed to decompress LZString data"): _ = eskin.abctools_url_to_abc(url) +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_bad_param_def(): url = "https://michaeleskin.com/abctools/abctools.html?def=hi" with pytest.raises(RuntimeError, match="Failed to decompress deflate data"): _ = eskin.abctools_url_to_abc(url) +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_bad(caplog): url = "https://michaeleski.com/deftools/abctools.html?lzw=BoLgjAUApFAuCWsA2BTAZgewHawAQAUBDJQhLDXMADmigGcBXAIwWXWzyJLIrAGZa8LJkw4CxUkN4CYAB0IAnWHVGcJPSjLgoAtrIyrx3KZtqwUAD1iGuk8qYAqIBwAsUuAIJMmKAJ64IABlwAHoaAFkQABYQqIgARRAAJliAXgBOAAYIACUQHJQUJGg6AHchAHNcTIA6SABpEABxaEImAGMAKzoAfToMBiwAE0M0UiYMX1pwgEkAERncWQUMCoVCHWrp+cWmQjo6ZdWtmFmF3HaXDAUho6rs053cPYOANwwkXAA2OMfzy+uQ3enx+rSGQx6xCQPVkJF8e3aAGsekghIi6BAAEQeHSYzx8XAAIU8SVwTQAorgAD6eDxNDy4TFNPGE8HEmnY3G0jzEunkgBi1MZzLJTXpRLZ1KxOLxHlJPJJZMpNI8dIZTJZko5MsVCr5go5IrF4tZQyqVKp0q5KAqttwhFJeyFGtwFTaVUIFRQQ2dOptdodrsIzuZTDdaFd7iGpMtnLx-o9FTDIcxnuTnu9vutto9pLdKbDhAjXqG7IAukA&format=noten&ssp=10&name=The_Abbey&play=1" with caplog.at_level("DEBUG"): @@ -460,6 +496,8 @@ def test_eskin_abc_url_bad(caplog): ] +@pytest.mark.web +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("use_lzw", [True, False]) def test_eskin_abc_url_creation(use_lzw): import requests @@ -476,11 +514,13 @@ def test_eskin_abc_url_creation(use_lzw): raise ValueError(f"URL {url} redirects to homepage") +@pytest.mark.xdist_group("eskin") def test_eskin_invalid_tunebook_key(): with pytest.raises(ValueError, match="Unknown Eskin tunebook key: 'asdf'"): _ = eskin.get_tunebook_info("asdf") +@pytest.mark.xdist_group("eskin") def test_eskin_inflate_invalid_length(): s = "eJyFjbEKwkAQRPv9iv2DQyvd7jbGK0wQJIVtktucJ4FIghZyH-abcdefg" with pytest.raises( @@ -490,11 +530,14 @@ def test_eskin_inflate_invalid_length(): _ = eskin._inflate(s) +@pytest.mark.xdist_group("eskin") def test_eskin_inflate_pad_3(): s = "abc" assert eskin._inflate(eskin._deflate(s)) == s +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") def test_bill_black_https(): session = bill_black._get_session() @@ -511,17 +554,22 @@ def test_bill_black_https(): @pytest.mark.xfail(reason="Bill Black tunefolders are currently in flux", strict=False) +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") @pytest.mark.parametrize("key", list(bill_black_tunefolders._KEY_TO_COLLECTION)) def test_bill_black_tunefolders(key): lst = bill_black_tunefolders.load_meta(key, redownload=True) assert len(lst) > 0 +@pytest.mark.xdist_group("bill_black") def test_bill_black_tunefolders_invalid_key(): with pytest.raises(ValueError, match="Unknown collection key: 'asdf'"): _ = bill_black_tunefolders.get_collection("asdf") +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") def test_bill_black_text_fns(): session = bill_black._get_session() @@ -537,17 +585,23 @@ def test_bill_black_text_fns(): assert bill_black.TXT_FNS == fns_web +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") def test_bill_black_load(): lst = bill_black.load_meta() assert len(lst) > 0 assert lst[0].startswith("X:") +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_tune_collections(): df = the_session.get_tune_collections(1) # Cooley's assert not df.empty +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_member_set(): tunes = the_session.get_member_set(65013, 106212) assert len(tunes) == 3 @@ -557,6 +611,8 @@ def test_the_session_get_member_set(): assert d["setting_id"] == 31341 +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_member_sets(): sets = the_session.get_member_sets(65013) assert len(sets) >= 1 @@ -566,6 +622,8 @@ def test_the_session_get_member_sets(): assert d["setting_id"] == 31341 +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_member_sets_multipage(): sets = the_session.get_member_sets(1, pages=3, size=2, max_threads=2, orderby="oldest") assert len(sets) == 6 @@ -575,6 +633,7 @@ def test_the_session_get_member_sets_multipage(): assert d["tune_id"] == 560 +@pytest.mark.xdist_group("the_session") def test_the_session_consume_validation(): f = the_session.get_member_sets @@ -588,11 +647,15 @@ def test_the_session_consume_validation(): _ = f(1, max_threads=0) +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_consume_auto_leading_slash(): (d,) = the_session._consume("tunes/22878") assert d["name"] == "Jack Farrell's" +@pytest.mark.web +@pytest.mark.xdist_group("hardy") @pytest.mark.parametrize("key", list(hardy._TUNEBOOK_KEY_TO_URL)) def test_hardy_load_meta(key): abcs = hardy.load_meta(key) @@ -604,6 +667,8 @@ def test_hardy_load_meta(key): assert "\n\n" not in abc, "no empty lines within a tune block" +@pytest.mark.web +@pytest.mark.xdist_group("hardy") def test_hardy_load_meta_remove_prefs(): # Default: no % lines key = "session" @@ -617,16 +682,20 @@ def test_hardy_load_meta_remove_prefs(): assert sum(line.lstrip().startswith("%") for abc in abcs_raw for line in abc.splitlines()) > 1 +@pytest.mark.xdist_group("hardy") def test_hardy_bad_key(): with pytest.raises(ValueError, match="Unknown Hardy tunebook key"): _ = hardy.load_meta("asdf") +@pytest.mark.xdist_group("hardy") def test_hardy_download_bad_key(): with pytest.raises(ValueError, match="Unknown Hardy tunebook key"): _ = hardy.download("asdf") +@pytest.mark.web +@pytest.mark.xdist_group("hardy") def test_hardy_annex_is_latest(): """Confirm the hardcoded 'annex' URL points to the current (non-superseded) annex file.""" import requests From 8c8e92ddc2c0a5b48c6facbf3d156b42b3d34732 Mon Sep 17 00:00:00 2001 From: zmoon Date: Wed, 5 Aug 2026 11:14:34 -0500 Subject: [PATCH 18/22] Normalize and fix decoding for new group names --- pyabc2/sources/eskin.py | 11 ++++++++++- tests/test_sources.py | 6 ++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index 94414e7..21e85cf 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -64,6 +64,14 @@ _URL_NETLOCS = {"michaeleskin.com", "www.michaeleskin.com"} +def _normalize_group_name(s: str, /) -> str: + """Normalize parsed Eskin group labels to expected style.""" + + s = s.replace(" · ", " ") + s = s.replace("–", "-") + return s + + @functools.lru_cache(1) def _get_session() -> requests.Session: import requests @@ -346,6 +354,7 @@ def _extract_data_from_html_2026(html: str, *, key: str): ): group_id, group_name_raw = m.groups() group_name = re.sub(r" *\([0-9]+\)$", "", unescape(group_name_raw)) + group_name = _normalize_group_name(group_name) logger.debug(f"Found group name: {group_id=}, {group_name=}") if group_name in {"Highland Schottish", "Highland Scottische"}: group_name_norm = "Highland Schottische" @@ -396,7 +405,7 @@ def _download_data(key: str): r = session.get(tb_info.url, timeout=5) r.raise_for_status() - html = r.text + html = r.content.decode("utf-8") try: data = _extract_data_from_html_2025(html, key=key) diff --git a/tests/test_sources.py b/tests/test_sources.py index 3d97792..5c68c94 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -536,6 +536,12 @@ def test_eskin_inflate_pad_3(): assert eskin._inflate(eskin._deflate(s)) == s +@pytest.mark.xdist_group("eskin") +def test_eskin_normalize_group_name(): + assert eskin._normalize_group_name("Reel M · 1–50") == "Reel M 1-50" + assert eskin._normalize_group_name("Reel A-E") == "Reel A-E" + + @pytest.mark.web @pytest.mark.xdist_group("bill_black") def test_bill_black_https(): From b5c9cccb31cc5435fd99bd057249f3f79cc9090a Mon Sep 17 00:00:00 2001 From: zmoon Date: Wed, 5 Aug 2026 11:16:20 -0500 Subject: [PATCH 19/22] No longer have Jig A-G in KSS, but A-B... --- docs/examples/sources.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/sources.ipynb b/docs/examples/sources.ipynb index 483b806..0dd8364 100644 --- a/docs/examples/sources.ipynb +++ b/docs/examples/sources.ipynb @@ -402,7 +402,7 @@ "metadata": {}, "outputs": [], "source": [ - "Tune(df.query(\"group == 'Jig A-G'\").iloc[0].abc)" + "Tune(df.query(\"group == 'Jig A-B'\").iloc[0].abc)" ] }, { From 8628b81a9ed16525ff2db011c7c24fdc32692b93 Mon Sep 17 00:00:00 2001 From: zmoon Date: Wed, 5 Aug 2026 11:17:45 -0500 Subject: [PATCH 20/22] Run sources example nb only for 3.14 case since it does hit Eskin/BB --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb30da3..1210360 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,10 @@ jobs: run: | cd docs/examples for f in *.ipynb; do + if [ "$f" = "sources.ipynb" ] && [ "${{ matrix.python-version }}" != "3.14" ]; then + echo "Skipping $f on Python ${{ matrix.python-version }}" + continue + fi jupyter nbconvert --to notebook --execute $f || exit 1 done From f196bd01ced3509aba04efde4cf87cad87e360e3 Mon Sep 17 00:00:00 2001 From: zmoon Date: Wed, 5 Aug 2026 11:57:46 -0500 Subject: [PATCH 21/22] More normalization for new Eskin groups --- docs/examples/sources.ipynb | 2 +- pyabc2/sources/eskin.py | 67 ++++++++++++++++-- tests/test_sources.py | 136 ++++++++++++++++++++++++++++-------- 3 files changed, 169 insertions(+), 36 deletions(-) diff --git a/docs/examples/sources.ipynb b/docs/examples/sources.ipynb index 0dd8364..3b2eac4 100644 --- a/docs/examples/sources.ipynb +++ b/docs/examples/sources.ipynb @@ -402,7 +402,7 @@ "metadata": {}, "outputs": [], "source": [ - "Tune(df.query(\"group == 'Jig A-B'\").iloc[0].abc)" + "Tune(df.query(\"group == 'Jig'\").iloc[0].abc)" ] }, { diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index 21e85cf..4ec8c22 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -63,12 +63,75 @@ _URL_NETLOCS = {"michaeleskin.com", "www.michaeleskin.com"} +_TUNE_TYPE_PATTERN = r"[A-Za-z][A-Za-z &'/-]*" +_TUNE_TYPE_RE = re.compile(rf"^{_TUNE_TYPE_PATTERN}$") + +_GROUP_ALPHA_SPLIT_RE = re.compile( + rf"^(?P{_TUNE_TYPE_PATTERN}?) " + r"(?P[A-Za-z](?:-[A-Za-z])?)" + r"(?: (?P[0-9]+-[0-9]+))?$" +) + +_GROUP_ALPHA_ONLY_RE = re.compile( + r"^(?P[A-Za-z](?:-[A-Za-z])?)(?: (?P[0-9]+-[0-9]+))?$" +) + + +def _is_valid_alpha_split_group_name(s: str, /) -> bool: + """Return whether `s` looks like an Eskin type + alpha split group label.""" + + return _GROUP_ALPHA_SPLIT_RE.fullmatch(s) is not None + + +def _is_valid_alpha_only_group_name(s: str, /) -> bool: + """Return whether `s` looks like an Eskin alpha-only split group label.""" + + return _GROUP_ALPHA_ONLY_RE.fullmatch(s) is not None + + +def _is_valid_tune_type_group_name(s: str, /) -> bool: + """Return whether `s` looks like a standalone tune type group label.""" + + return _TUNE_TYPE_RE.fullmatch(s) is not None + + +def _normalize_tune_type(s: str, /) -> str: + """Normalize tune type labels parsed from Eskin group names.""" + + if s in {"Highland Schottish", "Highland Scottische"}: + return "Highland Schottische" + + if s == "Set dance": + return "Set Dance" + + m = re.fullmatch(r"(.+?) or ([A-Za-z][A-Za-z &'/-]*)", s) + if m is not None: + first, second = m.groups() + return f"{first} or {second.title()}" + + return s + def _normalize_group_name(s: str, /) -> str: """Normalize parsed Eskin group labels to expected style.""" + s = s.strip() s = s.replace(" · ", " ") s = s.replace("–", "-") + + if _is_valid_alpha_split_group_name(s): + # Upstream alpha/number splits can churn; keep only tune type. + m = _GROUP_ALPHA_SPLIT_RE.fullmatch(s) + assert m is not None # for typing + return _normalize_tune_type(m.group("tune_type")) + + if _is_valid_alpha_only_group_name(s): + # Some tunebooks split by alpha range only; collapse to one generic group. + return "tunes" + + if _is_valid_tune_type_group_name(s): + return _normalize_tune_type(s) + return s @@ -356,10 +419,6 @@ def _extract_data_from_html_2026(html: str, *, key: str): group_name = re.sub(r" *\([0-9]+\)$", "", unescape(group_name_raw)) group_name = _normalize_group_name(group_name) logger.debug(f"Found group name: {group_id=}, {group_name=}") - if group_name in {"Highland Schottish", "Highland Scottische"}: - group_name_norm = "Highland Schottische" - logger.debug(f"Normalizing group name {group_name!r} to {group_name_norm!r}") - group_name = group_name_norm group_names[group_id] = group_name if not group_names: # pragma: no cover logger.debug("No group names found in HTML") diff --git a/tests/test_sources.py b/tests/test_sources.py index 5c68c94..30c62b7 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -396,49 +396,40 @@ def test_eskin_tunebook_data_load(key): "strathspeys", "waltzes", # New KSS - "Long Dance", + "Air", "Barndance", - "Reel O-R", - "Reel F-K", - "Set Dance", - "Reel A-E", - "Single Jig", - "Set dance", - "Jig or march", - "Quadrille", + "Bourree", + "Fling", "Highland Schottische", - "Pipe Reel", - "Strathspey", "Hornpipe", - "Reel S-Y", - "Reel L-N", + "Jig", + "Jig or March", + "Long Dance", "March", - "Waltz", - "Slip Jig", - "Jig A-G", - "Slide", + "Mazurka", "Other", + "Pipe Reel", + "Polka", + "Polka or March", "Polka or Reel", - "Bourree", + "Quadrille", + "Reel", "Schottische", - "Air", - "Polka or march", - "Song", - "Fling", + "Set Dance", + "Single Jig", + "Slide", + "Slip Jig", "Slow Air", - "Mazurka", - "Jig Q-Y", - "Polka", - "Jig H-P", + "Song", + "Strathspey", + "Waltz", } unique_groups = set(df.group.unique()) if key in {"kss"}: assert unique_groups <= tune_group_keys else: - assert unique_groups == {"tunes"} or all( - re.fullmatch(r"^[A-Z\-]+$", g) is not None for g in unique_groups - ) + assert unique_groups == {"tunes"} @pytest.mark.xdist_group("eskin") @@ -538,8 +529,91 @@ def test_eskin_inflate_pad_3(): @pytest.mark.xdist_group("eskin") def test_eskin_normalize_group_name(): - assert eskin._normalize_group_name("Reel M · 1–50") == "Reel M 1-50" - assert eskin._normalize_group_name("Reel A-E") == "Reel A-E" + assert eskin._normalize_group_name("Reel M · 1–50") == "Reel" + assert eskin._normalize_group_name("Reel A-E") == "Reel" + assert eskin._normalize_group_name("Highland Schottish") == "Highland Schottische" + assert eskin._normalize_group_name("Highland Scottische") == "Highland Schottische" + assert eskin._normalize_group_name("Set dance") == "Set Dance" + assert eskin._normalize_group_name("Jig or march") == "Jig or March" + assert eskin._normalize_group_name("Polka or march") == "Polka or March" + + +@pytest.mark.parametrize( + "s", + [ + "Reel M 51-56", + "Reel C", + "Jig C-E", + "Reel L", + "Reel T-Y", + "Hornpipe O-W", + "Reel S", + "Reel G-I", + "Reel J-K", + "Reel M 1-50", + "Jig I-L", + "Reel N-P", + "Reel D-F", + "Reel A-B", + "Jig A-B", + "Jig M-R", + "Jig F-H", + "Jig S-Y", + "Reel R", + "Hornpipe A-N", + ], +) +def test_eskin_alpha_split_group_validator(s): + assert eskin._is_valid_alpha_split_group_name(s) + + +@pytest.mark.parametrize( + "s, expected", + [ + ("Reel M 51-56", "Reel"), + ("Jig C-E", "Jig"), + ("Hornpipe O-W", "Hornpipe"), + ], +) +def test_eskin_alpha_split_group_normalization(s, expected): + assert eskin._normalize_group_name(s) == expected + + +@pytest.mark.parametrize( + "s", + [ + "T-V", + "A", + "D-E", + "J-L", + "S 51-80", + "C", + "N-P", + "W", + "M 51-54", + "F-G", + "M 1-50", + "B 1-50", + "S 1-50", + "B 51-51", + "R", + "H-I", + ], +) +def test_eskin_alpha_only_group_validator(s): + assert eskin._is_valid_alpha_only_group_name(s) + + +@pytest.mark.parametrize( + "s", + [ + "T-V", + "S 51-80", + "M 1-50", + ], +) +def test_eskin_alpha_only_group_normalization(s): + assert eskin._normalize_group_name(s) == "tunes" @pytest.mark.web From 5b2fc49e7f7eb4d0b11687c7387a5690dccde6b9 Mon Sep 17 00:00:00 2001 From: zmoon Date: Wed, 5 Aug 2026 12:02:22 -0500 Subject: [PATCH 22/22] Hardy 2026 --- pyabc2/sources/eskin.py | 3 ++- tests/test_sources.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index 4ec8c22..effda3a 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -41,6 +41,7 @@ "carp": f"{_TBWS}/carp_celtic_jam_tunebook_29jun2026.html", "hardy_2024": f"{_TBWS}/paul_hardy_2024_8feb2025.html", "hardy_2025": f"{_TBWS}/paul_hardy_2025_29jun2026.html", + "hardy_2026": f"{_TBWS}/paul_hardy_2026_1aug2026.html", "cce_dublin_2001": f"{_CCE_SD}/cce_dublin_2001_tunebook_29jun2026.html", "cce_san_diego_jan2025": f"{_CCE_SD}/cce_san_diego_tunes_31jan2025.html", "cce_san_diego_jun2026": f"{_TBWS}/comhaltas-san-diego-tunebook-24jun2026.html", @@ -509,7 +510,7 @@ def load_meta(key: str, *, redownload: bool = False) -> pandas.DataFrame: - CCE Dublin 2001 * - ``cce_san_diego`` - CCE San Diego - * - ``hardy_{2024,2025}`` + * - ``hardy_{2024,2025,2026}`` - Paul Hardy's Session Tunebook * - ``kss`` - King Street Sessions Tunebook diff --git a/tests/test_sources.py b/tests/test_sources.py index 30c62b7..b5c7b13 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -308,6 +308,7 @@ def test_load_url_invalid_domain(): @pytest.mark.web @pytest.mark.xdist_group("eskin") +@pytest.mark.xfail(reason="now it does 404", strict=False) def test_eskin_tunebook_bad_url_redirects(): session = eskin._get_session() @@ -361,6 +362,7 @@ def test_eskin_tunebook_url_current(): old_keys = { "cce_san_diego_jan2025", "hardy_2024", + "hardy_2025", } for key, tb_url in eskin._TUNEBOOK_KEY_TO_URL.items(): m = re.search(rf'href=["\']({tb_url})["\']', html)