Emergent:Blu/Patches/sideman-bandcamp
From PickiPedia: A knowledge base of bluegrass, old time psychedelic jams, and other public domain music
Jump to navigationJump to search
Replaces Emergent:Blu/Patches/bandcamp (pickipedia#178): the Bandcamp reader folded into Sideman as a third reading in --album, not a tool of its own (Justin, 7 Nov, block 26,365,089). One commit, on top of Emergent:Blu/Patches/ensemble-pipeline (#177). sha256 of the file: 10e6bedba35a29058e407dd2286500b2bc5bcd67566f8b3183a2e7966c409187. Round-trip checked on production f44da0e + the Sideman fixes + #177: 79 tests pass. tools/ only, nothing to redeploy. For magent: git am onto #177's branch, and close #178.
From da23678c6a1242c63459e100520a43bc84504e83 Mon Sep 17 00:00:00 2001
From: blu <blu@cryptograss.live>
Date: Sat, 7 Nov 2026 19:33:26 +0000
Subject: [PATCH] Sideman --album --bandcamp: a record's Bandcamp page as a
third reading
Folds the Bandcamp reader into Sideman instead of a tool of its own
(Justin: why a new tool?). Bandcamp is one more source answering "who is
on this record", so it is one more reading in --album.
Bandcamp's credits are prose, so the reading carries them word for word
(credits_text, about_text) and no people: splitting a line into a person
and an instrument is for the person who writes it up. Bandcamp challenges
searches, so the page is a URL a person hands in. Fetched with curl, since
Bandcamp challenges Python's urllib whatever its headers; an argument list,
no shell, one request every 1.5 s. main() moves to argparse for the new
flag. 79 tests.
---
tools/recording_credits.py | 142 ++++++++++++++++++++++++++++----
tools/test_recording_credits.py | 74 ++++++++++++++++-
2 files changed, 200 insertions(+), 16 deletions(-)
diff --git a/tools/recording_credits.py b/tools/recording_credits.py
index 9cc25c9..2c0358f 100644
--- a/tools/recording_credits.py
+++ b/tools/recording_credits.py
@@ -31,13 +31,27 @@ answer is printed under that source, every person with the roles and the
scope that source gives them. Nobody is merged across sources; a reader
sees two readings of the same record and decides. This is what Rambler
reads, and what a person verifies.
+
+--bandcamp URL adds a third reading: the record's Bandcamp page, where small
+bands put the credits MusicBrainz and Discogs never got. Bandcamp's credits
+are prose ("Joe Biglin - keys", "Recorded at 25 Murray Studios") with no
+fields for who or what, so Sideman copies them word for word and parses
+nothing: splitting a line into a person and an instrument is the judgement
+of the person who writes it up, with the text in front of them. Bandcamp
+challenges searches, so the page is a URL a person hands in.
+
+ python3 tools/recording_credits.py --album "Milkweed" "Milkweed EP" \
+ --bandcamp https://milkweed.bandcamp.com/album/milkweed-ep
"""
+import argparse
+import html
import json
import re
+import subprocess
import sys
import time
-from urllib.parse import urlencode
+from urllib.parse import urlencode, urlparse
from urllib.error import HTTPError
from urllib.request import Request, urlopen
@@ -409,7 +423,76 @@ def discogs_find_releases(artist, title):
return _one_per_group_earliest(hits, lambda r: r.get("master_id") or r["id"])
-def album(artist, title):
+# ---------------------------------------------------------------------------
+# Bandcamp: credits copied, never parsed
+# ---------------------------------------------------------------------------
+
+BANDCAMP_UA = "pickipedia-recording-credits/0.1 (https://pickipedia.xyz; Artificial Dumb)"
+BANDCAMP_INTERVAL = 1.5
+LD_JSON = re.compile(r'<script type="application/ld\+json">\s*(.*?)\s*</script>', re.S)
+
+
+class NotABandcampRecord(Exception):
+ pass
+
+
+def is_bandcamp(url):
+ host = urlparse(url).hostname or ""
+ return host == "bandcamp.com" or host.endswith(".bandcamp.com")
+
+
+def bandcamp_get(url):
+ """(final URL, page), fetched with curl, not urllib.
+
+ Bandcamp answers Python's own HTTPS client with a JavaScript challenge
+ page whatever its headers say, and answers curl with the album (tested
+ 7 Nov 2026, same User-Agent both ways). Same honest User-Agent; an
+ argument list, no shell.
+ """
+ wait = BANDCAMP_INTERVAL - (time.monotonic() - _last_call.get("bandcamp", 0.0))
+ if wait > 0:
+ time.sleep(wait)
+ try:
+ out = subprocess.run(
+ ["curl", "-sSL", "--max-time", "30", "-A", BANDCAMP_UA,
+ "-w", "\n%{url_effective}", "--", url],
+ capture_output=True, text=True, check=True).stdout
+ finally:
+ _last_call["bandcamp"] = time.monotonic()
+ page, _, final = out.rpartition("\n")
+ return final, page
+
+
+def _bc_text(value):
+ """Bandcamp's prose as written: entities decoded, line ends made plain."""
+ return html.unescape(value).replace("\r\n", "\n").strip() if value else ""
+
+
+def bandcamp_reading(url, get=bandcamp_get):
+ """A Bandcamp album or track page, from the schema.org JSON-LD it carries:
+ the credits and about text verbatim, and nothing parsed out of them."""
+ final, page = get(url)
+ m = LD_JSON.search(page)
+ d = json.loads(m.group(1)) if m else {}
+ if d.get("@type") not in ("MusicAlbum", "MusicRecording"):
+ raise NotABandcampRecord(f"{url}: no album or track data on the page "
+ "(an artist page, or Bandcamp's challenge page)")
+ return {
+ "source": "Bandcamp",
+ "url": final,
+ "id": final,
+ "group": f"bandcamp-{final}",
+ "artist": _bc_text((d.get("byArtist") or {}).get("name")),
+ "title": _bc_text(d.get("name")),
+ "date": re.sub(r"\s+00:00:00 GMT$", "", d.get("datePublished") or ""),
+ "tracks": len((d.get("track") or {}).get("itemListElement", [])),
+ "people": {},
+ "credits_text": _bc_text(d.get("creditText")),
+ "about_text": _bc_text(d.get("description")),
+ }
+
+
+def album(artist, title, bandcamp=None, bandcamp_get=bandcamp_get):
"""What each source says about who is on a record, source by source.
MusicBrainz and Discogs are searched independently, so a record one has
@@ -417,6 +500,7 @@ def album(artist, title):
release, that Discogs release is read under the MusicBrainz entry rather
than found again. Nothing is merged across sources: the same person
appears once per source that names them, with what that source says.
+ A Bandcamp URL, if given, is read last; its credits stay text.
"""
readings = []
linked_discogs = set()
@@ -441,6 +525,8 @@ def album(artist, title):
for hit in discogs_find_releases(artist, title):
if hit["id"] not in linked_discogs:
readings.append(_discogs_reading(discogs_get(f"releases/{hit['id']}"), hit["id"]))
+ if bandcamp:
+ readings.append(bandcamp_reading(bandcamp, get=bandcamp_get))
return readings
@@ -464,6 +550,16 @@ def render_album(artist, title, readings):
if not readings:
lines.append("Neither MusicBrainz nor Discogs has a release with this title and artist.")
for r in readings:
+ if r["source"] == "Bandcamp":
+ lines.append(f'Bandcamp: {r["artist"]}: {r["title"]} ({r["date"] or "undated"}), {r["tracks"]} tracks')
+ lines.append(f' {r["url"]}')
+ lines.append(" Credits, word for word (not parsed; a person reads who played what):")
+ lines += [f" {ln}" for ln in r["credits_text"].splitlines()] or [" (none)"]
+ if r["about_text"]:
+ lines.append(" About, word for word:")
+ lines += [f" {ln}" for ln in r["about_text"].splitlines()]
+ lines.append("")
+ continue
linked = f', linked from MusicBrainz release {r["linked_to"]}' if r.get("linked_to") else ""
lines.append(f'{r["source"]}: {r["artist"]}: {r["title"]} ({r["date"] or "undated"}), '
f'{r["tracks"]} tracks{linked}')
@@ -511,21 +607,37 @@ def _jsonable(o):
def main(argv):
- flags = {a for a in argv[1:] if a.startswith("--")}
- args = [a for a in argv[1:] if not a.startswith("--")]
- if len(args) != 2 or flags - {"--json", "--album"}:
- print(__doc__.strip().splitlines()[0])
- print('usage: recording_credits.py [--json] "ARTIST" "SONG TITLE"')
- print(' recording_credits.py [--json] --album "ARTIST" "RECORD TITLE"')
+ ap = argparse.ArgumentParser(
+ description=__doc__.strip().splitlines()[0],
+ usage='%(prog)s [--json] "ARTIST" "SONG TITLE"\n'
+ ' %(prog)s [--json] --album "ARTIST" "RECORD TITLE" [--bandcamp URL]')
+ ap.add_argument("artist")
+ ap.add_argument("title")
+ ap.add_argument("--album", action="store_true", help="who is on a whole record, source by source")
+ ap.add_argument("--bandcamp", metavar="URL", help="with --album: the record's Bandcamp page, read too")
+ ap.add_argument("--json", action="store_true")
+ try:
+ args = ap.parse_args(argv[1:])
+ except SystemExit as e:
+ return e.code
+ if args.bandcamp and not args.album:
+ ap.print_usage()
+ print("--bandcamp goes with --album")
return 2
- artist, title = args
- if "--album" in flags:
- readings = album(artist, title)
- print(json.dumps(_jsonable(readings), indent=2, ensure_ascii=False) if "--json" in flags
- else render_album(artist, title, readings))
+ if args.bandcamp and not is_bandcamp(args.bandcamp):
+ print(f"not a bandcamp.com URL: {args.bandcamp}")
+ return 2
+ if args.album:
+ try:
+ readings = album(args.artist, args.title, bandcamp=args.bandcamp)
+ except NotABandcampRecord as e:
+ print(e, file=sys.stderr)
+ return 1
+ print(json.dumps(_jsonable(readings), indent=2, ensure_ascii=False) if args.json
+ else render_album(args.artist, args.title, readings))
else:
- versions = lookup(artist, title)
- print(json.dumps(versions, indent=2) if "--json" in flags else render(artist, title, versions))
+ versions = lookup(args.artist, args.title)
+ print(json.dumps(versions, indent=2) if args.json else render(args.artist, args.title, versions))
return 0
diff --git a/tools/test_recording_credits.py b/tools/test_recording_credits.py
index bfe6a7e..41a0c18 100644
--- a/tools/test_recording_credits.py
+++ b/tools/test_recording_credits.py
@@ -8,6 +8,8 @@ on canned documents shaped like the real MusicBrainz and Discogs responses,
so no test touches the network.
"""
+import json
+
import pytest
import recording_credits as rc
@@ -336,7 +338,77 @@ def test_render_album_says_when_nobody_has_it():
def test_main_album_flag(monkeypatch, capsys):
- monkeypatch.setattr(rc, "album", lambda a, t: [])
+ monkeypatch.setattr(rc, "album", lambda a, t, bandcamp=None: [])
assert rc.main(["x", "--album", "A", "T"]) == 0
assert "Neither MusicBrainz nor Discogs" in capsys.readouterr().out
assert rc.main(["x", "--bogus", "A", "T"]) == 2
+
+
+# ---------------------------------------------------------------------------
+# --album --bandcamp: credits copied, never parsed
+# ---------------------------------------------------------------------------
+
+def bc_page(obj):
+ return '<html><script type="application/ld+json">\n' + json.dumps(obj) + "\n</script></html>"
+
+
+BC_ALBUM = {
+ "@type": "MusicAlbum", "name": "Workin's Too Hard", "byArtist": {"name": "Rayna Gellert"},
+ "datePublished": "20 Jan 2017 00:00:00 GMT",
+ "creditText": "Musicians: Rayna Gellert, Kieran Kane\r\nJoe - keys & vocals",
+ "description": "Cut live in a room.",
+ "track": {"itemListElement": [{"position": 1, "item": {"name": "Starch"}}]},
+}
+
+
+def test_bandcamp_credits_are_copied_not_parsed():
+ r = rc.bandcamp_reading("https://x.bandcamp.com/album/w",
+ get=lambda url: (url, bc_page(BC_ALBUM)))
+ assert r["source"] == "Bandcamp" and r["url"] == "https://x.bandcamp.com/album/w"
+ assert r["credits_text"] == "Musicians: Rayna Gellert, Kieran Kane\nJoe - keys & vocals"
+ assert r["title"] == "Workin's Too Hard" and r["date"] == "20 Jan 2017" and r["tracks"] == 1
+ assert r["people"] == {} # nobody is extracted from prose
+
+
+def test_bandcamp_challenge_page_is_an_error():
+ with pytest.raises(rc.NotABandcampRecord):
+ rc.bandcamp_reading("https://x.bandcamp.com/album/w",
+ get=lambda url: (url, "<title>Client Challenge</title>"))
+
+
+def test_album_reads_bandcamp_last_and_render_says_not_parsed(monkeypatch):
+ monkeypatch.setattr(rc, "find_releases", lambda a, t: [])
+ monkeypatch.setattr(rc, "discogs_find_releases", lambda a, t: [])
+ readings = rc.album("Rayna Gellert", "Workin's Too Hard", bandcamp="https://x.bandcamp.com/album/w",
+ bandcamp_get=lambda url: (url, bc_page(BC_ALBUM)))
+ assert [r["source"] for r in readings] == ["Bandcamp"]
+ text = rc.render_album("Rayna Gellert", "Workin's Too Hard", readings)
+ assert "Credits, word for word (not parsed" in text
+ assert " Joe - keys & vocals" in text
+ assert " About, word for word:\n Cut live in a room." in text
+
+
+def test_bandcamp_fetch_is_curl_with_an_argument_list(monkeypatch):
+ seen = {}
+
+ def run(cmd, **kw):
+ seen["cmd"] = cmd
+ return rc.subprocess.CompletedProcess(cmd, 0, stdout="<html/>\nhttps://x.bandcamp.com/album/w")
+ monkeypatch.setattr(rc.subprocess, "run", run)
+ monkeypatch.setattr(rc, "BANDCAMP_INTERVAL", 0)
+ assert rc.bandcamp_get("https://x.bandcamp.com/album/w") == ("https://x.bandcamp.com/album/w", "<html/>")
+ assert seen["cmd"][0] == "curl" and seen["cmd"][-2:] == ["--", "https://x.bandcamp.com/album/w"]
+
+
+@pytest.mark.parametrize("url,ok", [
+ ("https://milkweed.bandcamp.com/album/milkweed-ep", True),
+ ("https://evilbandcamp.com/album/x", False),
+ ("https://bandcamp.com.example.org/album/x", False),
+])
+def test_is_bandcamp(url, ok):
+ assert rc.is_bandcamp(url) is ok
+
+
+def test_main_bandcamp_needs_album_and_a_bandcamp_url():
+ assert rc.main(["x", "A", "T", "--bandcamp", "https://x.bandcamp.com/album/w"]) == 2
+ assert rc.main(["x", "--album", "A", "T", "--bandcamp", "https://example.org/album/w"]) == 2
--
2.53.0
