Emergent:Blu/Patches/bandcamp
From PickiPedia: A knowledge base of bluegrass, old time psychedelic jams, and other public domain music
Jump to navigationJump to search
A Bandcamp reader for the connectome (Justin, 7 Nov, block 26,365,064: "an ingest point that doesn't require inference"). One commit, on production f44da0e; independent of #172 and #177. sha256 of the file: 57718ecacab875a916566e5c54604db8d086edb1c45134f9deaa5f280aadf779. 12 tests pass on production. tools/ only, nothing to redeploy. Working name in the docstring is Liner; Justin names it. For magent: git am onto production and open a PR.
From 577dac987fd580fef6f72d8de7cd9c19c821c38a Mon Sep 17 00:00:00 2001
From: blu <blu@cryptograss.live>
Date: Sat, 7 Nov 2026 19:30:07 +0000
Subject: [PATCH] Add a Bandcamp reader: a page's credits, copied word for word
Records by small bands -- Milkweed from Binghamton, the Turpentine Ray --
are on neither MusicBrainz nor Discogs, and their credits live on
Bandcamp, in prose. Justin asked for an ingest point that needs no
inference (connectome, 7 Nov).
tools/bandcamp.py reads the schema.org JSON-LD on an album or track page
and prints the credits and about text verbatim, under the page URL, which
is the source a person cites. --tracks reads each track's page for credits
of its own. An artist URL lists their releases from /music. Nothing is
parsed into people or instruments: that is a person's judgement, made with
this text in front of them.
It fetches with curl: Bandcamp answers Python's urllib with a JavaScript
challenge page whatever its headers, and answers curl with the album
(same honest User-Agent both ways). Argument list, no shell. One request
every 1.5 s. 12 tests, all on canned pages.
---
tools/bandcamp.py | 213 +++++++++++++++++++++++++++++++++++++++++
tools/test_bandcamp.py | 131 +++++++++++++++++++++++++
2 files changed, 344 insertions(+)
create mode 100644 tools/bandcamp.py
create mode 100644 tools/test_bandcamp.py
diff --git a/tools/bandcamp.py b/tools/bandcamp.py
new file mode 100644
index 0000000..72ae422
--- /dev/null
+++ b/tools/bandcamp.py
@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+"""
+Liner: a Bandcamp page's credits, copied word for word.
+
+ python3 tools/bandcamp.py https://milkweed.bandcamp.com/album/milkweed-ep
+ python3 tools/bandcamp.py --tracks https://milkweed.bandcamp.com/album/milkweed-ep
+ python3 tools/bandcamp.py https://milkweed.bandcamp.com
+ python3 tools/bandcamp.py --json <url>
+
+An Artificial Dumb reader for the connectome. MusicBrainz and Discogs know
+little about a band from Main Street in Binghamton; Bandcamp is where bands
+like that put their credits. Bandcamp writes those credits in prose ("Joe
+Biglin - keys", "Recorded at 25 Murray Studios"), with no fields for who
+or what, so this tool does not try to read them. It copies them.
+
+What it reads: the schema.org JSON-LD every Bandcamp album and track page
+carries -- name, artist, label, release date, the credits (creditText), and
+the about text (description) -- and nothing else from the page.
+
+What it puts out: those fields, verbatim, under the page's URL, which is the
+source a person cites. With --tracks, each track's page is read too, since a
+track can carry credits of its own. Given an artist's page instead of an
+album, it lists the album and track URLs found there.
+
+What makes it dumb, deliberately:
+
+ - Credits are copied, never parsed. Splitting "Joe - keys, vocals" into a
+ person and two instruments is a judgement, and it belongs to the person
+ who writes the {{Record personnel}} lines, with this text in front of them.
+ - It names itself honestly in its User-Agent, and fetches with curl
+ (see fetch() for why). Needs curl; nothing else outside the stdlib.
+ - One request every 1.5 seconds.
+ - It writes nothing.
+"""
+
+import argparse
+import html
+import json
+import re
+import subprocess
+import sys
+import time
+from urllib.parse import urljoin, urlparse
+
+USER_AGENT = "pickipedia-liner/0.1 (https://pickipedia.xyz; Artificial Dumb for the connectome)"
+INTERVAL = 1.5
+_last = [0.0]
+
+LD_JSON = re.compile(r'<script type="application/ld\+json">\s*(.*?)\s*</script>', re.S)
+LINKS = re.compile(r'href="(/(?:album|track)/[A-Za-z0-9_-]+)"')
+
+
+class NotAnAlbum(Exception):
+ pass
+
+
+def fetch(url):
+ """(final URL after redirects, page). Fetched with curl, not urllib.
+
+ Bandcamp answers Python's own HTTPS client with a JavaScript challenge
+ page, whatever its headers say, and answers curl with the album (tested
+ 7 Nov 2026, same User-Agent both ways). So curl does the fetching: same
+ honest User-Agent, no pretending to be a browser. Arguments are a list;
+ no shell is involved.
+ """
+ wait = INTERVAL - (time.monotonic() - _last[0])
+ if wait > 0:
+ time.sleep(wait)
+ try:
+ out = subprocess.run(
+ ["curl", "-sSL", "--max-time", "30", "-A", USER_AGENT,
+ "-w", "\n%{url_effective}", "--", url],
+ capture_output=True, text=True, check=True).stdout
+ finally:
+ _last[0] = time.monotonic()
+ page, _, final = out.rpartition("\n")
+ return final, page
+
+
+def is_bandcamp(url):
+ host = urlparse(url).hostname or ""
+ return host == "bandcamp.com" or host.endswith(".bandcamp.com")
+
+
+def ld_json(page):
+ """The page's schema.org object, or None. A challenge page has none."""
+ m = LD_JSON.search(page)
+ return json.loads(m.group(1)) if m else None
+
+
+def text(value):
+ """Bandcamp's prose as written: entities decoded, Windows line ends made
+ plain, nothing else touched."""
+ if not value:
+ return ""
+ return html.unescape(value).replace("\r\n", "\n").strip()
+
+
+def date(value):
+ """'18 Jul 2013 00:00:00 GMT' -> '18 Jul 2013'. The time is always midnight."""
+ return re.sub(r"\s+00:00:00 GMT$", "", value or "")
+
+
+def read_page(url, page):
+ """One album or track page -> its credits and the facts around them."""
+ d = ld_json(page)
+ if not d or d.get("@type") not in ("MusicAlbum", "MusicRecording"):
+ raise NotAnAlbum(f"{url}: no album or track data on the page "
+ "(an artist page, a challenge page, or not Bandcamp)")
+ tracks = []
+ for entry in (d.get("track") or {}).get("itemListElement", []):
+ item = entry.get("item", {})
+ tracks.append({"position": entry.get("position"), "name": text(item.get("name")),
+ "url": item.get("@id") or item.get("mainEntityOfPage", "")})
+ return {
+ "url": url,
+ "kind": "album" if d["@type"] == "MusicAlbum" else "track",
+ "title": text(d.get("name")),
+ "artist": text((d.get("byArtist") or {}).get("name")),
+ "label": text((d.get("publisher") or {}).get("name")),
+ "album": text((d.get("inAlbum") or {}).get("name")),
+ "released": date(d.get("datePublished")),
+ "credits": text(d.get("creditText")),
+ "about": text(d.get("description")),
+ "tracks": tracks,
+ }
+
+
+def discography(url, page):
+ """Album and track URLs on an artist's page, in page order, once each."""
+ seen, found = set(), []
+ for path in LINKS.findall(page):
+ full = urljoin(url, path)
+ if full not in seen:
+ seen.add(full)
+ found.append(full)
+ return found
+
+
+def read(url, tracks=False, get=fetch):
+ """Read url. Returns ("album", record) or ("artist", [urls]).
+
+ An artist's URL (no /album/ or /track/ in it) is read as the artist's
+ /music page. Their front page often redirects to one featured album,
+ which would hide the rest.
+ """
+ path = urlparse(url).path
+ if "/album/" not in path and "/track/" not in path:
+ final, page = get(urljoin(url, "/music"))
+ found = discography(final, page)
+ if not found and ("/album/" in final or "/track/" in final):
+ found = [final] # one release: /music redirects straight to it
+ return "artist", found
+ final, page = get(url)
+ record = read_page(final, page)
+ if tracks:
+ for t in record["tracks"]:
+ if t["url"]:
+ turl, tpage = get(t["url"])
+ tr = read_page(turl, tpage)
+ t["credits"], t["about"] = tr["credits"], tr["about"]
+ return "album", record
+
+
+def render(record):
+ r = record
+ lines = [f'{r["artist"]}: {r["title"]} ({r["kind"]}{", on " + r["album"] if r["album"] else ""})',
+ f' Source: {r["url"]}']
+ if r["released"]:
+ lines.append(f' Released: {r["released"]}')
+ if r["label"] and r["label"] != r["artist"]:
+ lines.append(f' Label: {r["label"]}')
+ lines += ["", " Credits, as Bandcamp has them:"]
+ lines += [f" {ln}" for ln in r["credits"].splitlines()] if r["credits"] else [" (none)"]
+ if r["about"]:
+ lines += ["", " About, as Bandcamp has it:"]
+ lines += [f" {ln}" for ln in r["about"].splitlines()]
+ with_credits = [t for t in r["tracks"] if t.get("credits")]
+ if with_credits:
+ lines += ["", " Track credits:"]
+ for t in with_credits:
+ lines.append(f' {t["position"]}. {t["name"]} {t["url"]}')
+ lines += [f" {ln}" for ln in t["credits"].splitlines()]
+ elif r["tracks"] and any("credits" in t for t in r["tracks"]):
+ lines += ["", " No track carries credits of its own."]
+ lines += ["", " Nothing here is parsed. Who played what is for a person to read off the text above."]
+ return "\n".join(lines)
+
+
+def main(argv):
+ ap = argparse.ArgumentParser(description="Liner: a Bandcamp page's credits, word for word.")
+ ap.add_argument("url", help="a Bandcamp album, track or artist URL")
+ ap.add_argument("--tracks", action="store_true", help="read each track's page for its own credits")
+ ap.add_argument("--json", action="store_true")
+ args = ap.parse_args(argv[1:])
+ if not is_bandcamp(args.url):
+ ap.error("not a bandcamp.com URL")
+ try:
+ kind, result = read(args.url, tracks=args.tracks)
+ except NotAnAlbum as e:
+ print(e, file=sys.stderr)
+ return 1
+ if args.json:
+ print(json.dumps(result, indent=2, ensure_ascii=False))
+ elif kind == "artist":
+ print("\n".join(result) if result else "No albums or tracks found on that page.")
+ else:
+ print(render(result))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))
diff --git a/tools/test_bandcamp.py b/tools/test_bandcamp.py
new file mode 100644
index 0000000..5e1192a
--- /dev/null
+++ b/tools/test_bandcamp.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""
+Tests for the Bandcamp reader. What these guard: credits come out exactly
+as written, never split into people; a challenge page is an error, not an
+empty album; an artist's URL lists their releases. Pages are canned; no
+test touches the network.
+"""
+
+import json
+
+import pytest
+
+import bandcamp as bc
+
+
+def page(obj):
+ return ('<html><head><script type="application/ld+json">\n'
+ + json.dumps(obj) + "\n</script></head><body></body></html>")
+
+
+ALBUM = {
+ "@type": "MusicAlbum",
+ "name": "Workin's Too Hard",
+ "byArtist": {"name": "Rayna Gellert"},
+ "publisher": {"name": "Rayna Gellert"},
+ "datePublished": "20 Jan 2017 00:00:00 GMT",
+ "creditText": "Musicians: Rayna Gellert, Kieran Kane, Kai Welch, Jamie Dick\r\nJoe - keys & vocals",
+ "description": "Cut live in a room.",
+ "track": {"itemListElement": [
+ {"position": 1, "item": {"name": "Starch", "@id": "https://x.bandcamp.com/track/starch"}},
+ {"position": 2, "item": {"name": "Iron", "@id": "https://x.bandcamp.com/track/iron"}},
+ ]},
+}
+TRACK = {"@type": "MusicRecording", "name": "Starch", "byArtist": {"name": "Rayna Gellert"},
+ "inAlbum": {"name": "Workin's Too Hard"}, "creditText": "Fiddle: Rayna Gellert"}
+CHALLENGE = "<html><head><title>Client Challenge</title></head><body>enable JavaScript</body></html>"
+
+
+def fake_get(pages):
+ calls = []
+
+ def get(url):
+ calls.append(url)
+ final, body = pages[url]
+ return final, body
+ get.calls = calls
+ return get
+
+
+def test_credits_are_copied_not_parsed():
+ rec = bc.read_page("https://x.bandcamp.com/album/w", page(ALBUM))
+ assert rec["credits"] == "Musicians: Rayna Gellert, Kieran Kane, Kai Welch, Jamie Dick\nJoe - keys & vocals"
+ assert rec["title"] == "Workin's Too Hard"
+ assert rec["released"] == "20 Jan 2017"
+ assert rec["kind"] == "album"
+ assert [t["name"] for t in rec["tracks"]] == ["Starch", "Iron"]
+ # No field anywhere claims a person or an instrument.
+ assert not {"people", "personnel", "instruments"} & set(rec)
+
+
+def test_challenge_page_is_an_error_not_an_empty_album():
+ with pytest.raises(bc.NotAnAlbum):
+ bc.read_page("https://x.bandcamp.com/album/w", CHALLENGE)
+
+
+def test_tracks_flag_reads_each_track_page():
+ get = fake_get({
+ "https://x.bandcamp.com/album/w": ("https://x.bandcamp.com/album/w", page(ALBUM)),
+ "https://x.bandcamp.com/track/starch": ("https://x.bandcamp.com/track/starch", page(TRACK)),
+ "https://x.bandcamp.com/track/iron": ("https://x.bandcamp.com/track/iron",
+ page({**TRACK, "name": "Iron", "creditText": None})),
+ })
+ kind, rec = bc.read("https://x.bandcamp.com/album/w", tracks=True, get=get)
+ assert kind == "album"
+ assert rec["tracks"][0]["credits"] == "Fiddle: Rayna Gellert"
+ assert rec["tracks"][1]["credits"] == ""
+ text = bc.render(rec)
+ assert " 1. Starch https://x.bandcamp.com/track/starch\n Fiddle: Rayna Gellert" in text
+ assert "2. Iron" not in text
+
+
+def test_without_tracks_flag_only_the_album_is_fetched():
+ get = fake_get({"https://x.bandcamp.com/album/w": ("https://x.bandcamp.com/album/w", page(ALBUM))})
+ bc.read("https://x.bandcamp.com/album/w", get=get)
+ assert get.calls == ["https://x.bandcamp.com/album/w"]
+
+
+def test_artist_url_lists_the_discography_from_music():
+ music = '<a href="/album/one">1</a><a href="/track/single">s</a><a href="/album/one">again</a>'
+ get = fake_get({"https://x.bandcamp.com/music": ("https://x.bandcamp.com/music", music)})
+ kind, found = bc.read("https://x.bandcamp.com", get=get)
+ assert kind == "artist"
+ assert found == ["https://x.bandcamp.com/album/one", "https://x.bandcamp.com/track/single"]
+
+
+def test_artist_with_one_release_redirects_to_it():
+ get = fake_get({"https://x.bandcamp.com/music": ("https://x.bandcamp.com/album/only", page(ALBUM))})
+ assert bc.read("https://x.bandcamp.com/", get=get) == ("artist", ["https://x.bandcamp.com/album/only"])
+
+
+def test_render_names_the_source_and_says_nothing_is_parsed():
+ rec = bc.read_page("https://x.bandcamp.com/album/w", page({**ALBUM, "creditText": None}))
+ text = bc.render(rec)
+ assert " Source: https://x.bandcamp.com/album/w" in text
+ assert " Credits, as Bandcamp has them:\n (none)" in text
+ assert "Nothing here is parsed" in text
+ assert "Label:" not in text # self-released: label is the artist
+
+
+@pytest.mark.parametrize("url,ok", [
+ ("https://milkweed.bandcamp.com/album/milkweed-ep", True),
+ ("https://bandcamp.com/search?q=x", True),
+ ("https://evilbandcamp.com/album/x", False),
+ ("https://bandcamp.com.example.org/album/x", False),
+])
+def test_is_bandcamp(url, ok):
+ assert bc.is_bandcamp(url) is ok
+
+
+def test_fetch_uses_curl_with_an_argument_list(monkeypatch):
+ seen = {}
+
+ def run(cmd, **kw):
+ seen["cmd"] = cmd
+ return bc.subprocess.CompletedProcess(cmd, 0, stdout="<html>ok</html>\nhttps://x.bandcamp.com/album/w")
+ monkeypatch.setattr(bc.subprocess, "run", run)
+ monkeypatch.setattr(bc, "INTERVAL", 0)
+ final, body = bc.fetch("https://x.bandcamp.com/album/w")
+ assert final == "https://x.bandcamp.com/album/w" and body == "<html>ok</html>"
+ assert seen["cmd"][0] == "curl" and seen["cmd"][-2:] == ["--", "https://x.bandcamp.com/album/w"]
+ assert bc.USER_AGENT in seen["cmd"]
--
2.53.0
